From 1371d56dd04ae8a888df3afa8f2f2ef1a50533cb Mon Sep 17 00:00:00 2001
From: propcgamer20-png <propcgamer2.0@gmail.com>
Date: Thu, 10 Sep 2026 01:39:37 +0530
Subject: [PATCH] fix: significance_report's confidence parameter now affects
 the significant flag

significant was hardcoded p_value < 0.05 while confidence (default 0.95)
only widened or narrowed the returned CI. A caller passing a stricter
confidence (0.99) expecting a more conservative significance verdict got
the same p < 0.05 threshold, silently.

significant is now p_value < (1 - confidence): identical to the old
p < 0.05 at the default confidence=0.95 (so every audit's unfair.py /
fair.py output is unchanged), and correspondingly stricter for a caller
that asks for it. Documented in the docstring.

Closes #548
---
 faircode/significance.py   | 11 ++++++++++-
 tests/test_significance.py | 21 +++++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/faircode/significance.py b/faircode/significance.py
index db3c338..f61067f 100644
--- a/faircode/significance.py
+++ b/faircode/significance.py
@@ -93,6 +93,15 @@ def permutation_test(group_a, group_b, n_permutations=10000, random_state=42):
 def significance_report(group_a, group_b, n_resamples=10000,
                         n_permutations=10000, confidence=0.95,
                         random_state=42):
+    """Bootstrap CI + permutation p-value for the gap mean(a) - mean(b).
+
+    `confidence` sets both the width of the returned CI *and* the
+    significance threshold: `significant` is `p_value < (1 - confidence)`.
+    At the default `confidence=0.95` that is the usual `p < 0.05`; a caller
+    passing `confidence=0.99` gets the correspondingly stricter `p < 0.01`,
+    so the CI and the verdict move together instead of the verdict being
+    pinned at 0.05 regardless.
+    """
     a = _as_array(group_a)
     b = _as_array(group_b)
     gap, ci_low, ci_high = bootstrap_ci(a, b, n_resamples, confidence,
@@ -105,7 +114,7 @@ def significance_report(group_a, group_b, n_resamples=10000,
         "ci_low": ci_low,
         "ci_high": ci_high,
         "p_value": p_value,
-        "significant": p_value < 0.05,
+        "significant": p_value < (1.0 - confidence),
         "n_a": n_a,
         "n_b": n_b,
         "small_sample_warning": n_a < 30 or n_b < 30,
diff --git a/tests/test_significance.py b/tests/test_significance.py
index 337c911..601499c 100644
--- a/tests/test_significance.py
+++ b/tests/test_significance.py
@@ -37,6 +37,27 @@ def test_separated_groups_are_significant_and_ci_excludes_zero():
     assert rep["ci_low"] <= rep["gap"] <= rep["ci_high"]
 
 
+def test_confidence_tightens_the_significance_threshold_not_just_the_ci():
+    # A gap with 0.01 < p < 0.05 is significant at the default confidence
+    # (0.95 -> p < 0.05) but not at confidence=0.99 (-> p < 0.01). Before
+    # #548 the `significant` flag was pinned to p < 0.05 regardless of
+    # `confidence`, which only widened/narrowed the CI.
+    rng = np.random.default_rng(0)
+    for _ in range(5):
+        a = rng.binomial(1, 0.55, size=60).astype(float)
+        b = rng.binomial(1, 0.35, size=60).astype(float)
+
+    r95 = significance_report(a, b, n_resamples=3000, n_permutations=3000,
+                              confidence=0.95, random_state=4)
+    r99 = significance_report(a, b, n_resamples=3000, n_permutations=3000,
+                              confidence=0.99, random_state=4)
+
+    assert r95["p_value"] == r99["p_value"]          # same permutation test
+    assert 0.01 < r95["p_value"] < 0.05
+    assert r95["significant"] is True
+    assert r99["significant"] is False
+
+
 # ── Determinism ──────────────────────────────────────────────────────────────
 def test_random_state_makes_results_deterministic():
     a = [1] * 40 + [0] * 60