diff --git a/faircode/significance.py b/faircode/significance.py index db3c338..f61067f 100644 --- a/faircode/significance.py +++ b/faircode/significance.py @@ -93,6 +93,15 @@ def permutation_test(group_a, group_b, n_permutations=10000, random_state=42): def significance_report(group_a, group_b, n_resamples=10000, n_permutations=10000, confidence=0.95, random_state=42): + """Bootstrap CI + permutation p-value for the gap mean(a) - mean(b). + + `confidence` sets both the width of the returned CI *and* the + significance threshold: `significant` is `p_value < (1 - confidence)`. + At the default `confidence=0.95` that is the usual `p < 0.05`; a caller + passing `confidence=0.99` gets the correspondingly stricter `p < 0.01`, + so the CI and the verdict move together instead of the verdict being + pinned at 0.05 regardless. + """ a = _as_array(group_a) b = _as_array(group_b) gap, ci_low, ci_high = bootstrap_ci(a, b, n_resamples, confidence, @@ -105,7 +114,7 @@ def significance_report(group_a, group_b, n_resamples=10000, "ci_low": ci_low, "ci_high": ci_high, "p_value": p_value, - "significant": p_value < 0.05, + "significant": p_value < (1.0 - confidence), "n_a": n_a, "n_b": n_b, "small_sample_warning": n_a < 30 or n_b < 30, diff --git a/tests/test_significance.py b/tests/test_significance.py index 337c911..601499c 100644 --- a/tests/test_significance.py +++ b/tests/test_significance.py @@ -37,6 +37,27 @@ def test_separated_groups_are_significant_and_ci_excludes_zero(): assert rep["ci_low"] <= rep["gap"] <= rep["ci_high"] +def test_confidence_tightens_the_significance_threshold_not_just_the_ci(): + # A gap with 0.01 < p < 0.05 is significant at the default confidence + # (0.95 -> p < 0.05) but not at confidence=0.99 (-> p < 0.01). Before + # #548 the `significant` flag was pinned to p < 0.05 regardless of + # `confidence`, which only widened/narrowed the CI. + rng = np.random.default_rng(0) + for _ in range(5): + a = rng.binomial(1, 0.55, size=60).astype(float) + b = rng.binomial(1, 0.35, size=60).astype(float) + + r95 = significance_report(a, b, n_resamples=3000, n_permutations=3000, + confidence=0.95, random_state=4) + r99 = significance_report(a, b, n_resamples=3000, n_permutations=3000, + confidence=0.99, random_state=4) + + assert r95["p_value"] == r99["p_value"] # same permutation test + assert 0.01 < r95["p_value"] < 0.05 + assert r95["significant"] is True + assert r99["significant"] is False + + # ── Determinism ────────────────────────────────────────────────────────────── def test_random_state_makes_results_deterministic(): a = [1] * 40 + [0] * 60