From 65008c57603f978331244386317401c03f7b7a37 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sat, 14 Nov 2026 19:53:20 +0000 Subject: [PATCH 1/2] fix(stats): mean_ci and Cohen's d judge zero variance at the data's own scale (#640) `_is_degenerate` compares the sd with `1e-9 * max(1.0, abs(mean))`, so any sample with sd under 1e-9 counted as constant whatever its scale: `mean_ci([1e-300, 2e-300, 3e-300])` returned `degenerate` with `lo == hi == mean`, and `_cohens_d(2e-300, 1e-300)` returned None. `mean_ci` and `_cohens_d` now divide the mean and sd by a power of two taken from the data (the largest absolute value, and the larger of the mean and the sd) before the check, through one helper `_is_degenerate_at_scale`. The floor of 1.0 keeps its job of absorbing float noise on a constant sample, relative to the data. The returned mean and bounds are unchanged. Signed-off-by: Anton Volkov --- .../20261114_130000_tiny-scale-degenerate.md | 3 ++ dokimi/docs/benchmarking-and-comparison.md | 4 +- dokimi/dokimi/eval/stats.py | 16 +++++- dokimi/tests/test_stats_tiny_scale.py | 53 +++++++++++++++++++ 4 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 dokimi/changelog.d/20261114_130000_tiny-scale-degenerate.md create mode 100644 dokimi/tests/test_stats_tiny_scale.py diff --git a/dokimi/changelog.d/20261114_130000_tiny-scale-degenerate.md b/dokimi/changelog.d/20261114_130000_tiny-scale-degenerate.md new file mode 100644 index 000000000..52f198c57 --- /dev/null +++ b/dokimi/changelog.d/20261114_130000_tiny-scale-degenerate.md @@ -0,0 +1,3 @@ +### Fixed + +- **`mean_ci` and Cohen's d no longer read values that vary at a tiny scale as constant** — a sample such as `[1e-300, 2e-300, 3e-300]` or one whose values differ by less than 1e-9 in absolute terms got `method: degenerate` with `lo == hi == mean`, and deltas of that size got no `cohens_d`, because the check compared the sd with an absolute floor of 1.0. The check now divides by a power of two taken from the data scale first, so such a sample gets a Student-t interval and a d_z. A sample that is constant at its own scale (exact repeats, or float noise of 1e-9 relative) stays `degenerate`. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 9fa66e983..1b49eb65a 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -393,7 +393,9 @@ values are possible: A `—` is not a null result. A constant shift is not a `—`: eight pairs that all gain +20 have p = 2/2^8 = 0.0078 and the interval [+20, +20]. Cohen's d_z is descriptive (no verdict reads it) and renders `—` there, where it is *unbounded* -rather than negligible; it is 0.0 when every delta is 0. +rather than negligible; it is 0.0 when every delta is 0. "Constant" is judged at the +deltas' own scale (spread under 1e-9 of the larger of the mean and the sd), so deltas +of 1e-12 that differ get a d_z. The 95% CI inverts the same sign-flip test: it is every shift theta at which the test on the deltas minus theta does not reject at 5%, so it excludes zero exactly diff --git a/dokimi/dokimi/eval/stats.py b/dokimi/dokimi/eval/stats.py index 616a4f3bd..40efd1cd6 100644 --- a/dokimi/dokimi/eval/stats.py +++ b/dokimi/dokimi/eval/stats.py @@ -92,6 +92,18 @@ def _is_degenerate(mean: float, sd: float) -> bool: return sd <= ZERO_VARIANCE_REL_TOL * max(1.0, abs(mean)) +def _is_degenerate_at_scale(mean: float, sd: float, top: float) -> bool: + """:func:`_is_degenerate` after dividing ``mean`` and ``sd`` by the power of two at or below ``top`` (exact). + + The floor of 1.0 then absorbs float noise relative to the data, not to 1: a sample of 1e-300 values that + differ is not constant (#640). ``top`` is 0 only for a sample of zeros, which is constant. + """ + if top == 0: + return True + scale = _pow2_scale([top]) + return _is_degenerate(mean / scale, sd / scale) + + def _rnd(x: float, ndigits: Optional[int]) -> float: """``round(x, ndigits)``, or ``x`` itself when ``ndigits`` is None.""" return x if ndigits is None else round(x, ndigits) @@ -201,7 +213,7 @@ def mean_ci(values, confidence: float = 0.95, seed: int = 0, ndigits: Optional[i mu = _mean(vals) sd = _sample_sd(vals) - if _is_degenerate(mu, sd): + if _is_degenerate_at_scale(mu, sd, max(abs(v) for v in vals)): # Every value identical: the interval is the point itself, and a # bootstrap would report the same thing after 10k pointless resamples. return {"mean": _rnd(mu, ndigits), "lo": _rnd(mu, ndigits), "hi": _rnd(mu, ndigits), @@ -547,7 +559,7 @@ def point_biserial(flags: Sequence[bool], totals: Sequence[float]) -> Dict[str, def _cohens_d(d_mean: float, d_sd: float) -> Optional[float]: """Cohen's d_z of the deltas (sample SD), descriptive: no verdict reads it.""" - if not _is_degenerate(d_mean, d_sd): + if not _is_degenerate_at_scale(d_mean, d_sd, max(abs(d_mean), d_sd)): return round(d_mean / d_sd, 3) # Every pair identical: the limit of 0/0 here is a genuine null, and "0.0 (negligible)" says more than # "unknown"; a constant non-zero shift has an unbounded d_z, which None renders as "unknown". diff --git a/dokimi/tests/test_stats_tiny_scale.py b/dokimi/tests/test_stats_tiny_scale.py new file mode 100644 index 000000000..c170c651d --- /dev/null +++ b/dokimi/tests/test_stats_tiny_scale.py @@ -0,0 +1,53 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The zero-variance check reads a sample at its own scale, not against an absolute floor of 1.0 (#640).""" + +import pytest + +from dokimi.eval import stats + + +@pytest.mark.parametrize("scale", [1e-300, 1e-12]) +def test_values_that_vary_at_a_tiny_scale_get_a_student_t_interval(scale): + pytest.importorskip("scipy") + res = stats.mean_ci([1 * scale, 2 * scale, 3 * scale], ndigits=None) + assert res["method"] == "student_t" + assert res["lo"] < res["mean"] < res["hi"] + + +def test_mean_ci_is_scale_invariant_by_a_power_of_two(): + pytest.importorskip("scipy") + base = stats.mean_ci([1.0, 2.0, 4.0], ndigits=None) + small = stats.mean_ci([2.0**-40, 2.0**-39, 2.0**-38], ndigits=None) + assert small["method"] == base["method"] == "student_t" + assert small["lo"] / small["mean"] == base["lo"] / base["mean"] + assert small["hi"] / small["mean"] == base["hi"] / base["mean"] + + +def test_a_constant_tiny_sample_stays_degenerate(): + pytest.importorskip("scipy") + res = stats.mean_ci([1e-300] * 5, ndigits=None) + assert (res["method"], res["lo"], res["hi"]) == ("degenerate", 1e-300, 1e-300) + + +def test_float_noise_around_a_tiny_constant_stays_degenerate(): + pytest.importorskip("scipy") + noisy = [1e-12 * (1.0 + (1e-13 if i % 2 else -1e-13)) for i in range(6)] + assert stats.mean_ci(noisy, ndigits=None)["method"] == "degenerate" + assert stats.mean_ci([0.0] * 4)["method"] == "degenerate" + + +def test_cohens_d_of_tiny_deltas_is_a_number(): + assert stats._cohens_d(2e-300, 1e-300) == 2.0 + + +def test_cohens_d_of_a_constant_tiny_shift_is_unbounded_not_a_number(): + assert stats._cohens_d(1e-300, 1e-320) is None + assert stats._cohens_d(0.0, 0.0) == 0.0 + + +def test_paired_test_of_tiny_deltas_reports_an_effect_size(): + res = stats.paired_test([2e-300, 4e-300, 3e-300, 5e-300], [1e-300] * 4) + assert res["cohens_d"] == 1.936 + assert res["effect"] != "unknown" From d8b869d36dd9bb6566c16618f9a8e0f6d5fa4b30 Mon Sep 17 00:00:00 2001 From: vchamarthi <123653452+vchamarthi@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:49:27 -0600 Subject: [PATCH 2/2] fix(stats): drop unreachable top == 0 branch in degenerate check _pow2_scale([0.0]) is 0.5, so a sample of zeros has mean 0 and sd 0 after scaling and _is_degenerate already returns True. Removes the branch and its docstring sentence. Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com> --- dokimi/dokimi/eval/stats.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dokimi/dokimi/eval/stats.py b/dokimi/dokimi/eval/stats.py index 40efd1cd6..ddc217e33 100644 --- a/dokimi/dokimi/eval/stats.py +++ b/dokimi/dokimi/eval/stats.py @@ -96,10 +96,8 @@ def _is_degenerate_at_scale(mean: float, sd: float, top: float) -> bool: """:func:`_is_degenerate` after dividing ``mean`` and ``sd`` by the power of two at or below ``top`` (exact). The floor of 1.0 then absorbs float noise relative to the data, not to 1: a sample of 1e-300 values that - differ is not constant (#640). ``top`` is 0 only for a sample of zeros, which is constant. + differ is not constant (#640). """ - if top == 0: - return True scale = _pow2_scale([top]) return _is_degenerate(mean / scale, sd / scale)