diff --git a/dokimi/changelog.d/20261114_130000_tiny-scale-degenerate.md b/dokimi/changelog.d/20261114_130000_tiny-scale-degenerate.md new file mode 100644 index 000000000..52f198c57 --- /dev/null +++ b/dokimi/changelog.d/20261114_130000_tiny-scale-degenerate.md @@ -0,0 +1,3 @@ +### Fixed + +- **`mean_ci` and Cohen's d no longer read values that vary at a tiny scale as constant** — a sample such as `[1e-300, 2e-300, 3e-300]` or one whose values differ by less than 1e-9 in absolute terms got `method: degenerate` with `lo == hi == mean`, and deltas of that size got no `cohens_d`, because the check compared the sd with an absolute floor of 1.0. The check now divides by a power of two taken from the data scale first, so such a sample gets a Student-t interval and a d_z. A sample that is constant at its own scale (exact repeats, or float noise of 1e-9 relative) stays `degenerate`. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 9fa66e983..1b49eb65a 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -393,7 +393,9 @@ values are possible: A `—` is not a null result. A constant shift is not a `—`: eight pairs that all gain +20 have p = 2/2^8 = 0.0078 and the interval [+20, +20]. Cohen's d_z is descriptive (no verdict reads it) and renders `—` there, where it is *unbounded* -rather than negligible; it is 0.0 when every delta is 0. +rather than negligible; it is 0.0 when every delta is 0. "Constant" is judged at the +deltas' own scale (spread under 1e-9 of the larger of the mean and the sd), so deltas +of 1e-12 that differ get a d_z. The 95% CI inverts the same sign-flip test: it is every shift theta at which the test on the deltas minus theta does not reject at 5%, so it excludes zero exactly diff --git a/dokimi/dokimi/eval/stats.py b/dokimi/dokimi/eval/stats.py index 616a4f3bd..ddc217e33 100644 --- a/dokimi/dokimi/eval/stats.py +++ b/dokimi/dokimi/eval/stats.py @@ -92,6 +92,16 @@ def _is_degenerate(mean: float, sd: float) -> bool: return sd <= ZERO_VARIANCE_REL_TOL * max(1.0, abs(mean)) +def _is_degenerate_at_scale(mean: float, sd: float, top: float) -> bool: + """:func:`_is_degenerate` after dividing ``mean`` and ``sd`` by the power of two at or below ``top`` (exact). + + The floor of 1.0 then absorbs float noise relative to the data, not to 1: a sample of 1e-300 values that + differ is not constant (#640). + """ + scale = _pow2_scale([top]) + return _is_degenerate(mean / scale, sd / scale) + + def _rnd(x: float, ndigits: Optional[int]) -> float: """``round(x, ndigits)``, or ``x`` itself when ``ndigits`` is None.""" return x if ndigits is None else round(x, ndigits) @@ -201,7 +211,7 @@ def mean_ci(values, confidence: float = 0.95, seed: int = 0, ndigits: Optional[i mu = _mean(vals) sd = _sample_sd(vals) - if _is_degenerate(mu, sd): + if _is_degenerate_at_scale(mu, sd, max(abs(v) for v in vals)): # Every value identical: the interval is the point itself, and a # bootstrap would report the same thing after 10k pointless resamples. return {"mean": _rnd(mu, ndigits), "lo": _rnd(mu, ndigits), "hi": _rnd(mu, ndigits), @@ -547,7 +557,7 @@ def point_biserial(flags: Sequence[bool], totals: Sequence[float]) -> Dict[str, def _cohens_d(d_mean: float, d_sd: float) -> Optional[float]: """Cohen's d_z of the deltas (sample SD), descriptive: no verdict reads it.""" - if not _is_degenerate(d_mean, d_sd): + if not _is_degenerate_at_scale(d_mean, d_sd, max(abs(d_mean), d_sd)): return round(d_mean / d_sd, 3) # Every pair identical: the limit of 0/0 here is a genuine null, and "0.0 (negligible)" says more than # "unknown"; a constant non-zero shift has an unbounded d_z, which None renders as "unknown". diff --git a/dokimi/tests/test_stats_tiny_scale.py b/dokimi/tests/test_stats_tiny_scale.py new file mode 100644 index 000000000..c170c651d --- /dev/null +++ b/dokimi/tests/test_stats_tiny_scale.py @@ -0,0 +1,53 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The zero-variance check reads a sample at its own scale, not against an absolute floor of 1.0 (#640).""" + +import pytest + +from dokimi.eval import stats + + +@pytest.mark.parametrize("scale", [1e-300, 1e-12]) +def test_values_that_vary_at_a_tiny_scale_get_a_student_t_interval(scale): + pytest.importorskip("scipy") + res = stats.mean_ci([1 * scale, 2 * scale, 3 * scale], ndigits=None) + assert res["method"] == "student_t" + assert res["lo"] < res["mean"] < res["hi"] + + +def test_mean_ci_is_scale_invariant_by_a_power_of_two(): + pytest.importorskip("scipy") + base = stats.mean_ci([1.0, 2.0, 4.0], ndigits=None) + small = stats.mean_ci([2.0**-40, 2.0**-39, 2.0**-38], ndigits=None) + assert small["method"] == base["method"] == "student_t" + assert small["lo"] / small["mean"] == base["lo"] / base["mean"] + assert small["hi"] / small["mean"] == base["hi"] / base["mean"] + + +def test_a_constant_tiny_sample_stays_degenerate(): + pytest.importorskip("scipy") + res = stats.mean_ci([1e-300] * 5, ndigits=None) + assert (res["method"], res["lo"], res["hi"]) == ("degenerate", 1e-300, 1e-300) + + +def test_float_noise_around_a_tiny_constant_stays_degenerate(): + pytest.importorskip("scipy") + noisy = [1e-12 * (1.0 + (1e-13 if i % 2 else -1e-13)) for i in range(6)] + assert stats.mean_ci(noisy, ndigits=None)["method"] == "degenerate" + assert stats.mean_ci([0.0] * 4)["method"] == "degenerate" + + +def test_cohens_d_of_tiny_deltas_is_a_number(): + assert stats._cohens_d(2e-300, 1e-300) == 2.0 + + +def test_cohens_d_of_a_constant_tiny_shift_is_unbounded_not_a_number(): + assert stats._cohens_d(1e-300, 1e-320) is None + assert stats._cohens_d(0.0, 0.0) == 0.0 + + +def test_paired_test_of_tiny_deltas_reports_an_effect_size(): + res = stats.paired_test([2e-300, 4e-300, 3e-300, 5e-300], [1e-300] * 4) + assert res["cohens_d"] == 1.936 + assert res["effect"] != "unknown"