Draft: prove completeness before context reduction

benchmarks/tc1_real/fixtures/planning_phx_20260818/provider-truncation-evidence.json

@@ -0,0 +1,40 @@
+{
+  "evidence_status": "RECORDED_REAL_DIAGNOSTIC",
+  "scenario": "Phoenix public-library service-coverage context preparation",
+  "source_id": "phx-growth-projections",
+  "source_family": "City of Phoenix Growth Projections",
+  "representation": {
+    "layer_id": 2,
+    "layer_name": "FinalLUAUs_PopEmp",
+    "role": "joined Pop/Emp diagnostic representation",
+    "scoring_status": "EXCLUDED_WRONG_REPRESENTATION"
+  },
+  "frozen_scopes": {
+    "broad_bbox": [-112.20, 33.30, -111.90, 33.60],
+    "task_bbox": [-112.10, 33.40, -112.00, 33.50]
+  },
+  "ordinary_single_query": {
+    "observed_transfer_ceiling_count": 2000,
+    "broad": {
+      "feature_count": 2000,
+      "response_bytes": 4573559
+    },
+    "task": {
+      "feature_count": 2000,
+      "response_bytes": 2485291
+    }
+  },
+  "ids_first_complete_broad": {
+    "feature_count": 118190,
+    "unique_newluau_count": 471,
+    "page_count": 119,
+    "network_bytes": 172127458,
+    "complete": true
+  },
+  "interpretation_boundary": {
+    "ordinary_query_is_complete": false,
+    "ordinary_task_vs_broad_reduction_is_scoreable": false,
+    "reason": "The ordinary broad/task responses both hit the observed 2000-record transfer ceiling. IDs-first retrieval later proved 118190 complete features for the same broad joined representation."
+  },
+  "source_document": "docs/reference/task-context-tc1-real-spatial-planning-diagnostic-v0.1.md"
+}

benchmarks/tc1_real/spatial_planning/provider_truncation_proof.py

@@ -0,0 +1,96 @@
+"""Offline proof that provider truncation must be resolved before scoring reduction.
+
+This module replays recorded TC1-Real Phoenix planning evidence.  It does not
+query the provider and does not promote a new GeoTask Core semantic.  Its only
+purpose is to make one methodological failure reproducible:
+
+    smaller response != smaller task context
+
+when the response itself may have been truncated by the provider.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Mapping
+
+
+@dataclass(frozen=True)
+class TruncationProofResult:
+    naive_task_vs_broad_byte_reduction_ratio: float
+    naive_broad_return_fraction_of_complete: float
+    naive_broad_undercount_ratio: float
+    both_single_queries_hit_observed_ceiling: bool
+    complete_broad_exceeds_single_query: bool
+    reduction_scoreable: bool
+    verdict: str
+
+
+def _positive_int(value: object, name: str) -> int:
+    result = int(value)
+    if result <= 0:
+        raise ValueError(f"{name} must be > 0")
+    return result
+
+
+def analyze_provider_truncation_evidence(
+    evidence: Mapping[str, object],
+) -> TruncationProofResult:
+    """Compare ordinary bounded responses with the later complete broad retrieval.
+
+    Reduction is deliberately fail-closed.  It is scoreable only if the evidence
+    proves that ordinary responses did not hit the observed transfer ceiling and
+    the complete broad cardinality does not exceed the ordinary broad response.
+    The recorded real fixture should therefore reject the attractive naive byte
+    reduction rather than turn it into a benchmark headline.
+    """
+
+    ordinary = evidence.get("ordinary_single_query")
+    complete = evidence.get("ids_first_complete_broad")
+    if not isinstance(ordinary, Mapping) or not isinstance(complete, Mapping):
+        raise ValueError("evidence must contain ordinary and complete retrieval records")
+
+    ceiling = _positive_int(
+        ordinary.get("observed_transfer_ceiling_count"), "observed transfer ceiling"
+    )
+    broad = ordinary.get("broad")
+    task = ordinary.get("task")
+    if not isinstance(broad, Mapping) or not isinstance(task, Mapping):
+        raise ValueError("ordinary evidence must contain broad and task records")
+
+    broad_count = _positive_int(broad.get("feature_count"), "ordinary broad feature_count")
+    task_count = _positive_int(task.get("feature_count"), "ordinary task feature_count")
+    broad_bytes = _positive_int(broad.get("response_bytes"), "ordinary broad response_bytes")
+    task_bytes = _positive_int(task.get("response_bytes"), "ordinary task response_bytes")
+    complete_count = _positive_int(
+        complete.get("feature_count"), "complete broad feature_count"
+    )
+
+    if complete.get("complete") is not True:
+        raise ValueError("complete broad evidence must explicitly prove completeness")
+    if task_bytes > broad_bytes:
+        raise ValueError("recorded task response_bytes cannot exceed broad response_bytes")
+
+    both_hit_ceiling = broad_count == ceiling and task_count == ceiling
+    complete_exceeds_single = complete_count > broad_count
+
+    naive_reduction = 1.0 - (task_bytes / broad_bytes)
+    broad_return_fraction = broad_count / complete_count
+    undercount_ratio = 1.0 - broad_return_fraction
+
+    scoreable = not both_hit_ceiling and not complete_exceeds_single
+    verdict = (
+        "REJECT_NAIVE_REDUCTION_PROVIDER_TRUNCATED"
+        if not scoreable
+        else "ORDINARY_RESPONSE_COMPLETENESS_NOT_CONTRADICTED"
+    )
+
+    return TruncationProofResult(
+        naive_task_vs_broad_byte_reduction_ratio=naive_reduction,
+        naive_broad_return_fraction_of_complete=broad_return_fraction,
+        naive_broad_undercount_ratio=undercount_ratio,
+        both_single_queries_hit_observed_ceiling=both_hit_ceiling,
+        complete_broad_exceeds_single_query=complete_exceeds_single,
+        reduction_scoreable=scoreable,
+        verdict=verdict,
+    )

benchmarks/tc1_real/spatial_planning/run_provider_truncation_proof.py

@@ -0,0 +1,58 @@
+"""Replay the recorded Phoenix provider-truncation proof.
+
+Usage:
+    python -m benchmarks.tc1_real.spatial_planning.run_provider_truncation_proof
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from benchmarks.tc1_real.spatial_planning.provider_truncation_proof import (
+    analyze_provider_truncation_evidence,
+)
+
+
+FIXTURE = (
+    Path(__file__).resolve().parents[1]
+    / "fixtures"
+    / "planning_phx_20260818"
+    / "provider-truncation-evidence.json"
+)
+
+
+def main() -> int:
+    evidence = json.loads(FIXTURE.read_text(encoding="utf-8"))
+    result = analyze_provider_truncation_evidence(evidence)
+
+    print("GeoTask TC1 Provider Truncation Proof")
+    print("recorded_real_evidence=true")
+    print("live_network=false")
+    print(
+        "naive_task_vs_broad_byte_reduction_percent="
+        f"{result.naive_task_vs_broad_byte_reduction_ratio * 100:.5f}"
+    )
+    print(
+        "ordinary_broad_return_fraction_of_complete_percent="
+        f"{result.naive_broad_return_fraction_of_complete * 100:.5f}"
+    )
+    print(
+        "ordinary_broad_undercount_percent="
+        f"{result.naive_broad_undercount_ratio * 100:.5f}"
+    )
+    print(
+        "both_single_queries_hit_observed_ceiling="
+        f"{str(result.both_single_queries_hit_observed_ceiling).lower()}"
+    )
+    print(
+        "complete_broad_exceeds_single_query="
+        f"{str(result.complete_broad_exceeds_single_query).lower()}"
+    )
+    print(f"reduction_scoreable={str(result.reduction_scoreable).lower()}")
+    print(f"verdict={result.verdict}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

docs/reference/task-context-tc1-provider-truncation-proof-v0.1.md

@@ -0,0 +1,110 @@
+# TC1 Provider Truncation Proof v0.1
+
+**Status:** REAL RECORDED EVIDENCE / OFFLINE REPLAY  
+**Scenario:** Phoenix public-library service-coverage context preparation  
+**Scope:** Benchmark method evidence only; no GeoTask Core change
+
+## 1. Problem
+
+A bounded provider response can look smaller than a broader response even when
+neither response is complete. If a benchmark compares response bytes before
+proving completeness, provider pagination/transfer limits can masquerade as
+Task Context reduction.
+
+The first Phoenix planning acquisition appeared attractive:
+
+```text
+ordinary growth broad   2,000 features   4,573,559 bytes
+ordinary growth task    2,000 features   2,485,291 bytes
+```
+
+A naive byte comparison would report:
+
+```text
+task vs broad byte reduction ~= 45.66%
+```
+
+But both ordinary queries hit the same observed 2,000-record transfer ceiling.
+The later IDs-first complete retrieval for the same broad joined representation
+proved:
+
+```text
+complete broad features    118,190
+unique planning units           471
+pages                            119
+network bytes            172,127,458
+complete                       true
+```
+
+The ordinary broad response therefore returned only about 1.69% of the complete
+feature cardinality and undercounted it by about 98.31%.
+
+## 2. Method correction
+
+The benchmark rule is frozen as:
+
+```text
+Completeness Before Reduction
+
+scope
+  -> provider object-ID set
+  -> complete paged retrieval
+  -> prove retrieved IDs == source IDs
+  -> only then compare context burden
+```
+
+A smaller response is not admitted as evidence of better Task Context until the
+candidate source response is proven complete for the measured scope.
+
+## 3. Counterexample / anti-gaming condition
+
+The replay intentionally keeps the attractive naive reduction visible while
+rejecting it as a scoreable result.
+
+```text
+naive task-vs-broad byte reduction     ~45.66%
+both ordinary queries hit 2,000 cap     yes
+complete broad > ordinary broad         yes
+headline reduction scoreable             no
+verdict  REJECT_NAIVE_REDUCTION_PROVIDER_TRUNCATED
+```
+
+This prevents a pathological benchmark from becoming "better" merely because a
+provider silently returned fewer records than actually existed.
+
+## 4. Important separation
+
+This proof establishes only the completeness prerequisite. It does not make the
+joined layer 2 representation a valid benchmark baseline. A separate diagnostic
+showed that layer 2 repeats planning-unit geometry across Pop/Emp records and is
+therefore the wrong task representation. The formal planning benchmark uses base
+planning-unit layer 4 plus population table 13 instead.
+
+So the method order is:
+
+```text
+1. choose a task-appropriate representation
+2. prove provider completeness
+3. prove semantic/entity coverage
+4. only then measure context reduction
+```
+
+## 5. What this changes
+
+It does **not** add a new GeoTask Core object or algorithm.
+
+It strengthens the benchmark discipline:
+
+> **Reduction without completeness evidence is not a valid GeoTask result.**
+
+The method is potentially reusable beyond GIS/ArcGIS providers, including RAG,
+API pagination, database sampling, catalog search, and any system where a bounded
+response can be mistaken for the full candidate set.
+
+## 6. Files
+
+```text
+benchmarks/tc1_real/fixtures/planning_phx_20260818/provider-truncation-evidence.json
+benchmarks/tc1_real/spatial_planning/provider_truncation_proof.py
+tests/test_tc1_real_spatial_planning_provider_truncation.py
+```

tests/test_tc1_real_spatial_planning_provider_truncation.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from benchmarks.tc1_real.spatial_planning.provider_truncation_proof import (
+    analyze_provider_truncation_evidence,
+)
+
+
+FIXTURE = (
+    Path(__file__).resolve().parents[1]
+    / "benchmarks"
+    / "tc1_real"
+    / "fixtures"
+    / "planning_phx_20260818"
+    / "provider-truncation-evidence.json"
+)
+
+
+def _fixture() -> dict[str, object]:
+    return json.loads(FIXTURE.read_text(encoding="utf-8"))
+
+
+def test_recorded_provider_truncation_rejects_attractive_naive_reduction() -> None:
+    result = analyze_provider_truncation_evidence(_fixture())
+
+    # A naive comparison would advertise roughly a 45.66% smaller task response.
+    assert result.naive_task_vs_broad_byte_reduction_ratio == pytest.approx(
+        0.4565958370713049
+    )
+
+    # But both ordinary responses hit the same 2,000-record ceiling, and the
+    # later IDs-first retrieval proved 118,190 complete broad features.
+    assert result.both_single_queries_hit_observed_ceiling
+    assert result.complete_broad_exceeds_single_query
+    assert result.naive_broad_return_fraction_of_complete == pytest.approx(
+        0.016921905406548778
+    )
+    assert result.naive_broad_undercount_ratio == pytest.approx(
+        0.9830780945934512
+    )
+
+    assert not result.reduction_scoreable
+    assert result.verdict == "REJECT_NAIVE_REDUCTION_PROVIDER_TRUNCATED"
+
+
+def test_reduction_remains_fail_closed_when_only_one_truncation_signal_is_present() -> None:
+    evidence = _fixture()
+    ordinary = evidence["ordinary_single_query"]
+    assert isinstance(ordinary, dict)
+    task = ordinary["task"]
+    assert isinstance(task, dict)
+
+    # Even if the task response does not exactly hit the observed ceiling, the
+    # later complete broad cardinality still contradicts the ordinary broad
+    # response and must keep the reduction non-scoreable.
+    task["feature_count"] = 1500
+    result = analyze_provider_truncation_evidence(evidence)
+
+    assert not result.both_single_queries_hit_observed_ceiling
+    assert result.complete_broad_exceeds_single_query
+    assert not result.reduction_scoreable
+
+
+def test_explicitly_incomplete_ids_first_evidence_is_rejected() -> None:
+    evidence = _fixture()
+    complete = evidence["ids_first_complete_broad"]
+    assert isinstance(complete, dict)
+    complete["complete"] = False
+
+    with pytest.raises(ValueError, match="explicitly prove completeness"):
+        analyze_provider_truncation_evidence(evidence)