diff --git a/benchmarks/tc1_real/fixtures/planning_phx_20260818/provider-truncation-evidence.json b/benchmarks/tc1_real/fixtures/planning_phx_20260818/provider-truncation-evidence.json new file mode 100644 index 0000000..5a8da53 --- /dev/null +++ b/benchmarks/tc1_real/fixtures/planning_phx_20260818/provider-truncation-evidence.json @@ -0,0 +1,40 @@ +{ + "evidence_status": "RECORDED_REAL_DIAGNOSTIC", + "scenario": "Phoenix public-library service-coverage context preparation", + "source_id": "phx-growth-projections", + "source_family": "City of Phoenix Growth Projections", + "representation": { + "layer_id": 2, + "layer_name": "FinalLUAUs_PopEmp", + "role": "joined Pop/Emp diagnostic representation", + "scoring_status": "EXCLUDED_WRONG_REPRESENTATION" + }, + "frozen_scopes": { + "broad_bbox": [-112.20, 33.30, -111.90, 33.60], + "task_bbox": [-112.10, 33.40, -112.00, 33.50] + }, + "ordinary_single_query": { + "observed_transfer_ceiling_count": 2000, + "broad": { + "feature_count": 2000, + "response_bytes": 4573559 + }, + "task": { + "feature_count": 2000, + "response_bytes": 2485291 + } + }, + "ids_first_complete_broad": { + "feature_count": 118190, + "unique_newluau_count": 471, + "page_count": 119, + "network_bytes": 172127458, + "complete": true + }, + "interpretation_boundary": { + "ordinary_query_is_complete": false, + "ordinary_task_vs_broad_reduction_is_scoreable": false, + "reason": "The ordinary broad/task responses both hit the observed 2000-record transfer ceiling. IDs-first retrieval later proved 118190 complete features for the same broad joined representation." + }, + "source_document": "docs/reference/task-context-tc1-real-spatial-planning-diagnostic-v0.1.md" +} diff --git a/benchmarks/tc1_real/spatial_planning/provider_truncation_proof.py b/benchmarks/tc1_real/spatial_planning/provider_truncation_proof.py new file mode 100644 index 0000000..c9086f2 --- /dev/null +++ b/benchmarks/tc1_real/spatial_planning/provider_truncation_proof.py @@ -0,0 +1,96 @@ +"""Offline proof that provider truncation must be resolved before scoring reduction. + +This module replays recorded TC1-Real Phoenix planning evidence. It does not +query the provider and does not promote a new GeoTask Core semantic. Its only +purpose is to make one methodological failure reproducible: + + smaller response != smaller task context + +when the response itself may have been truncated by the provider. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Mapping + + +@dataclass(frozen=True) +class TruncationProofResult: + naive_task_vs_broad_byte_reduction_ratio: float + naive_broad_return_fraction_of_complete: float + naive_broad_undercount_ratio: float + both_single_queries_hit_observed_ceiling: bool + complete_broad_exceeds_single_query: bool + reduction_scoreable: bool + verdict: str + + +def _positive_int(value: object, name: str) -> int: + result = int(value) + if result <= 0: + raise ValueError(f"{name} must be > 0") + return result + + +def analyze_provider_truncation_evidence( + evidence: Mapping[str, object], +) -> TruncationProofResult: + """Compare ordinary bounded responses with the later complete broad retrieval. + + Reduction is deliberately fail-closed. It is scoreable only if the evidence + proves that ordinary responses did not hit the observed transfer ceiling and + the complete broad cardinality does not exceed the ordinary broad response. + The recorded real fixture should therefore reject the attractive naive byte + reduction rather than turn it into a benchmark headline. + """ + + ordinary = evidence.get("ordinary_single_query") + complete = evidence.get("ids_first_complete_broad") + if not isinstance(ordinary, Mapping) or not isinstance(complete, Mapping): + raise ValueError("evidence must contain ordinary and complete retrieval records") + + ceiling = _positive_int( + ordinary.get("observed_transfer_ceiling_count"), "observed transfer ceiling" + ) + broad = ordinary.get("broad") + task = ordinary.get("task") + if not isinstance(broad, Mapping) or not isinstance(task, Mapping): + raise ValueError("ordinary evidence must contain broad and task records") + + broad_count = _positive_int(broad.get("feature_count"), "ordinary broad feature_count") + task_count = _positive_int(task.get("feature_count"), "ordinary task feature_count") + broad_bytes = _positive_int(broad.get("response_bytes"), "ordinary broad response_bytes") + task_bytes = _positive_int(task.get("response_bytes"), "ordinary task response_bytes") + complete_count = _positive_int( + complete.get("feature_count"), "complete broad feature_count" + ) + + if complete.get("complete") is not True: + raise ValueError("complete broad evidence must explicitly prove completeness") + if task_bytes > broad_bytes: + raise ValueError("recorded task response_bytes cannot exceed broad response_bytes") + + both_hit_ceiling = broad_count == ceiling and task_count == ceiling + complete_exceeds_single = complete_count > broad_count + + naive_reduction = 1.0 - (task_bytes / broad_bytes) + broad_return_fraction = broad_count / complete_count + undercount_ratio = 1.0 - broad_return_fraction + + scoreable = not both_hit_ceiling and not complete_exceeds_single + verdict = ( + "REJECT_NAIVE_REDUCTION_PROVIDER_TRUNCATED" + if not scoreable + else "ORDINARY_RESPONSE_COMPLETENESS_NOT_CONTRADICTED" + ) + + return TruncationProofResult( + naive_task_vs_broad_byte_reduction_ratio=naive_reduction, + naive_broad_return_fraction_of_complete=broad_return_fraction, + naive_broad_undercount_ratio=undercount_ratio, + both_single_queries_hit_observed_ceiling=both_hit_ceiling, + complete_broad_exceeds_single_query=complete_exceeds_single, + reduction_scoreable=scoreable, + verdict=verdict, + ) diff --git a/benchmarks/tc1_real/spatial_planning/run_provider_truncation_proof.py b/benchmarks/tc1_real/spatial_planning/run_provider_truncation_proof.py new file mode 100644 index 0000000..fdb8533 --- /dev/null +++ b/benchmarks/tc1_real/spatial_planning/run_provider_truncation_proof.py @@ -0,0 +1,58 @@ +"""Replay the recorded Phoenix provider-truncation proof. + +Usage: + python -m benchmarks.tc1_real.spatial_planning.run_provider_truncation_proof +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from benchmarks.tc1_real.spatial_planning.provider_truncation_proof import ( + analyze_provider_truncation_evidence, +) + + +FIXTURE = ( + Path(__file__).resolve().parents[1] + / "fixtures" + / "planning_phx_20260818" + / "provider-truncation-evidence.json" +) + + +def main() -> int: + evidence = json.loads(FIXTURE.read_text(encoding="utf-8")) + result = analyze_provider_truncation_evidence(evidence) + + print("GeoTask TC1 Provider Truncation Proof") + print("recorded_real_evidence=true") + print("live_network=false") + print( + "naive_task_vs_broad_byte_reduction_percent=" + f"{result.naive_task_vs_broad_byte_reduction_ratio * 100:.5f}" + ) + print( + "ordinary_broad_return_fraction_of_complete_percent=" + f"{result.naive_broad_return_fraction_of_complete * 100:.5f}" + ) + print( + "ordinary_broad_undercount_percent=" + f"{result.naive_broad_undercount_ratio * 100:.5f}" + ) + print( + "both_single_queries_hit_observed_ceiling=" + f"{str(result.both_single_queries_hit_observed_ceiling).lower()}" + ) + print( + "complete_broad_exceeds_single_query=" + f"{str(result.complete_broad_exceeds_single_query).lower()}" + ) + print(f"reduction_scoreable={str(result.reduction_scoreable).lower()}") + print(f"verdict={result.verdict}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/reference/task-context-tc1-provider-truncation-proof-v0.1.md b/docs/reference/task-context-tc1-provider-truncation-proof-v0.1.md new file mode 100644 index 0000000..3b8621a --- /dev/null +++ b/docs/reference/task-context-tc1-provider-truncation-proof-v0.1.md @@ -0,0 +1,110 @@ +# TC1 Provider Truncation Proof v0.1 + +**Status:** REAL RECORDED EVIDENCE / OFFLINE REPLAY +**Scenario:** Phoenix public-library service-coverage context preparation +**Scope:** Benchmark method evidence only; no GeoTask Core change + +## 1. Problem + +A bounded provider response can look smaller than a broader response even when +neither response is complete. If a benchmark compares response bytes before +proving completeness, provider pagination/transfer limits can masquerade as +Task Context reduction. + +The first Phoenix planning acquisition appeared attractive: + +```text +ordinary growth broad 2,000 features 4,573,559 bytes +ordinary growth task 2,000 features 2,485,291 bytes +``` + +A naive byte comparison would report: + +```text +task vs broad byte reduction ~= 45.66% +``` + +But both ordinary queries hit the same observed 2,000-record transfer ceiling. +The later IDs-first complete retrieval for the same broad joined representation +proved: + +```text +complete broad features 118,190 +unique planning units 471 +pages 119 +network bytes 172,127,458 +complete true +``` + +The ordinary broad response therefore returned only about 1.69% of the complete +feature cardinality and undercounted it by about 98.31%. + +## 2. Method correction + +The benchmark rule is frozen as: + +```text +Completeness Before Reduction + +scope + -> provider object-ID set + -> complete paged retrieval + -> prove retrieved IDs == source IDs + -> only then compare context burden +``` + +A smaller response is not admitted as evidence of better Task Context until the +candidate source response is proven complete for the measured scope. + +## 3. Counterexample / anti-gaming condition + +The replay intentionally keeps the attractive naive reduction visible while +rejecting it as a scoreable result. + +```text +naive task-vs-broad byte reduction ~45.66% +both ordinary queries hit 2,000 cap yes +complete broad > ordinary broad yes +headline reduction scoreable no +verdict REJECT_NAIVE_REDUCTION_PROVIDER_TRUNCATED +``` + +This prevents a pathological benchmark from becoming "better" merely because a +provider silently returned fewer records than actually existed. + +## 4. Important separation + +This proof establishes only the completeness prerequisite. It does not make the +joined layer 2 representation a valid benchmark baseline. A separate diagnostic +showed that layer 2 repeats planning-unit geometry across Pop/Emp records and is +therefore the wrong task representation. The formal planning benchmark uses base +planning-unit layer 4 plus population table 13 instead. + +So the method order is: + +```text +1. choose a task-appropriate representation +2. prove provider completeness +3. prove semantic/entity coverage +4. only then measure context reduction +``` + +## 5. What this changes + +It does **not** add a new GeoTask Core object or algorithm. + +It strengthens the benchmark discipline: + +> **Reduction without completeness evidence is not a valid GeoTask result.** + +The method is potentially reusable beyond GIS/ArcGIS providers, including RAG, +API pagination, database sampling, catalog search, and any system where a bounded +response can be mistaken for the full candidate set. + +## 6. Files + +```text +benchmarks/tc1_real/fixtures/planning_phx_20260818/provider-truncation-evidence.json +benchmarks/tc1_real/spatial_planning/provider_truncation_proof.py +tests/test_tc1_real_spatial_planning_provider_truncation.py +``` diff --git a/tests/test_tc1_real_spatial_planning_provider_truncation.py b/tests/test_tc1_real_spatial_planning_provider_truncation.py new file mode 100644 index 0000000..878bf34 --- /dev/null +++ b/tests/test_tc1_real_spatial_planning_provider_truncation.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from benchmarks.tc1_real.spatial_planning.provider_truncation_proof import ( + analyze_provider_truncation_evidence, +) + + +FIXTURE = ( + Path(__file__).resolve().parents[1] + / "benchmarks" + / "tc1_real" + / "fixtures" + / "planning_phx_20260818" + / "provider-truncation-evidence.json" +) + + +def _fixture() -> dict[str, object]: + return json.loads(FIXTURE.read_text(encoding="utf-8")) + + +def test_recorded_provider_truncation_rejects_attractive_naive_reduction() -> None: + result = analyze_provider_truncation_evidence(_fixture()) + + # A naive comparison would advertise roughly a 45.66% smaller task response. + assert result.naive_task_vs_broad_byte_reduction_ratio == pytest.approx( + 0.4565958370713049 + ) + + # But both ordinary responses hit the same 2,000-record ceiling, and the + # later IDs-first retrieval proved 118,190 complete broad features. + assert result.both_single_queries_hit_observed_ceiling + assert result.complete_broad_exceeds_single_query + assert result.naive_broad_return_fraction_of_complete == pytest.approx( + 0.016921905406548778 + ) + assert result.naive_broad_undercount_ratio == pytest.approx( + 0.9830780945934512 + ) + + assert not result.reduction_scoreable + assert result.verdict == "REJECT_NAIVE_REDUCTION_PROVIDER_TRUNCATED" + + +def test_reduction_remains_fail_closed_when_only_one_truncation_signal_is_present() -> None: + evidence = _fixture() + ordinary = evidence["ordinary_single_query"] + assert isinstance(ordinary, dict) + task = ordinary["task"] + assert isinstance(task, dict) + + # Even if the task response does not exactly hit the observed ceiling, the + # later complete broad cardinality still contradicts the ordinary broad + # response and must keep the reduction non-scoreable. + task["feature_count"] = 1500 + result = analyze_provider_truncation_evidence(evidence) + + assert not result.both_single_queries_hit_observed_ceiling + assert result.complete_broad_exceeds_single_query + assert not result.reduction_scoreable + + +def test_explicitly_incomplete_ids_first_evidence_is_rejected() -> None: + evidence = _fixture() + complete = evidence["ids_first_complete_broad"] + assert isinstance(complete, dict) + complete["complete"] = False + + with pytest.raises(ValueError, match="explicitly prove completeness"): + analyze_provider_truncation_evidence(evidence)