From 5e946d1a42f87f89d4f97e9da4fd76cb05517caa Mon Sep 17 00:00:00 2001 From: DavidWallstructurallaw Date: Tue, 22 Sep 2026 19:48:36 -0700 Subject: [PATCH 01/12] Complete Phase 5 Step 1 contracts and verification consolidation --- .github/workflows/ci.yml | 119 +- .github/workflows/golden.yml | 22 +- .github/workflows/release.yml | 36 +- .github/workflows/security.yml | 23 +- PHASE_5_DECISIONS.md | 119 + PHASE_5_PLAN.md | 278 + PHASE_5_STEP_1.md | 116 + docs/architecture.md | 14 + docs/lineage_contract.md | 414 + docs/release_process.md | 153 +- docs/theory_traceability.md | 14 +- scripts/check_spec_consistency.py | 216 +- scripts/check_traceability.py | 5797 +-------- scripts/release_check.py | 10501 +--------------- tests/conftest.py | 709 +- tests/fixtures/lineage_complete/README.md | 41 +- tests/fixtures/lineage_complete/cases.json | 566 + .../lineage_complete/hero_expected.json | 60 + tests/fixtures/lineage_cycles/README.md | 23 +- tests/fixtures/lineage_cycles/cases.json | 530 + tests/fixtures/lineage_multi_root/README.md | 26 +- tests/fixtures/lineage_multi_root/cases.json | 583 + tests/integration/test_ci_workflows.py | 194 +- tests/integration/test_cli_validation.py | 19 +- .../integration/test_current_verification.py | 155 + tests/integration/test_no_algorithms.py | 606 +- .../test_phase3_metric_pipeline.py | 185 - tests/integration/test_phase4_gates.py | 1862 --- tests/performance/README.md | 6 + tests/unit/test_PR001_identity.py | 5 - tests/unit/test_PR002_loaders.py | 5 - tests/unit/test_PR003_schema_mapping.py | 3 +- tests/unit/test_PR004_coverage.py | 8 - tests/unit/test_PR005_source_shares.py | 8 - tests/unit/test_PR006_duplicates.py | 11 - tests/unit/test_PR008_parent_resolution.py | 5 - tests/unit/test_PR009_generation.py | 10 - tests/unit/test_PR012_evidence_classes.py | 7 - tests/unit/test_PR013_report_schema.py | 7 - tests/unit/test_PR014_unavailable.py | 7 - tests/unit/test_PR015_redaction.py | 7 - tests/unit/test_PR016_determinism.py | 16 - tests/unit/test_PR017_content_refs.py | 4 - tests/unit/test_PR018_language.py | 7 - tests/unit/test_T1_compatibility.py | 5 - tests/unit/test_T1_diversity.py | 7 - tests/unit/test_T1_resampling.py | 9 - tests/unit/test_T1_support.py | 7 - tests/unit/test_T2_tail.py | 7 - tests/unit/test_T3_bounds.py | 8 - tests/unit/test_T3_provenance.py | 8 - tests/unit/test_T4_ancestry.py | 17 +- tests/unit/test_T5_reopening.py | 3 +- tests/unit/test_T6_cycles.py | 17 +- tests/unit/test_lineage_fixture_inputs.py | 94 + tests/unit/test_phase3_contracts.py | 59 +- tests/unit/test_phase4_contracts.py | 1627 --- 57 files changed, 3652 insertions(+), 21713 deletions(-) create mode 100644 PHASE_5_DECISIONS.md create mode 100644 PHASE_5_PLAN.md create mode 100644 PHASE_5_STEP_1.md create mode 100644 docs/lineage_contract.md create mode 100644 tests/fixtures/lineage_complete/cases.json create mode 100644 tests/fixtures/lineage_complete/hero_expected.json create mode 100644 tests/fixtures/lineage_cycles/cases.json create mode 100644 tests/fixtures/lineage_multi_root/cases.json create mode 100644 tests/integration/test_current_verification.py delete mode 100644 tests/integration/test_phase4_gates.py create mode 100644 tests/unit/test_lineage_fixture_inputs.py delete mode 100644 tests/unit/test_phase4_contracts.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb5db74..83ffe4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,15 +1,29 @@ -name: Phase 4 Step 11 CI +name: Current verification on: pull_request: paths-ignore: - README.md - CHANGELOG.md - - "docs/**" + - docs/architecture.md + - docs/cli.md + - docs/data_schema.md + - docs/privacy.md + - docs/release_process.md + - docs/report_schema.md + - docs/theory_traceability.md - PHASE_4_COMPLETION.md - PHASE_4_VALIDATION_REPORT.md - PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md + - PHASE_5_STEP_1.md workflow_dispatch: + inputs: + gate: + description: Focused development checks or complete candidate verification + required: true + default: focused + type: choice + options: [focused, candidate] permissions: contents: read @@ -19,10 +33,36 @@ defaults: shell: bash jobs: + focused: + if: github.event_name != 'workflow_dispatch' || inputs.gate != 'candidate' + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + PYTHONDONTWRITEBYTECODE: "1" + RIT_TEST_PARQUET: "0" + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install ".[test]" + - name: Check Step 1 contracts and affected boundaries + run: | + set -euo pipefail + python scripts/check_spec_consistency.py + python scripts/check_traceability.py + python scripts/release_check.py + python -m pytest -p no:cacheprovider -q tests/integration/test_current_verification.py tests/integration/test_ci_workflows.py tests/unit/test_lineage_fixture_inputs.py tests/unit/test_phase3_contracts.py tests/integration/test_phase3_metric_pipeline.py tests/unit/test_PR016_determinism.py tests/integration/test_no_algorithms.py tests/integration/test_repository_structure.py + core: + if: github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate' name: Core (${{ matrix.dependencies }}) on ${{ matrix.os }} with Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} - timeout-minutes: 105 + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -66,32 +106,32 @@ jobs: fi - name: Verify actual optional dependency absence run: python -c "import importlib.util; assert importlib.util.find_spec('pyarrow') is None" - - name: Run the complete core suite + - name: Run complete canonical regression run: | set -euo pipefail mkdir -p "$RUNNER_TEMP/rit-core" - python -m pytest -p no:cacheprovider -q --junitxml="$RUNNER_TEMP/rit-core/core.xml" --maxfail=1 | tee "$RUNNER_TEMP/rit-core/phase4_test_results.log" - python scripts/release_check.py --phase 4 --step 11 --junit "$RUNNER_TEMP/rit-core/core.xml" --minimum-tests 3768 - - name: Check current authorized scope and canonical suite + python -m pytest -p no:cacheprovider -q --ignore=tests/performance --junitxml="$RUNNER_TEMP/rit-core/core.xml" --maxfail=1 | tee "$RUNNER_TEMP/rit-core/tests.log" + python scripts/release_check.py --junit "$RUNNER_TEMP/rit-core/core.xml" + - name: Check current source, specifications and traceability run: | set -euo pipefail - python scripts/check_spec_consistency.py --phase 4 --step 11 - python scripts/check_traceability.py --phase 4 --step 11 - python scripts/release_check.py --phase 4 --step 11 --diff - python scripts/release_check.py --phase 4 --step 11 --baseline-evidence "$RUNNER_TEMP/rit-core" + python scripts/check_spec_consistency.py + python scripts/check_traceability.py + python scripts/release_check.py - name: Retain core evidence without masking failure if: always() uses: actions/upload-artifact@v4 with: - name: phase4-current-core-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.dependencies }} + name: current-core-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.dependencies }} path: ${{ runner.temp }}/rit-core/ if-no-files-found: error retention-days: 30 parquet: + if: github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate' name: Real optional Parquet on Ubuntu with Python 3.12 runs-on: ubuntu-latest - timeout-minutes: 75 + timeout-minutes: 30 env: PYTHONDONTWRITEBYTECODE: "1" RIT_TEST_PARQUET: "1" @@ -108,7 +148,7 @@ jobs: python-version: "3.12" - name: Install approved real Parquet extra run: python -m pip install ".[test,parquet]" - - name: Run the complete extra-enabled suite + - name: Run complete canonical regression with Parquet run: | set -euo pipefail mkdir -p "$RUNNER_TEMP/rit-parquet" @@ -117,19 +157,60 @@ jobs: python -c "import pyarrow; print('PyArrow', pyarrow.__version__)" git rev-parse HEAD > "$RUNNER_TEMP/rit-parquet/source-commit.txt" free --bytes | tee "$RUNNER_TEMP/rit-parquet/host-memory-bytes.txt" - python -m pytest -p no:cacheprovider -q --junitxml="$RUNNER_TEMP/rit-parquet/parquet.xml" --maxfail=1 | tee "$RUNNER_TEMP/rit-parquet/phase4_parquet_results.log" - python scripts/release_check.py --phase 4 --step 11 --junit "$RUNNER_TEMP/rit-parquet/parquet.xml" --require-parquet --minimum-tests 3771 - python scripts/release_check.py --phase 4 --step 11 --baseline-evidence "$RUNNER_TEMP/rit-parquet" + python -m pytest -p no:cacheprovider -q --ignore=tests/performance --junitxml="$RUNNER_TEMP/rit-parquet/parquet.xml" --maxfail=1 | tee "$RUNNER_TEMP/rit-parquet/tests.log" + python scripts/release_check.py --junit "$RUNNER_TEMP/rit-parquet/parquet.xml" --require-parquet - name: Retain Parquet evidence without masking failure if: always() uses: actions/upload-artifact@v4 with: - name: phase4-current-real-parquet + name: current-real-parquet path: ${{ runner.temp }}/rit-parquet/ if-no-files-found: error retention-days: 30 + performance: + if: github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate' + runs-on: ubuntu-latest + timeout-minutes: 75 + env: + PYTHONDONTWRITEBYTECODE: "1" + RIT_TEST_PARQUET: "0" + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install ".[test]" + - name: Measure reference profile including required 100k workload + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/rit-performance" + python -m pip freeze > "$RUNNER_TEMP/rit-performance/dependencies.txt" + python -c "import platform; print(platform.platform(), platform.python_version())" + free --bytes + python -m pytest -p no:cacheprovider -q tests/performance --junitxml="$RUNNER_TEMP/rit-performance/performance.xml" | tee "$RUNNER_TEMP/rit-performance/tests.log" + python scripts/release_check.py --junit "$RUNNER_TEMP/rit-performance/performance.xml" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: current-reference-performance + path: ${{ runner.temp }}/rit-performance/ + if-no-files-found: error + retention-days: 30 + + hero: + if: github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate' + uses: ./.github/workflows/golden.yml + + security: + if: github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate' + uses: ./.github/workflows/security.yml + delivery: name: Build verified candidate - needs: [core, parquet] + needs: [core, parquet, performance, hero, security] uses: ./.github/workflows/release.yml diff --git a/.github/workflows/golden.yml b/.github/workflows/golden.yml index 5fea5d7..b0eec2a 100644 --- a/.github/workflows/golden.yml +++ b/.github/workflows/golden.yml @@ -1,14 +1,7 @@ -name: Phase 4 Step 11 Hero and Mathematical Contract +name: Hero and mathematical contract on: - pull_request: - paths-ignore: - - README.md - - CHANGELOG.md - - "docs/**" - - PHASE_4_COMPLETION.md - - PHASE_4_VALIDATION_REPORT.md - - PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md + workflow_call: workflow_dispatch: permissions: @@ -41,16 +34,15 @@ jobs: run: | set -euo pipefail mkdir -p "$RUNNER_TEMP/rit-math" - python scripts/check_spec_consistency.py --phase 4 --step 11 - python -m pytest -p no:cacheprovider -q tests/integration/test_hero_structure.py tests/integration/test_hero_end_to_end.py tests/integration/test_phase3_metric_pipeline.py tests/golden/test_phase3_math.py tests/golden/test_phase4_reports.py --junitxml="$RUNNER_TEMP/rit-math/math.xml" | tee "$RUNNER_TEMP/rit-math/phase4_math_validation.log" - python scripts/release_check.py --phase 4 --step 11 --golden-junit "$RUNNER_TEMP/rit-math/math.xml" - python scripts/release_check.py --phase 4 --step 11 --diff + python scripts/check_spec_consistency.py + python -m pytest -p no:cacheprovider -q tests/integration/test_hero_structure.py tests/integration/test_hero_end_to_end.py tests/integration/test_phase3_metric_pipeline.py tests/golden/test_phase3_math.py tests/golden/test_phase4_reports.py --junitxml="$RUNNER_TEMP/rit-math/math.xml" | tee "$RUNNER_TEMP/rit-math/tests.log" + python scripts/release_check.py --junit "$RUNNER_TEMP/rit-math/math.xml" + python scripts/release_check.py - name: Retain actual Hero and mathematical evidence if: always() uses: actions/upload-artifact@v4 with: - name: phase4-current-hero-mathematical + name: current-hero-mathematical path: ${{ runner.temp }}/rit-math/ if-no-files-found: error retention-days: 30 -# Frozen mathematical expectations are unchanged; Step 11 Hero reports use independent assertions. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1a9558f..55d243f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Phase 4 Step 11 Build and Candidate Evidence +name: Package and candidate verification on: workflow_call: @@ -33,19 +33,19 @@ jobs: - name: Reuse complete core results from this candidate run uses: actions/download-artifact@v4 with: - name: phase4-current-core-ubuntu-latest-3.12-current - path: ${{ runner.temp }}/rit-phase4-current/core-evidence/ + name: current-core-ubuntu-latest-3.12-current + path: ${{ runner.temp }}/rit-candidate/core-evidence/ - name: Reuse complete real-Parquet results from this candidate run uses: actions/download-artifact@v4 with: - name: phase4-current-real-parquet - path: ${{ runner.temp }}/rit-phase4-current/parquet-evidence/ + name: current-real-parquet + path: ${{ runner.temp }}/rit-candidate/parquet-evidence/ - name: Install approved build and test tools run: python -m pip install ".[release,test,parquet]" "setuptools>=82" - name: Check authority, declarations and current-step scope run: | set -euo pipefail - evidence="$RUNNER_TEMP/rit-phase4-current" + evidence="$RUNNER_TEMP/rit-candidate" test "$(cat "$evidence/core-evidence/source-commit.txt")" = "$(git rev-parse HEAD)" test "$(cat "$evidence/parquet-evidence/source-commit.txt")" = "$(git rev-parse HEAD)" cp "$evidence/core-evidence/core.xml" "$evidence/core.xml" @@ -53,17 +53,17 @@ jobs: python -m pip freeze > "$evidence/build-dependencies.txt" python -m pip check python -c "import pyarrow; print('PyArrow', pyarrow.__version__)" - python scripts/check_spec_consistency.py --phase 4 --step 11 - python scripts/check_traceability.py --phase 4 --step 11 - python scripts/release_check.py --phase 4 --step 11 --diff + python scripts/check_spec_consistency.py + python scripts/check_traceability.py + python scripts/release_check.py rit version - - name: Build and repeat the dev3 artifacts from the same committed source + - name: Build and repeat the development artifacts from the same committed source run: | set -euo pipefail - evidence="$RUNNER_TEMP/rit-phase4-current" + evidence="$RUNNER_TEMP/rit-candidate" export SOURCE_DATE_EPOCH="$(git show -s --format=%ct HEAD)" printf '%s\n' "$SOURCE_DATE_EPOCH" > "$evidence/source-date-epoch.txt" - python -m build --no-isolation --outdir "$evidence/dist" | tee "$evidence/phase4_build_results.log" + python -m build --no-isolation --outdir "$evidence/dist" | tee "$evidence/build.log" python -m twine check --strict "$evidence/dist/"* mkdir -p "$evidence/rebuild-source" git archive HEAD | tar -x -C "$evidence/rebuild-source" @@ -79,7 +79,7 @@ jobs: from pathlib import Path import tarfile - evidence = Path(os.environ["RUNNER_TEMP"]) / "rit-phase4-current" + evidence = Path(os.environ["RUNNER_TEMP"]) / "rit-candidate" first = {p.name: p for p in (evidence / "dist").iterdir() if p.is_file()} second = {p.name: p for p in (evidence / "rebuild-dist").iterdir() if p.is_file()} assert first.keys() == second.keys() and len(first) == 2 @@ -109,17 +109,17 @@ jobs: (evidence / "reproducibility.json").write_text(json.dumps(result, indent=2) + "\n") print(json.dumps(result, indent=2)) PY - - name: Verify installed packages and produce the byte-verified Step 11 candidate + - name: Verify installed packages and produce the byte-verified current candidate run: | set -euo pipefail - python scripts/release_check.py --phase 4 --step 11 --candidate "$RUNNER_TEMP/rit-phase4-current" | tee "$RUNNER_TEMP/rit-phase4-current/installed-candidate.log" + python scripts/release_check.py --candidate "$RUNNER_TEMP/rit-candidate" | tee "$RUNNER_TEMP/rit-candidate/installed-candidate.log" - name: Retain candidate or failure evidence without masking failure if: always() uses: actions/upload-artifact@v4 with: - name: recursive-integrity-toolkit-phase4-current-candidate + name: recursive-integrity-toolkit-current-candidate path: | - ${{ runner.temp }}/rit-phase4-current/ - !${{ runner.temp }}/rit-phase4-current/rebuild-source/ + ${{ runner.temp }}/rit-candidate/ + !${{ runner.temp }}/rit-candidate/rebuild-source/ if-no-files-found: error retention-days: 90 diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index e193294..4b291ca 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -1,14 +1,7 @@ -name: Phase 4 Step 11 Security Boundary +name: Security boundaries on: - pull_request: - paths-ignore: - - README.md - - CHANGELOG.md - - "docs/**" - - PHASE_4_COMPLETION.md - - PHASE_4_VALIDATION_REPORT.md - - PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md + workflow_call: workflow_dispatch: permissions: @@ -73,17 +66,15 @@ jobs: tests/integration/test_partial_provenance_report.py \ tests/unit/test_phase4_output_safety.py \ tests/integration/test_phase4_cli.py \ - tests/unit/test_phase4_contracts.py \ - tests/integration/test_phase4_gates.py \ - --junitxml="$RUNNER_TEMP/rit-security/security.xml" | tee "$RUNNER_TEMP/rit-security/phase4_current_security_results.log" - python scripts/release_check.py --phase 4 --step 11 --junit "$RUNNER_TEMP/rit-security/security.xml" - python scripts/check_traceability.py --phase 4 --step 11 - python scripts/release_check.py --phase 4 --step 11 + --junitxml="$RUNNER_TEMP/rit-security/security.xml" | tee "$RUNNER_TEMP/rit-security/tests.log" + python scripts/release_check.py --junit "$RUNNER_TEMP/rit-security/security.xml" + python scripts/check_traceability.py + python scripts/release_check.py - name: Retain actual security test evidence if: always() uses: actions/upload-artifact@v4 with: - name: phase4-current-security-boundary + name: current-security-boundary path: ${{ runner.temp }}/rit-security/ if-no-files-found: error retention-days: 30 diff --git a/PHASE_5_DECISIONS.md b/PHASE_5_DECISIONS.md new file mode 100644 index 0000000..074f9e1 --- /dev/null +++ b/PHASE_5_DECISIONS.md @@ -0,0 +1,119 @@ +# Phase 5 Decisions and Step 1 Authorization + +Status: **PLAN APPROVED; STEP 1 AUTHORIZED; PHASE INCOMPLETE**. + +On 2026-09-23 UTC the Theory Owner instructed: +`批准，Phase 5 step 1开始`. +This accepts the delivered Phase 5 plan and its eleven decisions without stated +exceptions, and authorizes Step 1. Execution remains one authorized step at a +time. Step 2, a main merge, a tag and software publication are not authorized by +this instruction. + +The accepted starting commit is +`1db3b1a460c233242ff37fbe45b8fac74d6101ef`, on `phase4-reports-cli`. +The Phase 5 implementation branch is `phase5-lineage`. The package remains +`0.1.0.dev3` and the executable report schema remains `1.0` during Step 1. +The original standalone plan records its draft submission; the repository copy +and this document record the subsequent actual approval. No new machine-readable +approval registry or historical source-binding chain is introduced. + +## Approved contract decisions + +| Decision | Status | Implementation boundary | +|---|---|---| +| P5-D01 | APPROVED | One primary target population; explicitly loaded context supplies paths and roots without entering target denominators. Empty targets have unavailable ratios and no invented version. | +| P5-D02 | APPROVED | Reuse canonical reference resolution, chronology, aliases and declaration states. Retain per-record failures and successful reference evidence in a bounded batch refactor. Original declarations determine reference coverage; unique edges determine graph behavior. | +| P5-D03 | APPROVED | Iterative cycle analysis; cycle count means cyclic strongly connected components. Distinguish cycle members and affected descendants. Depth measures fully resolved structural parent paths. | +| P5-D04 | APPROVED | Explicitly parentless grounded anchors; grounded single-parent carryover inherits roots. Unsupported grounded-with-parent forms remain unresolved. New external inputs use separately loaded explicit anchors. | +| P5-D05 | APPROVED | Complete root sets only for exact ancestry metrics; partition targets into grounded, closed and unresolved. Unknown required-path grounding and unresolved branches propagate uncertainty. | +| P5-D06 | APPROVED | Incidence shares use all targets N; fractional root weights use the complete nonempty-root population G. HHI and effective roots retain F-012/F-013 and their topological meaning. | +| P5-D07 | APPROVED | Lineage closure is `[C/N, (C+U)/N]`; direct closure is unchanged. Shared-root presence is descriptive and discloses partial coverage. | +| P5-D08 | APPROVED | Schema 1.1 in Step 7; typed immutable results, twelve existing report sections and equal capability mirrors. Protect every new identity; omitted detail remains schema-valid. | +| P5-D09 | APPROVED | Explicit `--lineage` on audit/example and repeatable local lineage context inputs. Primary/context versions stay disjoint; validation remains input-only. | +| P5-D10 | APPROVED | Finite graph, edge, root-membership and root-union work limits; bounded diagnostics; honest unavailable outcomes after exhaustion. | +| P5-D11 | APPROVED | Consolidate current verification around behavior and bounded candidate gates. Recover obsolete implementation checks from Git/history without requiring them in every execution. | + +The operational field names, shapes, defaults and edge cases are fixed in +[docs/lineage_contract.md](docs/lineage_contract.md). That document specifies +future implementation contracts; Step 1 adds no graph execution or schema 1.1 +runtime support. Implementation steps retain their dependencies in the approved +plan. + +## Step 1 clarifications within the approved decisions + +1. **Carryover aliases:** exactly one parent means one canonical resolved parent + after the existing resolver groups aliases/repeated declarations. Extra + unresolved declarations still prevent complete inheritance. Multiplicity is + retained in reference coverage, never ancestry mass. +2. **Depth versus grounding:** depth depends on complete structural parent paths. + An explicitly parentless unknown-grounding record has depth zero while its + external-root set remains unresolved. A whole-target maximum is unavailable if + any target's required structural path is incomplete. A resolved-subset maximum + is a separately labeled diagnostic. +3. **Proxy wire vocabulary:** the conceptual absent outcome uses the existing + `not_present` value. Unavailable uses `status: unavailable` with + `level: indeterminate`; no synonymous enum or risk scale is added. +4. **Diagnostic bounds:** each lineage detail table/key collection returns at most + 100 entries. A cycle witness contains at most 64 edges. Longer witnesses are + omitted with an explicit size reason; a nonclosed prefix is never presented as + a cycle witness. Exact aggregate counts survive detail omission. +5. **Resource interface:** the approved defaults are 200,000 loaded nodes, + 1,000,000 unique edges, 1,000,000 stored root memberships and 10,000,000 root-union + candidate visits. Limits reject booleans, nulls, nonintegers and nonpositive + values. Existing input limits retain their separate meaning. +6. **Empty scope:** N = G = C = U = 0 is a valid empty analytical scope. Coverage, + closure and concentration ratios are unavailable. Exact zero declared + references retain the executed-lineage convention of coverage 1.0 with + `no_declared_parents=true`; this does not establish external grounding or a + negative shared-root finding. + +These clarify implementation details requested by Step 1. They add no scientific +formula, inferred parent, external-input evidence schema or automatic longitudinal +behavior. A genuine conflict with the approved contracts must be surfaced before +dependent implementation. + +## Current verification boundary + +The user-approved Verification Governance and Complexity Control instruction +governs consolidation. Preserve direct mathematical, input, provenance, privacy, +CLI and packaging guarantees. Identify the behavior of each retired historical +check in the Step 1 change explanation and retain its equivalent current check. +Git history and archived phase records preserve prior bodies and receipts. + +Current consistency/dispatch checks continue to reject unauthorized product +changes where that boundary applies. Frozen Phase 0 specifications, the six +canonical Hero files, runtime modules, executable schemas and package version +remain unchanged in Step 1. Current canonical regression remains executable; +release verification retains compatibility, clean installs, reproducibility and +artifact integrity at suitable candidate gates. + +Do not add a new Tier D mechanism without identifying its exact failure, why +Tier A/B/C cannot detect it, why Git/archive cannot suffice, maintenance cost and +an exit/consolidation path. This step proposes no such mechanism. Historical +Phase 4 approval records retain their historical scope and are not rewritten as +new Phase 5 authority. + +The Step 1 consolidation retains these current protections while retiring the +corresponding historical execution machinery: + +| Historical mechanism consolidated | Protected behavior retained in current checks | +|---|---| +| Snapshot fixtures, staged path/migration assertions and historical-only Phase 4 gate suites | Current positive checks and negative tests reject unauthorized Step 1 product changes and frozen-specification tampering. | +| Exact owner-wrapper/docstring/placeholder assertions around implemented modules | Direct mathematical, representation, provenance, bounds, reference, generation, report, privacy and CLI tests exercise current supported behavior. The PR-003 operation inventory and T5 deferred public boundary remain explicit. | +| Historical package/schema/Hero source bindings | Current checks reject packaged schema or Hero corruption and protect frozen Hero inputs/expectations. | +| Repeated historical workflow/phase dispatch assertions | One current dispatch path and current workflow behavior checks retain focused ordinary gates and candidate compatibility/delivery roles. | + +Input-only validation and absence of analytical/network/IO side effects retain +their direct integration/security cases. Historical source forms remain +recoverable from the accepted starting commit and archived Phase 4 evidence. +This table explains consolidation; it is not a new executable registry. + +## Acceptance and handoff + +Step 1 requires reviewed typed contracts, independent fixture expectations and +affected consistency/dispatch/behavioral boundary checks. It does not require a +release matrix or demonstrate graph correctness. Actual changes, check results, +failures and unresolved limitations belong in the Step 1 completion report. +This approval record makes no claim that those checks have already passed. + +Stop after Step 1. Phase 5 Step 2 requires its own execution instruction. diff --git a/PHASE_5_PLAN.md b/PHASE_5_PLAN.md new file mode 100644 index 0000000..8c9faf0 --- /dev/null +++ b/PHASE_5_PLAN.md @@ -0,0 +1,278 @@ +# PHASE_5_PLAN + +## Document control + +| Field | Value | +|---|---| +| Project | Recursive Integrity Toolkit | +| Phase | Phase 5: lineage graph and ancestry | +| Document version / date | 1.0 / 2026-09-23 | +| Status | APPROVED; Step 1 authorized | +| Theory Owner | Xiangyu Guo | +| Accepted starting commit | `1db3b1a460c233242ff37fbe45b8fac74d6101ef` | +| Accepted source tree | `51d8854d9869b8a0aeb498230e02b03797ed8634` | +| Baseline branch / package | `phase4-reports-cli` / `0.1.0.dev3` | +| Proposed implementation branch | `phase5-lineage`, from the accepted commit after Step 1 authorization | +| Proposed completed-phase version | `0.1.0.dev4`; this is not a final v0.1 release | +| Authorization recorded | Plan approved and Step 1 started by the Theory Owner on 2026-09-23 | +| Merge, tag, publication authorization | None | + +The Theory Owner approved this plan and authorized Step 1 on 2026-09-23: “批准，Phase 5 step 1开始”. All P5-D decisions below are approved without stated exceptions. Execution continues one authorized step at a time; this approval does not automatically execute all ten steps. See `PHASE_5_DECISIONS.md` for the Step 1 contract record. + +## 1. Starting point and governing constraints + +The inspected checkout is clean at the accepted commit: 246 tracked files and 40 Python package modules. All sixteen frozen Phase 0 specifications match the existing approved baseline. PR #1 remains open and draft, with the accepted commit as its head. Starting the proposed Phase 5 branch does not require merging that PR. + +Phase 4 delivers input validation, existing metrics, JSON/Markdown reports, privacy modes, a local audit CLI, one explicitly requested comparison pair and an installed Hero example. General graph traversal and ancestry remain deferred. The lineage modules and their designated tests/fixtures provide the intended implementation locations. + +Historical Phase 4 evidence includes twelve successful candidate jobs: eight supported core profiles at 3,965 tests each, a real-Parquet profile at 3,968, Hero at 203 and security at 3,277, plus delivery checks. The verified code candidate was `ceb0cf702284dbdbe50b14722026ec26fbcc21ba`; the accepted successor changed three administrative reports. These are preservation evidence, not Phase 5 results. No product tests were rerun to write this plan. + +Apply `PROJECT_INSTRUCTIONS.md` section 4.4 and the approved UD decisions. The primary Phase 5 owners are **T4, T6, PR-008 full graph behavior, PR-009 lineage depth, and T3 lineage bounds**. P3-D01 reserves these functions for Phase 5 despite older overlapping roadmap wording. The plan introduces no new interpretation of the theory papers or new scientific formula. + +The user's Verification Governance and Complexity Control instruction governs this phase: + +- Strengthen direct product/scientific correctness and real security boundaries. +- Keep compatibility, packaging and reproducibility gates at suitable candidate milestones. +- Consolidate historical implementation checks when equivalent current behavioral coverage exists and Git/archive evidence remains recoverable. +- Add no new approval registry, source-body migration chain, execution-receipt framework or evidence-of-evidence layer by default. + +## 2. Objective and exclusions + +Deliver an explicitly requested, local lineage analysis over validated records: parent graph, cycle detection, ancestry resolution, external-root incidence and concentration, lineage coverage, depth and lineage-aware closure bounds. Integrate the results into typed reports and the CLI while preserving existing metric scopes and privacy guarantees. + +Phase 6A retains automatic multi-version orchestration, trajectories, lineage-change comparisons and additional longitudinal formulas. Phase 6B retains experiment/reopening workflows. Also excluded are network retrieval, inferred parents, causal root weights, semantic ancestry, HTML implementation, model-performance conclusions, universal risk thresholds, new runtime graph dependencies and unrestricted graph/path exports. + +## 3. Approved contract decisions + +### P5-D01. Separate the target population from the supporting graph + +Use canonical `RecordKey(dataset_version, record_id)` identities. The analyzed target contains N selected records from one primary version. The loaded graph may also contain explicitly supplied ancestor records. Only target records enter target counts, provenance shares, closure denominators, ancestry incidence shares and concentration allocation. Context records can supply roots and paths without becoming additional observations. + +Reports identify the target version/count and loaded graph counts separately. Existing representation eligibility exclusions remain specific to their metrics; they must not silently shrink the provenance or lineage denominator. An empty target produces explicit unavailable ratios and no invented version identity. + +### P5-D02. Reuse validated references and preserve declaration semantics + +Consume existing parent-resolution results, aliases, version order and declaration states. Do not implement a competing interpretation of parent IDs. Preserve the distinction between an absent/null declaration and explicit `parent_ids=[]`. + +The current bundle validator loses its aggregate generation/parent result after the first malformed, ambiguous, future-parent or self-parent exception. Step 2 therefore permits a narrow batch resolver refactor that retains per-record successes, failures and explicit self-reference evidence, reusing one identity lookup index. Preserve public validation/error and generation semantics. Invalid references never become accepted ancestry edges; self-reference evidence remains available for cycle diagnostics. This retention is necessary to report unaffected records honestly and must not require rebuilding an O(V) index for each record. + +Resolve cross-version edges only under the existing explicit chronology. Same-version edges are allowed when acyclic. Duplicate canonical identities fail validation. Missing parents remain unresolved and warning-level by default, with existing strict-mode escalation; ambiguity and future-parent violations remain errors. + +Reference coverage counts original declared references, including aliases or repeated references, using the existing resolver's denominator. Graph adjacency and root unions deduplicate canonical edges/roots. Thus repeated declarations cannot inflate ancestry mass. With no declared references, edge coverage is 1.0 and `no_declared_parents=true`; this alone proves neither grounding nor ancestry completeness. + +This zero-reference convention follows the frozen definition and deliberately replaces the current report's null/unavailable convention when lineage executes under schema 1.1. With lineage disabled, retain the existing immediate-reference observation and null reason for its zero denominator, clearly labeled as validation evidence; do not synthesize executed graph coverage. Test both modes explicitly. + +### P5-D03. Define cycles, affected records and depth precisely + +Use iterative graph algorithms. Report `cycle_count` as the number of **cyclic strongly connected components**, explicitly labeled with that method: a component has multiple nodes or a self-loop. This resolves the previously unspecified counting convention without enumerating every simple cycle. + +Distinguish cycle members from affected descendants along parent-to-child edges. Return a bounded deterministic witness per reported component, total counts and explicit detail-omission metadata. A detected cycle is a lineage validation error. Affected target records have unresolved ancestry; independent families and unaffected target ancestry may remain reportable with partial status and precise scope. Never label the entire loaded graph acyclic when a disconnected cycle exists, and never return a successful CLI audit status for a lineage error. + +Depth is the maximum number of edges to loaded parentless roots on fully resolved paths. An incomplete or invalid required path produces null with a reason. Known partial depth may be diagnostic only when labeled as a lower bound. Generation and lineage depth remain distinct: a carryover can have generation zero and positive depth. + +### P5-D04. Adopt a conservative, schema-preserving external-root rule + +An explicitly parentless record with validated `external_grounding=yes` anchors an external root. A parentless record with grounding `no` has a known-empty root set; grounding `unknown`, missing required provenance or an undeclared parent boundary makes root resolution incomplete. + +A grounded `transformation=carryover` record with exactly one resolved parent inherits that parent's external roots. It does not create a new root merely because its own grounding is `yes`. A carryover with additional unresolved declarations is incomplete. + +Represent a new independent external input as a separately loaded, explicitly parentless, grounded record and link it as an ordinary parent. No evidence URI, source label, review flag or transformation string alone certifies independence in Phase 5. This pattern supports truthfully ungrounded child transformations and unambiguous single-parent grounded carryovers. + +A grounded record with parents outside the single-parent carryover rule remains unresolved in Phase 5, even when a separate grounded anchor is present. A different rule would require an explicit contract decision. Multi-parent carryover is not silently collapsed. Never change a child's actual grounding label merely to fit the supported representation. + +Strict ancestors exclude the record itself. Separately, an external root supports itself: its support-root set is `{its own key}`. This convention must be named explicitly wherever root support is exposed. + +### P5-D05. Require complete root sets for exact ancestry metrics + +For each target record r, compute its unique external-root set A_r only when every required path is resolved under P5-D02 through P5-D04. Unknown grounding anywhere on a required path propagates uncertainty. One known root plus an unresolved branch is insufficient for an exact root distribution. + +Partition targets into grounded G, closed C and unresolved U: + +- G: complete, nonempty external-root set. +- C: complete, known-empty external-root set. +- U: incomplete or invalid ancestry, including missing/ambiguous links, cycles, unknown grounding, conflicting declarations and undeclared boundaries. + +N = |G| + |C| + |U|. `records_with_resolved_external_ancestry` includes G and C. Resolved lineage coverage is (|G|+|C|)/N; external ancestry coverage is |G|/N. Optional observed roots on incomplete paths are diagnostics only and cannot enter exact root metrics. + +### P5-D06. Preserve the approved incidence and fractional-mass formulas + +For each unique root a among G: + +```text +I_a = count of r in G for which a belongs to A_r +incidence_share_a = I_a / N +M_a = sum over r in G containing a of 1 / |A_r| +w_a = M_a / |G| +ancestry_HHI = sum_a w_a^2 +effective_external_root_count = 1 / ancestry_HHI +``` + +Each grounded target contributes total mass one, including a target that is itself an external root. Rank roots by decreasing incidence, breaking ties with canonical record key. Distinct roots are the union over G. All reported shares state their denominators. If G is empty, HHI and effective-root count are unavailable/undefined with reasons; they are not zero or infinity. + +These are topological allocation conventions. They do not estimate causal contribution, semantic error, independent information, biological relatedness or universal integrity. Multiple paths to the same root must not multiply that root's contribution. + +### P5-D07. Add lineage bounds without changing direct bounds + +For nonempty targets, lineage closure bounds are: + +```text +lower = |C| / N +upper = (|C| + |U|) / N +width = |U| / N +``` + +Keep direct closure calculations and their classifications unchanged. Report coverage and unresolved reasons alongside lineage bounds. A shared-root dependence signal is a descriptive proxy: `present` when a completely resolved root supports at least two targets; `absent` only when all target ancestry is resolved and no such root exists; otherwise `unavailable`. A witnessed `present` on partial coverage discloses that coverage. This is not a calibrated risk level. + +T4 retains its theory-guided operationalization label. T6 is an engineering graph-validity rule, not a theorem establishing the theory's scientific claims. + +### P5-D08. Use one report schema revision and protect new identifiers + +Use `report_schema_version: "1.1"` for all Phase 5 reports. Retain the twelve top-level sections, equal capability mirrors, finite values, explicit unavailable reasons and unchanged non-lineage numerical semantics. The root schema and packaged schema remain identical. + +Extend typed immutable results for lineage scope, resolution counts, roots/incidence, concentration, bounds, depth and cycles. Values are computed before rendering. JSON and Markdown share the same canonical result; renderers own no graph or mathematical logic. + +Lineage execution defaults to `not_requested`; explicit execution yields `completed`, `partial` or `failed` according to actual coverage/errors. Input observability remains distinct from execution. More loaded versions do not mean a comparison ran. + +Apply existing privacy modes to every new key, path and witness. Redacted output supports an explicit aggregate-only or omitted-detail representation that still validates against the schema. Removing required row fields while leaving invalid rows is unacceptable. Preserve no-network, safe-output and deterministic identifier behavior. + +Strict schema-1.0 consumers must explicitly adopt 1.1. This plan does not promise backward acceptance by those readers or add a report migration framework. + +### P5-D09. Make lineage opt-in and context inputs explicit + +Add `--lineage` to `audit` and `example`. Add repeatable `--lineage-records PATH`, backed by a dedicated `FileRole.LINEAGE_CONTEXT` and existing local loaders. For `audit`, context inputs require lineage opt-in. Record/provenance row schemas remain unchanged; config input-role and report inventory enums expand coherently. + +`--records` selects exactly one primary version. `--compare` retains its existing CLI limit of one explicit earlier comparison input and all current compatibility declarations. With lineage requested, that comparison input may also supply ancestors; supplying context does not automatically request comparison. + +Context versions must be disjoint from primary and comparison versions. This keeps the existing whole-version provenance join unambiguous. Same-version lineage is supported when the needed same-version records are in the primary input. Splitting one version across target and context roles is outside this phase. Duplicate keys across repeated context inputs are rejected. + +All ordinary metric calls explicitly select the primary population instead of pooling loaded versions. Preserve existing Python/config multiplicity for other supported input roles; a CLI restriction must not accidentally narrow those APIs. + +`validate` may ingest context inputs for existing input/reference validation, but rejects the lineage execution flag and never runs graph metrics. `validate_bundle(...)` remains input-only. `rit example --lineage --out ...` executes the frozen Hero with installed resources outside the checkout. Plain `rit example` retains ordinary calculations with lineage `not_requested` under schema 1.1. + +### P5-D10. Bound graph work and disclose resource failures + +Use standard-library iterative adjacency/SCC/topological algorithms. Graph traversal should be O(V+E); root-set propagation can grow with V times the number of roots. Sparse edges alone do not guarantee bounded root-set memory. + +Approved initial configurable limits for explicitly requested lineage are 200,000 loaded nodes, 1,000,000 unique edges, 1,000,000 stored record/root memberships and 10,000,000 root-union candidate visits. A visit means processing one candidate root identity into a child's union. These finite engineering guards bound algorithmic work; they do not guarantee a particular peak RSS or establish scientific thresholds. Check cumulative node admission across primary, comparison and context batches before graph materialization, edge admission before adjacency insertion, and root memberships/work before union insertion. Retain the separate inherited input byte/row/parent-list limits: graph guards do not bound allocations already incurred while loading a table. Caller overrides must be explicit positive finite integers. + +Guard details and counters must be deterministic. On exhaustion, provide a resource-limit reason and no exact lineage values based on truncated work. Preserve independently completed non-lineage families. Bound report diagnostics and witness output separately, disclosing omitted counts; do not export all paths or a full transitive closure. Step 1 fixes the diagnostic row cap using existing report conventions; Step 9 measures whether the proposed defaults need a documented adjustment. No silent limit increase is permitted to make a gate pass. + +### P5-D11. Consolidate current verification instead of extending its history + +Maintain three practical bodies: current canonical regression, candidate/release verification, and historical evidence recoverable from Git and archived phase records. Use existing test locations and tooling; this classification does not require new registries. + +At this phase boundary, replace obsolete placeholder, exact historical test-body and source-wrapper assertions with equivalent or stronger current behavioral tests. Identify the protected behavior in the existing decision/PR explanation, retain its direct test and keep the old evidence recoverable. For example, unauthorized product mutation must still be rejected where that boundary applies; the exact old Python method body need not remain executable forever. + +Consolidate the existing phase dispatch in spec/traceability/release checks and workflows into one current path. Do not clone the Phase 4 framework into a new permanent chain. Preserve frozen source specifications and historical evidence; do not rewrite them to manufacture approval. + +Any proposed new Tier D control needs all five justifications: exact failure; why A/B/C cannot detect it; why Git/archive is insufficient; maintenance cost; consolidation/exit path. Otherwise omit it. No new mechanism of that kind is proposed here. + +## 4. Ten implementation steps + +Each step ends with its actual changes, relevant checks and remaining limitations. Dependent work starts only after its preceding contracts are settled. Failed checks are repaired before dependent execution; no step is complete merely because its files exist. + +| Step | Deliverable and dependencies | Acceptance and smallest sufficient gate | +|---|---|---| +| **1. Contracts and current verification boundary** | Record actual approval/any exceptions in a concise Phase 5 decision document; finalize typed field names, bounded diagnostic detail and resource interfaces; specify independent fixture expectations. Consolidate obsolete current-phase dispatch/assertions under P5-D11. | Every behavior retired from historical execution has current equivalent coverage and recoverable evidence. Check affected consistency/dispatch and behavioral boundary tests only. No graph completion claims. | +| **2. Validated parent graph** | Implement graph types/construction in `lineage/graph.py`, consuming the resolver with the bounded retention refactor in P5-D02. Establish target/context scope and immutable adjacency. Depends on 1. | Identity, duplicate declarations, aliases, same-version edges, chronology, missing/ambiguous parents, retained partial results and role scope cases pass. Run parent-resolution and relevant generation neighbors. | +| **3. Cycles, topology and depth** | Implement iterative SCC detection, affected descendants, bounded witnesses, topological ordering and depth. Depends on 2. | Self-loop, two-node/long/disconnected cycles, descendant propagation and a chain beyond Python recursion depth behave as specified. Errors never claim completed whole-graph lineage. | +| **4. Root resolution and coverage** | Implement external anchors, carryover inheritance, unique root unions and G/C/U classification in `lineage/ancestry.py`. Depends on 2-3. | Parentless yes/no/unknown, absent versus empty declarations, multi-parent unions, incomplete branches and conservative grounded-parent cases match independent fixtures. | +| **5. Incidence and concentration** | Add typed root incidence, denominator metadata, fractional mass, HHI and effective roots. Depends on 4. | Hero and multi-root arithmetic match exact/rational oracles; renaming, permutation and duplicate-path invariance hold. Empty G has explicit unavailable values. | +| **6. Lineage closure and descriptive proxy** | Integrate lineage bounds and shared-root proxy, leaving direct bounds unchanged. Depends on 4-5. | G/C/U partition and interval identities hold; missing and unknown propagation are tested independently; direct-bound regressions remain unchanged. | +| **7. Canonical reports and privacy** | Integrate typed results, schema 1.1 and its packaged mirror, assembly, JSON/Markdown and redaction. Depends on 3-6. | Non-lineage values/scopes remain stable; complete/partial/failed/not-requested results validate; new keys and witnesses obey privacy; no renderer-owned calculation or invalid redacted row. | +| **8. Explicit CLI and installed Hero** | Wire `--lineage`, context role/flags and explicit primary selection; document usable commands. Depends on 7. | Audit opt-in, context-only ancestry, existing comparison behavior, input-only validate, meaningful exit codes and installed Hero pass. Exercise core inputs and real Parquet where the new role crosses that loader boundary. | +| **9. Bounded adversarial, mutation and scale work** | Resolve remaining graph/resource defects, measure lineage workloads, run a finite meaningful mutation set. Depends on 2-8. | Resource failure is honest; no recursion dependence; 100k sparse graph completes on the designated reference profile; end-to-end lineage is measured; meaningful seeded mutations are killed or remaining gaps explained and corrected. | +| **10. Candidate verification and handoff** | Update public docs/traceability and package to dev4; complete canonical regression, supported candidate matrix and delivery checks; record concise completion and limitations. Depends on 1-9. | Frozen Hero expectations hold, required owners implemented, supported installs and reproducibility verified, no Phase 6 execution or publication implied. Administrative report successors reuse unchanged candidate evidence. | + +The existing generation validator uses repeated propagation scans. If the deep-chain/100k workload demonstrates a bottleneck there, Steps 3 or 9 may replace that propagation with a behavior-preserving iterative dependency algorithm. Preserve existing generation semantics and run its direct/dependency-neighbor tests. This narrow allowance does not authorize unrelated input or metric redesign. + +## 5. Independent acceptance oracles + +### 5.1 Frozen Hero + +Keep the six canonical Hero input/expectation files unchanged. Construct current report goldens from their approved values rather than editing the oracle to fit output. + +| Quantity | Required value | +|---|---| +| Primary target / loaded records | v2: 8 / v1 plus v2: 16 | +| v1/v2 support | 8 / 5 | +| v1/v2 diversity / retention | 0.875 / 0.75 / 0.625 | +| v2 human/synthetic shares | 0.5 / 0.5 | +| Direct / lineage closure interval | [0.5, 0.5] / [0, 0] | +| Declared / resolved target references | 8 / 8 | +| Cycle detected / unresolved references | false / 0 | +| Distinct supporting external roots | 5 | +| Top root / incidence / share | `v1::v1_01` / 3 / 3/8 | +| Root masses / normalized shares | [3, 2, 1, 1, 1] / 8 | +| Ancestry HHI / effective roots | 0.25 / 4.0 | +| Input observability / lineage execution | Level 4 / completed when requested | +| Shared-ancestry proxy | present, with topological limitations | + +Required unavailable conclusions remain model-performance decline, causal ancestor effect, universal integrity and universal collapse prediction. No simulation runs by default. + +### 5.2 Proposed multi-root and uncertainty oracle + +Use four ungrounded target records in a later version. Context anchors a and b are explicitly parentless and grounded; context u is explicitly parentless with unknown grounding. Target t1 has parent a; t2 has parents a,b; t3 has explicit empty parents; t4 has parents a,u,and a missing key. + +| Quantity | Independent expected value | +|---|---| +| Complete root sets | t1: {a}; t2: {a,b}; t3: {}; t4: unresolved | +| N / G / C / U | 4 / 2 / 1 / 1 | +| Declared/resolved target references | 6 / 5; edge coverage 5/6 | +| Resolved lineage / external ancestry coverage | 3/4 / 1/2 | +| Incidence a,b / shares using N | 2,1 / 1/2,1/4 | +| Fractional masses a,b / normalized weights | 3/2,1/2 / 3/4,1/4 | +| Distinct roots / HHI / effective roots | 2 / 5/8 / 8/5 | +| Lineage lower / upper / width | 1/4 / 1/2 / 1/4 | + +Create separate variants removing the missing branch while retaining unknown grounding, and resolving unknown grounding while retaining the missing branch. Either defect independently keeps t4 unresolved. Additional compact fixtures cover carryover, repeated paths, tied ranks, all-closed, all-unresolved, empty targets, cycle components and context/target denominator isolation. + +## 6. Verification effort and performance policy + +| Change/risk | Gate | +|---|---| +| Documentation or non-authoritative metadata | Formatting and affected consistency only | +| Graph/root/math implementation | Affected unit/oracle tests plus resolver, generation or bounds neighbors | +| Input role, public schema, API/CLI or privacy boundary | Relevant integration, compatibility and boundary regression | +| Stable PR candidate | Complete canonical regression and supported OS/Python/dependency matrix | +| Release/delivery candidate | Candidate checks plus build, wheel/sdist, clean install, canonical examples, reproducibility and required security checks | +| Administrative successor with unchanged executable/authoritative bytes | Direct document consistency and evidence reuse; no automatic matrix rerun | + +Keep Ubuntu/Windows, Python 3.11/3.12, current/minimum dependency coverage and a real optional-Parquet profile. Adapt existing workflow triggers or dispatch so ordinary implementation updates use the affected gate and a stable candidate receives the full matrix. Reuse existing jobs instead of creating parallel governance workflows. + +Run the expensive 100,000-record sparse lineage benchmark on one designated reference profile; use bounded structural cases across the compatibility matrix. Preserve the required actual 100k execution without multiplying it across every environment. Candidate failures require rerunning affected gates; unchanged successful environments may be reused only when their tested candidate remains applicable and that relationship is clear. + +Measure elapsed time, peak RSS, graph size, target/context counts, total root memberships and output size. Test a deep sparse chain and a small high-fan-in graph, because root-set growth can dominate graph traversal. Include at least one full CLI lineage measurement through report serialization. Kernel timing alone cannot establish end-to-end performance. + +The existing Hero target is under five seconds on its stated reference environment, including input loading, validation, observability, required metrics and JSON/Markdown. Measure it afresh with lineage enabled. The sparse-lineage 100k requirement has no approved hard wall-time/RSS SLA. Establish a reproducible baseline and a measured operational CI timeout; an execution timeout is not a product performance target. + +Phase 4's metadata-only 100k audit took approximately 1,362.47 seconds and 7,640,743,936 bytes peak RSS. It did not execute general ancestry. Use that result to understand inherited report overhead, not to claim Phase 5 performance. A >20% time or >50% memory regression on a comparable workload requires review; changed workloads must be labeled. Reuse the existing scope pooling and avoid duplicating per-record graph evidence across report sections. + +Bound mutation work to meaningful deterministic defects: wrong N versus G denominator, set union replaced by duplicate counting, carryover minted as a root, unknown treated as closed, omitted cycle propagation, reversed bounds and incorrect HHI inversion. Record the attempted mutation and whether a current test detects it in the existing validation report. No permanent mutation registry or test-count target is required. + +## 7. Change scope and completion boundary + +Expected implementation locations are `lineage/`, narrowly shared models/config/errors, input-role adapters, bounded parent-result retention and necessary generation propagation, lineage branches of bounds, result/report assembly, JSON/Markdown, CLI, config/report schemas and their packaged mirrors. Relevant fixtures, tests, public docs, existing verification tooling and version declarations may change. These are ownership boundaries, not a new hash-based file authorization system. + +Do not change unrelated mathematical kernels, normalization semantics, direct provenance definitions or theory-source documents. Preserve frozen Phase 0 files. Approved Phase 5 clarifications are additive decisions with source references. An actual conflict or newly required independent-input evidence schema is surfaced before dependent implementation, with the affected step identified. + +Phase 5 is complete when its required owners, Hero and advanced oracles, privacy/resource boundaries, installed CLI, candidate compatibility and delivery checks pass, and limitations accurately describe partial evidence and topological meaning. Completion records identify the tested candidate and actual results without requiring historical implementation forms to execute forever. + +The handoff leaves Phase 6A and Phase 6B explicitly deferred. It does not merge a PR, tag a release, publish a package or claim final v0.1 completion. Future verification growth should primarily follow new product capabilities. + +## 8. Source basis for review + +Repository-relative references below identify the existing authority; they are not additional executable approval artifacts. + +- `PROJECT_INSTRUCTIONS.md`, Phase 5 and lineage/ancestry rules; section 4.4 authority order. +- `V0.1_PRODUCT_SPEC.md`, phase sequence and lineage requirements. +- `PHASE_3_DECISIONS.md`, P3-D01 scope reservation. +- `DEFINITIONS_AND_UNITS.md`, strict ancestry, roots, incidence, mass/concentration, coverage and bounds. +- `DATA_AND_PROVENANCE_SPEC.md`, sections 11.7-11.10 and version-order rules. +- `THEORY_TO_CODE_TRACEABILITY.md`, Phase 5 owners T4/T6/PR-008/PR-009/T3. +- `REPOSITORY_ARCHITECTURE.md`, lineage ownership and phase responsibilities. +- `OBSERVABILITY_AND_REPORTING.md`, typed evidence, explicit unavailable states and interpretive boundaries. +- `VALIDATION_PLAN.md`, sections 61.1, 62.3 and 81; `SUCCESS_CRITERIA.md`. +- `examples/hero/EXPECTED_OUTPUTS.md`, frozen full-product Hero expectations. +- `PHASE_4_COMPLETION.md`, `PHASE_4_VALIDATION_REPORT.md`, `PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md` and accepted candidate evidence. +- User instruction: Verification Governance and Complexity Control, supplied during Phase 4. diff --git a/PHASE_5_STEP_1.md b/PHASE_5_STEP_1.md new file mode 100644 index 0000000..1d300a2 --- /dev/null +++ b/PHASE_5_STEP_1.md @@ -0,0 +1,116 @@ +# Phase 5 Step 1 Completion + +Date: 2026-09-23 UTC. Status: **STEP 1 COMPLETE; PHASE 5 INCOMPLETE**. + +The Theory Owner approved the Phase 5 plan and instructed Step 1 to start. +Work begins from `1db3b1a460c233242ff37fbe45b8fac74d6101ef` on the new +`phase5-lineage` branch. This record covers contracts, fixtures and verification +consolidation only. It grants no subsequent-step execution, merge, tag or +publication. + +## Delivered + +- Approved `PHASE_5_PLAN.md` and P5-D01 through P5-D11 in + `PHASE_5_DECISIONS.md`. +- `docs/lineage_contract.md`: immutable type/field interfaces, explicit scope, + root/uncertainty/depth rules, status semantics, bounded privacy-safe detail and + resource interfaces for later implementation. +- Twelve independent acceptance cases across the three existing lineage fixture + directories, plus separate frozen-Hero lineage expectations. Exact oracle + arithmetic is recorded independently of future algorithms. +- One current path in the three existing verification scripts. Historical + source-body migrations, snapshot fixtures and staged authorization assertions + no longer run by default. Earlier forms remain recoverable from Git. +- Focused ordinary CI and explicitly selected candidate CI. The supported matrix, + real Parquet, Hero, security, installed packages and reproducibility remain + required at candidate gates; the expensive performance directory runs once on + a designated reference profile. + +The executable product is unchanged: forty Python modules, five root schemas, +seven packaged resource copies, six canonical Hero files and package metadata +match the accepted Phase 4 product. Package version remains `0.1.0.dev3`; report +schema remains `1.0`. General lineage algorithms are still deferred. + +## Contract details resolved + +One canonical parent remains one parent even when its declaration has multiple +aliases. Reference coverage retains those declarations, while adjacency/root mass +deduplicates them. Unknown grounding can coexist with known structural depth. +Other grounded-with-parent forms remain unresolved outside the approved +single-parent carryover rule. Empty analytical scopes do not imply current CLI +acceptance of an empty records file. + +Detail collections are capped at 100 rows; complete cycle witnesses at 64 edges. +Long witnesses are explicitly omitted rather than presented as nonclosed prefixes. +The four initial lineage budgets are 200,000 nodes, 1,000,000 edges, 1,000,000 +logical root memberships and 10,000,000 root-union visits. These are future graph +work guards, separate from ingestion limits and measured RSS. + +Current normalization preserves an absent parent field in +`CanonicalRow.field_states` while its normalized value is null. Step 2 must use +that retained evidence to preserve absent/null declaration semantics. + +## Verification performed + +| Check | Actual result | +|---|---| +| Three current source/specification/traceability commands | PASS: 59 protected product files unchanged, 16 frozen specifications, 7 exact resource copies and 40 owned modules | +| Exact focused CI test selection | 212 passed in 3.62 seconds | +| Affected current-control and neighboring tests | 263 passed in 3.01 seconds; overlaps the focused selection and is not added to it | +| Workflow scheduling/JUnit boundary tests after final path-filter fix | 18 passed | +| New fixture input validation | 12 passed; these also belong to the focused selection | +| Final canonical collection, excluding designated performance directory | 2,629 collected without errors; this is collection, not a full passing run | +| Independent fixture arithmetic | Hero and rational multi-root partitions, coverage, bounds, HHI and effective-root values checked | +| Four workflow files | YAML parsed; candidate dependencies and read-only permissions checked | +| Retained same-named test/helper definitions | 746 compared structurally; executable bodies unchanged, excluding explanatory docstrings | +| Diff formatting and independent review | PASS; no remaining Step 1 blocker | + +Checks used Python 3.12, pytest 9.1.1, NumPy 2.5.3 and pandas 3.0.6. Missing test +dependencies in the resumed local environment were installed before execution. +No full regression, cross-platform matrix, real-Parquet profile, release build, +installed-artifact check or 100k performance run was performed for Step 1. +Hosted execution of the changed workflows is not claimed by this record. + +The local focused command was: + +```bash +python -m pytest -p no:cacheprovider -q \ + tests/integration/test_current_verification.py \ + tests/integration/test_ci_workflows.py \ + tests/unit/test_lineage_fixture_inputs.py \ + tests/unit/test_phase3_contracts.py \ + tests/integration/test_phase3_metric_pipeline.py \ + tests/unit/test_PR016_determinism.py \ + tests/integration/test_no_algorithms.py \ + tests/integration/test_repository_structure.py +``` + +## Guarantees retained through consolidation + +Direct current tests continue to protect mathematical formulas, representation +semantics, provenance and direct bounds, input-only validation, unavailable +states, report schemas, privacy, no-network behavior, safe paths and CLI output. +Current negative checks reject unauthorized runtime mutation, self-authorized +specification edits, corrupted/extra packaged resources, damaged or incomplete +wheel/sdist payloads, duplicate archive entries and unsafe extraction members. +Existing installed behavioral checks remain in candidate verification. + +The three verification scripts shrink from 17,316 to 1,444 lines by removing +historical dispatch and repeated source-preservation machinery. This count +describes the consolidation and is not an acceptance target. The behavior map is +in `PHASE_5_DECISIONS.md`; historical bodies and evidence remain recoverable at +the accepted Phase 4 commit and its earlier history. No new baseline registry, +historical source-binding migration or evidence-of-evidence mechanism was added. + +Independent review found that a blanket documentation path exclusion would hide +changes to the new authoritative lineage contract. The workflow now excludes only +named nonauthoritative documents; a lineage-contract change receives focused +checks. The final workflow tests cover that distinction. + +## Handoff + +Step 2 will construct the validated parent graph and retain per-record reference +successes/failures using one identity lookup. Its algorithms and runtime changes +are not implemented here. Phase 6A and Phase 6B remain deferred. + +Continue only when Phase 5 Step 2 is instructed. diff --git a/docs/architecture.md b/docs/architecture.md index a1abb38..7a8e8cb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,5 +1,19 @@ # Architecture +## Current Phase 5 Step 1 boundary + +The approved [lineage contract](lineage_contract.md) fixes the interfaces for +future graph/ancestry implementation. Step 1 changes no runtime modules, executable +schemas, dependencies, canonical Hero inputs or package version. The following +Phase 3/4 sections describe historical implementation stages. + +Current verification uses one path in the existing consistency, traceability and +release scripts. Exact historical method bodies, snapshot fixtures and migration +chains are recoverable from Git, while direct current product/security tests +remain active. [Release process](release_process.md) defines focused and candidate +gates. The current source-scope check rejects product changes during this +administrative step without creating another approval registry. + Status: Phase 3 Step 11 development milestone. Actual acceptance is recorded in `PHASE_3_COMPLETION.md`. The input-layer sections below describe the preserved Phase 2 implementation. Subsequent step sections retain the boundary at their historical stage. The current boundary is summarized here and in the final Step 11 section. diff --git a/docs/lineage_contract.md b/docs/lineage_contract.md new file mode 100644 index 0000000..5682a3d --- /dev/null +++ b/docs/lineage_contract.md @@ -0,0 +1,414 @@ +# Phase 5 Lineage Contract + +Status: **approved implementation target, fixed in Phase 5 Step 1**. + +Authority: [Phase 5 decisions](../PHASE_5_DECISIONS.md), approved Phase 5 plan, +`PROJECT_INSTRUCTIONS.md` section 4.4 and the approved UD decisions. Relevant +frozen owners are T4, T6, PR-008, PR-009 and T3 lineage bounds. The original +definitions/specifications remain unchanged. + +This document defines interfaces to implement in Steps 2-8. Step 1 leaves the +package at `0.1.0.dev3`, the executable report schema at `1.0`, and general +lineage execution deferred. Code-like declarations below are contracts, not +claims about currently importable classes or functions. + +## 1. Scope and reference semantics + +Canonical identity is the existing `RecordKey(dataset_version, record_id)`; +transport spelling is `version::id`. Compare keys using their two string fields +in that order. Records in the primary version define the target population N. +Loaded comparison/context records support the graph without changing N. + +`LineageScope` has these immutable fields: + +| Field | Python type | Meaning | +|---|---|---| +| `target_dataset_version` | `str \| None` | Explicit selected primary version; null only when an empty input has no declared version. | +| `target_record_keys` | `tuple[RecordKey, ...]` | Unique sorted target identities; no representation eligibility exclusions. | +| `target_record_count` | `int` | N, equal to the tuple length. | +| `loaded_record_count` | `int` | Unique loaded nodes across all admitted roles. | +| `context_record_count` | `int` | Loaded nodes outside the target. | +| `loaded_dataset_versions` | `tuple[str, ...]` | Lexical inventory, not inferred chronological order. | + +The target keys must be a subset of the loaded keys. Counts are nonnegative +integers and never booleans. Context versions are disjoint from primary and +comparison versions. Same-version edges are supported within a loaded primary +version; split target/context roles for one version are outside this phase. +Duplicate canonical keys across any roles fail validation. + +Keep `ParentValidationResult.declaration_state` values `absent`, `null`, `empty` +and `declared`. The current generation adapter normalizes absent parent fields to null; Step 2 must recover the retained `CanonicalRow.field_states` distinction when constructing its batch result. Only `empty` explicitly declares no parents. Missing provenance +is separately identified. Reuse `ParentReference`, its canonical identity, +`source_references`, resolution status and chronology. The Step 2 batch resolver +retains a per-record result or per-record failure, original declaration counts +where knowable, and explicit self-parent evidence. It constructs one lookup +index for the batch. It never treats an ambiguous/future/invalid reference as an +accepted ancestry edge. + +For each syntactically valid reference list, declared count is the original +list length, including repeated aliases. Resolved count sums the lengths of +`source_references` for accepted resolved references. Unresolved count is +declared minus resolved and includes invalid references whose entries can be +counted. A malformed declaration with unknown reference cardinality makes the +whole-target reference totals/coverage unavailable; retain known-subset counts +only with an explicitly named subset. Never turn malformed input into zero +declared parents. + +Graph adjacency deduplicates canonical edges. Grounded carryover with two +spellings of the same resolved parent still has one parent. An additional +unresolved spelling that cannot resolve to that key prevents complete ancestry. +Cross-version edges require the existing declared chronology; missing chronology +cannot certify a resolved required path. Same-version edges require cycle +analysis. Default missing-parent warnings and existing strict-mode error +promotion remain unchanged. Ambiguity, future-parent and cycle errors remain +errors. + +## 2. Computation ownership and immutable targets + +The target high-level computation entry is: + +```python +analyze_lineage( + validation: BundleValidationResult, + *, + target_dataset_version: str | None, + limits: LineageLimits = LineageLimits(), +) -> LineageAnalysisResult +``` + +`validation` supplies retained record, provenance, chronology and parent evidence +after the Step 2 retention refactor. It must cover context as well as target +records. Selection is explicit, revalidated and independent of ordinary metric +eligibility. `validate_bundle` remains input-only and never calls this function. +CLI orchestration calls it only after an explicit lineage request. + +Ownership stays in existing modules: + +| Module | Typed handoff and responsibilities | +|---|---| +| `lineage/graph.py` | `LineageGraph`: scope, sorted node keys, deduplicated immutable `parents_by_child` and `children_by_parent`, retained per-record reference/declaration evidence, invalid-self-reference evidence and validation messages. | +| `lineage/cycles.py` | `CycleAnalysis`: cyclic components, member keys, affected descendant keys, topology for unaffected nodes and structural depth assessments. Iterative algorithms only. | +| `lineage/ancestry.py` | `RecordAncestry`, `RootContribution`, `LineageAnalysisResult`; complete root resolution, G/C/U partition, root incidence/mass and concentration. | +| `metrics/bounds.py` | Lineage closure envelope from the typed G/C/U counts, preserving the direct-bound branch. | +| `result.py`, `reports/assembly.py` | Validate/assemble supplied typed results, scope/status metadata and canonical schema 1.1 values. No graph traversal or metric recomputation. | +| `reports/json_report.py`, `reports/markdown_report.py` | Render the same privacy-safe canonical result. No computation ownership. | + +All result objects use frozen typed data. Sequences are tuples; sets of roots are +frozensets; mappings are detached read-only mappings whose contents are also +immutable. Raw row content, raw URIs and local paths are not copied into ancestry +results. Invalid handoffs are rejected instead of silently trusted. + +`RecordAncestry` contains `record_key: RecordKey`, +`classification: Literal["grounded", "closed", "unresolved"]`, +`external_root_keys: frozenset[RecordKey] | None`, +`reason_codes: tuple[str, ...]`, `lineage_depth: int | None`, and +`depth_reason_codes: tuple[str, ...]`. A null root set means unresolved; an empty +frozenset means complete known-empty ancestry. Unknown and empty never collapse +into the same value. Complete records have no root-resolution failure reasons. +Internal complete sets are bounded by the resource budget and are not exported +as a full transitive-closure table. + +`RootContribution` contains `record_key: RecordKey`, `incidence_count: int`, +`incidence_share: float`, `incidence_denominator: int`, `fractional_mass: float`, +`normalized_weight: float`, and `weight_denominator: int`. Runtime values must be +finite. Independent fixture expectations use exact fractions where possible; +deterministic numeric reduction uses canonical target/root order. + +`LineageAnalysisResult` contains `scope: LineageScope`, +`execution_status: ExecutionStatus`, `execution_reason_codes: tuple[str, ...]`, +`records: tuple[RecordAncestry, ...] | None`, `cycles: CycleAnalysis | None`, +`root_contributions: tuple[RootContribution, ...] | None`, typed counts/coverage, +concentration and depth values described below, `resource_usage: LineageResourceUsage` +and existing typed validation messages. Failed resource-dependent values are +nullable with reasons; the result never fills unprocessed records with a guessed +closed classification. A failed intermediate field uses null and its reason, +while the result identifies which computation stages actually completed. + +## 3. Cycles and structural depth + +Cycle analysis uses accepted resolved edges plus separately retained explicit +self-parent evidence. The self-edge remains invalid ancestry evidence while +permitting a truthful self-cycle diagnostic. Rejected ambiguous/future/malformed +references remain disclosed and cannot certify whole-input graph validity. + +`cycle_count` is the number of strongly connected components having more than one +node or a self-loop. The method label is `cyclic_strongly_connected_components`. +Components are ordered by their smallest canonical member. Members are the nodes +inside those components; affected records are the union of members and their +descendants along parent-to-child edges. Report total and target counts for both +sets. A disconnected loaded cycle remains detected even if no target descends +from it. + +A cycle witness is a closed directed sequence whose adjacent keys are actual +parent-to-child edges. Use a deterministic canonical-neighbor traversal within +each component. For a self-loop it is `(key, key)`. Return a witness only if it +contains at most 64 edges; otherwise return null with +`witness_reason="diagnostic_limit"`. A bounded search that does not find a short +witness may omit it; absence of a witness does not erase an SCC-based finding. +Do not enumerate all simple cycles or store every ancestry path. + +Depth is the maximum number of structural edges from a record to loaded, +explicitly parentless records, provided every required structural path is valid +and complete. Such a parentless node has depth zero even when its grounding is +unknown. A missing/invalid edge, unavailable parent declaration/chronology or +cycle on a required path makes depth null. Unknown grounding alone prevents an +exact external-root set but does not invalidate an otherwise complete structural +depth. Generation remains its existing non-grounding step count; it is not depth. + +`lineage_depth` is the whole-target maximum and is null for an empty target or +when any target depth is unavailable. `maximum_resolved_target_depth` and +`depth_resolved_record_count` may separately describe the complete-depth subset; +the former is null if that subset is empty. A subset maximum must not occupy the +whole-target field. + +## 4. External roots, coverage and values + +An explicit parentless grounded-yes record supports itself with root set `{key}`. +Strict ancestry still excludes self. Explicit parentless grounding-no produces a +known-empty set. Grounding-unknown, missing required provenance, conflicting +declarations or an undeclared boundary produces an unresolved set. + +A grounding-yes carryover with exactly one deduplicated resolved canonical parent +inherits the parent's set. Any unresolved additional declaration blocks it. +Other grounded-with-parent forms, including multi-parent grounded carryover, +remain unresolved under `GROUNDED_PARENT_RULE_UNSUPPORTED`. Evidence labels and +URIs do not mint independent roots. A new external input is a separately loaded, +explicitly parentless grounded anchor linked as an ordinary parent; actual child +grounding must never be relabeled to fit the supported rule. + +Grounding-no descendants union their parents' complete unique root sets. Unknown +grounding anywhere on a required path propagates unresolved ancestry. Every +required branch must resolve; observed roots on incomplete branches are excluded +from exact incidence and mass. Cycle-affected records remain unresolved. Topology +and roots are never inferred from record IDs, filenames or semantic content. + +For complete nonempty-root targets G, complete empty-root targets C and unresolved +targets U, `N=G+C+U`. The report field contracts are: + +| Field under `derived_metrics.lineage` unless stated | Type of envelope value | Exact meaning | +|---|---|---| +| `grounded_record_count` | integer | G, targets with complete nonempty roots. | +| `closed_record_count` | integer | C, targets with complete known-empty roots. | +| `unresolved_record_count` | integer | U, targets with incomplete/invalid ancestry. | +| `records_with_resolved_external_ancestry` | integer | G+C, including known-empty sets. | +| `resolved_lineage_coverage` | number or null | `(G+C)/N`. | +| `external_ancestry_coverage` | number or null | `G/N`. | +| `resolved_parent_edge_coverage` | number or null | Accepted resolved original target references / declared original target references. Exact zero declared count yields 1.0. | +| `distinct_external_root_count` | integer or null | Unique roots in the union over G, zero when the complete partition has G=0. | +| `top_shared_ancestors` | detail object or null | Bounded `RootContribution` rows, descending incidence then canonical key; despite the inherited name, roots with incidence one remain eligible. | +| `ancestry_concentration_hhi` | number or null | `sum((M_a/G)**2)`, where `M_a=sum(1/len(A_r))` over complete grounded targets containing a. | +| `effective_external_root_count` | number or null | Reciprocal HHI. | +| `lineage_depth` | integer or null | Exact whole-target maximum structural depth. | + +`no_declared_parents: bool` accompanies executed reference coverage. It is true +only when the exact total is zero. With lineage not requested, retain schema +1.0's immediate-validation observation semantics and its null ratio for zero +references; do not manufacture an executed coverage value. + +Root incidence shares divide by N; normalized fractional weights divide by G. +Each target in G contributes total fractional mass one. Multiple paths or alias +declarations do not multiply mass. Concentration is computed from all complete +root contributions before report-detail caps or redaction. If U>0, quantities +restricted to G carry partial status and explicit G/N coverage, even when their +conditional values are exact. No completeness of excluded targets is implied. + +Existing closure fields stay at +`derived_metrics.closure_exposure.lineage.{lower_bound,upper_bound,interval_width}` +with values `C/N`, `(C+U)/N`, `U/N`. G/C/U counts must cover all targets before +these values are emitted. A resource-aborted partial traversal provides no such +partition and therefore no numeric bounds. A fully classified unresolved record +may contribute to U and the conservative bound without having an exact root set. + +When N=0, counts are zero, version remains explicitly declared or null, and all +population ratios/bounds are null with `EMPTY_TARGET_SCOPE`. Distinct roots are +zero after successful empty analysis; HHI/effective roots are null with +`NO_RESOLVED_EXTERNAL_ROOTS`. When N>0 and G=0, those concentration fields remain +null with that reason, including all-closed and all-unresolved cases. No infinity, +NaN, fake version or zero-valued unavailable concentration is allowed. + +`proxy_signals.shared_ancestry_dependence` retains the current proxy vocabulary: + +| Condition | `status` | `level` | +|---|---|---| +| Some exact root incidence >=2, U=0 | `available` | `present` | +| Some exact root incidence >=2, U>0 | `partial` | `present` | +| N>0, U=0, every exact root incidence <=1 | `available` | `not_present` | +| N=0, incomplete evidence without a witness, or resource failure | `unavailable` | `indeterminate` | + +The proxy cites actual root/incidence and coverage fields. Topological allocation +does not certify causal contribution, semantic error, independent information, +biological relatedness, universal diversity, integrity or collapse. T4 retains +its theory-guided operationalization label; T6 is an engineering graph-validity +rule. Existing unavailable scientific conclusions remain explicit. + +## 5. Schema 1.1 placement, detail bounds and privacy + +Step 7 changes every Phase 5 output to report schema 1.1, including requests +without lineage. Strict 1.0 readers must adopt the new schema. Preserve the twelve +top-level sections, existing envelopes and all non-lineage numerical meanings. +Root and packaged report schemas must agree. No migration-loader framework is +introduced. + +New observed fields are `observed_facts.lineage.graph_scope` (the public scope +counts/version inventory), `cycle_analysis`, `depth_summary`, +`unresolved_record_details`, and `resource_usage`. Existing declared/resolved/ +unresolved parent counts remain in their current observed locations. +`cycle_status` remains `acyclic` or `cyclic` when a completed graph check can +establish that value; its scope and reasons disclose rejected declarations. +An acyclic accepted-edge subgraph never certifies validity of rejected input. +Incomplete graph traversal gives null/unavailable, not `acyclic`. + +Every bounded collection uses this exact detail object: + +```text +{ + items: tuple[T, ...] | null, + total_count: int, + returned_count: int, + omitted_count: int, + limit: 100, + detail_status: "complete" | "truncated" | "omitted", + omission_reasons: tuple["diagnostic_limit" | "redacted_identity_details", ...] +} +``` + +`total_count=returned_count+omitted_count`. For visible detail, `items` is an array, +`returned_count=len(items)`, and at most 100 items appear. Complete detail has zero +omissions/reasons, including an empty array. Truncated detail has positive +omissions and `diagnostic_limit`. Privacy omission uses `items:null`, returned +zero, omitted equal to total, status omitted and reason +`redacted_identity_details`. Applying privacy to a previously capped collection +retains that diagnostic-limit reason when applicable. These presentation counts +do not change analytical status or denominators. + +`cycle_analysis` includes `detected: bool`, `cycle_count: int`, +`counting_method: "cyclic_strongly_connected_components"`, total/target member and +affected counts, and three bounded collections: `components`, +`cycle_member_record_keys`, `affected_record_keys`. Component rows contain +`component_index: int` (one-based canonical order), `member_count: int`, +`target_member_count: int`, `witness_record_keys: tuple[RecordKey,...] | None`, +`witness_edge_count: int | None`, and +`witness_reason: Literal["diagnostic_limit"] | None`. A returned witness is closed +and has at most 65 keys; omitted witness edge count is null. The detail object +around components provides the privacy omission shape for the entire row list. + +`unresolved_record_details` rows contain only `record_key` and sorted unique +`reason_codes`, ordered by canonical key. Root rows are the seven fields in +`RootContribution`. `depth_summary` contains `depth_resolved_record_count`, +`maximum_resolved_target_depth` and the explicit target count. No arbitrary +message, URI, path or content payload is admitted into these shapes. + +In standard/hash views, identities use the existing protection transformation +appropriate to the selected privacy and record-ID modes. In redacted/hash, +dataset IDs, root keys, witness keys and dynamic identities use the existing +scoped HMAC mechanism consistently; sort/rank before privacy transformation. +In record-ID omit mode, each identity-bearing collection uses its whole-list +omission variant. Never remove a required `record_key` while retaining an invalid +row. Aggregate values, counts, severity and execution status remain visible. +Existing options for preserving only the declared record-ID field retain their +scope. Both renderers consume the same schema-valid safe result. + +## 6. Resource and invocation interfaces + +`LineageLimits` has four required positive integer fields with these defaults; +the existing `ResourceLimits` gains corresponding configuration fields when +the configuration integration is implemented: + +| `LineageLimits` field | `ResourceLimits` / JSON `resource_limits` field | Default | +|---|---|---| +| `max_nodes` | `max_lineage_nodes` | 200000 | +| `max_edges` | `max_lineage_edges` | 1000000 | +| `max_root_memberships` | `max_lineage_root_memberships` | 1000000 | +| `max_root_union_visits` | `max_lineage_root_union_visits` | 10000000 | + +Defaults apply only to explicitly requested lineage. A supplied override is an +exact integer greater than zero; bool, null, fractional, infinite and nonpositive +values are invalid. Existing input byte/row/parent-list/depth limits remain +independent and retain their existing optional semantics. + +Check cumulative node admission across all batches before graph materialization, +unique-edge admission before insertion, stored membership before storing each +new `(record, root)` association, and union work before each candidate root visit. +A union visit is processing one parent's candidate root into a child's union, +including a candidate already present there. An anchor self-membership consumes +one stored membership and zero union visits. Shared immutable set storage may +reduce actual memory but does not reduce the logical membership accounting. + +`LineageResourceUsage` contains `admitted_node_count`, `admitted_edge_count`, +`stored_root_membership_count`, `root_union_visit_count`, `limits: LineageLimits`, +and `exhausted_limit: str | None`. On exhaustion, include +`attempted_value: int | None` for the rejected next admission/visit, stop dependent +work, and emit a typed error with `LINEAGE_RESOURCE_LIMIT_EXCEEDED`. Distinguish +these operational reason codes from the existing `ErrorCode` namespace; add the +smallest needed error constant during implementation without renaming existing +errors. A successfully completed earlier stage may retain its exact observations, +but no truncated partition, root distribution or bound is reported as exact. +Independently completed non-lineage families survive. + +Iteration order for budgeting is canonical nodes, canonical parent keys and +canonical root keys. Budget exhaustion must be deterministic for permuted input. +Limits bound defined work units, not peak RSS, loader allocations, decompression +or scientific thresholds. Step 9 measures actual time/RSS and may propose an +explicitly documented limit adjustment. It must not silently raise limits to +turn a failing check into a pass. + +The future inert configuration field is `ResolvedConfig.lineage: bool = False`, +serialized as top-level `lineage: false` when explicit. The dedicated input role +is `FileRole.LINEAGE_CONTEXT` with value `lineage_context`; it permits repeated +existing local input declarations. Preserve all existing multi-role Python/config +behavior. Config resolution neither reads inputs nor starts analysis. + +`audit --lineage` and `example --lineage` enable computation. Repeatable +`--lineage-records PATH` supplies context to audit only with lineage enabled. +`validate` may ingest a configured/context input role for input/reference +validation, but rejects any true lineage execution request. Ordinary metric calls +select the primary version explicitly. Context does not request comparison, +trajectories, simulations or network access. Existing CLI comparison stays one +explicit earlier input; broader Python/config multiplicity stays unchanged. + +## 7. Execution and error states + +Input observability stays separate from actual computation. Completed/partial +execution names performed operations. All other execution states have explicit +reasons. Use these rules in order: + +| Situation | Lineage execution | Value handling | +|---|---|---| +| No explicit request | `not_requested` | No graph execution claim; ordinary validation evidence may remain. | +| Fatal admission/resource failure | `failed` | No exact result from incomplete stages; earlier completed observations and other families may survive. | +| Invalid graph/input, with useful unaffected target results | `partial` | Preserve errors, complete subset values and precise target coverage. | +| Invalid graph/input, with no usable target ancestry result | `failed` | Preserve valid diagnostics; exact root metrics unavailable. | +| No lineage error, with U>0 | `partial` | Complete partition/bounds may be available; G-only metrics disclose partial coverage. | +| No lineage error and U=0, including a successful empty analysis | `completed` | Defined values available; empty-denominator metrics remain unavailable. | + +A disconnected loaded cycle remains a lineage error even with complete target +ancestry. Missing parents promoted by strict mode follow the error rows; ordinary +missing-parent warnings follow the unresolved-coverage row. A known all-closed +target can complete successfully while HHI remains unavailable. Root uncertainty +does not erase independently valid graph observations. + +Existing run-status behavior is preserved: fatal errors fail the run; other +errors yield partial when useful evidence survives and failed otherwise. +Warnings alone retain their existing run-status behavior. Any surviving lineage +error produces a non-success CLI audit exit under the existing error policy. +Redaction and capped detail do not themselves turn completed analysis into +partial execution. + +## 8. Direct verification and completion boundary + +Independent fixture expectations cover the frozen Hero, rational multi-root +allocation, missing and unknown branches separately, carryover aliases, known +empty versus unknown ancestry, empty targets, disconnected cycles, affected +descendants and target/context isolation. They describe future behavior and are +not successful runtime results in Step 1. + +Implementation checks protect those behaviors directly. New graph boundaries +receive bounded adversarial/resource cases. Candidate verification retains the +supported OS/Python/dependency and packaging gates. The actual 100k sparse lineage +measurement runs on the designated reference profile; ordinary step checks do +not repeat a release matrix. Historical behavior is consolidated under P5-D11, +without new source-body chains or evidence-of-evidence frameworks. + +No automatic longitudinal analysis, new independent-input evidence schema, +causal weighting, semantic inference, HTML implementation or theory-source change +is authorized by this contract. diff --git a/docs/release_process.md b/docs/release_process.md index 074aae5..a7f9c14 100644 --- a/docs/release_process.md +++ b/docs/release_process.md @@ -1,75 +1,110 @@ # Development and Release Verification -`0.1.0.dev3` is the Phase 4 report/CLI development milestone. Completion requires recorded candidate evidence and does not authorize stable v0.1 publication, a tag, main merge or Phase 5 implementation. The three `PHASE_4_*` milestone reports distinguish actual acceptance, remaining limits and unexecuted work. Prior phase records remain historical evidence. +## Current Phase 5 boundary -## Choose the gate by impact +Phase 5 Step 1 is approved. Runtime remains at `0.1.0.dev3`, with executable +report schema 1.0 and general lineage deferred. The Phase 5 contracts describe +future implementation. Stable v0.1 publication, a tag and main merge require +separate authorization. -| Change or milestone | Required scope | -|---|---| -| Documentation or nonauthoritative metadata | Formatting/static checks and affected document consistency | -| Isolated implementation | Affected unit tests, dependency neighbors and applicable mathematical/security checks | -| Schema, public API, mathematics, provenance, privacy or package boundary | Relevant integration, compatibility and broader regression appropriate to impact | -| PR candidate | Complete canonical regression and required supported-environment matrix | -| Release candidate or phase delivery candidate | Complete regression, supported matrix, build/install, canonical examples, reproducibility, artifact integrity and applicable security checks | - -Documentation that participates in executable authority receives the applicable authority checks. Administrative successors after an accepted code candidate do not automatically repeat the full matrix when executable and authoritative bytes are unchanged. Preserve the accepted source identity and state exactly what changed. If an executable or authoritative change invalidates prior evidence, rerun the affected gate before acceptance. - -Keep three concerns distinct: the canonical suite protects current supported behavior; release verification covers environment/package/reproducibility boundaries; Git and archived records preserve superseded evidence. Preserve historical behavioral guarantees without automatically adding test-body migrations, registries or evidence-of-evidence layers. A new verification mechanism needs a concrete failure that existing controls cannot adequately detect. New governance mechanisms additionally require the reason Git/archive evidence is insufficient, maintenance cost and an exit or consolidation path. - -## Supported candidate profiles - -| Profile | Environment and evidence | -|---|---| -| Core, current dependencies | Ubuntu and Windows, Python 3.11 and 3.12; resolved versions retained; PyArrow genuinely absent | -| Core, minimum jointly compatible dependencies | Same OS/Python matrix, NumPy 2.0.0 and pandas 2.2.2; PyArrow absent | -| Real Parquet | Ubuntu/Python 3.12 with actual PyArrow; real roundtrip, row-limit and invalid-file regressions | -| Security | No-network/import, declarative mapping, safe input/output paths, privacy/redaction and protected-owner boundaries | -| Hero and mathematics | Frozen mathematical cases, literal standard/redacted report goldens and installed Hero behavior | -| Package and delivery | Wheel/sdist integrity, strict metadata checks, clean installed execution outside the checkout, resource equality and tracked-source archive | +Use one current verification path: -The package still declares `numpy>=2.0` and `pandas>=2.2`. NumPy 2.0.0 with pandas 2.2.0 cannot resolve because that pandas version requires NumPy below 2. The retained minimum profile uses pandas 2.2.2. This covers that jointly compatible pair, without claiming every intervening version or all optional-dependency minimums. Current profiles record the versions actually resolved at execution time. - -Retain Python/platform/dependency details, `pip freeze`, `pip check`, commands, exit codes, JUnit identities and workflow logs. Core absence and real-PyArrow presence must be checked explicitly. A skipped, failed, duplicate or uncollected required test cannot stand in for an executed passing test. Do not add repeated jobs/subsets into a fictitious distinct-test total. +```bash +python scripts/check_spec_consistency.py +python scripts/check_traceability.py +python scripts/release_check.py +``` -## Local checks and builds +Historical `--phase` and `--step` dispatch is retired. Earlier source forms, +migrations and receipts remain recoverable from Git and archived phase records. +The accepted Phase 4 commit supplies the existing historical authority; no new +Phase 5 hash registry is needed. During Step 1 the source check rejects changes +to runtime, schemas, package metadata and canonical Hero files. Sixteen frozen +specifications and seven packaged resource copies remain protected. -From a prepared full Git checkout: +## Risk-sensitive gates -```bash -python -m pip install ".[test,release]" -python scripts/check_spec_consistency.py --phase 4 --step 11 -python scripts/check_traceability.py --phase 4 --step 11 -python scripts/release_check.py --phase 4 --step 11 --diff -``` +Ordinary PR checks use the focused job for the active step. Documentation-only +changes require formatting and affected consistency checks. As implementation +scope changes, update the focused selection to affected owners and dependency +neighbors. It does not establish candidate acceptance. -The Phase 4 Step 11 candidate runs `python -m pytest -p no:cacheprovider -q` in all eight core matrix cells and the real-Parquet cell, retaining the complete suite including the 100,000-record report case. The release job reuses the exact same-commit Ubuntu/Python 3.12 current-core and real-Parquet results rather than running both suites again. The Hero/mathematics and security roles run separately. Full Git history is needed for the bounded retained historical checks. Scope guards protect the fixed authority files, supported runtime behavior, schemas, Hero resources and mathematical/report oracles. The active manifest cannot expand its own permissions. +Dispatch `ci.yml` with `gate=candidate` for a stable candidate. The workflow runs: -Build into a fresh evidence directory: +| Role | Required execution | +|---|---| +| Core | Ubuntu/Windows, Python 3.11/3.12, current/minimum compatible dependencies; PyArrow absent | +| Optional Parquet | Ubuntu/Python 3.12, real PyArrow with roundtrip, row-limit and invalid-file regressions | +| Canonical Hero and mathematics | Frozen mathematical cases, report goldens and packaged Hero behavior | +| Security | No-network, input/path, privacy/redaction and output-publication boundaries | +| Reference performance | One Ubuntu/Python 3.12 profile, including required actual 100k workloads | +| Package/delivery | Wheel/sdist integrity, clean installed execution, reproducibility and tracked-source archive | + +Compatibility cells run complete canonical regression with +`--ignore=tests/performance`. The designated performance job runs +`tests/performance` once, avoiding nine repetitions of the costly 100k report +workload. The Phase 5 sparse-lineage test is still a placeholder in Step 1; +it becomes an executable graph benchmark in Step 9. No placeholder is counted as +a successful lineage measurement. + +The existing minimum compatible pair remains NumPy 2.0.0/pandas 2.2.2. Record +actual resolved dependencies, `pip check`, Python/platform, commands, results +and material limitations. A skipped, failed, duplicate or uncollected required +test cannot stand in for a passing result. No raw test-count growth target is used. + +Hero and security are reusable candidate roles and may also be dispatched +explicitly for a relevant change. Delivery requires core, Parquet, performance, +Hero and security success. A failing command cannot be masked by evidence upload. + +## Package and candidate checks ```bash python -m build --outdir /absolute/evidence/dist python -m twine check --strict /absolute/evidence/dist/* -python scripts/release_check.py --phase 4 --step 11 --dist /absolute/evidence/dist +python scripts/release_check.py --junit /absolute/evidence/core.xml +python scripts/release_check.py --junit /absolute/evidence/parquet.xml --require-parquet +python scripts/release_check.py --dist /absolute/evidence/dist +python scripts/release_check.py --candidate /absolute/evidence ``` -The candidate build role performs two default builds from sdist with the same recorded build environment and `SOURCE_DATE_EPOCH`, comparing wheel bytes and sdist payloads. The package checks compare wheel/sdist module and resource bytes with the tested tree. Installed checks must import the installed artifact, with checkout source excluded. Keep import/input-only checks with NumPy/pandas/PyArrow blocked separate from numerical/audit checks with required dependencies available. Both operate under the declared no-network checks. Exercise installed `example`, `audit`, `validate`, help/version and the documented Python examples. Wheel and sdist must include the six exact Hero files and report schema, without adding Python modules. - -A source archive contains one project root and the complete tracked tree. Compare archived member bytes with the accepted checkout. Exclude Git internals, environments, caches, generated bytecode, private inputs and full theory PDFs. Wheel, sdist and source archive have different documented scopes. Retain artifact hashes with actual source commit/tree identity in the external receipt; committed documents need not encode their own eventual hash. - -## Performance evidence and limits - -Keep complete-report timing separate from the CLI's pre-publication `run.duration_seconds`. Use untraced wall time for the Hero under-five-second target and record hardware/environment. Allocation tracing is a separate observation with measured overhead. RSS describes fresh-process peak resident size and includes interpreter/native allocation; it is not a delta or a Python-only allocation measure. - -The accepted Step 10 reference-container 100,000-record metadata run took 1,362.4708 seconds, peaked at 7.12 GiB RSS and produced approximately 443 MB JSON and 165 MB Markdown. That run checked independent aggregate expectations through actual published outputs. It measures the declared synthetic workload without certifying other data shapes, common-laptop throughput, general linear scaling or deferred lineage behavior. Its full-scale Python allocation peak was not measured; a separately labeled bounded run characterized tracing overhead. - -The Step 10 observation supplies the comparison baseline; the Step 11 full candidate matrix also records fresh complete-suite performance observations. For later changes, reuse accepted observations when product/report bytes and relevant conditions are unchanged. Repeat expensive performance measurements only for a concrete impact or required gate, with comparable dataset, tracing mode, environment and resources. Preserve all attempts and explain target misses. A timeout is a resource guard and a failed attempt, never a successful throughput result. The [performance README](../tests/performance/README.md) defines the generator, expectations and measurement scope. - -## Acceptance and handoff - -The active CI workflow calls the reusable build/delivery workflow after core and Parquet success. Together with the separately dispatched Hero and security workflows, this supplies four verification roles from three dispatchable workflows. There is no additional push-triggered full matrix or standalone performance job. Record the actual immutable candidate commit and required workflow roles. Completion can be claimed only after their required checks pass and deliverable identities are known. If the final record changes only nonauthoritative documentation, verify those changes and reuse the identified code-candidate evidence under the policy above. Changes to executable/authoritative content require the corresponding checks. Do not claim a matrix was rerun when it was reused. - -The external receipt identifies candidate/final source, actual workflow attempts, commands, artifacts, hashes, failed attempts, fixes and limitations. Hosted artifacts have finite retention. If a hosted binary cannot be retrieved and verified, disclose that gap and distinguish locally reproduced bytes from the hosted artifact. Archive previous evidence without expanding permanent current execution merely to preserve old source forms. - -GitHub evaluates PR path filters against the complete three-dot diff. A documentation-only successor on an existing code PR can therefore trigger the matrix despite `paths-ignore`. After every required candidate gate passes, an acceptance-record successor may use `[skip ci]` when an exact diff confirms that only nonauthoritative documentation changed and executable, workflow, test and authority bytes are unchanged. Retain the accepted candidate identity and results, run focused document/source checks, and record evidence reuse. This cannot excuse a failed or outstanding product gate. Required checks may remain pending on the skipped head, so this handoff keeps the PR draft and unmerged; a later merge must satisfy its applicable checks. See GitHub's [path-filter semantics](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/trigger-a-workflow#git-diff-comparisons) and [skip instructions](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/skip-workflow-runs). - -Stop at the approved development milestone. Stable release, merge, tag, publication and the next phase remain separate actions. +Delivery reuses same-commit core/Parquet XML instead of rerunning those suites. +Candidate validation checks complete current canonical test identities against +collection, exact committed source, wheel/sdist module and resource bytes, safe +archive handling and clean installed behavior outside the checkout. Synthetic +archive/XML negatives test corruption detection; they never count as a real +build or installed execution. + +Build twice from the same committed source and recorded environment with +`SOURCE_DATE_EPOCH`; compare wheel bytes and every sdist file payload. Sdist archive +timestamps may differ. Keep installed import/input-only checks with numerical +and optional imports blocked separate from actual mathematical/audit execution. +Installed checks cover the existing public APIs, CLI, privacy, safe output and +Hero. Schema and Hero resources must match their canonical source bytes. + +The source archive contains one project root and the complete tracked tree. +Exclude Git internals, environments, caches, generated bytecode, private inputs +and full theory PDFs. Wheel, sdist and source archives retain their distinct scopes. + +## Performance and evidence reuse + +Measure complete output publication separately from report +`run.duration_seconds`. Untraced Hero wall time supports the under-five-second +target on the stated reference environment. Traced allocation and process peak +RSS are separate observations. Do not substitute a small workload or extrapolation +for required actual 100k execution. See [performance details](../tests/performance/README.md). + +Phase 4's metadata-only 100k run took about 1,362.47 seconds and 7.12 GiB peak RSS. +It did not execute general ancestry. Phase 5 must measure its own enabled graph +workloads; comparable regression review uses the approved >20% time or >50% +memory thresholds. Operational timeouts are resource guards, not product SLAs. + +Reuse unchanged successful candidate evidence for nonauthoritative administrative +successors and disclose that reuse. Repeat affected gates when executable or +authoritative content changes. Preserve failed attempts and material limitations +in the existing completion/validation record; do not generate another permanent +receipt framework. A hosted artifact that cannot be retrieved is distinct from +locally reproduced bytes and must be described accurately. + +The historical Phase 4 procedure and full evidence are recoverable at +`1db3b1a460c233242ff37fbe45b8fac74d6101ef` and its phase completion records. +Stop at the authorized step. Verification never merges, tags or publishes by itself. diff --git a/docs/theory_traceability.md b/docs/theory_traceability.md index 4bd2dd6..9dc732f 100644 --- a/docs/theory_traceability.md +++ b/docs/theory_traceability.md @@ -1,6 +1,18 @@ # Theory Traceability -Status: Phase 3 mathematical development milestone. Final acceptance is recorded in the three Phase 3 milestone reports. +## Current Phase 5 Step 1 status + +The Phase 5 plan and decisions are approved. T4, T6, full PR-008, PR-009 depth and +T3 lineage bounds now have fixed [implementation contracts](lineage_contract.md) +and independent fixtures. Their general lineage algorithms remain unimplemented. +The existing Phase 3 mathematics and Phase 4 reports/CLI remain the current runtime. + +The sections below retain historical implementation explanations. Current checks +protect owner/layer boundaries and current behavior; they no longer require +historical test-body or source-binding migrations to execute. Frozen specifications +and mathematical/Hero expectations remain unchanged. + +Historical Phase 3 status: mathematical development milestone. Final acceptance is recorded in the three Phase 3 milestone reports. The controlling maps are: diff --git a/scripts/check_spec_consistency.py b/scripts/check_spec_consistency.py index 85fe9d4..9f9c13e 100644 --- a/scripts/check_spec_consistency.py +++ b/scripts/check_spec_consistency.py @@ -1,18 +1,22 @@ -"""Check Phase 2 specification and repository presence without analytical work.""" +"""Check current specifications, schema shape and canonical packaged resources. +Historical phase/step gates remain recoverable in Git. This command performs no +product calculations and does not certify a test run or release candidate. +""" from __future__ import annotations +import argparse import json from pathlib import Path +import runpy +import sys ROOT = Path(__file__).resolve().parents[1] REQUIRED = { "README.md", "LICENSE", "NOTICE", "pyproject.toml", "PHASE_0_APPROVAL.md", - "PHASE_1_PLAN.md", "REPOSITORY_ARCHITECTURE.md", "DEPENDENCY_STRATEGY.md", + "REPOSITORY_ARCHITECTURE.md", "DEPENDENCY_STRATEGY.md", "DATA_AND_PROVENANCE_SPEC.md", "OBSERVABILITY_AND_REPORTING.md", - "THEORY_TO_CODE_TRACEABILITY.md", "PHASE_1_COMPLETION.md", - "PHASE_2_COMPLETION.md", "PHASE_2_VALIDATION_REPORT.md", - "PHASE_2_ARCHITECTURE_COMPLIANCE_REPORT.md", + "THEORY_TO_CODE_TRACEABILITY.md", "PHASE_5_PLAN.md", "PHASE_5_DECISIONS.md", } SCHEMAS = {"report.schema.json", "config.schema.json", "schema_mapping.schema.json", "version_order.schema.json", "normalized_manifest.schema.json"} @@ -21,188 +25,34 @@ WORKFLOWS = {"ci.yml", "golden.yml", "security.yml", "release.yml"} -def main() -> int: - missing = sorted(name for name in REQUIRED if not (ROOT / name).is_file()) +def audit(root: Path = ROOT) -> dict: + missing = sorted(name for name in REQUIRED if not (root / name).is_file()) if missing: - raise SystemExit(f"Missing required files: {missing}") - schema_root = ROOT / "schemas" - if {path.name for path in schema_root.glob("*.json")} != SCHEMAS: - raise SystemExit("Schema file set differs from approved architecture") - for path in sorted(schema_root.glob("*.json")): + raise ValueError(f"Missing required files: {missing}") + if {path.name for path in (root / "schemas").glob("*.json")} != SCHEMAS: + raise ValueError("Schema file set differs from the supported architecture") + for path in sorted((root / "schemas").glob("*.json")): schema = json.loads(path.read_text(encoding="utf-8")) if not isinstance(schema, dict) or schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema": - raise SystemExit(f"Unexpected schema dialect or root: {path.name}") - if {p.name for p in (ROOT / "examples/hero").iterdir() if p.is_file()} != HERO: - raise SystemExit("Hero file set differs from approved architecture") - if {p.name for p in (ROOT / ".github/workflows").glob("*.yml")} != WORKFLOWS: - raise SystemExit("Workflow file set differs from approved architecture") - print(f"required root files: {len(REQUIRED)}") - print(f"schemas parsed: {len(SCHEMAS)}") - print(f"hero files found: {len(HERO)}") - print(f"workflows found: {len(WORKFLOWS)}") - print("specification structure: PASS") + raise ValueError(f"Unexpected schema dialect or root: {path.name}") + if {path.name for path in (root / "examples/hero").iterdir() if path.is_file()} != HERO: + raise ValueError("Canonical Hero file set differs") + if {path.name for path in (root / ".github/workflows").glob("*.yml")} != WORKFLOWS: + raise ValueError("Workflow file set differs") + checker = runpy.run_path(str(ROOT / "scripts/release_check.py"), run_name="current_specification_checks") + return {"schemas_parsed": len(SCHEMAS), "hero_files": len(HERO), + **checker["verify_frozen_specifications"](root), **checker["verify_resources"](root)} + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + if any(arg in {"--phase", "--step"} or arg.startswith(("--phase=", "--step=")) for arg in argv): + raise SystemExit("Historical phase dispatch has been retired. Run this current check without --phase/--step; prior gates are recoverable from Git history.") + argparse.ArgumentParser(description=__doc__).parse_args(argv) + print(json.dumps(audit(), indent=2)) + print("Current specification consistency: PASS") return 0 -def phase4_main(step: int = 1) -> int: - """Validate Step 1 control and preserve the inherited structure checks.""" - import runpy - - if type(step) is not int or step != 1: - raise ValueError("Only authorized Phase 4 Step 1 specification checking is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_specification_control") - control["audit_phase4"](step=step) - result = main() - print("Phase 4 Step 1: approved governance and frozen specification structure: PASS") - return result - - -def phase4_step2_main(step: int = 2) -> int: - """Check the authorized canonical contract and the frozen repository layout.""" - import runpy - - if type(step) is not int or step != 2: - raise ValueError("Only authorized Phase 4 Step 2 specification checking is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step2_specification_control") - control["audit_phase4_step2"](step=step) - result = main() - print("Phase 4 Step 2: canonical report schema and preserved specification structure: PASS") - return result - - -def phase4_step3_main(step: int = 3) -> int: - """Check authorized evidence assembly against the frozen public contract.""" - import runpy - - if type(step) is not int or step != 3: - raise ValueError("Only authorized Phase 4 Step 3 specification checking is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step3_specification_control") - control["audit_phase4_step3"](step=step) - result = main() - print("Phase 4 Step 3: explicit evidence assembly and preserved specification structure: PASS") - return result - - -def phase4_step4_main(step: int = 4) -> int: - """Check authorized evidence assembly against the frozen public contract.""" - import runpy - - if type(step) is not int or step != 4: - raise ValueError("Only authorized Phase 4 Step 4 specification checking is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step4_specification_control") - control["audit_phase4_step4"](step=step) - result = main() - print("Phase 4 Step 4: privacy views and safe diagnostics and preserved specification structure: PASS") - return result - - -def phase4_step5_main(step: int = 5) -> int: - """Check pure renderers against the frozen report and repository contracts.""" - import runpy - - if type(step) is not int or step != 5: - raise ValueError("Only authorized Phase 4 Step 5 specification checking is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step5_specification_control") - control["audit_phase4_step5"](step=step) - result = main() - print("Phase 4 Step 5: JSON and Markdown renderers and preserved specification structure: PASS") - return result - - -def phase4_step6_main(step: int = 6) -> int: - """Check safe output publication against the frozen report and repository contracts.""" - import runpy - - if type(step) is not int or step != 6: - raise ValueError("Only authorized Phase 4 Step 6 specification checking is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step6_specification_control") - control["audit_phase4_step6"](step=step) - result = main() - print("Phase 4 Step 6: safe JSON and Markdown publication and preserved specification structure: PASS") - return result - - -def phase4_step7_main(step: int = 7) -> int: - """Check CLI orchestration against the frozen report and repository contracts.""" - import runpy - - if type(step) is not int or step != 7: - raise ValueError("Only authorized Phase 4 Step 7 specification checking is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step7_specification_control") - control["audit_phase4_step7"](step=step) - result = main() - print("Phase 4 Step 7: local audit and input-only validation and preserved specification structure: PASS") - return result - - -def phase4_step8_main(step: int = 8) -> int: - """Check CLI orchestration against the frozen report and repository contracts.""" - import runpy - - if type(step) is not int or step != 8: - raise ValueError("Only authorized Phase 4 Step 8 specification checking is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step8_specification_control") - control["audit_phase4_step8"](step=step) - result = main() - print("Phase 4 Step 8: local audit and input-only validation and preserved specification structure: PASS") - return result - - -def phase4_current_main(step: int = 11) -> int: - """Check CLI orchestration against the frozen report and repository contracts.""" - import runpy - - if type(step) is not int or step != 11: - raise ValueError("Only authorized Phase 4 Step 11 specification checking is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_current_specification_control") - control["audit_phase4_current"](step=step) - result = main() - print("Phase 4 Step 11: final documentation and preserved specification structure: PASS") - return result - - -def cli_main(argv: list[str] | None = None) -> int: - """Keep the argument-free historical gate and add explicit Phase 4 dispatch.""" - import argparse - - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--phase", type=int, choices=(3, 4)) - parser.add_argument("--step", type=int) - args = parser.parse_args(argv) - if args.phase is None and args.step is None: - return main() - if args.phase == 3 and args.step == 11: - return main() - if args.phase == 4 and args.step == 1: - return phase4_main(step=args.step) - if args.phase == 4 and args.step == 2: - return phase4_step2_main(step=args.step) - if args.phase == 4 and args.step == 3: - return phase4_step3_main(step=args.step) - if args.phase == 4 and args.step == 4: - return phase4_step4_main(step=args.step) - if args.phase == 4 and args.step == 5: - return phase4_step5_main(step=args.step) - if args.phase == 4 and args.step == 6: - return phase4_step6_main(step=args.step) - if args.phase == 4 and args.step == 7: - return phase4_step7_main(step=args.step) - if args.phase == 4 and args.step == 8: - return phase4_step8_main(step=args.step) - if args.phase == 4 and args.step == 11: - return phase4_current_main(step=args.step) - parser.error("Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5 or --phase 4 --step 6 or --phase 4 --step 7 or --phase 4 --step 8 or --phase 4 --step 11") - return 2 - - if __name__ == "__main__": - raise SystemExit(cli_main()) + raise SystemExit(main()) diff --git a/scripts/check_traceability.py b/scripts/check_traceability.py index 098d4cd..e78a575 100644 --- a/scripts/check_traceability.py +++ b/scripts/check_traceability.py @@ -1,5749 +1,88 @@ -"""Check inherited Phase 2 rules and approved Phase 3 Step 8 boundaries. +"""Check current module ownership and local-only architectural boundaries. -The Theory Owner authorized synchronized checker maintenance for each explicitly -approved Phase 2 step on 2026-09-16. Step 9 adds explicit local bundle orchestration only. Earlier -validation and explicit local content-read definitions stay intact. -General graphs, cycles, roots, ancestors, metrics and later layers stay protected. - -This script inspects source syntax without importing the package or loading -hero data. Definition and import allowlists are structural checks; behavioral -correctness remains covered by the unit and security tests. +Behavioral tests own formulas and product outcomes. This static check retains +owner declarations, dependency boundaries, input-only validation and pure +rendering. It does not pin function bodies or replay historical source forms. """ - from __future__ import annotations +import argparse import ast +import json from pathlib import Path - +import sys ROOT = Path(__file__).resolve().parents[1] -PACKAGE = ROOT / "src" / "recursive_integrity_toolkit" -BOOTSTRAP = {"__init__.py", "__main__.py", "cli.py"} -STEP1_CORE = {"models.py", "errors.py", "config.py"} -STEP2_IO = {"io/loaders.py", "utils/hashing.py", "utils/paths.py"} -STEP3_MAPPING = {"io/schema_mapping.py"} -STEP8_OBSERVABILITY = {"observability/levels.py"} -ALLOWED_FUNCTIONS = { - "__init__.py": set(), - "__main__.py": set(), - "cli.py": {"build_parser", "main"}, - "config.py": { - "_bool_value", "_enum_value", "_frozen_mapping", "_invalid", - "_optional_positive_int", "_optional_string", "_parse_input_source", - "_parse_inputs", "_parse_representation", "_parse_resource_limits", - "_parse_simulation", "_parse_version_order", "_string_mapping", - "_string_sequence", "load_config", "resolve_config", - }, - "errors.py": {"IngestionError.__init__", "ToolkitError.__init__", "MappingError.__init__"}, - "io/loaders.py": { - "_check_csv_quotes", "_check_depth", "_failure", "_finite_float", "_headers", - "_limits", "_parse_csv", "_parse_jsonl", "_parse_parquet", "_read_source", - "_reject_constant", "_row_limit", "_select_format", "_text", - "_unique_object", "inventory_source", "load_table", - }, - "models.py": { - "ObservabilityAssessment.__post_init__", "RecordKey.__post_init__", - "RecordKey.__str__", "RecordKey.parse", "ValidationCoverage.__post_init__", - "_validate_identifier", - }, - "utils/hashing.py": {"sha256_bytes"}, - "utils/paths.py": {"local_input_path"}, - "io/schema_mapping.py": { - "_error", "_copy_value", "_unique_object", "_reject_constant", "_finite_float", - "_json_value", "_name", "_validate_operation", "_validate_document", - "compile_mapping", "parse_mapping_json", "load_mapping", "_select", - "_coalesce", "_apply_operation", "map_row", - }, -} -ALLOWED_CLASSES = { - "config.py": {"RepresentationConfig", "ResourceLimits", "ScenarioConfig", "ResolvedConfig"}, - "errors.py": { - "ConfigurationError", "ErrorCode", "IngestionError", "InputError", - "SchemaError", "SecurityError", "ToolkitError", "WarningCode", "MappingError", - }, - "models.py": { - "AuditBundle", "Capability", "CapabilityKey", "CapabilityStatus", "FileFormat", - "FileInventoryEntry", "InputSource", "LoadedTable", "FileRole", - "ObservabilityAssessment", "ParentResolutionStatus", "PrivacyMode", "RawRow", - "RecordKey", "ValidationCoverage", "ValidationMessage", "ValidationSeverity", - }, - "io/schema_mapping.py": {"MappingPlan", "FieldMappingTrace", "MappingNotice", "MappedRow"}, -} -ALLOWED_CORE_IMPORTS = { - "config.py": {".errors", ".models", "__future__", "dataclasses", "json", "pathlib", "tomllib", "typing"}, - "errors.py": {"enum", "__future__"}, - "io/loaders.py": { - "..config", "..errors", "..models", "..utils.hashing", "..utils.paths", - "__future__", "csv", "io", "json", "math", "os", "pathlib", "pyarrow", - "pyarrow.parquet", "stat", "threading", - }, - "models.py": {"__future__", "dataclasses", "typing", "enum", "pathlib"}, - "utils/hashing.py": {"__future__", "hashlib"}, - "utils/paths.py": {"__future__", "re", "os", "pathlib", "..errors"}, - "io/schema_mapping.py": { - "..config", "..models", "..errors", "..utils.hashing", ".loaders", "__future__", - "dataclasses", "datetime", "json", "math", "pathlib", "re", - }, -} - -# Step 4 permission covers row-local normalization, base validation and ordering. -# Step 5 join definitions are authorized individually below, never by wildcard. -STEP4_ROWS = {"io/normalization.py", "io/validation.py", "utils/ordering.py"} -ALLOWED_FUNCTIONS.update({ - "io/normalization.py": { - "_policy", "_location", "_csv_quotes", "_input", "_state", "_timestamp", - "_parent_list", "_field_value", "_freeze", "normalize_row", "normalize_table", - }, - "io/validation.py": { - "_fail", "canonical_fields", "validate_canonical_values", "validate_unique_keys", - }, - "utils/ordering.py": {"_record_order_key", "stable_record_order"}, -}) -ALLOWED_FUNCTIONS["errors.py"].add("CanonicalValidationError.__init__") -ALLOWED_CLASSES["errors.py"].add("CanonicalValidationError") -ALLOWED_CLASSES["models.py"].update({ - "SourceType", "ProvenanceConfidence", "ExternalGrounding", "Transformation", - "ContentMode", "NormalizationOptions", "RowLocation", "CanonicalRow", -}) -ALLOWED_CORE_IMPORTS.update({ - "io/normalization.py": { - "__future__", "math", "re", "datetime", "zoneinfo", "types", "json", "..config", - "..errors", "..models", "..utils.ordering", ".schema_mapping", ".loaders", ".validation", - }, - "io/validation.py": {"__future__", "math", "datetime", "..errors", "..models"}, - "utils/ordering.py": {"__future__", "..models"}, -}) - -# Step 5 changes no module-level allowlist. Only these exact definitions are added. -ALLOWED_FUNCTIONS["io/validation.py"].update({ - "_validate_present_fields", "_join_location", "_join_fields", "_join_identity", - "assess_provenance_row", "_join_message", "_join_options", "join_provenance", -}) -ALLOWED_FUNCTIONS["models.py"].update({"ValidationCoverage.ratio", "ProvenanceJoinResult.has_errors"}) -ALLOWED_CLASSES["models.py"].update({"ProvenanceAssessment", "ProvenanceMatch", "ProvenanceJoinResult"}) -ALLOWED_CORE_IMPORTS["io/validation.py"].add("types") - -# Step 6 adds only chronology, immediate references and generation validation. -# The executable module set is unchanged; no future module is opened. -ALLOWED_FUNCTIONS["io/validation.py"].update({ - "_version_failure", "_version_labels", "_version_map", "_version_timestamp", - "resolve_version_order", "_checked_order", "_parent_keys", "parse_parent_ids", - "_reference_key", "_parent_notice", "_parent_lookup", "_resolve_parent_list", - "resolve_parent_references", "validate_generation_declarations", -}) -ALLOWED_FUNCTIONS["models.py"].add("GenerationValidationResult.has_errors") -ALLOWED_CLASSES["models.py"].update({ - "VersionOrderResult", "ParentReference", "ParentValidationResult", - "GenerationAssessment", "GenerationValidationResult", -}) -ALLOWED_CORE_IMPORTS["io/validation.py"].add("json") - -# Step 7 opens no new module. Content paths and explicit reads stay in IO helpers. -ALLOWED_FUNCTIONS["utils/paths.py"].update({ - "_content_error", "_content_lexical", "_content_base", "_inside_base", - "resolve_content_reference", "_open_content_fd", -}) -ALLOWED_FUNCTIONS["io/loaders.py"].add("load_content_reference") -ALLOWED_CORE_IMPORTS["utils/paths.py"].add("stat") - -# Step 8 opens exactly one classifier. No analytical or IO permission is added. -ALLOWED_FUNCTIONS["observability/levels.py"] = { - "_invalid", "_cap", "_records", "_row_key", "_content_keys", "_representation", - "_content_capability", "_provenance_capability", "_lineage_capability", - "_compatibility", "_longitudinal_capability", "_distribution", - "_scenario_capability", "classify_observability", -} -ALLOWED_CLASSES["models.py"].add("ScenarioParameters") -ALLOWED_CORE_IMPORTS["observability/levels.py"] = { - "__future__", "math", "types", "..config", "..errors", "..io.validation", "..models", -} - - -# Step 9 delegates to existing input layers. Exact call sites keep all reads -# within explicit invocation; no eager orchestration or later-layer permission. -ALLOWED_FUNCTIONS["io/validation.py"].update({ - "_bundle_source", "_bundle_source_key", "_bundle_control", "_bundle_plain", - "_bundle_document", "_bundle_setup", "_bundle_incomplete_provenance", - "_bundle_table", "_bundle_order", "_bundle_message_key", "_bundle_messages", - "_bundle_error_message", "validate_bundle", "_bundle_row_key", "_bundle_inventory_key", -}) -ALLOWED_FUNCTIONS["models.py"].add("BundleValidationResult.has_errors") -ALLOWED_CLASSES["models.py"].update({"RowMappingEvidence", "BundleValidationResult"}) -ALLOWED_CORE_IMPORTS["io/validation.py"].update({ - "pathlib", "dataclasses", "tomllib", "..config", "..utils.paths", "..utils.hashing", - ".loaders", ".normalization", ".schema_mapping", "..observability.levels", -}) -PIPELINE_CALL_SITES = { - "_read_source": {"_bundle_control"}, - "load_table": {"_bundle_table"}, - "inventory_source": {"validate_bundle"}, - "load_content_reference": {"validate_bundle"}, - "parse_mapping_json": {"validate_bundle"}, - "map_row": {"_bundle_table"}, - "normalize_row": {"_bundle_table"}, - "classify_observability": {"validate_bundle"}, - "_bundle_control": {"_bundle_setup", "validate_bundle"}, - "_bundle_table": {"validate_bundle"}, - "_bundle_setup": {"validate_bundle"}, - "validate_bundle": set(), -} -PIPELINE_IMPORT_NAMES = { - ".loaders": {"_limits", "_read_source", "_select_format", "_text", "_check_depth", - "_finite_float", "_reject_constant", "_unique_object", "load_table", - "inventory_source", "load_content_reference"}, - ".normalization": {"_field_value", "_input", "_policy", "_state", "normalize_row"}, - ".schema_mapping": {"map_row", "parse_mapping_json"}, - "..observability.levels": {"classify_observability"}, -} - -# Preserve every import and file restriction from the Phase 1 checker. -FORBIDDEN_IMPORTS = { - "numpy", "pandas", "pyarrow", "networkx", "scipy", "sklearn", "torch", - "tensorflow", "transformers", -} +FORBIDDEN_IMPORTS = {"socket", "urllib", "http", "ftplib", "requests", "httpx", + "aiohttp", "subprocess", "networkx", "scipy", "sklearn", "torch", + "tensorflow", "transformers"} FORBIDDEN_FILES = {"collapse_score.py", "integrity_score.py", "universal_score.py"} - - - -# Phase 3 Step 1: declarations and validation contracts only. -PHASE3_ACTIVE_STEP = 11 -PHASE3_CONTRACT_CLASSES = { - "CalculationEvidenceClass", "CalculationStatus", "CalculationReason", "NumericalPolicy", - "RepresentationDescriptor", "RecordStateAssignment", "CalculationScope", "WeightingOptions", - "TailSelectionOptions", "CalculationMetadata", "ScalarCalculation", "ExplicitPairContext", - "ClosedResamplingMetadata", -} -PHASE3_CONTRACT_FUNCTIONS = { - "_calculation_text", "_calculation_strings", "_calculation_integer", "_calculation_number", - "NumericalPolicy.__post_init__", "RepresentationDescriptor.__post_init__", - "RecordStateAssignment.__post_init__", "CalculationScope.__post_init__", - "WeightingOptions.__post_init__", "TailSelectionOptions.__post_init__", - "CalculationMetadata.__post_init__", "ScalarCalculation.__post_init__", - "ExplicitPairContext.__post_init__", "ClosedResamplingMetadata.__post_init__", -} -ALLOWED_FUNCTIONS["models.py"].update(PHASE3_CONTRACT_FUNCTIONS) -ALLOWED_CLASSES["models.py"].update(PHASE3_CONTRACT_CLASSES) -ALLOWED_CORE_IMPORTS["models.py"].add("math") - - -# Step 2 opens only literal-field representation, not any mathematical metric. -PHASE3_FIELD_MODULES = {"representations/base.py", "representations/field.py"} -ALLOWED_FUNCTIONS["representations/base.py"] = { - "RepresentationResult.selected_count", "RepresentationResult.included_count", - "RepresentationResult.excluded_count", "RepresentationProtocol.__call__", - "_representation_error", "_literal_text", "_selected_records", "_selected_key", -} -ALLOWED_CLASSES["representations/base.py"] = { - "RepresentationSelection", "RepresentationResult", "RepresentationProtocol", -} -ALLOWED_FUNCTIONS["representations/field.py"] = { - "_selection", "select_field_representation", "assign_field_states", -} -REPRESENTATION_IMPORT_NAMES = { - "representations/base.py": { - "__future__": {"annotations"}, "dataclasses": {"dataclass", "field"}, - "types": {"MappingProxyType"}, "typing": {"Protocol"}, "..config": {"RepresentationConfig"}, - "..errors": {"CanonicalValidationError", "ErrorCode"}, "..io.validation": {"validate_canonical_values"}, - "..models": {"CalculationReason", "CalculationScope", "CalculationStatus", "CanonicalRow", - "FileRole", "RecordKey", "RecordStateAssignment", "RepresentationDescriptor", - "RowLocation", "ValidationCoverage", "ValidationMessage"}, - }, - "representations/field.py": { - "__future__": {"annotations"}, "..config": {"RepresentationConfig"}, - "..errors": {"ErrorCode", "WarningCode"}, - "..models": {"CalculationReason", "CalculationScope", "CalculationStatus", "CanonicalRow", - "RecordKey", "RecordStateAssignment", "RepresentationDescriptor", - "ValidationCoverage", "ValidationMessage", "ValidationSeverity"}, - ".base": {"RepresentationResult", "RepresentationSelection", "_literal_text", - "_representation_error", "_selected_records"}, - }, -} -ALLOWED_CORE_IMPORTS["representations/base.py"] = set(REPRESENTATION_IMPORT_NAMES["representations/base.py"]) -ALLOWED_CORE_IMPORTS["representations/field.py"] = set(REPRESENTATION_IMPORT_NAMES["representations/field.py"]) - - -def _phase3_representation_boundary(tree: ast.Module, relative: str) -> None: - """Reject hidden computation, I/O, aliases and user callback execution.""" - direct_calls = { - "representations/base.py": { - "dataclass", "field", "len", "type", "bool", "any", "set", "tuple", "dict", "sorted", "str", - "CanonicalValidationError", "RowLocation", "_representation_error", "_literal_text", - "validate_canonical_values", - }, - "representations/field.py": { - "type", "any", "len", "tuple", "sorted", "_representation_error", "_literal_text", - "_selected_records", "_selection", "RepresentationDescriptor", "RepresentationSelection", - "ValidationMessage", "RecordStateAssignment", "CalculationScope", - "RepresentationResult", "ValidationCoverage", - }, - } - method_calls = { - "representations/base.py": {("value", "encode"), ("version", "strip"), - ("seen", "add"), ("loaded", "add"), ("selected", "append")}, - "representations/field.py": {("values", "get"), ("value", "encode"), - ("assignments", "append"), ("states", "append"), - ("included", "append"), ("excluded", "append")}, - } - for top in tree.body: - doc = (isinstance(top, ast.Expr) and isinstance(top.value, ast.Constant) - and isinstance(top.value.value, str)) - if not doc and not isinstance(top, (ast.ImportFrom, ast.FunctionDef, ast.ClassDef)): - raise SystemExit("Unexpected eager representation operation") - for node in ast.walk(tree): - if isinstance(node, (ast.Lambda, ast.AsyncFunctionDef, ast.Await, ast.Yield, ast.YieldFrom)): - raise SystemExit("Unexpected callback in field representations") - if isinstance(node, ast.Import): - raise SystemExit("Import must use exact representation symbols") - if isinstance(node, ast.ImportFrom): - module = "." * node.level + (node.module or "") - allowed = REPRESENTATION_IMPORT_NAMES[relative].get(module, set()) - if any(a.name not in allowed or a.asname is not None for a in node.names): - raise SystemExit("Import symbol outside representation scope") - if isinstance(node, ast.BinOp): - difference = (relative == "representations/base.py" and isinstance(node.op, ast.Sub) - and isinstance(node.left, ast.Call) and isinstance(node.left.func, ast.Name) - and node.left.func.id == "set" and len(node.left.args) == 1 - and isinstance(node.left.args[0], ast.Name) and node.left.args[0].id == "dataset_versions" - and isinstance(node.right, ast.Name) and node.right.id == "loaded") - if not isinstance(node.op, ast.BitOr) and not difference: - raise SystemExit("Unexpected metric arithmetic in representations") - if isinstance(node, ast.Call): - direct = isinstance(node.func, ast.Name) and node.func.id in direct_calls[relative] - method = (isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) - and (node.func.value.id, node.func.attr) in method_calls[relative]) - if not direct and not method: - raise SystemExit("Unexpected call in pure field representations") - - - -# Step 3 opens only exact record-form states and PR-006 duplicate counts. -PHASE3_EXACT_MODULES = {"representations/content_hash.py", "metrics/duplicates.py"} -ALLOWED_FUNCTIONS["representations/content_hash.py"] = { - "exact_content_bytes", "_payload_snapshot", "assign_content_states", -} -ALLOWED_CLASSES["representations/content_hash.py"] = {"ExactContentRepresentation"} -ALLOWED_FUNCTIONS["metrics/duplicates.py"] = { - "ExactDuplicateGroup.representative", "ExactDuplicateGroup.record_count", "detect_exact_duplicates", -} -ALLOWED_CLASSES["metrics/duplicates.py"] = {"ExactDuplicateGroup", "ExactDuplicateResult"} -EXACT_IMPORT_NAMES = { - "representations/content_hash.py": { - "__future__": {"annotations"}, "dataclasses": {"dataclass", "field"}, - "types": {"MappingProxyType"}, "..errors": {"ErrorCode"}, - "..models": {"CalculationReason", "CalculationScope", "CalculationStatus", "CanonicalRow", - "ContentMode", "RecordKey", "RecordStateAssignment", "RepresentationDescriptor", "ValidationCoverage"}, - "..utils.hashing": {"sha256_bytes"}, - ".base": {"RepresentationResult", "RepresentationSelection", "_literal_text", - "_representation_error", "_selected_records"}, - }, - "metrics/duplicates.py": { - "__future__": {"annotations"}, "dataclasses": {"dataclass", "field"}, - "..models": {"CalculationEvidenceClass", "CalculationMetadata", "CalculationScope", "CalculationStatus", - "CanonicalRow", "ContentMode", "RecordKey", "RepresentationDescriptor", "ScalarCalculation", - "ValidationCoverage"}, - "..representations.content_hash": {"assign_content_states"}, - }, -} -for _exact_module, _exact_imports in EXACT_IMPORT_NAMES.items(): - ALLOWED_CORE_IMPORTS[_exact_module] = set(_exact_imports) - - -def _phase3_exact_boundary(tree: ast.Module, relative: str) -> None: - """Exact imports and calls; only PR-006 counting arithmetic is admitted.""" - direct_calls = { - "representations/content_hash.py": { - "dataclass", "field", "type", "len", "set", "tuple", "_literal_text", "_representation_error", - "_selected_records", "_payload_snapshot", "exact_content_bytes", "sha256_bytes", - "RepresentationDescriptor", "RepresentationSelection", "RepresentationResult", - "RecordStateAssignment", "CalculationScope", "ValidationCoverage", "ExactContentRepresentation", - }, - "metrics/duplicates.py": { - "dataclass", "field", "assign_content_states", "sorted", "tuple", "len", - "ExactDuplicateGroup", "CalculationMetadata", "ScalarCalculation", "ExactDuplicateResult", - }, - } - methods = { - "representations/content_hash.py": { - ("text", "strip"), ("text", "encode"), ("resolved_content", "items"), - ("assignments", "append"), ("snapshots", "append"), ("keys", "append"), ("states", "append"), - }, - "metrics/duplicates.py": { - ("members", "setdefault"), ("bucket", "append"), ("groups", "append"), ("counts", "append"), - }, - } - expected_subtractions = { - "representations/content_hash.py": {ast.dump(ast.parse(expr, mode="eval").body) for expr in - ("set(payloads) - selected_keys", "selected_keys - set(payloads)")}, - "metrics/duplicates.py": {ast.dump(ast.parse("len(keys) - 1", mode="eval").body)}, - } - for top in tree.body: - doc = (isinstance(top, ast.Expr) and isinstance(top.value, ast.Constant) - and isinstance(top.value.value, str)) - if not doc and not isinstance(top, (ast.ImportFrom, ast.FunctionDef, ast.ClassDef)): - raise SystemExit("Unexpected eager exact-content operation") - for node in ast.walk(tree): - if isinstance(node, (ast.Lambda, ast.AsyncFunctionDef, ast.Await, ast.Yield, ast.YieldFrom, ast.Import)): - raise SystemExit("Unexpected dynamic or import operation in exact-content scope") - if isinstance(node, ast.ImportFrom): - module = "." * node.level + (node.module or "") - allowed = EXACT_IMPORT_NAMES[relative].get(module, set()) - if any(a.name not in allowed or a.asname is not None for a in node.names): - raise SystemExit("Import symbol outside exact-content scope") - if isinstance(node, ast.BinOp) and not isinstance(node.op, ast.BitOr): - if ast.dump(node) not in expected_subtractions[relative]: - raise SystemExit("Unexpected arithmetic outside exact duplicate definitions") - if isinstance(node, ast.AugAssign): - expected = ast.parse("duplicates += len(keys) - 1").body[0] - if relative != "metrics/duplicates.py" or ast.dump(node) != ast.dump(expected): - raise SystemExit("Unexpected augmented arithmetic outside PR-006") - if isinstance(node, ast.Call): - direct = isinstance(node.func, ast.Name) and node.func.id in direct_calls[relative] - method = (isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) - and (node.func.value.id, node.func.attr) in methods[relative]) - if not direct and not method: - raise SystemExit("Unexpected call in pure exact-content scope") - - - -# Step 4 opens only the reviewed single-scope F-001 through F-004 kernels. -PHASE3_DISTRIBUTION_MODULES = {"metrics/diversity.py"} -ALLOWED_FUNCTIONS["metrics/diversity.py"] = { - "_invalid", "_text", "_key", "_context", "_number", "_pairs", "_probabilities", - "_metadata", "_metrics", "distribution_from_counts", "distribution_from_probabilities", - "_assignments", "_weights", "calculate_state_distribution", -} -ALLOWED_CLASSES["metrics/diversity.py"] = {"StateFrequency", "DistributionMetrics", "StateDistributionResult"} -ALLOWED_CORE_IMPORTS["metrics/diversity.py"] = { - "__future__", "dataclasses", "math", "types", "..errors", "..models", "..representations.base", -} -DISTRIBUTION_SOURCE_SHA256 = "34b1b77d00b9a2169aa7680d0edf77b05d5aec0bdc26b3bd141d38853f9728a9" - - -def _phase3_distribution_boundary(tree: ast.Module) -> None: - """Pin the Step 9 additive body and compare in this interpreter, not across AST formats.""" - import hashlib - raw = (PACKAGE / "metrics/diversity.py").read_bytes() - if hashlib.sha256(raw).hexdigest() != DISTRIBUTION_SOURCE_SHA256: - raise SystemExit("Unexpected distribution source outside the reviewed Step 9 body") - if ast.dump(tree, include_attributes=False) != ast.dump(ast.parse(raw), include_attributes=False): - raise SystemExit("Unexpected distribution AST outside the reviewed Step 9 body") - - -ALLOWED_FUNCTIONS["metrics/diversity.py"].update({'_pair_distribution', '_pair_scalar_check', 'compare_support', '_pair_metadata_check', '_pair_metadata', '_harmonize_distribution'}) -ALLOWED_CLASSES["metrics/diversity.py"].update({'SupportComparison'}) -ALLOWED_CORE_IMPORTS["metrics/diversity.py"].add("..representations.compatibility") - - -# Step 5 opens only declared provenance composition and direct classification. -PHASE3_PROVENANCE_MODULES = {"metrics/provenance.py"} -ALLOWED_FUNCTIONS["metrics/provenance.py"] = set(['_checked_join', '_composition', '_direct', '_invalid', '_key', '_keys', '_messages', '_metadata', '_scalar', '_text', '_weighted', '_weights', 'classify_direct_grounding', 'summarize_provenance']) -ALLOWED_CLASSES["metrics/provenance.py"] = set(['DeclaredComposition', 'DirectGroundingAssignment', 'DirectGroundingBasis', 'ProvenanceCompositionResult', 'WeightedSourceComposition']) -ALLOWED_CORE_IMPORTS["metrics/provenance.py"] = { - "__future__", "dataclasses", "math", "types", "..errors", "..models", "..io.validation", -} -PROVENANCE_AST_SHA256 = "554e40896edf5c0e30432015ad357a82e3b3419b6f7529d4e22a09894b4569c2" - - -def _phase3_provenance_boundary(tree: ast.Module) -> None: - """Pin the reviewed Step 5 body, never learn a new body at gate execution. - - Independent rational tests cover mathematics; mutation tests cover change - control. Only empty Python-version type_params metadata is omitted. - """ - import hashlib - for node in ast.walk(tree): - if "type_params" in node._fields: - if getattr(node, "type_params", []): - raise SystemExit("Unexpected generic declaration in provenance metrics") - node._fields = tuple(name for name in node._fields if name != "type_params") - try: - serialized = ast.dump(tree, include_attributes=False, show_empty=True) - except TypeError: - serialized = ast.dump(tree, include_attributes=False) - if hashlib.sha256(serialized.encode("utf-8")).hexdigest() != PROVENANCE_AST_SHA256: - raise SystemExit("Unexpected provenance implementation outside reviewed Step 5 body") - - - - -# Step 6 opens only direct F-009/F-010 bounds, with no lineage branch. -PHASE3_BOUNDS_MODULES = {"metrics/bounds.py"} -ALLOWED_FUNCTIONS["metrics/bounds.py"] = {'_metric', 'closure_exposure_bounds', '_messages', '_confidence', '_text', '_invalid', '_key', '_scope', '_coverage', '_count', '_basis', 'DirectClosureExposureBounds.status', 'direct_closure_exposure', '_observed'} -ALLOWED_CLASSES["metrics/bounds.py"] = {'DirectClosureExposureBounds'} -ALLOWED_CORE_IMPORTS["metrics/bounds.py"] = {"__future__", "dataclasses", "math", "..errors", "..models", ".provenance"} -BOUNDS_SOURCE_SHA256 = "01693a92ab9c0a78cba5addf7bd778fca5d14051599d7878dd682ddc5c9707a4" - - -def _phase3_bounds_boundary(tree: ast.Module) -> None: - """Pin reviewed bytes, then compare ASTs in the same interpreter. - - The digest is a reviewed literal. It is never reset from inspected input. - Both actual file edits and injected executable nodes are rejected. - """ - import hashlib - raw = (PACKAGE / "metrics/bounds.py").read_bytes() - if hashlib.sha256(raw).hexdigest() != BOUNDS_SOURCE_SHA256: - raise SystemExit("Direct bounds source differs from the reviewed Step 6 body") - if ast.dump(tree, include_attributes=False) != ast.dump(ast.parse(raw), include_attributes=False): - raise SystemExit("Unexpected direct bounds executable syntax outside Step 6") - - -# Step 7 opens only explicit observed-tail diagnostics and analytic F-014. -PHASE3_TAIL_MODULES = {"metrics/tail.py"} -ALLOWED_FUNCTIONS["metrics/tail.py"] = {'_state', '_metadata', 'select_tail', '_distribution', 'one_step_extinction_probability', '_rarity_key', '_invalid', '_options'} -ALLOWED_CLASSES["metrics/tail.py"] = {'TailSelectionResult', 'RarityEntry', 'ExtinctionProbabilityResult'} -ALLOWED_CORE_IMPORTS["metrics/tail.py"] = {"__future__", "dataclasses", "math", "..errors", "..models", ".diversity"} -TAIL_SOURCE_SHA256 = "b7618dea6a28fd69aa951d7ae5170c772bc2bea2999906ed61bca9079b8d5cf7" - - -def _phase3_tail_boundary(tree: ast.Module) -> None: - """Fixed reviewed bytes and same-interpreter syntax; no later T2 behaviors.""" - import hashlib - raw = (PACKAGE / "metrics/tail.py").read_bytes() - if hashlib.sha256(raw).hexdigest() != TAIL_SOURCE_SHA256: - raise SystemExit("Tail source differs from the reviewed Step 7 body") - if ast.dump(tree, include_attributes=False) != ast.dump(ast.parse(raw), include_attributes=False): - raise SystemExit("Unexpected tail/scenario executable syntax outside Step 7") - - -# Step 8 opens only explicit closed resampling, never T5 reopening. -PHASE3_RESAMPLING_MODULES = {"metrics/resampling.py"} -ALLOWED_FUNCTIONS["metrics/resampling.py"] = set(['_diversity', '_generation', '_inputs', '_integer', '_invalid', '_metadata', '_resources', '_sample_counts', '_state', 'expected_diversity_after_steps', 'simulate_closed_resampling']) -ALLOWED_CLASSES["metrics/resampling.py"] = set(['ExpectedDiversityResult', 'ResamplingInput', 'ResamplingReplicate', 'ResamplingSimulation', 'SampledGeneration']) -ALLOWED_CORE_IMPORTS["metrics/resampling.py"] = {"__future__", "dataclasses", "math", "types", "..errors", "..models", ".diversity", "numpy"} -RESAMPLING_SOURCE_SHA256 = "0896b80c26ad0b9865e24957202b54c2ab9217921d1c19e84ab22d84928f2e40" - - -def _phase3_resampling_boundary(tree: ast.Module) -> None: - """Fixed reviewed source and exact AST; lazy NumPy only in the sampler.""" - import hashlib - raw = (PACKAGE / "metrics/resampling.py").read_bytes() - if hashlib.sha256(raw).hexdigest() != RESAMPLING_SOURCE_SHA256: - raise SystemExit("Resampling source differs from the reviewed Step 8 body") - if ast.dump(tree, include_attributes=False) != ast.dump(ast.parse(raw), include_attributes=False): - raise SystemExit("Unexpected resampling syntax outside Step 8") - - -def _phase3_contract_boundary(tree: ast.Module) -> None: - """Reject calculation or execution inside the newly authorized declarations.""" - permitted = {"type", "len", "set", "any", "ValueError", "TypeError", "field", "dataclass", - "WeightingOptions", "_calculation_text", "_calculation_strings", - "_calculation_integer", "_calculation_number"} - roots = {name.split(".")[0] for name in PHASE3_CONTRACT_FUNCTIONS} | PHASE3_CONTRACT_CLASSES - for top in tree.body: - if not isinstance(top, (ast.FunctionDef, ast.ClassDef)) or top.name not in roots: - continue - for node in ast.walk(top): - if isinstance(node, (ast.Lambda, ast.AsyncFunctionDef, ast.Await, ast.Yield, ast.YieldFrom)): - raise SystemExit("Unexpected callback in Phase 3 contracts") - if isinstance(node, ast.BinOp) and not isinstance(node.op, (ast.BitOr, ast.BitAnd)): - raise SystemExit("Unexpected metric arithmetic in Phase 3 contracts") - if isinstance(node, ast.Call): - direct = isinstance(node.func, ast.Name) and node.func.id in permitted - finite = (isinstance(node.func, ast.Attribute) and node.func.attr == "isfinite" - and isinstance(node.func.value, ast.Name) and node.func.value.id == "math" - and isinstance(top, ast.FunctionDef) and top.name == "_calculation_number") - if not (direct or finite): - raise SystemExit("Unexpected call in Phase 3 contracts") - for node in ast.walk(tree): - if isinstance(node, ast.Import) and any(alias.name == "math" for alias in node.names): - if len(node.names) != 1 or node.names[0].asname is not None or not any( - isinstance(fn, ast.FunctionDef) and fn.name == "_calculation_number" - and any(child is node for child in ast.walk(fn)) for fn in tree.body): - raise SystemExit("Unexpected math import outside finite-type validation") - - -def _definitions(tree: ast.AST, prefix: str = "") -> tuple[list[str], list[str]]: - """Collect qualified definitions without executing inspected source.""" - functions: list[str] = [] - classes: list[str] = [] - for node in ast.iter_child_nodes(tree): - child_prefix = prefix - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - name = f"{prefix}.{node.name}" if prefix else node.name - if isinstance(node, ast.AsyncFunctionDef): - raise SystemExit(f"Async implementation is outside Step 8: {name}") - if isinstance(node, ast.ClassDef): - classes.append(name) - else: - functions.append(name) - child_prefix = name - child_functions, child_classes = _definitions(node, child_prefix) - functions.extend(child_functions) - classes.extend(child_classes) - return functions, classes - - -# Step 9 opens only explicit pair compatibility; no metric dispatch here. -PHASE3_PAIR_MODULES = {"representations/compatibility.py"} -ALLOWED_FUNCTIONS["representations/compatibility.py"] = {'_scope', '_mapping', '_descriptor', 'validate_representation_compatibility', '_invalid', '_context', '_text'} -ALLOWED_CLASSES["representations/compatibility.py"] = {'StateMappingDeclaration', 'RepresentationCompatibility'} -ALLOWED_CORE_IMPORTS["representations/compatibility.py"] = {'..models', 'types', '..io.validation', '..errors', '__future__', 'dataclasses'} -PAIR_SOURCE_SHA256 = "c4ce8e30d8026dd7ac1e10ac09e7b70a23ae2c4fc451f2e37890b919965d4509" - - -def _phase3_pair_boundary(tree: ast.Module) -> None: - """Exact reviewed declaration checks; reject source changes and injected bodies.""" - import hashlib - raw = (PACKAGE / "representations/compatibility.py").read_bytes() - if hashlib.sha256(raw).hexdigest() != PAIR_SOURCE_SHA256: - raise SystemExit("Unexpected compatibility source outside reviewed Step 9") - if ast.dump(tree, include_attributes=False) != ast.dump(ast.parse(raw), include_attributes=False): - raise SystemExit("Unexpected compatibility AST outside reviewed Step 9") - - -def main() -> int: - paths = sorted(PACKAGE.rglob("*.py")) - if len(paths) != 40: - raise SystemExit(f"Expected 40 package modules, found {len(paths)}") - relative_paths = {path.relative_to(PACKAGE).as_posix() for path in paths} - missing = (BOOTSTRAP | STEP1_CORE | STEP2_IO | STEP3_MAPPING | STEP4_ROWS | STEP8_OBSERVABILITY | PHASE3_FIELD_MODULES | PHASE3_EXACT_MODULES | PHASE3_DISTRIBUTION_MODULES | PHASE3_PROVENANCE_MODULES | PHASE3_BOUNDS_MODULES | PHASE3_TAIL_MODULES | PHASE3_RESAMPLING_MODULES | PHASE3_PAIR_MODULES) - relative_paths - if missing: - raise SystemExit(f"Required startup or Step 8 modules missing: {sorted(missing)}") - - imported_roots: set[str] = set() - forbidden_locations: list[str] = [] - placeholder_count = 0 - +DYNAMIC_CALLS = {"eval", "exec", "compile", "__import__"} +INPUT_ONLY = {"io/validation.py", "io/normalization.py", "io/schema_mapping.py", "io/loaders.py", + "observability/levels.py"} +RENDERERS = {"reports/json_report.py", "reports/markdown_report.py"} + + +def _module_name(node: ast.ImportFrom, relative: str) -> str: + """Resolve relative imports sufficiently to check the package layer.""" + if not node.level: + return node.module or "" + parent = ["recursive_integrity_toolkit", *Path(relative).parts[:-1]] + prefix = parent[:len(parent) - node.level + 1] + return ".".join(prefix + ([node.module] if node.module else [])) + + +def audit(root: Path = ROOT) -> dict: + package = root / "src/recursive_integrity_toolkit" + paths = sorted(package.rglob("*.py")) + if not paths: + raise ValueError("Package source is missing") for path in paths: - relative = path.relative_to(PACKAGE).as_posix() - if path.name in FORBIDDEN_FILES: - raise SystemExit(f"Forbidden module found: {path}") + relative = path.relative_to(package).as_posix() + if path.is_symlink() or path.name in FORBIDDEN_FILES: + raise ValueError(f"Aliased or prohibited module: {relative}") tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - if relative == "models.py": - _phase3_contract_boundary(tree) - if relative in PHASE3_FIELD_MODULES: - _phase3_representation_boundary(tree, relative) - if relative in PHASE3_EXACT_MODULES: - _phase3_exact_boundary(tree, relative) - if relative in PHASE3_DISTRIBUTION_MODULES: - _phase3_distribution_boundary(tree) - if relative in PHASE3_PROVENANCE_MODULES: - _phase3_provenance_boundary(tree) - if relative in PHASE3_BOUNDS_MODULES: - _phase3_bounds_boundary(tree) - if relative in PHASE3_TAIL_MODULES: - _phase3_tail_boundary(tree) - if relative in PHASE3_RESAMPLING_MODULES: - _phase3_resampling_boundary(tree) - if relative in PHASE3_PAIR_MODULES: - _phase3_pair_boundary(tree) doc = ast.get_docstring(tree) or "" if "Owner IDs:" not in doc or "Current phase status:" not in doc: - raise SystemExit(f"Owner or phase metadata missing: {path}") - - if relative not in BOOTSTRAP | STEP1_CORE | STEP2_IO | STEP3_MAPPING | STEP4_ROWS | STEP8_OBSERVABILITY | PHASE3_FIELD_MODULES | PHASE3_EXACT_MODULES | PHASE3_DISTRIBUTION_MODULES | PHASE3_PROVENANCE_MODULES | PHASE3_BOUNDS_MODULES | PHASE3_TAIL_MODULES | PHASE3_RESAMPLING_MODULES | PHASE3_PAIR_MODULES: - if not ( - len(tree.body) == 1 - and isinstance(tree.body[0], ast.Expr) - and isinstance(tree.body[0].value, ast.Constant) - and isinstance(tree.body[0].value.value, str) - ): - raise SystemExit(f"Protected non-Step-8 executable body found: {path}") - placeholder_count += 1 - else: - functions, classes = _definitions(tree) - if set(functions) != ALLOWED_FUNCTIONS[relative] or len(functions) != len(set(functions)): - raise SystemExit(f"Unexpected Step 9 functions in {relative}: {functions}") - if set(classes) != ALLOWED_CLASSES.get(relative, set()) or len(classes) != len(set(classes)): - raise SystemExit(f"Unexpected Step 9 classes in {relative}: {classes}") - + raise ValueError(f"Owner or implementation status missing: {relative}") for node in ast.walk(tree): - if relative == "io/validation.py": - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id in PIPELINE_CALL_SITES: - if not any(isinstance(fn, ast.FunctionDef) and fn.name in PIPELINE_CALL_SITES[node.func.id] - and any(child is node for child in ast.walk(fn)) for fn in tree.body): - raise SystemExit(f"Unexpected bundle read or orchestration call site: {node.func.id}") - if isinstance(node, ast.ImportFrom): - imported_path = "." * node.level + (node.module or "") - if imported_path in PIPELINE_IMPORT_NAMES: - if any(alias.name not in PIPELINE_IMPORT_NAMES[imported_path] or alias.asname is not None for alias in node.names): - raise SystemExit(f"Unexpected adjacent-layer import in {relative}") - if not any(isinstance(fn, ast.FunctionDef) and fn.name in ALLOWED_FUNCTIONS["io/validation.py"] - and (fn.name.startswith("_bundle_") or fn.name == "validate_bundle") - and any(child is node for child in ast.walk(fn)) for fn in tree.body): - raise SystemExit(f"Unexpected eager adjacent-layer import in {relative}") - if relative in STEP8_OBSERVABILITY: - if isinstance(node, ast.BinOp) and isinstance(node.op, (ast.Mult, ast.Div, ast.FloorDiv, ast.Mod, ast.Pow, ast.MatMult)): - raise SystemExit(f"Unexpected analytical arithmetic in {relative}") - if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) and node.func.value.id == "math": - if node.func.attr not in {"isfinite", "fsum", "isclose"} or not any( - isinstance(fn, ast.FunctionDef) and fn.name == "_distribution" - and any(child is node for child in ast.walk(fn)) for fn in tree.body - ): - raise SystemExit(f"Unexpected math use outside input validation in {relative}") - if relative in STEP2_IO: - if isinstance(node, ast.Lambda): - raise SystemExit(f"Unexpected dynamic callback in {relative}") - if isinstance(node, ast.Call): - blocked_calls = {"eval", "exec", "compile", "__import__", "globals", "locals", "vars", "input", "breakpoint"} - blocked_methods = {"write_text", "write_bytes", "mkdir", "unlink", "remove", "system", "popen", "connect", "urlopen", "urlretrieve", "send", "sendall"} - if (isinstance(node.func, ast.Name) and node.func.id in blocked_calls) or ( - isinstance(node.func, ast.Attribute) and node.func.attr in blocked_methods - ): - raise SystemExit(f"Unexpected executable, write or network capability in {relative}") - if relative in STEP3_MAPPING | STEP4_ROWS | STEP8_OBSERVABILITY | PHASE3_FIELD_MODULES | PHASE3_EXACT_MODULES | PHASE3_DISTRIBUTION_MODULES | PHASE3_PROVENANCE_MODULES | PHASE3_BOUNDS_MODULES | PHASE3_TAIL_MODULES | PHASE3_RESAMPLING_MODULES | PHASE3_PAIR_MODULES | {"models.py"}: - if isinstance(node, ast.Lambda): - raise SystemExit(f"Unexpected dynamic callback in {relative}") - if isinstance(node, ast.Call): - dangerous = {"eval", "exec", "compile", "__import__", "getattr", "setattr", - "delattr", "globals", "locals", "vars", "open", "input", "breakpoint"} - io_methods = {"open", "read_text", "read_bytes", "write_text", "write_bytes", - "mkdir", "unlink", "system", "popen", "connect", "urlopen"} - if (isinstance(node.func, ast.Name) and node.func.id in dangerous) or ( - isinstance(node.func, ast.Attribute) and node.func.attr in io_methods - ): - raise SystemExit(f"Unexpected executable or IO capability in {relative}") - if isinstance(node.func, ast.Name) and node.func.id == "_read_source": - if not ((relative in STEP3_MAPPING and any( - isinstance(fn, ast.FunctionDef) and fn.name == "load_mapping" - and any(child is node for child in ast.walk(fn)) for fn in tree.body - )) or (relative == "io/validation.py" and any( - isinstance(fn, ast.FunctionDef) and fn.name == "_bundle_control" - and any(child is node for child in ast.walk(fn)) for fn in tree.body - ))): - raise SystemExit(f"Unexpected file read outside explicit mapping loader in {relative}") - imports: set[str] = set() + imports = [] if isinstance(node, ast.Import): - imports.update(alias.name for alias in node.names) - imported_roots.update(alias.name.split(".")[0] for alias in node.names) + imports = [alias.name for alias in node.names] elif isinstance(node, ast.ImportFrom): - imports.add("." * node.level + (node.module or "")) - if node.module: - imported_roots.add(node.module.split(".")[0]) - for imported in imports: - root = imported.lstrip(".").split(".")[0] - if root in FORBIDDEN_IMPORTS: - lazy_parquet = ( - relative == "io/loaders.py" - and imported in {"pyarrow", "pyarrow.parquet"} - and any( - isinstance(function, ast.FunctionDef) - and function.name == "_parse_parquet" - and any(child is node for child in ast.walk(function)) - for function in tree.body - ) - ) - lazy_numpy = ( - relative == "metrics/resampling.py" and imported == "numpy" - and isinstance(node, ast.Import) - and len(node.names) == 1 and node.names[0].name == "numpy" and node.names[0].asname == "np" - and any(isinstance(fn, ast.FunctionDef) and fn.name == "simulate_closed_resampling" - and any(child is node for child in ast.walk(fn)) for fn in tree.body) - ) - if not (lazy_parquet or lazy_numpy): - forbidden_locations.append(f"{relative}: {imported}") - if relative in STEP1_CORE | STEP2_IO | STEP3_MAPPING | STEP4_ROWS | STEP8_OBSERVABILITY | PHASE3_FIELD_MODULES | PHASE3_EXACT_MODULES | PHASE3_DISTRIBUTION_MODULES | PHASE3_PROVENANCE_MODULES | PHASE3_BOUNDS_MODULES | PHASE3_TAIL_MODULES | PHASE3_RESAMPLING_MODULES | PHASE3_PAIR_MODULES: - unexpected = imports - ALLOWED_CORE_IMPORTS[relative] - if unexpected: - raise SystemExit(f"Import outside Step 9 contracts in {relative}: {sorted(unexpected)}") - - if forbidden_locations: - raise SystemExit(f"Dependency outside the approved scope: {forbidden_locations}") - - print(f"package modules checked: {len(paths)}") - print(f"allowed startup functions: {sorted(ALLOWED_FUNCTIONS['cli.py'])}") - print(f"authorized Step 2 contract modules: {sorted(STEP1_CORE)}") - print(f"authorized Step 2 file modules: {sorted(STEP2_IO)}") - print(f"authorized Step 3 mapping modules: {sorted(STEP3_MAPPING)}") - print(f"authorized Step 4 row modules: {sorted(STEP4_ROWS)}") - print(f"authorized Step 8 observability modules: {sorted(STEP8_OBSERVABILITY)}") - print(f"protected docstring-only modules: {placeholder_count}") - print("owner metadata: PASS") - print("Phase 3 Step 11: retained Step 9 explicit-pair and inherited definition/import boundaries: PASS") - print("no-algorithm phase boundary: PASS") - return 0 - - -def phase4_main(step: int = 1) -> int: - """Apply the approved Step 1 control before the frozen Phase 3 AST gate.""" - import runpy - - if type(step) is not int or step != 1: - raise ValueError("Only authorized Phase 4 Step 1 traceability is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_traceability_control") - control["audit_phase4"](step=step) - result = main() - print("Phase 4 Step 1: approved governance and frozen Phase 3 traceability: PASS") - return result - - -PHASE4_STEP2_REPORT_SCHEMA_SHA256 = "34b7edb8e8672fca3f3d4f6fe647b4d8967acb6f0435544ba449c1c44f7bc1f6" -PHASE4_STEP2_RESULT_IMPORTS = ["Import(names=[alias(name='math')])", - "Import(names=[alias(name='re')])", - "Import(names=[alias(name='sys')])", - "ImportFrom(module='__future__', names=[alias(name='annotations')], level=0)", - "ImportFrom(module='collections.abc', names=[alias(name='Mapping')], level=0)", - "ImportFrom(module='dataclasses', names=[alias(name='dataclass')], level=0)", - "ImportFrom(module='enum', names=[alias(name='StrEnum')], level=0)", - "ImportFrom(module='types', names=[alias(name='MappingProxyType')], level=0)"] -PHASE4_STEP2_RESULT_FUNCTIONS = ['CanonicalReport.__init__', - 'CanonicalReport.from_dict', - 'CanonicalReport.to_dict', - '_array', - '_base_envelope', - '_build_contract', - '_check', - '_check_nullable_reason', - '_enum', - '_equality_key', - '_fail', - '_field', - '_freeze', - '_integer', - '_json_input', - '_map', - '_matches', - '_metric', - '_nullable', - '_number', - '_object', - '_observed', - '_ref', - '_resolved_contract', - '_same', - '_semantic_checks', - '_strings', - '_text', - '_thaw', - '_walk', - 'report_schema', - 'validate_report'] -PHASE4_STEP2_RESULT_CLASSES = ['CanonicalReport', - 'ExecutionStatus', - 'FieldDefinition', - 'ReportEvidenceClass', - 'ReportStatus', - 'ReportValidationError', - 'RunStatus'] -PHASE4_STEP2_RESULT_CALLS = {'': ['_build_contract', '_freeze', 'tuple'], - 'CanonicalReport': ['dataclass'], - 'CanonicalReport.__init__': ['MappingProxyType', - 'MappingProxyType', - '_freeze', - 'object.__setattr__', - 'validate_report'], - 'CanonicalReport.from_dict': ['CanonicalReport'], - 'CanonicalReport.to_dict': ['_thaw'], - 'FieldDefinition': ['dataclass'], - '_base_envelope': ['_array', - '_array', - '_enum', - '_enum', - '_enum', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_number', - '_number', - '_ref', - '_ref', - '_strings', - '_strings', - '_strings', - '_strings', - '_text', - '_text'], - '_build_contract': ['FieldDefinition', - 'FieldDefinition', - 'FieldDefinition', - 'FieldDefinition', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_base_envelope', - '_base_envelope', - '_base_envelope', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_field_entries.append', - '_field_entries.append', - '_field_entries.append', - '_field_entries.append', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_map', - '_map', - '_map', - '_map', - '_map', - '_map', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_observed', - '_observed', - '_observed', - '_observed', - '_observed', - '_observed', - '_observed', - '_observed', - '_observed', - '_observed', - '_observed', - '_observed', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - 'list', - 'list', - 'props.update', - 'props.update', - 'props.update', - 'props.update', - 'tuple', - 'unavailable_owners.items', - 'unavailable_variants.append'], - '_check': ['_check', - '_check', - '_check', - '_check', - '_check', - '_check', - '_check', - '_equality_key', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_matches', - '_matches', - '_matches', - '_same', - '_same', - 'any', - 'any', - 'any', - 'enumerate', - 'isinstance', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'math.isfinite', - 're.search', - 'reference.removeprefix', - 'reference.startswith', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'sum', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'value.is_integer', - 'value.items'], - '_check_nullable_reason': ['_fail'], - '_enum': ['list'], - '_equality_key': ['_equality_key', - '_equality_key', - 'frozenset', - 'tuple', - 'type', - 'type', - 'type', - 'type', - 'value.items'], - '_fail': ['ReportValidationError'], - '_field': ['FieldDefinition', - '_base_envelope', - '_field_entries.append', - '_nullable', - '_object', - '_ref', - '_ref', - '_ref', - '_text', - '_text', - 'properties.update', - "result['allOf'].append", - 'tuple'], - '_freeze': ['MappingProxyType', '_freeze', '_freeze', 'tuple', 'type', 'type', 'value.items'], - '_json_input': ['_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_json_input', - '_json_input', - 'active.add', - 'active.remove', - 'enumerate', - 'id', - 'id', - 'id', - 'math.isfinite', - 'set', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'value.encode', - 'value.items'], - '_map': ['_text'], - '_matches': ['_check'], - '_metric': ['_field'], - '_object': ['list', 'list'], - '_observed': ['_field'], - '_resolved_contract': ['_matches', - '_resolved_contract', - '_resolved_contract', - 'contract.get', - 'contract.get', - "contract['$ref'].removeprefix"], - '_same': ['_same', - '_same', - 'all', - 'all', - 'isinstance', - 'isinstance', - 'len', - 'len', - 'set', - 'set', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'zip'], - '_semantic_checks': ['_check_nullable_reason', - '_check_nullable_reason', - '_check_nullable_reason', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_walk', - 'all', - 'all', - 'any', - 'any', - 'any', - 'any', - 'any', - 'bool', - 'contract.get', - 'int', - 'item.get', - 'item.get', - 'item.get', - 'item.get', - 'item.get', - "item['owner_ids'][0].startswith", - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'math.isclose', - 'math.isclose', - 'observability.get', - "payload['capabilities'].items", - "payload['derived_metrics'].get", - "payload['derived_metrics'].get", - "payload['derived_metrics'].get('closure_exposure', {}).get", - "payload['simulations'].items", - 'set', - 'set', - 'set', - 'set', - 'type', - 'type', - 'type'], - '_strings': ['_array', '_text'], - '_thaw': ['_thaw', '_thaw', 'isinstance', 'type', 'type', 'value.items'], - '_walk': ['_resolved_contract', - '_walk', - '_walk', - 'contract.get', - 'contract.get', - 'enumerate', - 'isinstance', - 'properties.get', - 'type', - 'type', - 'value.items'], - 'report_schema': ['_freeze', '_thaw'], - 'validate_report': ['_check', '_json_input', '_semantic_checks']} -PHASE4_STEP2_RESULT_ARITHMETIC = {'_build_contract': ["'PR-004.' + name", - "'PR-004.' + name", - "'PR-006.' + name", - "'PR-008.' + name", - "'derived_metrics.closure_exposure.direct.' + name", - "'derived_metrics.closure_exposure.lineage.' + name", - "'derived_metrics.diversity.by_version.*.' + name", - "'derived_metrics.diversity.by_version.*.' + name", - "'derived_metrics.lineage.' + name", - "'derived_metrics.provenance.' + name", - "'derived_metrics.support.' + name", - "'derived_metrics.support.by_version.*.' + name", - "'observed_facts.content.' + name", - "'observed_facts.lineage.' + name", - "'observed_facts.provenance.' + name", - "'observed_facts.provenance.' + name", - "'observed_facts.provenance.' + name", - "'observed_facts.provenance.' + name", - "'proxy_signals.' + name", - "'simulations.' + name", - "'unavailable_conclusions[].' + name", - '-1', - '-sys.float_info.max', - "owner + '.'", - "owner + '.'", - "owner + '.'", - "owner + '.'", - "owner + '.'", - "owner + '.' + name", - "owner + '.' + name", - "owner + '.' + name", - "owner + '.' + name", - "owner + '.' + name", - "owner + '.unavailable_conclusion'", - "owner + '.unavailable_conclusion'", - "required += ('by_state',)"], - '_check': ["not _same(value, schema['const'])", - "not any((_matches(value, item) for item in schema['anyOf']))", - "not any((_same(value, choice) for choice in schema['enum']))", - "not reference.startswith('#/$defs/')", - 'not valid_type[expected]', - "path + '.'", - "path + '.' + key", - "path + '.'", - "path + '.'", - "path + f'[{index}]'"], - '_json_input': ['not finite', "path + '.'", "path + '.'", 'set | None'], - '_number': ['-sys.float_info.max', 'int | float', 'int | float'], - '_object': ['tuple | list', 'tuple | list | None'], - '_ref': ["'#/$defs/' + name"], - '_semantic_checks': ['-1', - 'included + excluded', - 'included_ids & excluded_ids', - "item['ratio'] * item['denominator']", - "not capability['execution_reason_codes']", - "not capability['execution_scope']", - "not item['reason']", - "not item['reason_codes']", - "not item['required_evidence']", - "not math.isclose(interval['interval_width']['value'], upper - lower, rel_tol=1e-12, " - 'abs_tol=1e-12)', - "not math.isclose(item['ratio'] * item['denominator'], item['numerator'], " - 'rel_tol=1e-12, abs_tol=1e-12)', - 'not present', - "parameters['simulation_horizon'] + 1", - "parameters['simulation_horizon'] + 1", - 'upper - lower'], - '_walk': ['path + (index,)', 'path + (key,)']} -PHASE4_STEP2_RESULT_MUTATIONS = [('CanonicalReport.__init__', "object.__setattr__(self, 'sections', frozen)")] -PHASE4_STEP2_RESULT_DECLARATIONS = [('', - "CAPABILITY_KEYS = ('ingestion', 'content_diagnostics', 'provenance', 'lineage', 'dataset_longitudinal', " - "'model_longitudinal', 'intervention_simulation')"), - ('', 'FIELD_REGISTRY = tuple(_field_entries)'), - ('', - "LEVEL_LABELS = ('ingest_observability', 'content_or_representation_observability', " - "'provenance_observability', 'lineage_observability', 'longitudinal_dataset_observability', " - "'experimental_intervention_or_scenario_observability')"), - ('', - "RUN_NULLABLE_FIELDS = ('started_at', 'completed_at', 'duration_seconds', 'python_version', 'platform', " - "'command', 'config_hash', 'random_seed')"), - ('', - "SECTION_ORDER = ('run', 'inputs', 'observability', 'capabilities', 'observed_facts', 'derived_metrics', " - "'proxy_signals', 'simulations', 'unavailable_conclusions', 'recommended_next_metadata', 'warnings', " - "'errors')"), - ('', '_REPORT_SCHEMA = _build_contract()'), - ('', '_REPORT_SCHEMA = _freeze(_REPORT_SCHEMA)'), - ('', '_field_entries: list[FieldDefinition] = []'), - ('', 'del _field_entries'), - ('CanonicalReport', 'sections: Mapping'), - ('ExecutionStatus', "COMPLETED = 'completed'"), - ('ExecutionStatus', "DEFERRED = 'deferred'"), - ('ExecutionStatus', "FAILED = 'failed'"), - ('ExecutionStatus', "NOT_REQUESTED = 'not_requested'"), - ('ExecutionStatus', "PARTIAL = 'partial'"), - ('FieldDefinition', 'evidence_class: str'), - ('FieldDefinition', 'method_id: str'), - ('FieldDefinition', 'minimum_level: int'), - ('FieldDefinition', 'owner: str'), - ('FieldDefinition', 'path: str'), - ('FieldDefinition', 'requires_representation: bool = False'), - ('FieldDefinition', "status_boundary: str = 'current'"), - ('FieldDefinition', 'unit: str'), - ('FieldDefinition', 'value_type: str'), - ('ReportEvidenceClass', "DERIVED_METRIC = 'derived_metric'"), - ('ReportEvidenceClass', "OBSERVED_FACT = 'observed_fact'"), - ('ReportEvidenceClass', "PROXY_SIGNAL = 'proxy_signal'"), - ('ReportEvidenceClass', "SIMULATION = 'simulation'"), - ('ReportEvidenceClass', "UNAVAILABLE_CONCLUSION = 'unavailable_conclusion'"), - ('ReportStatus', "AVAILABLE = 'available'"), - ('ReportStatus', "EXPERIMENTAL = 'experimental'"), - ('ReportStatus', "PARTIAL = 'partial'"), - ('ReportStatus', "UNAVAILABLE = 'unavailable'"), - ('RunStatus', "COMPLETE = 'complete'"), - ('RunStatus', "FAILED = 'failed'"), - ('RunStatus', "PARTIAL = 'partial'")] -PHASE4_STEP2_RESULT_TRAVERSAL = [('', 'del _field_entries'), - ('_walk', '(yield (path, value, contract))'), - ('_walk', "(yield from _walk(item, path + (index,), contract['items']))"), - ('_walk', '(yield from _walk(item, path + (key,), child))')] -PHASE4_STEP2_RESULT_ATTRIBUTES = {'CanonicalReport.__init__': ['object.__setattr__'], - 'CanonicalReport.to_dict': ['self.sections'], - '_build_contract': ['_field_entries.append', - '_field_entries.append', - '_field_entries.append', - '_field_entries.append', - 'item.value', - 'props.update', - 'props.update', - 'props.update', - 'props.update', - 'sys.float_info', - 'sys.float_info', - 'sys.float_info.max', - 'sys.float_info.max', - 'unavailable_owners.items', - 'unavailable_variants.append'], - '_check': ['math.isfinite', - 're.search', - 'reference.removeprefix', - 'reference.startswith', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'value.is_integer', - 'value.items'], - '_equality_key': ['value.items'], - '_field': ['_field_entries.append', 'properties.update', "result['allOf'].append"], - '_freeze': ['value.items'], - '_integer': ['sys.float_info', 'sys.float_info.max'], - '_json_input': ['active.add', 'active.remove', 'math.isfinite', 'value.encode', 'value.items'], - '_number': ['sys.float_info', 'sys.float_info', 'sys.float_info.max', 'sys.float_info.max'], - '_resolved_contract': ['contract.get', 'contract.get', "contract['$ref'].removeprefix"], - '_semantic_checks': ['contract.get', - 'item.get', - 'item.get', - 'item.get', - 'item.get', - 'item.get', - "item['owner_ids'][0].startswith", - 'math.isclose', - 'math.isclose', - 'observability.get', - "payload['capabilities'].items", - "payload['derived_metrics'].get", - "payload['derived_metrics'].get", - "payload['derived_metrics'].get('closure_exposure', {}).get", - "payload['simulations'].items"], - '_thaw': ['value.items'], - '_walk': ['contract.get', 'contract.get', 'properties.get', 'value.items']} -PHASE4_STEP2_RESULT_HEADERS = [('CanonicalReport', (), (), ('dataclass(frozen=True, slots=True, init=False)',)), - ('CanonicalReport.__init__', - "arguments(posonlyargs=[], args=[arg(arg='self'), arg(arg='payload', annotation=Name(id='dict', " - 'ctx=Load()))], kwonlyargs=[], kw_defaults=[], defaults=[])', - (), - 'None'), - ('CanonicalReport.from_dict', - "arguments(posonlyargs=[], args=[arg(arg='cls'), arg(arg='payload', annotation=Name(id='dict', " - 'ctx=Load()))], kwonlyargs=[], kw_defaults=[], defaults=[])', - ('classmethod',), - "'CanonicalReport'"), - ('CanonicalReport.to_dict', - "arguments(posonlyargs=[], args=[arg(arg='self')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - 'dict'), - ('ExecutionStatus', ('StrEnum',), (), ()), - ('FieldDefinition', (), (), ('dataclass(frozen=True, slots=True)',)), - ('ReportEvidenceClass', ('StrEnum',), (), ()), - ('ReportStatus', ('StrEnum',), (), ()), - ('ReportValidationError', ('ValueError',), (), ()), - ('RunStatus', ('StrEnum',), (), ()), - ('_array', - "arguments(posonlyargs=[], args=[arg(arg='item', annotation=Name(id='dict', ctx=Load()))], " - "kwonlyargs=[arg(arg='minimum', annotation=Name(id='int', ctx=Load())), arg(arg='unique', " - "annotation=Name(id='bool', ctx=Load()))], kw_defaults=[Constant(value=0), Constant(value=False)], " - 'defaults=[])', - (), - 'dict'), - ('_base_envelope', - "arguments(posonlyargs=[], args=[arg(arg='evidence', annotation=Name(id='str', ctx=Load())), " - "arg(arg='unit', annotation=Name(id='str', ctx=Load())), arg(arg='owner', annotation=Name(id='str', " - "ctx=Load())), arg(arg='method', annotation=Name(id='str', ctx=Load()))], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - 'dict'), - ('_build_contract', - 'arguments(posonlyargs=[], args=[], kwonlyargs=[], kw_defaults=[], defaults=[])', - (), - 'dict'), - ('_check', - "arguments(posonlyargs=[], args=[arg(arg='value'), arg(arg='schema', annotation=Name(id='dict', " - "ctx=Load())), arg(arg='path', annotation=Name(id='str', ctx=Load()))], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - 'None'), - ('_check_nullable_reason', - "arguments(posonlyargs=[], args=[arg(arg='value', annotation=Name(id='dict', ctx=Load())), arg(arg='name', " - "annotation=Name(id='str', ctx=Load())), arg(arg='reason_name', annotation=Name(id='str', ctx=Load())), " - "arg(arg='path', annotation=Name(id='str', ctx=Load()))], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - 'None'), - ('_enum', - "arguments(posonlyargs=[], args=[], vararg=arg(arg='values', annotation=Name(id='str', ctx=Load())), " - 'kwonlyargs=[], kw_defaults=[], defaults=[])', - (), - 'dict'), - ('_equality_key', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_fail', - "arguments(posonlyargs=[], args=[arg(arg='path', annotation=Name(id='str', ctx=Load())), arg(arg='rule', " - "annotation=Name(id='str', ctx=Load()))], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - 'None'), - ('_field', - "arguments(posonlyargs=[], args=[arg(arg='path', annotation=Name(id='str', ctx=Load())), arg(arg='value', " - "annotation=Name(id='dict', ctx=Load()))], kwonlyargs=[arg(arg='owner', annotation=Name(id='str', " - "ctx=Load())), arg(arg='evidence', annotation=Name(id='str', ctx=Load())), arg(arg='unit', " - "annotation=Name(id='str', ctx=Load())), arg(arg='method', annotation=Name(id='str', ctx=Load())), " - "arg(arg='level', annotation=Name(id='int', ctx=Load())), arg(arg='value_type', annotation=Name(id='str', " - "ctx=Load())), arg(arg='representation', annotation=Name(id='bool', ctx=Load())), arg(arg='boundary', " - "annotation=Name(id='str', ctx=Load()))], kw_defaults=[None, None, None, None, None, None, " - "Constant(value=False), Constant(value='current')], defaults=[])", - (), - 'dict'), - ('_freeze', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_integer', - "arguments(posonlyargs=[], args=[], kwonlyargs=[arg(arg='minimum', annotation=Name(id='int', " - 'ctx=Load()))], kw_defaults=[Constant(value=0)], defaults=[])', - (), - 'dict'), - ('_json_input', - "arguments(posonlyargs=[], args=[arg(arg='value'), arg(arg='path', annotation=Name(id='str', ctx=Load())), " - "arg(arg='active', annotation=BinOp(left=Name(id='set', ctx=Load()), op=BitOr(), " - "right=Constant(value=None)))], kwonlyargs=[], kw_defaults=[], defaults=[Constant(value='$'), " - 'Constant(value=None)])', - (), - 'None'), - ('_map', - "arguments(posonlyargs=[], args=[arg(arg='values', annotation=Name(id='dict', ctx=Load()))], " - 'kwonlyargs=[], kw_defaults=[], defaults=[])', - (), - 'dict'), - ('_matches', - "arguments(posonlyargs=[], args=[arg(arg='value'), arg(arg='schema', annotation=Name(id='dict', " - 'ctx=Load()))], kwonlyargs=[], kw_defaults=[], defaults=[])', - (), - 'bool'), - ('_metric', - "arguments(posonlyargs=[], args=[arg(arg='path', annotation=Name(id='str', ctx=Load())), arg(arg='value', " - "annotation=Name(id='dict', ctx=Load())), arg(arg='owner', annotation=Name(id='str', ctx=Load())), " - "arg(arg='unit', annotation=Name(id='str', ctx=Load())), arg(arg='method', annotation=Name(id='str', " - "ctx=Load())), arg(arg='level', annotation=Name(id='int', ctx=Load())), arg(arg='value_type', " - "annotation=Name(id='str', ctx=Load()))], kwonlyargs=[arg(arg='representation', annotation=Name(id='bool', " - "ctx=Load())), arg(arg='boundary', annotation=Name(id='str', ctx=Load()))], " - "kw_defaults=[Constant(value=False), Constant(value='current')], defaults=[])", - (), - 'dict'), - ('_nullable', - "arguments(posonlyargs=[], args=[arg(arg='schema', annotation=Name(id='dict', ctx=Load()))], " - 'kwonlyargs=[], kw_defaults=[], defaults=[])', - (), - 'dict'), - ('_number', - "arguments(posonlyargs=[], args=[], kwonlyargs=[arg(arg='minimum', annotation=BinOp(left=Name(id='int', " - "ctx=Load()), op=BitOr(), right=Name(id='float', ctx=Load()))), arg(arg='maximum', " - "annotation=BinOp(left=Name(id='int', ctx=Load()), op=BitOr(), right=Name(id='float', ctx=Load())))], " - "kw_defaults=[UnaryOp(op=USub(), operand=Attribute(value=Attribute(value=Name(id='sys', ctx=Load()), " - "attr='float_info', ctx=Load()), attr='max', ctx=Load())), Attribute(value=Attribute(value=Name(id='sys', " - "ctx=Load()), attr='float_info', ctx=Load()), attr='max', ctx=Load())], defaults=[])", - (), - 'dict'), - ('_object', - "arguments(posonlyargs=[], args=[arg(arg='properties', annotation=Name(id='dict', ctx=Load())), " - "arg(arg='required', annotation=BinOp(left=BinOp(left=Name(id='tuple', ctx=Load()), op=BitOr(), " - "right=Name(id='list', ctx=Load())), op=BitOr(), right=Constant(value=None)))], kwonlyargs=[], " - 'kw_defaults=[], defaults=[Constant(value=None)])', - (), - 'dict'), - ('_observed', - "arguments(posonlyargs=[], args=[arg(arg='path', annotation=Name(id='str', ctx=Load())), arg(arg='value', " - "annotation=Name(id='dict', ctx=Load())), arg(arg='owner', annotation=Name(id='str', ctx=Load())), " - "arg(arg='unit', annotation=Name(id='str', ctx=Load())), arg(arg='method', annotation=Name(id='str', " - "ctx=Load())), arg(arg='level', annotation=Name(id='int', ctx=Load())), arg(arg='value_type', " - "annotation=Name(id='str', ctx=Load()))], kwonlyargs=[arg(arg='boundary', annotation=Name(id='str', " - "ctx=Load())), arg(arg='representation', annotation=Name(id='bool', ctx=Load()))], " - "kw_defaults=[Constant(value='current'), Constant(value=False)], defaults=[])", - (), - 'dict'), - ('_ref', - "arguments(posonlyargs=[], args=[arg(arg='name', annotation=Name(id='str', ctx=Load()))], kwonlyargs=[], " - 'kw_defaults=[], defaults=[])', - (), - 'dict'), - ('_resolved_contract', - "arguments(posonlyargs=[], args=[arg(arg='value'), arg(arg='contract')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - ''), - ('_same', - "arguments(posonlyargs=[], args=[arg(arg='left'), arg(arg='right')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - 'bool'), - ('_semantic_checks', - "arguments(posonlyargs=[], args=[arg(arg='payload', annotation=Name(id='dict', ctx=Load()))], " - 'kwonlyargs=[], kw_defaults=[], defaults=[])', - (), - 'None'), - ('_strings', - "arguments(posonlyargs=[], args=[], kwonlyargs=[arg(arg='minimum', annotation=Name(id='int', " - 'ctx=Load()))], kw_defaults=[Constant(value=0)], defaults=[])', - (), - 'dict'), - ('_text', - "arguments(posonlyargs=[], args=[], kwonlyargs=[arg(arg='empty', annotation=Name(id='bool', ctx=Load()))], " - 'kw_defaults=[Constant(value=False)], defaults=[])', - (), - 'dict'), - ('_thaw', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_walk', - "arguments(posonlyargs=[], args=[arg(arg='value'), arg(arg='path'), arg(arg='contract')], kwonlyargs=[], " - 'kw_defaults=[], defaults=[Tuple(elts=[], ctx=Load()), Constant(value=None)])', - (), - ''), - ('report_schema', - 'arguments(posonlyargs=[], args=[], kwonlyargs=[], kw_defaults=[], defaults=[])', - (), - 'dict'), - ('validate_report', - "arguments(posonlyargs=[], args=[arg(arg='payload', annotation=Name(id='dict', ctx=Load()))], " - 'kwonlyargs=[], kw_defaults=[], defaults=[])', - (), - 'None')] - - -def phase4_step2_result_boundary(path: Path) -> None: - """Inspect only the supplied canonical-result source without executing it. - - Fixed reviewed inventories constrain definitions, imports, call targets and - validation arithmetic. They are never regenerated from inspected source. - The release controller separately freezes the other 39 runtime modules. - """ - try: - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - except (OSError, UnicodeError, SyntaxError) as exc: - raise ValueError("Phase 4 Step 2 result source cannot be inspected") from exc - doc = ast.get_docstring(tree) or "" - if "Owner IDs:" not in doc or "Current phase status:" not in doc: - raise ValueError("Phase 4 Step 2 result ownership metadata is missing") - allowed_top = (ast.Import, ast.ImportFrom, ast.Assign, ast.AnnAssign, - ast.FunctionDef, ast.ClassDef, ast.Delete) - for index, node in enumerate(tree.body): - if index == 0 and isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str): - continue - if not isinstance(node, allowed_top): - raise ValueError("Phase 4 Step 2 result has an unapproved module statement") - imports = [] - functions = [] - classes = [] - calls = {} - arithmetic = {} - mutations = [] - declarations = [] - traversal = [] - attributes = {} - headers = [] - call_names = {node.func.id for node in ast.walk(tree) - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)} - forbidden = (ast.Lambda, ast.AsyncFunctionDef, ast.Await, ast.With, - ast.AsyncWith, ast.Global, ast.Nonlocal, ast.NamedExpr) - forbidden_names = {"eval", "exec", "compile", "__import__", "getattr", - "setattr", "delattr", "globals", "locals", "vars", - "open", "input", "breakpoint", "print", "exit", "quit"} - forbidden_attributes = {"__builtins__", "__globals__", "__code__", "__subclasses__", - "__class__", "__dict__", "__getattribute__", "__getattr__", - "open", "read", "read_text", "read_bytes", "write", "write_text", - "write_bytes", "mkdir", "unlink", "system", "popen", "connect", - "connect_ex", "getaddrinfo", "urlopen", "urlretrieve", "send", "sendall"} - stack = [(tree, "")] - while stack: - node, scope = stack.pop() - if isinstance(node, forbidden): - raise ValueError("Phase 4 Step 2 result has a callback or executable control outside scope") - if isinstance(node, (ast.FunctionDef, ast.ClassDef)): - scope = node.name if scope == "" else scope + "." + node.name - if isinstance(node, ast.FunctionDef): - functions.append(scope) - headers.append((scope, ast.dump(node.args, include_attributes=False), - tuple(ast.unparse(item) for item in node.decorator_list), - ast.unparse(node.returns) if node.returns is not None else "")) - else: - classes.append(scope) - headers.append((scope, tuple(ast.unparse(item) for item in node.bases), - tuple(ast.unparse(item) for item in node.keywords), - tuple(ast.unparse(item) for item in node.decorator_list))) - if isinstance(node, (ast.Import, ast.ImportFrom)): - if scope != "" or node not in tree.body: - raise ValueError("Phase 4 Step 2 result imports must be explicit and at module scope") - imports.append(ast.dump(node, include_attributes=False)) - if isinstance(node, ast.Name) and node.id in forbidden_names: - raise ValueError("Phase 4 Step 2 result names a forbidden execution capability") - if scope != "" and ((isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store) and node.id in call_names) - or (isinstance(node, ast.arg) and node.arg in call_names)): - raise ValueError("Phase 4 Step 2 result shadows an approved call target") - if isinstance(node, ast.Attribute): - attributes.setdefault(scope, []).append(ast.unparse(node)) - if isinstance(node, ast.Attribute) and node.attr in forbidden_attributes: - raise ValueError("Phase 4 Step 2 result names a forbidden IO or dynamic capability") - if isinstance(node, ast.Call): - if not isinstance(node.func, (ast.Name, ast.Attribute)): - raise ValueError("Phase 4 Step 2 result invokes a dynamic call target") - calls.setdefault(scope, []).append(ast.unparse(node.func)) - if isinstance(node.func, ast.Attribute) and node.func.attr == "__setattr__": - mutations.append((scope, ast.unparse(node))) - if isinstance(node, (ast.BinOp, ast.UnaryOp, ast.AugAssign)): - arithmetic.setdefault(scope, []).append(ast.unparse(node)) - if (scope == "" or scope in classes) and isinstance(node, (ast.Assign, ast.AnnAssign, ast.Delete)): - declarations.append((scope, ast.unparse(node))) - if isinstance(node, (ast.Yield, ast.YieldFrom, ast.Delete)): - traversal.append((scope, ast.unparse(node))) - if isinstance(node, ast.Attribute) and isinstance(node.ctx, (ast.Store, ast.Del)): - raise ValueError("Phase 4 Step 2 result has an unapproved attribute mutation") - stack.extend((child, scope) for child in ast.iter_child_nodes(node)) - if sorted(imports) != PHASE4_STEP2_RESULT_IMPORTS: - raise ValueError("Phase 4 Step 2 result imports differ from the reviewed stdlib inventory") - if sorted(functions) != PHASE4_STEP2_RESULT_FUNCTIONS or sorted(classes) != PHASE4_STEP2_RESULT_CLASSES: - raise ValueError("Phase 4 Step 2 result definitions differ from the approved contract") - if {name: sorted(values) for name, values in calls.items()} != PHASE4_STEP2_RESULT_CALLS: - raise ValueError("Phase 4 Step 2 result call targets differ from the reviewed inventory") - if {name: sorted(values) for name, values in arithmetic.items()} != PHASE4_STEP2_RESULT_ARITHMETIC: - raise ValueError("Phase 4 Step 2 result arithmetic exceeds reviewed semantic validation") - if sorted(mutations) != PHASE4_STEP2_RESULT_MUTATIONS: - raise ValueError("Phase 4 Step 2 result mutation differs from reviewed immutable construction") - if sorted(declarations) != PHASE4_STEP2_RESULT_DECLARATIONS: - raise ValueError("Phase 4 Step 2 result module declarations differ from the approved contract") - if sorted(traversal) != PHASE4_STEP2_RESULT_TRAVERSAL: - raise ValueError("Phase 4 Step 2 result generator or teardown differs from reviewed local traversal") - if {name: sorted(values) for name, values in attributes.items()} != PHASE4_STEP2_RESULT_ATTRIBUTES: - raise ValueError("Phase 4 Step 2 result attributes exceed the reviewed pure contract") - if sorted(headers) != PHASE4_STEP2_RESULT_HEADERS: - raise ValueError("Phase 4 Step 2 result definition headers differ from the reviewed contract") - - -def phase4_step2_main(step: int = 2) -> int: - """Validate Step 2 control, reviewed schema bytes and the result AST only.""" - import hashlib - import runpy - - if type(step) is not int or step != 2: - raise ValueError("Only authorized Phase 4 Step 2 traceability is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step2_traceability_control") - control["audit_phase4_step2"](step=step) - schema_path = ROOT / "schemas/report.schema.json" - if hashlib.sha256(schema_path.read_bytes()).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Phase 4 Step 2 report schema differs from the reviewed contract") - phase4_step2_result_boundary(PACKAGE / "result.py") - print("Phase 4 Step 2: canonical results and reviewed local schema: PASS") - return 0 - - - - -# Step 3 reviewed pure adapters. These independent inventories are never -# regenerated by the active gate from inspected source. -PHASE4_STEP3_ASSEMBLY_IMPORTS = ["ImportFrom(module='__future__', names=[alias(name='annotations')], level=0)", - "ImportFrom(module='dataclasses', names=[alias(name='dataclass')], level=0)", - "ImportFrom(module='math', names=[alias(name='isfinite'), alias(name='isclose')], level=0)", - "ImportFrom(module='metrics.bounds', names=[alias(name='DirectClosureExposureBounds')], level=2)", - "ImportFrom(module='metrics.diversity', names=[alias(name='DistributionMetrics'), " - "alias(name='StateDistributionResult'), alias(name='SupportComparison'), alias(name='StateFrequency')], " - 'level=2)', - "ImportFrom(module='metrics.duplicates', names=[alias(name='ExactDuplicateResult'), " - "alias(name='ExactDuplicateGroup')], level=2)", - "ImportFrom(module='metrics.provenance', names=[alias(name='ProvenanceCompositionResult'), " - "alias(name='DeclaredComposition'), alias(name='WeightedSourceComposition'), " - "alias(name='DirectGroundingBasis'), alias(name='DirectGroundingAssignment')], level=2)", - "ImportFrom(module='metrics.resampling', names=[alias(name='ExpectedDiversityResult'), " - "alias(name='ResamplingSimulation'), alias(name='ResamplingInput'), alias(name='ResamplingReplicate'), " - "alias(name='SampledGeneration')], level=2)", - "ImportFrom(module='metrics.tail', names=[alias(name='TailSelectionResult'), " - "alias(name='ExtinctionProbabilityResult'), alias(name='RarityEntry')], level=2)", - "ImportFrom(module='models', names=[alias(name='BundleValidationResult'), " - "alias(name='CalculationEvidenceClass'), alias(name='CalculationMetadata'), " - "alias(name='CalculationReason'), alias(name='CalculationScope'), alias(name='CalculationStatus'), " - "alias(name='CapabilityKey'), alias(name='CanonicalRow'), alias(name='NumericalPolicy'), " - "alias(name='RecordKey'), alias(name='RepresentationDescriptor'), alias(name='ScalarCalculation'), " - "alias(name='ValidationCoverage'), alias(name='ValidationMessage'), alias(name='ValidationSeverity'), " - "alias(name='WeightingOptions'), alias(name='Capability'), alias(name='CapabilityStatus'), " - "alias(name='ObservabilityAssessment'), alias(name='FileInventoryEntry'), alias(name='FileRole'), " - "alias(name='FileFormat'), alias(name='VersionOrderResult'), alias(name='ProvenanceJoinResult'), " - "alias(name='ProvenanceMatch'), alias(name='ProvenanceAssessment'), alias(name='ExplicitPairContext'), " - "alias(name='RowMappingEvidence'), alias(name='RowLocation'), alias(name='GenerationValidationResult'), " - "alias(name='RecordStateAssignment'), alias(name='TailSelectionOptions')], level=2)", - "ImportFrom(module='pathlib', names=[alias(name='PosixPath'), alias(name='WindowsPath')], level=0)", - "ImportFrom(module='representations.compatibility', names=[alias(name='RepresentationCompatibility'), " - "alias(name='StateMappingDeclaration')], level=2)", - "ImportFrom(module='result', names=[alias(name='CanonicalReport'), alias(name='FIELD_REGISTRY'), " - "alias(name='LEVEL_LABELS'), alias(name='SECTION_ORDER'), alias(name='ReportValidationError')], level=2)", - "ImportFrom(module='types', names=[alias(name='MappingProxyType')], level=0)"] - -PHASE4_STEP3_ASSEMBLY_FUNCTIONS = ['FamilyFailure.__post_init__', - '_base', - '_bundle_check', - '_bundle_scope', - '_capabilities', - '_closed_simulation', - '_closure', - '_comparison', - '_coverage', - '_definition', - '_diagnostics', - '_disclosure_base', - '_disclosure_recommend', - '_disclosure_scope', - '_disclosure_signal', - '_disclosure_unavailable', - '_disclosures', - '_distribution', - '_distribution_exclusions', - '_duplicates', - '_envelope', - '_extinction', - '_inputs', - '_key', - '_lineage_observations', - '_message_family', - '_metadata', - '_metadata_text', - '_number', - '_policy', - '_provenance', - '_provenance_consistency', - '_representation', - '_require', - '_scalar', - '_scope', - '_simulation_inputs', - '_table', - '_tail', - '_typed', - 'assemble_report'] - -PHASE4_STEP3_ASSEMBLY_CLASSES = ['FamilyFailure', 'ReportAssemblyError'] - -PHASE4_STEP3_ASSEMBLY_MUTATIONS = [] - -PHASE4_STEP3_ASSEMBLY_DECLARATIONS = [('FamilyFailure', 'capability: CapabilityKey'), ('FamilyFailure', 'messages: tuple[ValidationMessage, ...]')] - -PHASE4_STEP3_ASSEMBLY_TRAVERSAL = [] - -PHASE4_STEP3_ASSEMBLY_HEADERS = [('FamilyFailure', (), (), ('dataclass(frozen=True, slots=True)',)), - ('FamilyFailure.__post_init__', - "arguments(posonlyargs=[], args=[arg(arg='self')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('ReportAssemblyError', ('ReportValidationError',), (), ()), - ('_base', - "arguments(posonlyargs=[], args=[arg(arg='path'), arg(arg='scope')], " - "kwonlyargs=[arg(arg='representation'), arg(arg='coverage'), arg(arg='denominator'), " - "arg(arg='assumptions'), arg(arg='limitations'), arg(arg='status'), arg(arg='reasons'), " - "arg(arg='required')], kw_defaults=[Constant(value=None), Constant(value=None), Constant(value=None), " - "Tuple(elts=[], ctx=Load()), Tuple(elts=[], ctx=Load()), Constant(value='available'), Tuple(elts=[], " - 'ctx=Load()), Tuple(elts=[], ctx=Load())], defaults=[])', - (), - ''), - ('_bundle_check', - "arguments(posonlyargs=[], args=[arg(arg='bundle')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_bundle_scope', - "arguments(posonlyargs=[], args=[arg(arg='bundle')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_capabilities', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='bundle'), arg(arg='operations'), " - "arg(arg='failures')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_closed_simulation', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='value')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - ''), - ('_closure', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='value'), arg(arg='bundle'), " - "arg(arg='provenance')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_comparison', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='value'), arg(arg='bundle')], kwonlyargs=[], " - 'kw_defaults=[], defaults=[])', - (), - ''), - ('_coverage', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_definition', - "arguments(posonlyargs=[], args=[arg(arg='path')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_diagnostics', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='messages'), arg(arg='family')], " - 'kwonlyargs=[], kw_defaults=[], defaults=[Constant(value=None)])', - (), - ''), - ('_disclosure_base', - "arguments(posonlyargs=[], args=[arg(arg='owner'), arg(arg='evidence_class'), arg(arg='method'), " - "arg(arg='scope'), arg(arg='basis')], kwonlyargs=[], kw_defaults=[], defaults=[Constant(value=None)])", - (), - ''), - ('_disclosure_recommend', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='priority'), arg(arg='metadata'), " - "arg(arg='scope'), arg(arg='unlock'), arg(arg='reason')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_disclosure_scope', - "arguments(posonlyargs=[], args=[arg(arg='payload')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_disclosure_signal', - "arguments(posonlyargs=[], args=[arg(arg='name'), arg(arg='owner'), arg(arg='basis'), arg(arg='paths'), " - "arg(arg='present'), arg(arg='rule'), arg(arg='limitations')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_disclosure_unavailable', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='name'), arg(arg='owner'), " - "arg(arg='capability'), arg(arg='statement'), arg(arg='reasons'), arg(arg='blockers'), " - "arg(arg='next_metadata'), arg(arg='limit')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_disclosures', - "arguments(posonlyargs=[], args=[arg(arg='payload')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_distribution', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='value'), arg(arg='bundle')], " - "kwonlyargs=[arg(arg='weighted'), arg(arg='coverage')], kw_defaults=[Constant(value=False), " - 'Constant(value=None)], defaults=[])', - (), - ''), - ('_distribution_exclusions', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='value')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - ''), - ('_duplicates', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='value'), arg(arg='bundle')], kwonlyargs=[], " - 'kw_defaults=[], defaults=[])', - (), - ''), - ('_envelope', - "arguments(posonlyargs=[], args=[arg(arg='path'), arg(arg='value'), arg(arg='scope')], kwonlyargs=[], " - "kw_defaults=[], kwarg=arg(arg='kwargs'), defaults=[])", - (), - ''), - ('_extinction', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='values')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - ''), - ('_inputs', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='bundle')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - ''), - ('_key', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_lineage_observations', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='bundle')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - ''), - ('_message_family', - "arguments(posonlyargs=[], args=[arg(arg='message')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_metadata', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[arg(arg='name'), arg(arg='scope'), " - "arg(arg='representation'), arg(arg='owner'), arg(arg='evidence'), arg(arg='unit'), arg(arg='formula'), " - "arg(arg='weighting')], kw_defaults=[None, None, None, None, None, None, Constant(value=None), " - 'Constant(value=None)], defaults=[])', - (), - ''), - ('_metadata_text', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_number', - "arguments(posonlyargs=[], args=[arg(arg='value'), arg(arg='name')], kwonlyargs=[arg(arg='integer'), " - "arg(arg='minimum'), arg(arg='maximum')], kw_defaults=[Constant(value=False), Constant(value=None), " - 'Constant(value=None)], defaults=[])', - (), - ''), - ('_policy', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_provenance', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='value'), arg(arg='bundle')], kwonlyargs=[], " - 'kw_defaults=[], defaults=[])', - (), - ''), - ('_provenance_consistency', - "arguments(posonlyargs=[], args=[arg(arg='value'), arg(arg='bundle')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - ''), - ('_representation', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_require', - "arguments(posonlyargs=[], args=[arg(arg='condition'), arg(arg='message')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - ''), - ('_scalar', - "arguments(posonlyargs=[], args=[arg(arg='path'), arg(arg='value'), arg(arg='scope'), " - "arg(arg='representation'), arg(arg='bundle')], kwonlyargs=[arg(arg='name'), arg(arg='denominator'), " - "arg(arg='coverage'), arg(arg='weighting'), arg(arg='limitations'), arg(arg='formula')], " - 'kw_defaults=[Constant(value=None), Constant(value=None), Constant(value=None), Constant(value=None), ' - 'Tuple(elts=[], ctx=Load()), Constant(value=None)], defaults=[])', - (), - ''), - ('_scope', - "arguments(posonlyargs=[], args=[arg(arg='value'), arg(arg='bundle')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - ''), - ('_simulation_inputs', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_table', - "arguments(posonlyargs=[], args=[arg(arg='path'), arg(arg='value'), arg(arg='metadata'), arg(arg='scope'), " - "arg(arg='representation'), arg(arg='bundle')], kwonlyargs=[arg(arg='name'), arg(arg='status'), " - "arg(arg='reasons'), arg(arg='denominator'), arg(arg='coverage'), arg(arg='weighting'), " - "arg(arg='limitations'), arg(arg='formula')], kw_defaults=[None, None, Tuple(elts=[], ctx=Load()), " - 'Constant(value=None), Constant(value=None), Constant(value=None), Tuple(elts=[], ctx=Load()), ' - 'Constant(value=None)], defaults=[])', - (), - ''), - ('_tail', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='value'), arg(arg='bundle')], kwonlyargs=[], " - 'kw_defaults=[], defaults=[])', - (), - ''), - ('_typed', - "arguments(posonlyargs=[], args=[arg(arg='value'), arg(arg='expected'), arg(arg='name')], kwonlyargs=[], " - 'kw_defaults=[], defaults=[])', - (), - ''), - ('assemble_report', - "arguments(posonlyargs=[], args=[arg(arg='bundle', annotation=Name(id='BundleValidationResult', " - "ctx=Load()))], kwonlyargs=[arg(arg='run', annotation=Name(id='dict', ctx=Load())), " - "arg(arg='distributions', annotation=Subscript(value=Name(id='tuple', ctx=Load()), " - "slice=Tuple(elts=[BinOp(left=Name(id='StateDistributionResult', ctx=Load()), op=BitOr(), " - "right=Name(id='DistributionMetrics', ctx=Load())), Constant(value=Ellipsis)], ctx=Load()), ctx=Load())), " - "arg(arg='provenance', annotation=BinOp(left=Name(id='ProvenanceCompositionResult', ctx=Load()), " - "op=BitOr(), right=Constant(value=None))), arg(arg='duplicates', " - "annotation=BinOp(left=Name(id='ExactDuplicateResult', ctx=Load()), op=BitOr(), " - "right=Constant(value=None))), arg(arg='tail', annotation=BinOp(left=Name(id='TailSelectionResult', " - "ctx=Load()), op=BitOr(), right=Constant(value=None))), arg(arg='comparison', " - "annotation=BinOp(left=Name(id='SupportComparison', ctx=Load()), op=BitOr(), right=Constant(value=None))), " - "arg(arg='closure', annotation=BinOp(left=Name(id='DirectClosureExposureBounds', ctx=Load()), op=BitOr(), " - "right=Constant(value=None))), arg(arg='expected_diversity', " - "annotation=BinOp(left=Name(id='ExpectedDiversityResult', ctx=Load()), op=BitOr(), " - "right=Constant(value=None))), arg(arg='resampling', annotation=BinOp(left=Name(id='ResamplingSimulation', " - "ctx=Load()), op=BitOr(), right=Constant(value=None))), arg(arg='extinction', " - "annotation=Subscript(value=Name(id='tuple', ctx=Load()), " - "slice=Tuple(elts=[Name(id='ExtinctionProbabilityResult', ctx=Load()), Constant(value=Ellipsis)], " - "ctx=Load()), ctx=Load())), arg(arg='family_errors', annotation=Subscript(value=Name(id='tuple', " - "ctx=Load()), slice=Tuple(elts=[Name(id='FamilyFailure', ctx=Load()), Constant(value=Ellipsis)], " - 'ctx=Load()), ctx=Load()))], kw_defaults=[None, Tuple(elts=[], ctx=Load()), Constant(value=None), ' - 'Constant(value=None), Constant(value=None), Constant(value=None), Constant(value=None), ' - 'Constant(value=None), Constant(value=None), Tuple(elts=[], ctx=Load()), Tuple(elts=[], ctx=Load())], ' - 'defaults=[])', - (), - 'CanonicalReport')] - -PHASE4_STEP3_ASSEMBLY_LITERAL_DECLARATIONS = [('_metadata_text', "allowed = ('sampled_path; sequential_binomial_complement_v1',)"), - ('_metadata_text', 'allowed = methods.get(name, ())'), - ('_metadata_text', - "assumptions = ('Approved direct grounding partition; uncertainty remains unresolved.',)"), - ('_metadata_text', - "assumptions = ('Categorical multinomial sampling; positive explicit n; one-step horizon; no external " - "reopening.' if value.evidence_class is CalculationEvidenceClass.SIMULATION else 'Explicit rule over " - "positive observed unweighted count support.',)"), - ('_metadata_text', - "assumptions = ('Earlier and later scopes explicitly selected and independently validated.', 'State " - "identity uses the declared common basis, including any disclosed map.')"), - ('_metadata_text', - "assumptions = ('Exact explicit single-version Phase 2 join scope; no representation exclusions.',)"), - ('_metadata_text', - "assumptions = ('Fixed finite declared state space and constant positive integer resample size.', 'X_t " - "conditional on p_t is Multinomial(n,p_t); p_(t+1)=X_t/n.', 'No mutation, migration, independent real data " - "or external corrective input.')"), - ('_metadata_text', - "assumptions = ('One explicitly selected version and declared representation.', 'No implicit pooling, " - "probability repair, confidence weighting or sampling.')"), - ('_metadata_text', - "assumptions = ('exact_utf8_v1 and SHA-256; equal digests verified against exact bytes', 'one explicitly " - "selected version; unweighted records')"), - ('_metadata_text', - "basis = ('empirical_assignments', 'explicit_counts_divided_by_included_records', 'weighted_record_mass', " - "'explicit_probability_vector')"), - ('_metadata_text', - "limitations = ('Exact record form does not establish semantic identity or independent origin.',)"), - ('_metadata_text', - "limitations = ('Experimental conditional simulation; no calibrated production-failure probability.', " - "'Diversity contraction holds in expectation, not monotonically on every sampled path.', 'Simulated steps " - "are not record generations, training epochs or dataset releases.', 'No external-reference loss, " - "reopening, lineage, risk score or audit workflow is implemented.', 'Floating-point and pseudorandom " - "sampling are numerical realizations of the declared model.')"), - ('_metadata_text', - "limitations = ('Observed/supplied support only; no permanent extinction or causal/model-performance " - "verdict.', 'A coarsened comparison cannot recover distinctions lost through its mapping.')"), - ('_metadata_text', - "limitations = ('Representation-bound; does not establish functional failure or semantic completeness.',)"), - ('_metadata_text', - "limitations = ('Representation-bound; no calibrated production-failure or universal risk conclusion.',)"), - ('_metadata_text', - "limitations = ('Supplied declarations only; no truth or source-independence certification.',)"), - ('_metadata_text', - "limitations = ('Toolkit operationalization relative to supplied metadata, without lineage or truth " - "certification.',)"), - ('_metadata_text', - "methods = {'support_size': basis, 'gini_simpson_diversity': basis, 'simpson_concentration': basis, " - "'state_frequency': basis + ('empirical_assignments; n_i/N',), 'state_count': tuple(('Definitions 7.1; ' + " - "item for item in basis)) + ('count included record-state assignments',), 'state_mass': ('Definitions 9.5; " - "canonical record weights summed within each state',), 'support_delta': ('later support_size minus earlier " - "support_size',), 'support_loss_count': ('cardinality of earlier support minus later support',), " - "'support_added_count': ('cardinality of later support minus earlier support',), " - "'support_retention_ratio': ('intersection support size / earlier positive-mass support size',), " - "'gini_simpson_diversity_delta': ('later Gini-Simpson diversity minus earlier diversity',), " - "'extinct_states': ('earlier support minus later support',), 'added_states': ('later support minus earlier " - "support',), 'retained_states': ('earlier support intersect later support',), 'source_type_counts': " - "('Definitions 3.3/11.1; count declared canonical categories',), 'provenance_confidence_counts': " - "('Definitions 3.3/11.1; count declared canonical categories',), 'source_type_shares': ('category count / " - "all selected valid records',), 'source_type_field_coverage': ('Definitions 3.13; nonmissing valid field / " - "all selected valid records',), 'provenance_confidence_field_coverage': ('Definitions 3.13; nonmissing " - "valid field / all selected valid records',), 'provenance_row_coverage': ('Reuse Phase 2 coverage; " - "Definitions 3.10-3.12',), 'provenance_required_field_coverage': ('Reuse Phase 2 coverage; Definitions " - "3.10-3.12',), 'grounding_field_coverage': ('Reuse Phase 2 coverage; Definitions 3.10-3.12',), " - "'analyzed_record_count': ('All selected valid records',), 'records_with_matching_rows': ('Phase 2 " - "matched-row inventory',), 'missing_provenance_count': ('Phase 2 missing-row inventory',), " - "'missing_provenance_share': ('Definitions 11.4; missing rows / all selected valid records',), " - "'known_open_count': ('toolkit_operationalization; Definitions 3.7-3.9; P3-D08',), 'known_closed_count': " - "('toolkit_operationalization; Definitions 3.7-3.9; P3-D08',), 'unresolved_grounding_count': " - "('toolkit_operationalization; Definitions 3.7-3.9; P3-D08',), 'weighted_source_type_masses': " - "('Definitions 19; sum explicit weights by source',), 'weighted_source_type_shares': ('F-007 weighted " - "variant; category mass / all selected record weight mass',), 'total_weight': ('Definitions 19; all " - "selected weights',), 'missing_provenance_weight': ('Definitions 19; missing-row mass',), " - "'weighted_missing_provenance_share': ('Definitions 11.4/19; missing-row mass / all selected record weight " - "mass',), 'duplicate_record_count': ('sum_group_size_minus_one; DEFINITIONS_AND_UNITS:8.3',), " - "'duplicate_group_count': ('count_groups_of_size_greater_than_one; DEFINITIONS_AND_UNITS:8.4',), " - "'rarity_rank': ('ascending frequency, then count, then Unicode state ID; 1-based ordinal',), " - "'tail_support_size': tuple(('Definitions 10.1-10.6; explicit ' + rule for rule in ('singleton_count', " - "'count_at_or_below', 'frequency_at_or_below', 'state_list'))), 'tail_membership': tuple(('Definitions " - "10.1-10.6; explicit ' + rule for rule in ('singleton_count', 'count_at_or_below', " - "'frequency_at_or_below', 'state_list'))), 'tail_record_share': tuple(('Definitions 10.1-10.6; explicit ' " - "+ rule + '; selected counts / included records' for rule in ('singleton_count', 'count_at_or_below', " - "'frequency_at_or_below', 'state_list'))), 'direct_closure_exposure_lower_bound': ('known_closed_count / " - "total_record_count', 'no usable required-provenance row'), 'direct_closure_exposure_upper_bound': " - "('(known_closed_count + unresolved_grounding_count) / total_record_count', 'no usable required-provenance " - "row'), 'direct_closure_exposure_interval_width': ('upper_bound - lower_bound = unresolved_grounding_count " - "/ total_record_count', 'no usable required-provenance row'), 'one_step_extinction_probability': ('F-014; " - "(1-p_i)^n; analytic one-step closed multinomial',), 'expected_gini_simpson_diversity': " - "('analytic_expectation; D0*(1-1/n)**t; t=0..steps; constant n',)}"), - ('_metadata_text', 'name = value.metric_name'), - ('_metadata_text', - "pair_names = ('support_delta', 'support_loss_count', 'support_added_count', 'support_retention_ratio', " - "'gini_simpson_diversity_delta', 'extinct_states', 'added_states', 'retained_states')")] - -PHASE4_STEP3_ASSEMBLY_LITERAL_CONTAINERS = [('_bundle_check', "('source_type', 'external_grounding', 'provenance_confidence')"), - ('_capabilities', - "('Input eligibility is distinct from executed analysis; Phase 5 lineage remains deferred.',)"), - ('_capabilities', "('partial', ['R_LONGITUDINAL_FAMILIES_DEFERRED'])"), - ('_capabilities', "['R_INPUT_VALIDATION_ERROR']"), - ('_capabilities', "['R_LONGITUDINAL_FAMILIES_DEFERRED']"), - ('_capabilities', - "{'content': 'record_coverage', 'representation': 'representation_coverage', 'row': " - "'provenance_row_coverage', 'required_fields': 'provenance_required_field_coverage', 'grounding': " - "'grounding_field_coverage'}"), - ('_closed_simulation', - "('Fixed finite declared state space and constant positive integer resample size.', 'X_t conditional on " - "p_t is Multinomial(n,p_t); p_(t+1)=X_t/n.', 'No mutation, migration, independent real data or external " - "corrective input.')"), - ('_closed_simulation', "('gini_simpson_diversity', 'F-003', 'ratio')"), - ('_closed_simulation', "('state_count', None, 'sampled_records')"), - ('_closed_simulation', "('state_frequency', 'F-001', 'ratio')"), - ('_closed_simulation', "('support_size', 'F-002', 'states')"), - ('_closed_simulation', - "(('state_count', None, 'sampled_records'), ('state_frequency', 'F-001', 'ratio'), ('support_size', " - "'F-002', 'states'), ('gini_simpson_diversity', 'F-003', 'ratio'))"), - ('_closed_simulation', - "['Extinction events are not separately inferred by the assembly adapter; inspect the supplied sampled " - "support paths.']"), - ('_comparison', "('earlier', 'later')"), - ('_comparison', "('earlier_to_later', 'later_to_earlier')"), - ('_comparison', "('extinct_states', 'added_states', 'retained_states')"), - ('_comparison', "{'observed_facts': {}, 'derived_metrics': {}}"), - ('_comparison', "{'observed_facts': {}, 'derived_metrics': {}}"), - ('_disclosure_base', "('T1', 'T2', 'T3', 'T4', 'T5', 'T6')"), - ('_disclosure_recommend', "['PR-014']"), - ('_disclosure_scope', - "{'dataset_versions': [], 'record_count': None, 'excluded_record_count': None, 'denominator_basis': " - "'no_selected_record_scope_supplied', 'scope_id': 'report_inputs'}"), - ('_disclosure_unavailable', - "['Unavailable means the supplied evidence does not support this conclusion; it does not establish that " - "the conclusion is false.']"), - ('_disclosures', - "('R_PARENT_UNRESOLVED', 'R_PARENT_AMBIGUOUS', 'R_PARENT_DECLARATION_MISSING', 'R_PARENT_INVALID')"), - ('_disclosures', "('available', 'partial')"), - ('_disclosures', "('available', 'partial')"), - ('_disclosures', "('available', 'partial')"), - ('_disclosures', "('available', 'partial')"), - ('_disclosures', "('external_ancestry', 'T4', 'External ancestry results are deferred to Phase 5.')"), - ('_disclosures', - "('grounding_field_coverage', 'external_grounding', 'records whose grounding remains unresolved', 'a " - "narrower direct closure exposure interval when the additional evidence resolves grounding')"), - ('_disclosures', "('lineage_analysis', 'PR-014', 'General lineage graph analysis is deferred to Phase 5.')"), - ('_disclosures', "('lineage_closure_exposure', 'T3', 'Lineage closure exposure is deferred to Phase 5.')"), - ('_disclosures', - "('provenance_required_field_coverage', 'required_provenance_fields', 'records lacking usable required " - "provenance', 'usable declared provenance without discarding independently valid calculations')"), - ('_disclosures', - "('provenance_row_coverage', 'provenance_required_field_coverage', 'grounding_field_coverage')"), - ('_disclosures', "('support', 'diversity', 'tail')"), - ('_disclosures', "('support', 'diversity', 'tail')"), - ('_disclosures', - "(('lineage_analysis', 'PR-014', 'General lineage graph analysis is deferred to Phase 5.'), " - "('lineage_closure_exposure', 'T3', 'Lineage closure exposure is deferred to Phase 5.'), " - "('external_ancestry', 'T4', 'External ancestry results are deferred to Phase 5.'))"), - ('_disclosures', - "(('provenance_required_field_coverage', 'required_provenance_fields', 'records lacking usable required " - "provenance', 'usable declared provenance without discarding independently valid calculations'), " - "('grounding_field_coverage', 'external_grounding', 'records whose grounding remains unresolved', 'a " - "narrower direct closure exposure interval when the additional evidence resolves grounding'))"), - ('_disclosures', "['A bounded operational definition and evidence matched to that definition.']"), - ('_disclosures', "['A controlled empirical intervention design with measured comparable outcomes.']"), - ('_disclosures', - "['A separately defined bounded outcome and appropriately validated predictive evidence.']"), - ('_disclosures', - "['A stated production-failure criterion and corresponding versioned outcome measurements.']"), - ('_disclosures', - "['An explicit pipeline boundary and evidence for its relevant external inputs and transformations.']"), - ('_disclosures', - "['An identified ancestor, measured outcomes and a controlled or otherwise justified causal design.']"), - ('_disclosures', - "['Direct closure exposure describes the selected declared grounding scope, without tracing every pipeline " - "input.']"), - ('_disclosures', - "['Each coverage keeps its own field meaning and denominator.', 'This signal reports unresolved supplied " - "provenance evidence and does not estimate factual truth or source independence.', 'A not_present signal " - "is limited to the cited fields and does not certify universal integrity.']"), - ('_disclosures', "['No accepted versioned model-outcome analysis is supplied.']"), - ('_disclosures', "['No operational failure outcome is measured by these structural fields.']"), - ('_disclosures', "['No universal collapse prediction is defined by the approved toolkit.']"), - ('_disclosures', - "['Only existing reference validation observations can be reported; no graph traversal, root tracing or " - "ancestry metric executes.']"), - ('_disclosures', - "['Phase 4 does not compute ancestry; topology alone would not identify a causal contribution.']"), - ('_disclosures', - "['Preserve explicit composite parent references, chronology and external-grounding metadata for the " - "future Phase 5 analysis.']"), - ('_disclosures', "['R_CAUSAL_EVIDENCE_MISSING', 'R_LINEAGE_EXECUTION_DEFERRED']"), - ('_disclosures', "['R_CONTROLLED_EMPIRICAL_DESIGN_MISSING']"), - ('_disclosures', "['R_LINEAGE_EXECUTION_DEFERRED']"), - ('_disclosures', "['R_MODEL_EVIDENCE_MISSING']"), - ('_disclosures', "['R_OUTCOME_EVIDENCE_MISSING']"), - ('_disclosures', "['R_OUTSIDE_PRODUCT_SCOPE']"), - ('_disclosures', "['R_PIPELINE_BOUNDARY_EVIDENCE_MISSING']"), - ('_disclosures', "['R_UNIVERSAL_OPERATIONAL_DEFINITION_ABSENT']"), - ('_disclosures', "['Supplied scenario outputs are conditional mathematical or stochastic results.']"), - ('_disclosures', - "['Tail membership depends on the explicit rule and selected representation.', 'This signal is not a " - "calibrated forecast of the production pipeline.', 'No simulation is run by this signal, and tail " - "membership does not establish importance or harm.']"), - ('_disclosures', - "['The signal is restricted to the selected versions and their declared common state meaning.', 'An " - 'observed support decrease does not establish model-performance decline, production failure or universal ' - "collapse.']"), - ('_disclosures', "['This product defines no universal integrity scalar or operational test.']"), - ('_disclosures', "['Versioned model evaluation outcomes with comparable tasks and evaluation conditions.']"), - ('_disclosures', "['derived_metrics.support.support_delta']"), - ('_disclosures', "['derived_metrics.tail.tail_support_size', 'derived_metrics.tail.selection']"), - ('_disclosures', - "['eligibility for an explicitly requested comparison after state-meaning compatibility is supplied']"), - ('_disclosures', "['evidence needed by a future model-longitudinal implementation']"), - ('_disclosures', "['more complete declared provenance coverage']"), - ('_disclosures', "['more complete immediate-parent validation; future Phase 5 analysis remains deferred']"), - ('_disclosures', "['representation-dependent support, diversity and tail diagnostics']"), - ('_distribution', "('empirical_assignments', 'explicit_counts_divided_by_included_records')"), - ('_distribution', - "('empirical_assignments', 'explicit_counts_divided_by_included_records', 'weighted_record_mass', " - "'explicit_probability_vector')"), - ('_distribution_exclusions', "('derived_metrics', 'diversity')"), - ('_distribution_exclusions', "('derived_metrics', 'support')"), - ('_distribution_exclusions', "('state_counts', 'supplied_state_probabilities')"), - ('_distribution_exclusions', "(('derived_metrics', 'support'), ('derived_metrics', 'diversity'))"), - ('_duplicates', "('usable_exact_content',)"), - ('_inputs', "('R_CALC_EMPTY_SCOPE',)"), - ('_inputs', "('Validation coverage describes supplied declarations without certifying their truth.',)"), - ('_inputs', "('nonempty_validated_record_scope',)"), - ('_inputs', - "['Input hashes identify supplied bytes and do not certify authenticity.', 'Only allowlisted evidence " - "fields are assembled; raw content, extras, notes and embeddings are not exported.']"), - ('_inputs', "['R_INVENTORY_ONLY_ROLE']"), - ('_lineage_observations', - "('Counts preserve validation reference-entry multiplicity; they do not describe an ancestry graph.',)"), - ('_lineage_observations', - "('Immediate-reference validation coverage does not establish resolved ancestry or external roots.',)"), - ('_lineage_observations', "('No general graph-cycle traversal executes in Phase 4.',)"), - ('_lineage_observations', "('Phase_5_graph_analysis',)"), - ('_lineage_observations', "('R_GRAPH_EXECUTION_DEFERRED',)"), - ('_lineage_observations', "('R_NO_DECLARED_PARENT_ENTRIES',)"), - ('_lineage_observations', - "('This retained earlier-version ordering certificate is not general graph-cycle traversal.',)"), - ('_lineage_observations', "('declared_parent_reference_entries',)"), - ('_message_family', - "('E_PARENT_AMBIGUOUS', 'E_PARENT_FORMAT', 'E_PARENT_FUTURE_VERSION', 'E_LINEAGE_CYCLE', " - "'W_GENERATION_MISMATCH')"), - ('_message_family', "('parent_ids', 'generation')"), - ('_message_family', "('representation', 'topic', 'embedding_cluster', 'content', 'content_ref')"), - ('_message_family', "('source_type', 'provenance_confidence', 'external_grounding')"), - ('_message_family', "('version_order', 'pair_order', 'representation_compatibility')"), - ('_message_family', "['content_diagnostics']"), - ('_message_family', "['dataset_longitudinal']"), - ('_message_family', "['ingestion']"), - ('_message_family', "['lineage']"), - ('_message_family', "['provenance']"), - ('_metadata_text', - "('(known_closed_count + unresolved_grounding_count) / total_record_count', 'no usable required-provenance " - "row')"), - ('_metadata_text', "('All selected valid records',)"), - ('_metadata_text', "('Approved direct grounding partition; uncertainty remains unresolved.',)"), - ('_metadata_text', "('Definitions 11.4/19; missing-row mass / all selected record weight mass',)"), - ('_metadata_text', "('Definitions 11.4; missing rows / all selected valid records',)"), - ('_metadata_text', "('Definitions 19; all selected weights',)"), - ('_metadata_text', "('Definitions 19; missing-row mass',)"), - ('_metadata_text', "('Definitions 19; sum explicit weights by source',)"), - ('_metadata_text', "('Definitions 3.13; nonmissing valid field / all selected valid records',)"), - ('_metadata_text', "('Definitions 3.13; nonmissing valid field / all selected valid records',)"), - ('_metadata_text', "('Definitions 3.3/11.1; count declared canonical categories',)"), - ('_metadata_text', "('Definitions 3.3/11.1; count declared canonical categories',)"), - ('_metadata_text', "('Definitions 9.5; canonical record weights summed within each state',)"), - ('_metadata_text', - "('Earlier and later scopes explicitly selected and independently validated.', 'State identity uses the " - "declared common basis, including any disclosed map.')"), - ('_metadata_text', "('Exact explicit single-version Phase 2 join scope; no representation exclusions.',)"), - ('_metadata_text', "('Exact record form does not establish semantic identity or independent origin.',)"), - ('_metadata_text', - "('Experimental conditional simulation; no calibrated production-failure probability.', 'Diversity " - "contraction holds in expectation, not monotonically on every sampled path.', 'Simulated steps are not " - "record generations, training epochs or dataset releases.', 'No external-reference loss, reopening, " - "lineage, risk score or audit workflow is implemented.', 'Floating-point and pseudorandom sampling are " - "numerical realizations of the declared model.')"), - ('_metadata_text', "('F-007 weighted variant; category mass / all selected record weight mass',)"), - ('_metadata_text', "('F-014; (1-p_i)^n; analytic one-step closed multinomial',)"), - ('_metadata_text', - "('Fixed finite declared state space and constant positive integer resample size.', 'X_t conditional on " - "p_t is Multinomial(n,p_t); p_(t+1)=X_t/n.', 'No mutation, migration, independent real data or external " - "corrective input.')"), - ('_metadata_text', - "('Observed/supplied support only; no permanent extinction or causal/model-performance verdict.', 'A " - "coarsened comparison cannot recover distinctions lost through its mapping.')"), - ('_metadata_text', - "('One explicitly selected version and declared representation.', 'No implicit pooling, probability " - "repair, confidence weighting or sampling.')"), - ('_metadata_text', "('Phase 2 matched-row inventory',)"), - ('_metadata_text', "('Phase 2 missing-row inventory',)"), - ('_metadata_text', - "('Representation-bound; does not establish functional failure or semantic completeness.',)"), - ('_metadata_text', - "('Representation-bound; no calibrated production-failure or universal risk conclusion.',)"), - ('_metadata_text', "('Reuse Phase 2 coverage; Definitions 3.10-3.12',)"), - ('_metadata_text', "('Reuse Phase 2 coverage; Definitions 3.10-3.12',)"), - ('_metadata_text', "('Reuse Phase 2 coverage; Definitions 3.10-3.12',)"), - ('_metadata_text', "('Supplied declarations only; no truth or source-independence certification.',)"), - ('_metadata_text', - "('Toolkit operationalization relative to supplied metadata, without lineage or truth certification.',)"), - ('_metadata_text', "('analytic_expectation; D0*(1-1/n)**t; t=0..steps; constant n',)"), - ('_metadata_text', "('ascending frequency, then count, then Unicode state ID; 1-based ordinal',)"), - ('_metadata_text', "('cardinality of earlier support minus later support',)"), - ('_metadata_text', "('cardinality of later support minus earlier support',)"), - ('_metadata_text', "('category count / all selected valid records',)"), - ('_metadata_text', "('count included record-state assignments',)"), - ('_metadata_text', "('count_groups_of_size_greater_than_one; DEFINITIONS_AND_UNITS:8.4',)"), - ('_metadata_text', "('earlier support intersect later support',)"), - ('_metadata_text', "('earlier support minus later support',)"), - ('_metadata_text', - "('empirical_assignments', 'explicit_counts_divided_by_included_records', 'weighted_record_mass', " - "'explicit_probability_vector')"), - ('_metadata_text', "('empirical_assignments; n_i/N',)"), - ('_metadata_text', - "('exact_utf8_v1 and SHA-256; equal digests verified against exact bytes', 'one explicitly selected " - "version; unweighted records')"), - ('_metadata_text', "('intersection support size / earlier positive-mass support size',)"), - ('_metadata_text', "('known_closed_count / total_record_count', 'no usable required-provenance row')"), - ('_metadata_text', "('later Gini-Simpson diversity minus earlier diversity',)"), - ('_metadata_text', "('later support minus earlier support',)"), - ('_metadata_text', "('later support_size minus earlier support_size',)"), - ('_metadata_text', "('sampled_path; sequential_binomial_complement_v1',)"), - ('_metadata_text', "('singleton_count', 'count_at_or_below', 'frequency_at_or_below', 'state_list')"), - ('_metadata_text', "('singleton_count', 'count_at_or_below', 'frequency_at_or_below', 'state_list')"), - ('_metadata_text', "('singleton_count', 'count_at_or_below', 'frequency_at_or_below', 'state_list')"), - ('_metadata_text', "('state_count', 'state_frequency', 'support_size', 'gini_simpson_diversity')"), - ('_metadata_text', "('sum_group_size_minus_one; DEFINITIONS_AND_UNITS:8.3',)"), - ('_metadata_text', - "('support_delta', 'support_loss_count', 'support_added_count', 'support_retention_ratio', " - "'gini_simpson_diversity_delta', 'extinct_states', 'added_states', 'retained_states')"), - ('_metadata_text', "('toolkit_operationalization; Definitions 3.7-3.9; P3-D08',)"), - ('_metadata_text', "('toolkit_operationalization; Definitions 3.7-3.9; P3-D08',)"), - ('_metadata_text', "('toolkit_operationalization; Definitions 3.7-3.9; P3-D08',)"), - ('_metadata_text', - "('upper_bound - lower_bound = unresolved_grounding_count / total_record_count', 'no usable " - "required-provenance row')"), - ('_provenance_consistency', "('confirmed', 'log_derived', 'estimated', 'unknown')"), - ('_provenance_consistency', "('human', 'synthetic', 'mixed', 'sensor', 'unknown')"), - ('_provenance_consistency', "('known_closed', 'valid_direct_grounding_no', ())"), - ('_provenance_consistency', "('known_open', 'valid_direct_grounding_yes', ())"), - ('_provenance_consistency', "('unresolved_grounding', 'declared_unknown_grounding', ())"), - ('_provenance_consistency', "('unresolved_grounding', 'missing_provenance_row', ())"), - ('_scalar', "('valid_nonempty_input_for_declared_scope',)"), - ('_table', "('valid_nonempty_input_for_declared_scope',)"), - ('assemble_report', "['existing_bundle_validation']")] - -PHASE4_STEP3_ASSEMBLY_CALLS = {'FamilyFailure': ['dataclass'], - 'FamilyFailure.__post_init__': ['ReportAssemblyError', 'ReportAssemblyError', 'any', 'type', 'type', 'type'], - '_base': ['_definition', - 'dict.fromkeys', - 'dict.fromkeys', - 'dict.fromkeys', - 'dict.fromkeys', - 'field.owner.startswith', - 'list', - 'list', - 'list', - 'list'], - '_bundle_check': ['_coverage', - '_coverage', - '_coverage', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - 'all', - 'all', - 'all', - 'all', - 'all', - 'all', - 'all', - 'all', - 'all', - 'all', - 'assessment.capabilities.items', - 'capability.coverage_details.values', - 'len', - 'len', - 'len', - 'len', - 'len', - 'row.values.get', - 'row.values.get', - 'set', - 'set', - 'set', - 'set', - 'set', - 'set', - 'sum', - 'sum', - 'sum', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type'], - '_bundle_scope': ['_key', 'len', 'list'], - '_capabilities': ['_coverage', - '_coverage', - '_coverage', - '_coverage', - '_require', - '_require', - '_typed', - '_typed', - 'any', - 'dict.fromkeys', - 'dict.fromkeys', - 'dict.fromkeys', - "entry['notes'].append", - "entry['notes'].append", - 'list', - 'list', - 'list', - 'list', - 'list', - 'list', - 'list', - 'list', - 'list', - 'matrix.values', - 'original.coverage_details.items', - 'reasons.append', - 'set', - 'set', - 'type'], - '_closed_simulation': ['WeightingOptions', - 'WeightingOptions', - '_base', - '_metadata', - '_metadata', - '_policy', - '_representation', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_scope', - '_simulation_inputs', - '_typed', - '_typed', - 'dict.fromkeys', - 'generations.append', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'list', - 'list', - 'list', - 'list', - 'list', - 'list', - 'list', - 'list', - 'paths.append', - 'range', - 'range', - 'set', - 'set', - 'target.update', - 'target.update', - 'trajectories.append', - 'tuple', - 'type', - 'type', - 'zip'], - '_closure': ['_coverage', - '_number', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_scalar', - '_scope', - '_typed', - 'len', - "payload['derived_metrics'].setdefault", - 'set', - 'set', - 'sum'], - '_comparison': ['_distribution', - '_distribution', - '_envelope', - '_representation', - '_representation', - '_representation', - '_representation', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_scalar', - '_scalar', - '_scope', - '_table', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - 'effects.append', - 'len', - 'len', - 'len', - 'list', - 'list', - 'list', - 'list', - 'list', - 'list', - 'list', - 'mapping.state_mapping.items', - 'order.index', - 'order.index', - "payload['derived_metrics'].setdefault", - "payload['derived_metrics'].setdefault", - 'support.values', - 'type', - 'type', - 'zip'], - '_coverage': ['_number', '_number', '_require', '_typed'], - '_definition': ['ReportAssemblyError'], - '_diagnostics': ['_key', - '_key', - '_message_family', - '_require', - '_require', - '_require', - '_typed', - '_typed', - "payload['errors'].append", - "payload['inputs']['limitations'].append", - "payload['warnings'].append", - 'type', - 'type', - 'type', - 'type', - 'type'], - '_disclosure_recommend': ["payload['recommended_next_metadata'].append"], - '_disclosure_scope': ["payload['inputs'].get"], - '_disclosure_signal': ['_disclosure_base', 'dict.fromkeys', 'list', 'list', 'list', 'result.update'], - '_disclosure_unavailable': ['_disclosure_base', - '_disclosure_scope', - "payload['unavailable_conclusions'].append", - 'result.update'], - '_disclosures': ['_disclosure_recommend', - '_disclosure_recommend', - '_disclosure_recommend', - '_disclosure_recommend', - '_disclosure_recommend', - '_disclosure_recommend', - '_disclosure_signal', - '_disclosure_signal', - '_disclosure_signal', - '_disclosure_unavailable', - '_disclosure_unavailable', - '_disclosure_unavailable', - '_disclosure_unavailable', - '_disclosure_unavailable', - '_disclosure_unavailable', - '_disclosure_unavailable', - '_disclosure_unavailable', - 'any', - 'any', - 'any', - 'basis.append', - 'basis.append', - 'content_cap.get', - 'derived.get', - 'derived.get', - 'derived.get', - "derived.get('closure_exposure', {}).get", - 'direct.get', - 'list', - 'longitudinal_cap.get', - 'paths.append', - 'paths.append', - "payload['capabilities'].get", - "payload['capabilities'].get", - "payload['capabilities'].get", - "payload['observed_facts'].get", - 'provenance.get', - 'provenance.get', - 'provenance.get', - 'provenance.get', - 'reasons.extend', - 'set', - 'set', - 'support.get', - 'tail.get'], - '_distribution': ['WeightingOptions', - '_envelope', - '_number', - '_number', - '_number', - '_number', - '_policy', - '_representation', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_scalar', - '_scope', - '_table', - '_table', - '_table', - '_table', - '_typed', - '_typed', - '_typed', - 'all', - 'all', - 'all', - 'all', - 'isclose', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - "payload['derived_metrics'].setdefault", - "payload['derived_metrics'].setdefault", - "payload['derived_metrics'].setdefault('diversity', {}).setdefault", - "payload['derived_metrics'].setdefault('diversity', {}).setdefault('by_version', " - '{}).setdefault', - "payload['derived_metrics'].setdefault('support', {}).setdefault", - "payload['derived_metrics'].setdefault('support', {}).setdefault('by_version', " - '{}).setdefault', - "payload['observed_facts'].setdefault", - "payload['observed_facts'].setdefault", - "payload['observed_facts'].setdefault('state_counts', {}).setdefault", - "payload['observed_facts'].setdefault('supplied_state_probabilities', {}).setdefault", - 'set', - 'sum', - 'sum', - 'type', - 'type', - 'type', - 'type', - 'type'], - '_distribution_exclusions': ['_key', - '_require', - '_require', - '_require', - 'all', - 'exclusions.append', - 'len', - 'len', - "payload['observed_facts'].get", - "payload['observed_facts'].get(family, {}).get", - "payload['observed_facts'].get(family, {}).get('by_version', {}).get", - 'payload[section].get', - 'payload[section].get(family, {}).get', - "payload[section].get(family, {}).get('by_version', {}).get", - "payload[section].get(family, {}).get('by_version', {}).get(version, " - '{}).values', - 'set', - 'type', - 'type', - 'type'], - '_duplicates': ['_coverage', - '_envelope', - '_key', - '_representation', - '_require', - '_require', - '_require', - '_require', - '_require', - '_scalar', - '_scope', - '_typed', - '_typed', - 'enumerate', - 'groups.append', - 'len', - 'len', - 'len', - 'len', - 'len', - "payload['observed_facts'].setdefault", - 'seen.intersection', - 'seen.update', - 'set', - 'set', - 'set', - 'set', - 'str', - 'sum', - 'tuple'], - '_envelope': ['_base'], - '_extinction': ['ScalarCalculation', - 'WeightingOptions', - '_base', - '_metadata', - '_policy', - '_representation', - '_require', - '_require', - '_require', - '_require', - '_require', - '_scope', - '_typed', - '_typed', - 'target.update', - "target['initial_distribution'].append", - "target['parameters']['state_order'].append", - 'type'], - '_inputs': ['_bundle_scope', - '_coverage', - '_diagnostics', - '_diagnostics', - '_diagnostics', - '_diagnostics', - '_envelope', - '_envelope', - '_key', - '_key', - '_require', - '_require', - '_require', - 'affected.append', - 'any', - 'dict.fromkeys', - 'enumerate', - 'hashes.add', - 'iter', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'list', - 'list', - 'list', - 'list', - 'messages_by_location.get', - 'messages_by_location.get', - 'messages_by_location.get', - 'messages_by_location.setdefault', - 'messages_by_location.setdefault((message.file_role, message.file_path), []).append', - 'next', - 'operations.append', - "payload['inputs']['artifacts'].append", - "payload['inputs']['file_hashes'].append", - 'records_by_version.get', - 'records_by_version.setdefault', - 'records_by_version.setdefault(row.record_key.dataset_version, []).append', - 'set', - 'set', - 'set', - 'sorted', - 'str', - 'str', - 'str', - 'tuple', - 'tuple', - 'tuple', - 'type', - 'unmapped.update', - 'versions_by_location.get', - 'versions_by_location.setdefault', - 'versions_by_location.setdefault((row.location.file_role, row.location.file_path), set()).add'], - '_key': ['_typed'], - '_lineage_observations': ['_coverage', '_envelope', '_envelope', '_envelope', '_envelope', 'list'], - '_metadata': ['CalculationMetadata', - 'WeightingOptions', - 'WeightingOptions', - '_metadata_text', - '_require', - '_require', - '_require', - '_require', - '_require', - '_typed', - '_typed', - 'type', - 'type'], - '_metadata_text': ['_require', - '_require', - '_require', - '_require', - 'all', - 'all', - 'methods.get', - 'name.startswith', - 'tuple', - 'tuple', - 'tuple', - 'tuple', - 'type', - 'type', - 'type', - 'type'], - '_number': ['_require', '_require', '_require', '_require', 'isfinite', 'type', 'type'], - '_policy': ['NumericalPolicy', '_typed'], - '_provenance': ['WeightingOptions', - '_coverage', - '_coverage', - '_provenance_consistency', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_scalar', - '_scalar', - '_scalar', - '_scalar', - '_scope', - '_table', - '_table', - '_table', - '_table', - '_table', - '_typed', - '_typed', - '_typed', - '_typed', - 'all', - 'dict', - 'dict', - 'dict', - 'len', - 'len', - 'len', - 'len', - "payload['derived_metrics'].setdefault", - "payload['observed_facts'].setdefault", - 'set', - 'sum', - 'type', - 'type', - 'zip'], - '_provenance_consistency': ['_coverage', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_typed', - '_typed', - 'all', - 'any', - 'dict', - 'expected.values', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'match.provenance.values.get', - 'match.provenance.values.get', - 'set', - 'set', - 'set', - 'set', - 'set', - 'set', - 'set', - 'set', - 'sum', - 'sum', - 'sum', - 'sum', - 'sum', - 'sum', - 'tuple', - 'tuple', - 'tuple', - 'tuple', - 'type', - 'type', - 'type', - 'zip'], - '_representation': ['RepresentationDescriptor', '_typed'], - '_require': ['ReportAssemblyError'], - '_scalar': ['CalculationEvidenceClass', - 'ScalarCalculation', - '_definition', - '_envelope', - '_metadata', - '_representation', - '_scope', - '_typed', - 'field.method_id.startswith', - 'path.rsplit', - 'tuple', - 'tuple'], - '_scope': ['CalculationScope', - '_key', - '_key', - '_require', - '_require', - '_typed', - 'len', - 'len', - 'list', - 'set', - 'set', - 'set'], - '_simulation_inputs': ['_number', - '_policy', - '_representation', - '_require', - '_require', - '_require', - '_scope', - '_typed', - 'all', - 'len', - 'len', - 'len', - 'type', - 'type', - 'type', - 'type', - 'type'], - '_table': ['CalculationEvidenceClass', - '_definition', - '_envelope', - '_metadata', - '_representation', - '_require', - '_require', - '_require', - '_scope', - 'all', - 'field.method_id.startswith', - 'tuple', - 'tuple', - 'type', - 'type', - 'type'], - '_tail': ['_require', - '_require', - '_scalar', - '_scope', - '_table', - '_typed', - '_typed', - 'list', - 'list', - 'ranking.append', - 'set'], - '_typed': ['_require', 'type'], - 'assemble_report': ['CanonicalReport.from_dict', - 'FamilyFailure', - '_bundle_check', - '_capabilities', - '_closed_simulation', - '_closed_simulation', - '_closure', - '_comparison', - '_coverage', - '_diagnostics', - '_diagnostics', - '_diagnostics', - '_diagnostics', - '_disclosures', - '_distribution', - '_distribution', - '_distribution', - '_distribution_exclusions', - '_duplicates', - '_extinction', - '_inputs', - '_lineage_observations', - '_provenance', - '_representation', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_tail', - '_typed', - 'all', - 'all', - 'any', - 'dict', - 'empirical_representations.append', - 'empirical_representations.append', - 'empirical_representations.append', - 'empirical_representations.append', - 'enumerate', - 'failures.setdefault', - 'failures.setdefault(failure.capability.value, []).extend', - 'len', - 'len', - 'len', - 'len', - "operations['content_diagnostics'].append", - "operations['content_diagnostics'].append", - "operations['content_diagnostics'].append", - "operations['dataset_longitudinal'].append", - "operations['intervention_simulation'].append", - "operations['intervention_simulation'].append", - "operations['intervention_simulation'].append", - "operations['provenance'].append", - "operations['provenance'].append", - "payload['inputs']['limitations'].append", - 'run.get', - 'run.get', - 'set', - 'set', - 'set', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type']} - -PHASE4_STEP3_ASSEMBLY_ARITHMETIC = {'FamilyFailure.__post_init__': ['not self.messages'], - '_capabilities': ["bundle.observability.limitations + ('Input eligibility is distinct from executed " - "analysis; Phase 5 lineage remains deferred.',)", - 'not completed', - 'reasons + family_codes'], - '_closed_simulation': ["target['limitations'] + ['Extinction events are not separately inferred by the " - "assembly adapter; inspect the supplied sampled support paths.']", - 'value.simulation_horizon + 1'], - '_closure': ["'derived_metrics.closure_exposure.direct.' + name"], - '_comparison': ["'derived_metrics.support.' + name", - "'derived_metrics.support.' + name", - 'context.earlier_scope.excluded_record_keys + context.later_scope.excluded_record_keys', - 'context.earlier_scope.included_record_keys + context.later_scope.included_record_keys', - "context.earlier_scope.scope_id + ' -> '", - "context.earlier_scope.scope_id + ' -> ' + context.later_scope.scope_id", - 'earlier + later', - 'value.limitations + compatibility.limitations'], - '_diagnostics': ["'Validation notice ' + message.code", - "'Validation notice ' + message.code + ': '", - "'Validation notice ' + message.code + ': ' + message.message"], - '_disclosure_signal': ["list(basis['limitations']) + limitations", "owner + '.'", "owner + '.' + name"], - '_disclosure_unavailable': ["owner + '.unavailable_conclusion'"], - '_disclosures': ["'observed_facts.provenance.' + key", - "not any((key in derived for key in ('support', 'diversity', 'tail')))", - 'not basis', - "set(content_cap.get('reason_codes', ())) | set(longitudinal_cap.get('reason_codes', ()))"], - '_distribution': ["'derived_metrics.' + family", - "'derived_metrics.' + family + '.by_version.*.'", - "'derived_metrics.' + family + '.by_version.*.' + prefix", - "'derived_metrics.' + family + '.by_version.*.' + prefix + name", - "'derived_metrics.diversity.by_version.*.' + prefix", - "'derived_metrics.diversity.by_version.*.' + prefix + 'state_frequencies'", - 'not weighted', - 'not weighted', - 'not weighted', - "prefix + 'state_frequencies'", - "prefix + 'support_size'", - 'prefix + name', - 'prefix + name'], - '_duplicates': ["'exact_duplicate_group_' + str(index + 1)", - "'observed_facts.content.' + name", - "group['record_count'] - 1", - 'index + 1', - 'not seen.intersection(group.record_keys)'], - '_inputs': ["'observed_facts.provenance.' + name", - "'validated_version:' + version", - 'bundle.records + (() if bundle.provenance is None else bundle.provenance)', - 'messages + uncertain', - 'messages + uncertain', - "tuple(messages_by_location.get((artifact.role, '[redacted]'), ())) + " - 'tuple(messages_by_location.get((artifact.role, None), ()))'], - '_lineage_observations': ["'observed_facts.lineage.' + name", 'coverage.denominator - coverage.numerator'], - '_metadata_text': ["'Definitions 10.1-10.6; explicit ' + rule", - "'Definitions 10.1-10.6; explicit ' + rule", - "'Definitions 10.1-10.6; explicit ' + rule", - "'Definitions 10.1-10.6; explicit ' + rule + '; selected counts / included records'", - "'Definitions 7.1; ' + item", - "basis + ('empirical_assignments; n_i/N',)", - "tuple(('Definitions 7.1; ' + item for item in basis)) + ('count included record-state " - "assignments',)"], - '_number': ["name + ' is above its accepted bound'", - "name + ' is below its accepted bound'", - "name + ' must be a built-in number'", - "name + ' must be finite'"], - '_provenance': ["'derived_metrics.provenance.' + name", - "'derived_metrics.provenance.' + name", - "'observed_facts.provenance.' + field", - "'observed_facts.provenance.' + field", - "'observed_facts.provenance.' + field + '_counts'", - "'observed_facts.provenance.' + field + '_field_coverage'", - "'observed_facts.provenance.' + name", - "'observed_facts.provenance.' + name", - "'observed_facts.provenance.' + name", - "field + '_counts'", - "field + '_counts'", - "field + '_field_coverage'", - "field + '_field_coverage'"], - '_provenance_consistency': ['len(keys) - len(missing)', - 'not row.required_fields_valid', - 'not value.scope.excluded_record_keys'], - '_require': ['not condition'], - '_scalar': ['-1', 'value.metadata.limitations + tuple(limitations)'], - '_scope': ['value.included_record_keys + value.excluded_record_keys'], - '_table': ['metadata.limitations + tuple(limitations)'], - '_tail': ["'derived_metrics.tail.' + name", "'derived_metrics.tail.' + name"], - '_typed': ["name + ' requires its exact accepted result type'"], - 'assemble_report': ["'supplied_declared_tail:' + tail.scope.scope_id", - "'supplied_direct_closure_interval:' + closure.scope.scope_id", - "'supplied_distribution:' + (supplied.unweighted.scope.scope_id if type(supplied) is " - 'StateDistributionResult else supplied.scope.scope_id)', - "'supplied_exact_duplicates:' + duplicates.scope.scope_id", - "'supplied_provenance_composition:' + provenance.scope.scope_id", - 'DirectClosureExposureBounds | None', - 'ExactDuplicateResult | None', - 'ExpectedDiversityResult | None', - 'ProvenanceCompositionResult | None', - 'ResamplingSimulation | None', - 'StateDistributionResult | DistributionMetrics', - 'SupportComparison | None', - 'TailSelectionResult | None', - 'supplied.unweighted.scope.included_record_keys + ' - 'supplied.unweighted.scope.excluded_record_keys']} - -PHASE4_STEP3_ASSEMBLY_ATTRIBUTES = {'FamilyFailure.__post_init__': ['ValidationSeverity.ERROR', - 'ValidationSeverity.FATAL', - 'item.severity', - 'self.capability', - 'self.messages', - 'self.messages', - 'self.messages'], - '_base': ['dict.fromkeys', - 'dict.fromkeys', - 'dict.fromkeys', - 'dict.fromkeys', - 'field.evidence_class', - 'field.method_id', - 'field.owner', - 'field.owner', - 'field.owner', - 'field.owner.startswith', - 'field.unit'], - '_bundle_check': ['assessment.capabilities', - 'assessment.capabilities', - 'assessment.capabilities', - 'assessment.capabilities.items', - 'assessment.maximum_level', - 'assessment.maximum_level', - 'bundle.generation', - 'bundle.generation', - 'bundle.inventory', - 'bundle.inventory', - 'bundle.inventory', - 'bundle.mapping_traces', - 'bundle.mapping_traces', - 'bundle.observability', - 'bundle.observability', - 'bundle.provenance', - 'bundle.provenance', - 'bundle.provenance', - 'bundle.provenance', - 'bundle.provenance', - 'bundle.provenance', - 'bundle.provenance_join', - 'bundle.provenance_join', - 'bundle.records', - 'bundle.records', - 'bundle.records', - 'bundle.records', - 'bundle.validation_messages', - 'bundle.validation_messages', - 'bundle.version_order', - 'bundle.version_order', - 'bundle.version_order', - 'bundle.version_order.loaded_versions', - 'bundle.version_order.order', - 'capability.coverage', - 'capability.coverage', - 'capability.coverage_details', - 'capability.coverage_details', - 'capability.coverage_details', - 'capability.coverage_details.values', - 'capability.notes', - 'capability.reason_codes', - 'capability.requirements_met', - 'capability.requirements_missing', - 'capability.status', - 'coverage.denominator', - 'coverage.numerator', - 'item.file_format', - 'item.path', - 'item.role', - 'join.grounding_field_coverage', - 'join.matches', - 'join.matches', - 'join.matches', - 'join.matches', - 'join.matches', - 'join.matches', - 'join.matches', - 'join.matches', - 'join.matches', - 'join.missing_record_keys', - 'join.missing_record_keys', - 'join.provenance_required_field_coverage', - 'join.provenance_row_coverage', - 'join.provenance_supplied', - 'join.scope_record_keys', - 'join.scope_record_keys', - 'join.scope_record_keys', - 'join.scope_record_keys', - 'join.scope_record_keys', - 'join.scope_record_keys', - 'join.scope_record_keys', - 'join.selected_dataset_versions', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance.grounding_known', - 'match.provenance.required_fields_valid', - 'match.record_key', - 'match.record_key', - 'match.record_key', - 'match.record_key', - 'row.grounding_known', - 'row.location', - 'row.location', - 'row.record_key', - 'row.record_key', - 'row.record_key', - 'row.record_key', - 'row.required_fields_valid', - 'row.values', - 'row.values', - 'row.values', - 'row.values', - 'row.values.get', - 'row.values.get'], - '_bundle_scope': ['bundle.records', - 'bundle.records', - 'bundle.version_order', - 'bundle.version_order.loaded_versions', - 'row.record_key'], - '_capabilities': ['CapabilityKey.DATASET_LONGITUDINAL', - 'CapabilityKey.DATASET_LONGITUDINAL', - 'CapabilityKey.INGESTION', - 'CapabilityKey.INTERVENTION_SIMULATION', - 'CapabilityKey.LINEAGE', - 'CapabilityKey.LINEAGE', - 'CapabilityKey.LINEAGE', - 'CapabilityKey.LINEAGE', - 'CapabilityKey.MODEL_LONGITUDINAL', - 'CapabilityKey.MODEL_LONGITUDINAL', - 'CapabilityKey.PROVENANCE', - 'bundle.observability', - 'bundle.observability', - 'bundle.observability', - 'bundle.observability', - 'bundle.observability', - 'bundle.observability.basis', - 'bundle.observability.capabilities', - 'bundle.observability.limitations', - 'bundle.observability.maximum_level', - 'bundle.observability.maximum_level', - 'bundle.provenance_join', - 'bundle.provenance_join', - 'bundle.provenance_join', - 'bundle.provenance_join.grounding_field_coverage', - 'bundle.provenance_join.provenance_required_field_coverage', - 'bundle.provenance_join.provenance_row_coverage', - 'bundle.records', - 'dict.fromkeys', - 'dict.fromkeys', - 'dict.fromkeys', - "entry['notes'].append", - "entry['notes'].append", - 'key.value', - 'key.value', - 'key.value', - 'matrix.values', - 'original.coverage', - 'original.coverage', - 'original.coverage', - 'original.coverage', - 'original.coverage_details', - 'original.coverage_details', - 'original.coverage_details', - 'original.coverage_details.items', - 'original.notes', - 'original.reason_codes', - 'original.requirements_met', - 'original.requirements_missing', - 'original.status', - 'original.status', - 'original.status.value', - 'reasons.append'], - '_closed_simulation': ['CalculationEvidenceClass.SIMULATION', - 'CalculationEvidenceClass.SIMULATION', - 'CalculationEvidenceClass.SIMULATION', - 'dict.fromkeys', - 'generation.gini_simpson_diversity', - 'generation.state_counts', - 'generation.state_counts', - 'generation.state_counts', - 'generation.state_counts', - 'generation.state_counts', - 'generation.state_frequencies', - 'generation.state_frequencies', - 'generation.step', - 'generation.step', - 'generation.support', - 'generation.support', - 'generation.support', - 'generation.support_size', - 'generation.support_size', - 'generation.support_size', - 'generations.append', - 'inputs.effective_distribution', - 'inputs.effective_distribution', - 'inputs.input_basis', - 'inputs.numerical_policy', - 'inputs.representation', - 'inputs.representation', - 'inputs.representation', - 'inputs.scope', - 'inputs.scope', - 'inputs.scope', - 'paths.append', - 'replicate.generations', - 'replicate.generations', - 'replicate.replicate_index', - 'replicate.replicate_index', - 'target.update', - 'target.update', - 'trajectories.append', - 'value.assumptions', - 'value.assumptions', - 'value.contraction_factor', - 'value.evidence_class', - 'value.expected_diversity', - 'value.expected_diversity_metadata', - 'value.experimental', - 'value.initial_gini_simpson_diversity', - 'value.inputs', - 'value.limitations', - 'value.method', - 'value.method', - 'value.method_version', - 'value.method_version', - 'value.model_name', - 'value.model_name', - 'value.numerical_underflow_steps', - 'value.numpy_version', - 'value.random_seed', - 'value.replicate_schedule', - 'value.resample_size', - 'value.resample_size', - 'value.resample_size', - 'value.rng_name', - 'value.sampled_paths', - 'value.sampler_algorithm', - 'value.simulation_horizon', - 'value.simulation_horizon', - 'value.simulation_replicates', - 'value.simulation_replicates', - 'value.state_order', - 'value.state_order', - 'value.state_order', - 'value.state_order', - 'value.state_schedule', - 'value.trajectory_metadata', - 'value.trajectory_metadata'], - '_closure': ["payload['derived_metrics'].setdefault", - 'provenance.confidence', - 'provenance.confidence', - 'provenance.confidence', - 'provenance.confidence.counts', - 'provenance.confidence.field_coverage', - 'provenance.confidence.status', - 'provenance.direct_grounding', - 'provenance.direct_grounding', - 'provenance.direct_grounding', - 'provenance.direct_grounding.known_closed_count', - 'provenance.direct_grounding.known_closed_count.value', - 'provenance.direct_grounding.known_open_count', - 'provenance.direct_grounding.known_open_count.value', - 'provenance.direct_grounding.unresolved_grounding_count', - 'provenance.direct_grounding.unresolved_grounding_count.value', - 'provenance.grounding_field_coverage', - 'provenance.input_has_errors', - 'provenance.provenance_required_field_coverage', - 'provenance.provenance_row_coverage', - 'provenance.scope', - 'provenance.validation_messages', - 'value.classification_basis', - 'value.classification_basis', - 'value.confidence_counts', - 'value.confidence_disclosure', - 'value.confidence_disclosure', - 'value.confidence_field_coverage', - 'value.confidence_status', - 'value.denominator', - 'value.denominator', - 'value.denominator', - 'value.denominator_basis', - 'value.grounding_field_coverage', - 'value.grounding_field_coverage', - 'value.grounding_field_coverage', - 'value.input_has_errors', - 'value.interval_width', - 'value.known_closed_count', - 'value.known_closed_count', - 'value.known_closed_count', - 'value.known_open_count', - 'value.known_open_count', - 'value.known_open_count', - 'value.limitations', - 'value.lower_bound', - 'value.operationalization_label', - 'value.provenance_required_field_coverage', - 'value.provenance_row_coverage', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope.denominator_basis', - 'value.scope.included_record_keys', - 'value.unresolved_grounding_count', - 'value.unresolved_grounding_count', - 'value.unresolved_grounding_count', - 'value.upper_bound', - 'value.validation_messages'], - '_comparison': ['bundle.version_order', - 'compatibility.collision_groups', - 'compatibility.collision_groups', - 'compatibility.context', - 'compatibility.earlier_state_semantics', - 'compatibility.earlier_state_semantics', - 'compatibility.earlier_state_semantics', - 'compatibility.earlier_state_semantics', - 'compatibility.earlier_state_semantics', - 'compatibility.harmonized_representation', - 'compatibility.harmonized_representation', - 'compatibility.harmonized_representation', - 'compatibility.harmonized_representation', - 'compatibility.harmonized_representation', - 'compatibility.harmonized_state_semantics', - 'compatibility.harmonized_state_semantics', - 'compatibility.harmonized_state_semantics', - 'compatibility.harmonized_state_semantics', - 'compatibility.later_state_semantics', - 'compatibility.later_state_semantics', - 'compatibility.later_state_semantics', - 'compatibility.later_state_semantics', - 'compatibility.later_state_semantics', - 'compatibility.limitations', - 'compatibility.mapping', - 'compatibility.mapping', - 'compatibility.mapping', - 'compatibility.method', - 'compatibility.method', - 'compatibility.method', - 'context.earlier_representation', - 'context.earlier_representation', - 'context.earlier_representation', - 'context.earlier_representation', - 'context.earlier_representation', - 'context.earlier_scope', - 'context.earlier_scope', - 'context.earlier_scope', - 'context.earlier_scope', - 'context.earlier_scope', - 'context.earlier_scope', - 'context.earlier_scope.dataset_versions', - 'context.earlier_scope.excluded_record_keys', - 'context.earlier_scope.included_record_keys', - 'context.earlier_scope.scope_id', - 'context.later_representation', - 'context.later_representation', - 'context.later_representation', - 'context.later_representation', - 'context.later_representation', - 'context.later_scope', - 'context.later_scope', - 'context.later_scope', - 'context.later_scope', - 'context.later_scope', - 'context.later_scope', - 'context.later_scope.dataset_versions', - 'context.later_scope.excluded_record_keys', - 'context.later_scope.included_record_keys', - 'context.later_scope.scope_id', - 'context.version_order', - 'context.version_order', - 'context.version_order', - 'context.version_order.order', - 'context.version_order.order_source', - 'effects.append', - 'harmonized.denominator_basis', - 'harmonized.frequency_denominator', - 'harmonized.input_basis', - 'harmonized.weighting', - 'item.weighting', - 'item.weighting', - 'item.weighting.weighting_mode', - 'item.weighting.weighting_mode', - 'mapping.direction', - 'mapping.direction', - 'mapping.direction', - 'mapping.direction', - 'mapping.direction', - 'mapping.source_representation', - 'mapping.source_state_semantics', - 'mapping.state_mapping', - 'mapping.state_mapping', - 'mapping.state_mapping.items', - 'mapping.target_representation', - 'mapping.target_state_semantics', - 'order.index', - 'order.index', - 'original.denominator_basis', - 'original.frequency_denominator', - 'original.input_basis', - 'original.weighting', - "payload['derived_metrics'].setdefault", - "payload['derived_metrics'].setdefault", - 'scalar.reason_codes', - 'scalar.status', - 'scalar_scope.dataset_versions', - 'scalar_scope.denominator_basis', - 'scalar_scope.excluded_record_keys', - 'scalar_scope.included_record_keys', - 'scalar_scope.scope_id', - 'support.values', - 'value.added_states', - 'value.compatibility', - 'value.extinct_states', - 'value.gini_simpson_diversity_delta', - 'value.gini_simpson_diversity_delta', - 'value.gini_simpson_diversity_delta', - 'value.gini_simpson_diversity_delta.reason_codes', - 'value.gini_simpson_diversity_delta.status', - 'value.harmonized_earlier', - 'value.harmonized_earlier', - 'value.harmonized_earlier', - 'value.harmonized_earlier', - 'value.harmonized_earlier', - 'value.harmonized_earlier', - 'value.harmonized_earlier', - 'value.harmonized_earlier', - 'value.harmonized_earlier', - 'value.harmonized_earlier.representation', - 'value.harmonized_earlier.scope', - 'value.harmonized_earlier.support', - 'value.harmonized_earlier.support_size', - 'value.harmonized_earlier.support_size.value', - 'value.harmonized_earlier.weighting', - 'value.harmonized_later', - 'value.harmonized_later', - 'value.harmonized_later', - 'value.harmonized_later', - 'value.harmonized_later', - 'value.harmonized_later', - 'value.harmonized_later', - 'value.harmonized_later', - 'value.harmonized_later', - 'value.harmonized_later.representation', - 'value.harmonized_later.scope', - 'value.harmonized_later.support', - 'value.harmonized_later.support_size', - 'value.harmonized_later.support_size.value', - 'value.harmonized_later.weighting', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.mapping_effect', - 'value.mapping_effect', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier.denominator_basis', - 'value.original_earlier.input_basis', - 'value.original_earlier.input_basis', - 'value.original_earlier.input_basis', - 'value.original_earlier.input_basis', - 'value.original_earlier.input_basis', - 'value.original_earlier.representation', - 'value.original_earlier.scope', - 'value.original_earlier.support', - 'value.original_earlier.support_size', - 'value.original_earlier.support_size.value', - 'value.original_earlier.weighting', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later.denominator_basis', - 'value.original_later.input_basis', - 'value.original_later.input_basis', - 'value.original_later.representation', - 'value.original_later.scope', - 'value.original_later.support', - 'value.original_later.support_size', - 'value.original_later.support_size.value', - 'value.original_later.weighting', - 'value.reason_codes', - 'value.reason_codes', - 'value.reason_codes', - 'value.retained_states', - 'value.retention_denominator', - 'value.retention_denominator', - 'value.retention_denominator', - 'value.retention_denominator', - 'value.retention_denominator', - 'value.retention_denominator_basis', - 'value.set_metadata', - 'value.set_metadata', - 'value.status', - 'value.status', - 'value.status', - 'value.support_added_count', - 'value.support_delta', - 'value.support_delta', - 'value.support_delta', - 'value.support_delta.metadata', - 'value.support_delta.metadata', - 'value.support_delta.metadata.assumptions', - 'value.support_delta.metadata.scope', - 'value.support_loss_count', - 'value.support_retention_ratio'], - '_coverage': ['value.denominator', - 'value.denominator', - 'value.denominator', - 'value.denominator', - 'value.denominator_name', - 'value.numerator', - 'value.numerator', - 'value.numerator', - 'value.ratio'], - '_definition': ['definition.path'], - '_diagnostics': ['ValidationSeverity.INFO', - 'ValidationSeverity.WARNING', - 'message.code', - 'message.code', - 'message.code', - 'message.code', - 'message.field', - 'message.field', - 'message.field', - 'message.field', - 'message.file_role', - 'message.file_role', - 'message.file_role', - 'message.file_role', - 'message.file_role', - 'message.file_role', - 'message.file_role.value', - 'message.file_role.value', - 'message.line_number', - 'message.message', - 'message.message', - 'message.message', - 'message.message', - 'message.record_key', - 'message.record_key', - 'message.record_key', - 'message.record_key', - 'message.row_number', - 'message.row_number', - 'message.severity', - 'message.severity', - 'message.severity', - 'message.severity', - 'message.severity.value', - "payload['errors'].append", - "payload['inputs']['limitations'].append", - "payload['warnings'].append"], - '_disclosure_recommend': ["payload['recommended_next_metadata'].append"], - '_disclosure_scope': ["payload['inputs'].get"], - '_disclosure_signal': ['dict.fromkeys', 'result.update'], - '_disclosure_unavailable': ["payload['unavailable_conclusions'].append", 'result.update'], - '_disclosures': ['basis.append', - 'basis.append', - 'content_cap.get', - 'derived.get', - 'derived.get', - 'derived.get', - "derived.get('closure_exposure', {}).get", - 'direct.get', - 'longitudinal_cap.get', - 'paths.append', - 'paths.append', - "payload['capabilities'].get", - "payload['capabilities'].get", - "payload['capabilities'].get", - "payload['observed_facts'].get", - 'provenance.get', - 'provenance.get', - 'provenance.get', - 'provenance.get', - 'reasons.extend', - 'support.get', - 'tail.get'], - '_distribution': ['CalculationStatus.AVAILABLE', - 'CalculationStatus.AVAILABLE', - 'CalculationStatus.AVAILABLE', - 'CalculationStatus.AVAILABLE', - "payload['derived_metrics'].setdefault", - "payload['derived_metrics'].setdefault", - "payload['derived_metrics'].setdefault('diversity', {}).setdefault", - "payload['derived_metrics'].setdefault('diversity', {}).setdefault('by_version', " - '{}).setdefault', - "payload['derived_metrics'].setdefault('support', {}).setdefault", - "payload['derived_metrics'].setdefault('support', {}).setdefault('by_version', " - '{}).setdefault', - "payload['observed_facts'].setdefault", - "payload['observed_facts'].setdefault", - "payload['observed_facts'].setdefault('state_counts', {}).setdefault", - "payload['observed_facts'].setdefault('supplied_state_probabilities', {}).setdefault", - 'row.state_count', - 'row.state_count', - 'row.state_count', - 'row.state_count', - 'row.state_count', - 'row.state_frequency', - 'row.state_frequency', - 'row.state_frequency', - 'row.state_frequency', - 'row.state_id', - 'row.state_id', - 'row.state_id', - 'row.state_id', - 'row.state_id', - 'row.state_id', - 'row.state_id', - 'row.state_mass', - 'row.state_mass', - 'row.state_mass', - 'row.state_mass', - 'row.state_mass', - 'scalar.reason_codes', - 'scalar.status', - 'value.analyzed_record_count', - 'value.analyzed_record_count', - 'value.count_metadata', - 'value.count_metadata', - 'value.count_metadata', - 'value.count_metadata', - 'value.denominator_basis', - 'value.denominator_basis', - 'value.denominator_basis', - 'value.denominator_basis', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_metadata', - 'value.frequency_metadata', - 'value.gini_simpson_diversity', - 'value.gini_simpson_diversity', - 'value.input_basis', - 'value.input_basis', - 'value.input_basis', - 'value.input_basis', - 'value.input_basis', - 'value.input_basis', - 'value.input_basis', - 'value.input_basis', - 'value.input_basis', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.mass_metadata', - 'value.mass_metadata', - 'value.mass_metadata', - 'value.mass_metadata', - 'value.numerical_policy', - 'value.probability_residual', - 'value.reason_codes', - 'value.reason_codes', - 'value.reason_codes', - 'value.reason_codes', - 'value.reason_codes', - 'value.representation', - 'value.representation', - 'value.representation', - 'value.representation', - 'value.representation', - 'value.representation', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope.dataset_versions', - 'value.scope.dataset_versions', - 'value.scope.denominator_basis', - 'value.scope.included_record_keys', - 'value.scope.included_record_keys', - 'value.simpson_concentration', - 'value.simpson_concentration', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.status', - 'value.status', - 'value.status', - 'value.status', - 'value.status', - 'value.status', - 'value.status', - 'value.status', - 'value.status', - 'value.status', - 'value.supplied_probability_total', - 'value.support', - 'value.support', - 'value.support', - 'value.support_size', - 'value.support_size', - 'value.support_size', - 'value.support_size.value', - 'value.weighting', - 'value.weighting', - 'value.weighting', - 'value.weighting', - 'value.weighting', - 'value.weighting', - 'value.weighting.weighting_mode'], - '_distribution_exclusions': ['exclusions.append', - 'item.exclusion_reason', - 'item.exclusion_reason', - 'item.exclusion_reason.value', - 'item.record_key', - 'item.record_key', - 'item.state_id', - "payload['observed_facts'].get", - "payload['observed_facts'].get(family, {}).get", - "payload['observed_facts'].get(family, {}).get('by_version', {}).get", - 'payload[section].get', - 'payload[section].get(family, {}).get', - "payload[section].get(family, {}).get('by_version', {}).get", - "payload[section].get(family, {}).get('by_version', {}).get(version, " - '{}).values', - 'scope.dataset_versions', - 'scope.excluded_record_keys', - 'scope.excluded_record_keys', - 'value.excluded_assignments', - 'value.excluded_assignments', - 'value.excluded_assignments', - 'value.excluded_assignments', - 'value.excluded_assignments', - 'value.unweighted', - 'value.unweighted.scope'], - '_duplicates': ['CalculationEvidenceClass.OBSERVED_FACT', - 'CalculationStatus.AVAILABLE', - 'CalculationStatus.UNAVAILABLE', - 'group.record_keys', - 'group.record_keys', - 'group.record_keys', - 'group.record_keys', - 'group.record_keys', - 'group.record_keys', - 'group.record_keys', - 'group.record_keys', - 'groups.append', - 'item.value', - "payload['observed_facts'].setdefault", - 'seen.intersection', - 'seen.update', - 'status.value', - 'value.coverage', - 'value.duplicate_group_count', - 'value.duplicate_group_count', - 'value.duplicate_group_count', - 'value.duplicate_group_count', - 'value.duplicate_group_count', - 'value.duplicate_group_count.reason_codes', - 'value.duplicate_group_count.status', - 'value.duplicate_group_count.value', - 'value.duplicate_group_count.value', - 'value.duplicate_record_count', - 'value.duplicate_record_count', - 'value.duplicate_record_count', - 'value.duplicate_record_count.value', - 'value.duplicate_record_count.value', - 'value.evidence_class', - 'value.exact_duplicate_groups', - 'value.limitations', - 'value.limitations', - 'value.representation', - 'value.representation', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope.included_record_keys'], - '_extinction': ['CalculationEvidenceClass.SIMULATION', - 'CalculationEvidenceClass.SIMULATION', - 'CalculationStatus.AVAILABLE', - 'first.input_basis', - 'first.limitations', - 'first.numerical_policy', - 'first.numerical_policy', - 'first.one_step_extinction_probability', - 'first.one_step_extinction_probability.metadata', - 'first.one_step_extinction_probability.metadata.assumptions', - 'first.representation', - 'first.representation', - 'first.resample_size', - 'first.resample_size', - 'first.resample_size', - 'first.resample_size', - 'first.scope', - 'first.scope', - 'scalar.metadata', - 'scalar.metadata', - 'scalar.reason_codes', - 'scalar.status', - 'scalar.status', - 'scalar.value', - 'scalar.value', - 'target.update', - "target['initial_distribution'].append", - "target['parameters']['state_order'].append", - 'value.evaluation_method', - 'value.evidence_class', - 'value.experimental', - 'value.input_basis', - 'value.method', - 'value.model_name', - 'value.numerical_policy', - 'value.numerical_underflow', - 'value.one_step_extinction_probability', - 'value.random_seed', - 'value.representation', - 'value.representation', - 'value.resample_size', - 'value.scope', - 'value.scope', - 'value.simulation_horizon', - 'value.state_frequency', - 'value.state_frequency', - 'value.state_id', - 'value.state_id', - 'value.state_id', - 'value.state_id'], - '_inputs': ['FileRole.EMBEDDING_DATA', - 'FileRole.EXTERNAL_REFERENCE', - 'ValidationSeverity.ERROR', - 'ValidationSeverity.FATAL', - 'affected.append', - 'artifact.fields', - 'artifact.file_format', - 'artifact.file_format.value', - 'artifact.path', - 'artifact.path', - 'artifact.path', - 'artifact.role', - 'artifact.role', - 'artifact.role', - 'artifact.role', - 'artifact.role', - 'artifact.role', - 'artifact.role.value', - 'artifact.row_count', - 'artifact.sha256', - 'artifact.sha256', - 'artifact.size_bytes', - 'bundle.generation', - 'bundle.generation', - 'bundle.generation.messages', - 'bundle.inventory', - 'bundle.mapping_traces', - 'bundle.mapping_traces', - 'bundle.observability', - 'bundle.observability.validation_messages', - 'bundle.provenance', - 'bundle.provenance', - 'bundle.provenance_join', - 'bundle.records', - 'bundle.records', - 'bundle.validation_messages', - 'bundle.validation_messages', - 'bundle.version_order', - 'bundle.version_order', - 'bundle.version_order', - 'bundle.version_order.loaded_versions', - 'bundle.version_order.order', - 'bundle.version_order.order_source', - 'coverage.denominator', - 'coverage.denominator', - 'dict.fromkeys', - 'hashes.add', - 'join.grounding_field_coverage', - 'join.messages', - 'join.provenance_required_field_coverage', - 'join.provenance_row_coverage', - 'join.provenance_row_coverage', - 'join.provenance_row_coverage.denominator_name', - 'join.scope_record_keys', - 'join.scope_record_keys', - 'join.scope_record_keys', - 'join.selected_dataset_versions', - 'message.code', - 'message.file_path', - 'message.file_role', - 'message.severity', - 'messages_by_location.get', - 'messages_by_location.get', - 'messages_by_location.get', - 'messages_by_location.setdefault', - 'messages_by_location.setdefault((message.file_role, message.file_path), []).append', - 'operations.append', - "payload['inputs']['artifacts'].append", - "payload['inputs']['file_hashes'].append", - 'records_by_version.get', - 'records_by_version.setdefault', - 'records_by_version.setdefault(row.record_key.dataset_version, []).append', - 'row.location', - 'row.location', - 'row.location.file_path', - 'row.location.file_role', - 'row.record_key', - 'row.record_key', - 'row.record_key', - 'row.record_key.dataset_version', - 'row.record_key.dataset_version', - 'trace.fields', - 'trace.mapping_sha256', - 'trace.unmapped_fields', - 'unmapped.update', - 'versions_by_location.get', - 'versions_by_location.setdefault', - 'versions_by_location.setdefault((row.location.file_role, row.location.file_path), set()).add'], - '_key': ['value.dataset_version', 'value.record_id'], - '_lineage_observations': ['CapabilityKey.LINEAGE', - 'bundle.observability', - 'bundle.observability.capabilities', - 'bundle.version_order', - 'bundle.version_order.order', - 'capability.coverage', - 'capability.requirements_met', - 'coverage.denominator', - 'coverage.denominator', - 'coverage.denominator', - 'coverage.denominator', - 'coverage.numerator', - 'coverage.numerator'], - '_message_family': ['message.code', - 'message.field', - 'message.field', - 'message.field', - 'message.field', - 'message.file_role', - 'message.file_role', - 'message.file_role.value'], - '_metadata': ['value.assumptions', - 'value.assumptions', - 'value.evidence_class', - 'value.evidence_class', - 'value.formula_id', - 'value.formula_id', - 'value.limitations', - 'value.limitations', - 'value.method', - 'value.metric_name', - 'value.metric_name', - 'value.owner_id', - 'value.owner_id', - 'value.representation', - 'value.representation', - 'value.scope', - 'value.scope', - 'value.unit', - 'value.unit', - 'value.weighting', - 'value.weighting', - 'value.weighting', - 'value.weighting', - 'value.weighting', - 'value.weighting.weight_field', - 'value.weighting.weighting_mode'], - '_metadata_text': ['CalculationEvidenceClass.SIMULATION', - 'CalculationEvidenceClass.SIMULATION', - 'CalculationEvidenceClass.SIMULATION', - 'methods.get', - 'name.startswith', - 'value.assumptions', - 'value.assumptions', - 'value.assumptions', - 'value.assumptions', - 'value.evidence_class', - 'value.evidence_class', - 'value.evidence_class', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.method', - 'value.metric_name', - 'value.owner_id', - 'value.owner_id', - 'value.owner_id', - 'value.owner_id', - 'value.owner_id', - 'value.owner_id'], - '_policy': ['value.absolute_tolerance', - 'value.absolute_tolerance', - 'value.probability_mass_tolerance', - 'value.probability_mass_tolerance', - 'value.relative_tolerance', - 'value.relative_tolerance'], - '_provenance': ['CalculationReason.EMPTY_SCOPE', - 'CalculationReason.EMPTY_SCOPE', - 'CalculationStatus.AVAILABLE', - 'CalculationStatus.AVAILABLE', - 'CalculationStatus.UNAVAILABLE', - 'CalculationStatus.UNAVAILABLE', - 'bundle.provenance_join', - 'coverage.denominator', - 'coverage.ratio', - 'coverage.ratio', - 'coverage.ratio', - 'coverage.ratio', - 'grounding.assignments', - 'grounding.assignments', - 'grounding.assignments', - 'grounding.known_closed_count', - 'grounding.known_open_count', - 'grounding.limitations', - 'grounding.scope', - 'grounding.unresolved_grounding_count', - 'item.counts', - 'item.counts', - 'item.counts', - 'item.counts', - 'item.counts_metadata', - 'item.field_coverage', - 'item.field_coverage', - 'item.field_coverage', - 'item.field_coverage', - 'item.field_coverage', - 'item.field_coverage', - 'item.field_coverage', - 'item.field_coverage', - 'item.field_coverage.denominator', - 'item.field_coverage.ratio', - 'item.field_coverage.ratio', - 'item.field_coverage.ratio', - 'item.field_coverage.ratio', - 'item.field_coverage.ratio', - 'item.field_coverage.ratio', - 'item.field_coverage_metadata', - 'item.field_name', - 'item.reason_codes', - 'item.reason_codes', - 'item.record_key', - 'item.shares', - 'item.shares', - 'item.shares_metadata', - 'item.status', - 'item.status', - 'join.grounding_field_coverage', - 'join.provenance_required_field_coverage', - 'join.provenance_row_coverage', - 'match.provenance', - "payload['derived_metrics'].setdefault", - "payload['observed_facts'].setdefault", - 'value.analyzed_record_count', - 'value.analyzed_record_count', - 'value.analyzed_record_count.value', - 'value.confidence', - 'value.coverage_metadata', - 'value.coverage_metadata', - 'value.direct_grounding', - 'value.grounding_field_coverage', - 'value.grounding_field_coverage', - 'value.grounding_field_coverage.ratio', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.missing_provenance_count', - 'value.missing_provenance_count', - 'value.missing_provenance_count.value', - 'value.missing_provenance_share', - 'value.provenance_required_field_coverage', - 'value.provenance_row_coverage', - 'value.records_with_matching_rows', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope.included_record_keys', - 'value.scope.included_record_keys', - 'value.source', - 'value.weighted_source', - 'value.weighted_source', - 'weighted.mass_metadata', - 'weighted.missing_provenance_weight', - 'weighted.reason_codes', - 'weighted.shares_metadata', - 'weighted.status', - 'weighted.total_weight', - 'weighted.total_weight', - 'weighted.total_weight.value', - 'weighted.weighted_missing_provenance_share', - 'weighted.weighted_source_type_masses', - 'weighted.weighted_source_type_shares'], - '_provenance_consistency': ['CalculationStatus.AVAILABLE', - 'CalculationStatus.UNAVAILABLE', - 'ValidationSeverity.ERROR', - 'ValidationSeverity.FATAL', - 'assignment.basis', - 'assignment.classification', - 'assignment.missing_required_fields', - 'bundle.provenance_join', - 'bundle.provenance_join', - 'bundle.provenance_join', - 'bundle.provenance_join.matches', - 'bundle.provenance_join.messages', - 'bundle.provenance_join.provenance_supplied', - 'bundle.records', - 'composition.counts', - 'composition.counts', - 'composition.counts', - 'composition.field_coverage', - 'composition.field_coverage', - 'composition.field_coverage', - 'composition.field_coverage.denominator', - 'composition.field_coverage.denominator_name', - 'composition.field_coverage.numerator', - 'composition.shares', - 'composition.status', - 'composition.status', - 'composition.unavailable_record_keys', - 'coverage.denominator', - 'coverage.denominator_name', - 'coverage.numerator', - 'expected.values', - 'grounding.assignments', - 'grounding.assignments', - 'grounding.assignments', - 'grounding.assignments', - 'grounding.assignments', - 'grounding.input_has_errors', - 'grounding.known_closed_count', - 'grounding.known_open_count', - 'grounding.unresolved_grounding_count', - 'grounding.validation_messages', - 'item.classification', - 'item.record_key', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance.grounding_known', - 'match.provenance.required_fields_valid', - 'match.provenance.values', - 'match.provenance.values', - 'match.provenance.values.get', - 'match.provenance.values.get', - 'match.record_key', - 'match.record_key', - 'match.record_key', - 'match.record_key', - 'message.record_key', - 'message.record_key', - 'message.severity', - 'row.missing_required_fields', - 'row.record_key', - 'row.record_key', - 'row.record_key.dataset_version', - 'row.required_fields_valid', - 'row.values', - 'row.values', - 'scalar.value', - 'value.analyzed_record_count', - 'value.analyzed_record_count.value', - 'value.confidence', - 'value.direct_grounding', - 'value.grounding_field_coverage', - 'value.input_has_errors', - 'value.input_has_errors', - 'value.missing_provenance_count', - 'value.missing_provenance_count.value', - 'value.provenance_required_field_coverage', - 'value.provenance_row_coverage', - 'value.provenance_supplied', - 'value.records_with_matching_rows', - 'value.records_with_matching_rows.value', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope.dataset_versions', - 'value.scope.dataset_versions', - 'value.scope.denominator_basis', - 'value.scope.denominator_basis', - 'value.scope.excluded_record_keys', - 'value.scope.included_record_keys', - 'value.source', - 'value.validation_messages', - 'value.validation_messages'], - '_representation': ['value.binning_or_mapping_rule', - 'value.binning_or_mapping_rule', - 'value.field_name', - 'value.field_name', - 'value.missing_state_id', - 'value.missing_state_id', - 'value.missing_value_policy', - 'value.missing_value_policy', - 'value.normalization_profile', - 'value.normalization_profile', - 'value.representation_name', - 'value.representation_name', - 'value.representation_source', - 'value.representation_source', - 'value.representation_version', - 'value.representation_version'], - '_scalar': ['CalculationStatus.UNAVAILABLE', - 'field.evidence_class', - 'field.method_id', - 'field.method_id', - 'field.method_id.startswith', - 'field.owner', - 'field.unit', - 'path.rsplit', - 'reason.value', - 'value.metadata', - 'value.metadata', - 'value.metadata', - 'value.metadata', - 'value.metadata', - 'value.metadata', - 'value.metadata', - 'value.metadata.assumptions', - 'value.metadata.limitations', - 'value.metadata.method', - 'value.metadata.weighting', - 'value.metadata.weighting', - 'value.metadata.weighting.weight_field', - 'value.metadata.weighting.weighting_mode', - 'value.reason_codes', - 'value.reason_codes', - 'value.status', - 'value.status', - 'value.status', - 'value.status.value', - 'value.value', - 'value.value'], - '_scope': ['bundle.records', - 'bundle.version_order', - 'bundle.version_order.loaded_versions', - 'row.record_key', - 'value.dataset_versions', - 'value.dataset_versions', - 'value.dataset_versions', - 'value.denominator_basis', - 'value.denominator_basis', - 'value.excluded_record_keys', - 'value.excluded_record_keys', - 'value.excluded_record_keys', - 'value.excluded_record_keys', - 'value.included_record_keys', - 'value.included_record_keys', - 'value.included_record_keys', - 'value.included_record_keys', - 'value.scope_id', - 'value.scope_id'], - '_simulation_inputs': ['value.correction_applied', - 'value.correction_method', - 'value.effective_distribution', - 'value.effective_distribution', - 'value.effective_probability_total', - 'value.normalization_divisor', - 'value.numerical_policy', - 'value.probability_corrections', - 'value.probability_residual', - 'value.representation', - 'value.scope', - 'value.supplied_distribution', - 'value.supplied_distribution', - 'value.supplied_probability_total'], - '_table': ['CalculationStatus.UNAVAILABLE', - 'CalculationStatus.UNAVAILABLE', - 'field.evidence_class', - 'field.method_id', - 'field.method_id', - 'field.method_id.startswith', - 'field.owner', - 'field.unit', - 'metadata.assumptions', - 'metadata.limitations', - 'metadata.method', - 'metadata.weighting', - 'metadata.weighting', - 'metadata.weighting.weight_field', - 'metadata.weighting.weighting_mode', - 'reason.value', - 'status.value'], - '_tail': ['CalculationStatus.AVAILABLE', - 'item.in_tail', - 'item.in_tail', - 'item.rarity_rank', - 'item.state_count', - 'item.state_frequency', - 'item.state_id', - 'item.state_id', - 'ranking.append', - 'scalar.reason_codes', - 'scalar.status', - 'value.denominator', - 'value.denominator', - 'value.limitations', - 'value.limitations', - 'value.membership_metadata', - 'value.options', - 'value.options', - 'value.options', - 'value.options', - 'value.options', - 'value.options.count_threshold', - 'value.options.frequency_threshold', - 'value.options.rule', - 'value.options.rule', - 'value.options.state_ids', - 'value.ranking_metadata', - 'value.ranking_rule', - 'value.rarity_ranking', - 'value.rarity_ranking', - 'value.reason_codes', - 'value.reason_codes', - 'value.representation', - 'value.representation', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.status', - 'value.status', - 'value.status', - 'value.tail_membership', - 'value.tail_membership', - 'value.tail_record_share', - 'value.tail_support_size'], - 'assemble_report': ['CanonicalReport.from_dict', - 'bundle.inventory', - 'bundle.observability', - 'bundle.observability.capabilities', - 'bundle.records', - 'bundle.records', - 'bundle.records', - 'bundle.records', - 'bundle.records', - 'bundle.records', - 'bundle.records', - 'bundle.version_order', - 'bundle.version_order.loaded_versions', - 'closure.scope', - 'closure.scope.scope_id', - 'closure.validation_messages', - 'duplicates.representation', - 'duplicates.scope', - 'duplicates.scope.scope_id', - 'empirical_representations.append', - 'empirical_representations.append', - 'empirical_representations.append', - 'empirical_representations.append', - 'failure.capability', - 'failure.capability', - 'failure.capability', - 'failure.capability.value', - 'failure.capability.value', - 'failure.messages', - 'failure.messages', - 'failure.messages', - 'failures.setdefault', - 'failures.setdefault(failure.capability.value, []).extend', - 'key.value', - "operations['content_diagnostics'].append", - "operations['content_diagnostics'].append", - "operations['content_diagnostics'].append", - "operations['dataset_longitudinal'].append", - "operations['intervention_simulation'].append", - "operations['intervention_simulation'].append", - "operations['intervention_simulation'].append", - "operations['provenance'].append", - "operations['provenance'].append", - "payload['inputs']['limitations'].append", - 'provenance.scope', - 'provenance.scope.scope_id', - 'provenance.validation_messages', - 'row.record_key', - 'row.record_key', - 'row.record_key.dataset_version', - 'run.get', - 'run.get', - 'supplied.coverage', - 'supplied.coverage', - 'supplied.coverage', - 'supplied.coverage', - 'supplied.coverage', - 'supplied.coverage.denominator', - 'supplied.coverage.numerator', - 'supplied.coverage.ratio', - 'supplied.coverage.ratio', - 'supplied.input_basis', - 'supplied.representation', - 'supplied.scope', - 'supplied.scope.scope_id', - 'supplied.selection_messages', - 'supplied.unweighted', - 'supplied.unweighted', - 'supplied.unweighted', - 'supplied.unweighted', - 'supplied.unweighted', - 'supplied.unweighted', - 'supplied.unweighted', - 'supplied.unweighted', - 'supplied.unweighted.representation', - 'supplied.unweighted.representation', - 'supplied.unweighted.scope', - 'supplied.unweighted.scope', - 'supplied.unweighted.scope', - 'supplied.unweighted.scope', - 'supplied.unweighted.scope', - 'supplied.unweighted.scope.excluded_record_keys', - 'supplied.unweighted.scope.included_record_keys', - 'supplied.unweighted.scope.included_record_keys', - 'supplied.unweighted.scope.scope_id', - 'supplied.weighted', - 'supplied.weighted', - 'supplied.weighted', - 'supplied.weighted', - 'supplied.weighted.representation', - 'supplied.weighted.scope', - 'tail.representation', - 'tail.scope', - 'tail.scope.scope_id']} - - -def phase4_step3_assembly_boundary(path: Path) -> None: - """Inspect only the supplied assembly source without executing it. - - Fixed reviewed inventories constrain definitions, imports, call targets and - validation arithmetic. They are never regenerated from inspected source. - The release controller separately freezes result and the other runtime modules. - """ - try: - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - except (OSError, UnicodeError, SyntaxError) as exc: - raise ValueError("Phase 4 Step 3 assembly source cannot be inspected") from exc - doc = ast.get_docstring(tree) or "" - if "Owner IDs:" not in doc or "Current phase status:" not in doc: - raise ValueError("Phase 4 Step 3 assembly ownership metadata is missing") - allowed_top = (ast.Import, ast.ImportFrom, ast.Assign, ast.AnnAssign, - ast.FunctionDef, ast.ClassDef) - for index, node in enumerate(tree.body): - if index == 0 and isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str): - continue - if not isinstance(node, allowed_top): - raise ValueError("Phase 4 Step 3 assembly has an unapproved module statement") - imports = [] - functions = [] - classes = [] - calls = {} - arithmetic = {} - mutations = [] - declarations = [] - traversal = [] - attributes = {} - headers = [] - literal_declarations = [] - literal_containers = [] - call_names = {node.func.id for node in ast.walk(tree) - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)} - forbidden = (ast.Lambda, ast.AsyncFunctionDef, ast.Await, ast.With, - ast.AsyncWith, ast.Global, ast.Nonlocal, ast.NamedExpr) - forbidden_names = {"eval", "exec", "compile", "__import__", "getattr", - "setattr", "delattr", "globals", "locals", "vars", - "open", "input", "breakpoint", "print", "exit", "quit", - "asdict", "astuple"} - forbidden_attributes = {"__builtins__", "__globals__", "__code__", "__subclasses__", - "__class__", "__dict__", "__getattribute__", "__getattr__", "__setattr__", - "__reduce__", "__reduce_ex__", "__mro__", "__bases__", - "open", "read", "read_text", "read_bytes", "write", "write_text", - "write_bytes", "mkdir", "unlink", "system", "popen", "connect", - "connect_ex", "getaddrinfo", "urlopen", "urlretrieve", "send", "sendall"} - stack = [(tree, "")] - while stack: - node, scope = stack.pop() - if isinstance(node, forbidden): - raise ValueError("Phase 4 Step 3 assembly has a callback or executable control outside scope") - if isinstance(node, (ast.FunctionDef, ast.ClassDef)): - scope = node.name if scope == "" else scope + "." + node.name - if isinstance(node, ast.FunctionDef): - functions.append(scope) - headers.append((scope, ast.dump(node.args, include_attributes=False), - tuple(ast.unparse(item) for item in node.decorator_list), - ast.unparse(node.returns) if node.returns is not None else "")) - else: - classes.append(scope) - headers.append((scope, tuple(ast.unparse(item) for item in node.bases), - tuple(ast.unparse(item) for item in node.keywords), - tuple(ast.unparse(item) for item in node.decorator_list))) - if isinstance(node, (ast.Import, ast.ImportFrom)): - if scope != "" or node not in tree.body: - raise ValueError("Phase 4 Step 3 assembly imports must be explicit and at module scope") - imports.append(ast.dump(node, include_attributes=False)) - if isinstance(node, ast.Name) and node.id in forbidden_names: - raise ValueError("Phase 4 Step 3 assembly names a forbidden execution capability") - if scope != "" and ((isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store) and node.id in call_names) - or (isinstance(node, ast.arg) and node.arg in call_names)): - raise ValueError("Phase 4 Step 3 assembly shadows an approved call target") - if isinstance(node, ast.Attribute): - attributes.setdefault(scope, []).append(ast.unparse(node)) - if isinstance(node, ast.Attribute) and node.attr in forbidden_attributes: - raise ValueError("Phase 4 Step 3 assembly names a forbidden IO or dynamic capability") - if isinstance(node, ast.Call): - if not isinstance(node.func, (ast.Name, ast.Attribute)): - raise ValueError("Phase 4 Step 3 assembly invokes a dynamic call target") - calls.setdefault(scope, []).append(ast.unparse(node.func)) - if isinstance(node.func, ast.Attribute) and node.func.attr == "__setattr__": - mutations.append((scope, ast.unparse(node))) - if isinstance(node, (ast.BinOp, ast.UnaryOp, ast.AugAssign)): - arithmetic.setdefault(scope, []).append(ast.unparse(node)) - if (scope == "" or scope in classes) and isinstance(node, (ast.Assign, ast.AnnAssign, ast.Delete)): - declarations.append((scope, ast.unparse(node))) - if scope == "_metadata_text" and isinstance(node, (ast.Assign, ast.AnnAssign)): - literal_declarations.append((scope, ast.unparse(node))) - if isinstance(node, (ast.Dict, ast.Tuple, ast.List, ast.Set)): - try: - literal = ast.literal_eval(node) - except (ValueError, TypeError, SyntaxError): - literal = None - if literal: - literal_containers.append((scope, ast.unparse(node))) - if isinstance(node, (ast.Yield, ast.YieldFrom, ast.Delete)): - traversal.append((scope, ast.unparse(node))) - if isinstance(node, ast.Attribute) and isinstance(node.ctx, (ast.Store, ast.Del)): - raise ValueError("Phase 4 Step 3 assembly has an unapproved attribute mutation") - stack.extend((child, scope) for child in ast.iter_child_nodes(node)) - if sorted(imports) != PHASE4_STEP3_ASSEMBLY_IMPORTS: - raise ValueError("Phase 4 Step 3 assembly imports differ from the reviewed import-safe typed-result inventory") - if sorted(functions) != PHASE4_STEP3_ASSEMBLY_FUNCTIONS or sorted(classes) != PHASE4_STEP3_ASSEMBLY_CLASSES: - raise ValueError("Phase 4 Step 3 assembly definitions differ from the approved contract") - if {name: sorted(values) for name, values in calls.items()} != PHASE4_STEP3_ASSEMBLY_CALLS: - raise ValueError("Phase 4 Step 3 assembly call targets differ from the reviewed inventory") - if {name: sorted(values) for name, values in arithmetic.items()} != PHASE4_STEP3_ASSEMBLY_ARITHMETIC: - raise ValueError("Phase 4 Step 3 assembly arithmetic exceeds reviewed semantic validation") - if sorted(mutations) != PHASE4_STEP3_ASSEMBLY_MUTATIONS: - raise ValueError("Phase 4 Step 3 assembly mutation differs from reviewed immutable construction") - if sorted(declarations) != PHASE4_STEP3_ASSEMBLY_DECLARATIONS: - raise ValueError("Phase 4 Step 3 assembly module declarations differ from the approved contract") - if sorted(traversal) != PHASE4_STEP3_ASSEMBLY_TRAVERSAL: - raise ValueError("Phase 4 Step 3 assembly generator or teardown differs from reviewed local traversal") - if {name: sorted(values) for name, values in attributes.items()} != PHASE4_STEP3_ASSEMBLY_ATTRIBUTES: - raise ValueError("Phase 4 Step 3 assembly attributes exceed the reviewed pure contract") - if sorted(literal_declarations) != PHASE4_STEP3_ASSEMBLY_LITERAL_DECLARATIONS: - raise ValueError("Phase 4 Step 3 assembly method registry differs from reviewed owner declarations") - if sorted(literal_containers) != PHASE4_STEP3_ASSEMBLY_LITERAL_CONTAINERS: - raise ValueError("Phase 4 Step 3 assembly literal evidence boundaries differ from reviewed declarations") - if sorted(headers) != PHASE4_STEP3_ASSEMBLY_HEADERS: - raise ValueError("Phase 4 Step 3 assembly definition headers differ from the reviewed contract") - - - -def phase4_step3_main(step: int = 3) -> int: - """Validate Step 3 control, the frozen canonical contract and pure assembly.""" - import hashlib - import runpy - - if type(step) is not int or step != 3: - raise ValueError("Only authorized Phase 4 Step 3 traceability is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step3_traceability_control") - control["audit_phase4_step3"](step=step) - schema_path = ROOT / "schemas/report.schema.json" - if hashlib.sha256(schema_path.read_bytes()).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Phase 4 Step 3 report schema differs from the frozen Step 2 contract") - phase4_step2_result_boundary(PACKAGE / "result.py") - phase4_step3_assembly_boundary(PACKAGE / "reports/assembly.py") - print("Phase 4 Step 3: frozen canonical results and reviewed evidence assembly: PASS") - return 0 - - -# Step 4 adds privacy after canonical assembly. Historical stage checkers above -# retain their original source. Fixed identities below are review data; none is -# derived from the current source when the boundary runs. -PHASE4_STEP4_INHERITED_AST_SHA256 = { - "src/recursive_integrity_toolkit/result.py": "4cbf8f28cb0aa04edc0a931c8a27b64c18ee1975b7e93a51edf62d34f6615ca9", - "src/recursive_integrity_toolkit/reports/assembly.py": "23b0574b56baa9b147a221742c53a37d995352d52d51f528bda0316eef30fd72", - "src/recursive_integrity_toolkit/utils/hashing.py": "d263ee776c8c2c9751106b959589cb7c49f5bbc3be0cbf7634717c88dccb8b98", - "src/recursive_integrity_toolkit/utils/logging.py": "35ad89e78bf13acf4ca8adea239b5c79db8001f5cbabbf2cfc7e27680e9c3e57", - "src/recursive_integrity_toolkit/config.py": "a85a8bab70aa2e2625be4943d1769ac33301d8a6725396925c7dbf44ac28f3b2", -} -PHASE4_STEP4_RUNTIME_AST_SHA256 = { - "src/recursive_integrity_toolkit/config.py": "92fb35fe0895f799c0b90c20d80f0262c92da8afb2b96e17154a870837ed71d5", - "src/recursive_integrity_toolkit/reports/assembly.py": "e96f66d5fcaa45e2eea53f19e78db345d703b4d565b4d635944682c728eb2329", - "src/recursive_integrity_toolkit/result.py": "a5f9f85b2e10128202feb1bc122f912315b0b46ccf1e2c67e0c7127229160a12", - "src/recursive_integrity_toolkit/utils/hashing.py": "6774dbc0177e876731da2d163f85acd583430dee1747a092528e0edaa869c0da", - "src/recursive_integrity_toolkit/utils/logging.py": "973a746221ea3d83fd1ab42a5687075bed6ad962a0c727cfa6350e94f71c94d3" -} -PHASE4_STEP4_ADDITIVE_BINDINGS = { - "src/recursive_integrity_toolkit/result.py": [ - "PrivacyMode", - "RecordIdMode", - "SafeReportView" - ], - "src/recursive_integrity_toolkit/reports/assembly.py": [ - "_PRIVACY_SAFE_TEXT", - "_privacy_contract", - "_privacy_alias", - "_privacy_text", - "_privacy_value", - "_privacy_run_fields", - "privacy_view", - "build_run_metadata" - ], - "src/recursive_integrity_toolkit/utils/hashing.py": [ - "canonical_json_bytes", - "sha256_canonical", - "_phase4_secret_file", - "IdentifierProtection" - ], - "src/recursive_integrity_toolkit/utils/logging.py": [ - "_TEMPLATES", - "_FIELDS", - "_EXCEPTION_TYPES", - "_severity", - "_code", - "_protection", - "_policy", - "safe_code", - "safe_diagnostic_text", - "safe_remediation", - "safe_field", - "safe_role", - "safe_effects", - "safe_record_key", - "_position", - "safe_diagnostic", - "format_diagnostic", - "emit_diagnostic" - ], - "src/recursive_integrity_toolkit/config.py": [ - "Phase4Options", - "_phase4_config_data", - "_phase4_option_path", - "_phase4_literal", - "resolve_phase4_options", - "phase4_config_summary", - "phase4_config_hash" - ] -} - - -def _phase4_step4_ast_digest(nodes: list) -> str: - """Hash a location-free AST consistently on supported Python 3.11/3.12.""" - import hashlib - import json - - def canonical(value): - if isinstance(value, ast.AST): - return [type(value).__name__, [[name, canonical(item)] - for name, item in ast.iter_fields(value) - if name != "type_params" or item]] - if isinstance(value, list): - return [canonical(item) for item in value] - return [type(value).__name__, repr(value)] - - # Python 3.12 adds empty type_params fields. Ignoring only empty ones makes - # the same non-generic source portable; nonempty type parameters remain data. - module = ast.Module(body=nodes, type_ignores=[]) - payload = json.dumps(canonical(module), ensure_ascii=True, separators=(",", ":")) - return hashlib.sha256(payload.encode("utf-8")).hexdigest() - - -def _phase4_step4_bound_names(node) -> set[str]: - """Return only explicit top-level definition or assignment targets.""" - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - return {node.name} - if isinstance(node, ast.Assign): - targets = node.targets - elif isinstance(node, ast.AnnAssign): - targets = [node.target] - else: - return set() - return {item.id for target in targets for item in ast.walk(target) - if isinstance(item, ast.Name) and isinstance(item.ctx, ast.Store)} - - -def phase4_step4_runtime_boundary(path: Path, relative: str) -> None: - """Check one reviewed Step 4 runtime module without importing its code. - - The complete reviewed AST constrains calls, imports, declarations, arithmetic, - sink use and control flow. A separate accepted-Step-3 identity freezes all - inherited non-import declarations after the explicitly named additions are - removed. Only the module ownership documentation is outside the AST digest. - """ - if relative not in PHASE4_STEP4_INHERITED_AST_SHA256: - raise ValueError("Phase 4 Step 4 runtime path is outside the approved five modules") - try: - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - except (OSError, UnicodeError, SyntaxError) as exc: - raise ValueError("Phase 4 Step 4 runtime source cannot be inspected") from exc - doc = ast.get_docstring(tree) or "" - if "Owner IDs:" not in doc or "Current phase status:" not in doc: - raise ValueError("Phase 4 Step 4 runtime ownership metadata is missing") - body = tree.body[1:] if ast.get_docstring(tree) is not None else tree.body - if _phase4_step4_ast_digest(body) != PHASE4_STEP4_RUNTIME_AST_SHA256.get(relative): - raise ValueError("Phase 4 Step 4 runtime differs from the reviewed privacy implementation") - additions = set(PHASE4_STEP4_ADDITIVE_BINDINGS.get(relative, ())) - inherited = [] - observed_additions = set() - for node in body: - if isinstance(node, (ast.Import, ast.ImportFrom)): - continue - names = _phase4_step4_bound_names(node) - if names & additions: - if not names <= additions or names & observed_additions: - raise ValueError("Phase 4 Step 4 additions rebind an inherited or duplicate name") - observed_additions.update(names) - else: - inherited.append(node) - if observed_additions != additions: - raise ValueError("Phase 4 Step 4 reviewed additive declaration is missing") - if _phase4_step4_ast_digest(inherited) != PHASE4_STEP4_INHERITED_AST_SHA256[relative]: - raise ValueError("Phase 4 Step 4 changed an inherited runtime helper or contract") - - -def phase4_step4_privacy_boundary(root: Path) -> None: - """Enforce the five explicit privacy modules and frozen public schema.""" - import hashlib - - for relative in sorted(PHASE4_STEP4_INHERITED_AST_SHA256): - phase4_step4_runtime_boundary(root / relative, relative) - schema = root / "schemas/report.schema.json" - if hashlib.sha256(schema.read_bytes()).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Phase 4 Step 4 changed the frozen public report schema") - - -def phase4_step4_main(step: int = 4) -> int: - """Validate Step 4 control, additive privacy and inherited helper identities.""" - import runpy - - if type(step) is not int or step != 4: - raise ValueError("Only authorized Phase 4 Step 4 traceability is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step4_traceability_control") - control["audit_phase4_step4"](step=step) - phase4_step4_privacy_boundary(ROOT) - print("Phase 4 Step 4: inherited contracts and reviewed privacy, metadata and diagnostics: PASS") - return 0 - - -# Step 5 opens only two rendering placeholders. These review identities are -# fixed checker data, independent of the source being inspected at gate time. -PHASE4_STEP5_RUNTIME_AST_SHA256 = { - "src/recursive_integrity_toolkit/reports/json_report.py": "04c46cdd644021b01b285dd6e62a2b026be27ac60607cc6117582d9581c0a283", - "src/recursive_integrity_toolkit/reports/markdown_report.py": "08ece6fc0ba3ea57dab335508978a974439c7f37aa84e4dcd652615fa6628e84", -} -PHASE4_STEP5_RUNTIME_OWNERS = { - "src/recursive_integrity_toolkit/reports/json_report.py": ["PR-013", "PR-016"], - "src/recursive_integrity_toolkit/reports/markdown_report.py": ["PR-013", "PR-018"], -} - - -def phase4_step5_runtime_boundary(path: Path, relative: str) -> None: - """Inspect the complete reviewed renderer AST without importing its code. - - A fixed identity covers imports, function signatures, literal declarations, - calls, effects and control flow. The two accepted placeholders contained no - inherited functions or assignments. Every other runtime module is frozen by - the separately verified accepted Step 4 file inventory. - """ - if relative not in PHASE4_STEP5_RUNTIME_AST_SHA256: - raise ValueError("Phase 4 Step 5 runtime path is outside the two approved renderers") - try: - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - except (OSError, UnicodeError, SyntaxError) as exc: - raise ValueError("Phase 4 Step 5 runtime source cannot be inspected") from exc - doc = ast.get_docstring(tree) or "" - if ("Owner IDs:" not in doc or "Current phase status:" not in doc - or not all(owner in doc for owner in PHASE4_STEP5_RUNTIME_OWNERS[relative])): - raise ValueError("Phase 4 Step 5 renderer ownership metadata is missing") - body = tree.body[1:] if ast.get_docstring(tree) is not None else tree.body - if _phase4_step4_ast_digest(body) != PHASE4_STEP5_RUNTIME_AST_SHA256[relative]: - raise ValueError("Phase 4 Step 5 runtime differs from the reviewed rendering implementation") - - -def phase4_step5_renderers_boundary(root: Path) -> None: - """Enforce both pure renderer implementations and the frozen public schema.""" - import hashlib - - for relative in sorted(PHASE4_STEP5_RUNTIME_AST_SHA256): - phase4_step5_runtime_boundary(root / relative, relative) - if hashlib.sha256((root / "schemas/report.schema.json").read_bytes()).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Phase 4 Step 5 changed the frozen public report schema") - - -def phase4_step5_main(step: int = 5) -> int: - """Validate explicit Step 5 authorization and reviewed renderer boundaries.""" - import runpy - - if type(step) is not int or step != 5: - raise ValueError("Only authorized Phase 4 Step 5 traceability is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step5_traceability_control") - control["audit_phase4_step5"](step=step) - phase4_step5_renderers_boundary(ROOT) - print("Phase 4 Step 5: reviewed JSON and Markdown rendering and frozen prior contracts: PASS") - return 0 - - -PHASE4_STEP6_RUNTIME_AST_SHA256 = {'src/recursive_integrity_toolkit/utils/logging.py': '1ce0d0c77339f4c63e9f091fa218cfcb0202c709b235643f550a8c3d4c0160b5', - 'src/recursive_integrity_toolkit/utils/paths.py': 'f9e77ed3638b520f81769ae2c09095572913b501383d53124d51cde2826a5efa'} - -PHASE4_STEP6_INHERITED_AST_SHA256 = {'src/recursive_integrity_toolkit/utils/logging.py': '973a746221ea3d83fd1ab42a5687075bed6ad962a0c727cfa6350e94f71c94d3', - 'src/recursive_integrity_toolkit/utils/paths.py': '0010c7bbf86e389ac2599eae023a89caaabe48b10a65cfaeddb01aa619112824'} - -PHASE4_STEP6_INHERITED_NODES = {'src/recursive_integrity_toolkit/utils/logging.py': 24, - 'src/recursive_integrity_toolkit/utils/paths.py': 17} - -def phase4_step6_runtime_boundary(path: Path, relative: str) -> None: - """Enforce reviewed output code and unchanged inherited input/diagnostic AST.""" - if relative not in PHASE4_STEP6_RUNTIME_AST_SHA256: - raise ValueError("Step 6 runtime is limited to output paths and diagnostics") - try: - tree = ast.parse(path.read_text(encoding="utf-8")) - except (OSError, UnicodeError, SyntaxError) as error: - raise ValueError("Step 6 output helper cannot be inspected") from error - doc = ast.get_docstring(tree) or "" - if "Owner IDs:" not in doc or "Current phase status:" not in doc or "PR-015" not in doc: - raise ValueError("Step 6 output ownership metadata is missing") - body = tree.body[1:] if ast.get_docstring(tree) is not None else tree.body - if _phase4_step4_ast_digest(body) != PHASE4_STEP6_RUNTIME_AST_SHA256[relative]: - raise ValueError("Step 6 output implementation differs from the reviewed AST") - if _phase4_step4_ast_digest(body[:PHASE4_STEP6_INHERITED_NODES[relative]]) != PHASE4_STEP6_INHERITED_AST_SHA256[relative]: - raise ValueError("Step 6 changed an inherited input or diagnostic helper") - - -def phase4_step6_output_boundary(root: Path) -> None: - for relative in sorted(PHASE4_STEP6_RUNTIME_AST_SHA256): - phase4_step6_runtime_boundary(root / relative, relative) - phase4_step5_renderers_boundary(root) - - -def phase4_step6_main(step: int = 6) -> int: - import runpy - if type(step) is not int or step != 6: - raise ValueError("Only authorized Phase 4 Step 6 traceability is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), run_name="phase4_step6_traceability_control") - control["audit_phase4_step6"](step=step) - phase4_step6_output_boundary(ROOT) - print("Phase 4 Step 6: safe publication and preserved input/report contracts: PASS") - return 0 - - -PHASE4_STEP7_RUNTIME_AST_SHA256 = {'src/recursive_integrity_toolkit/cli.py': 'c1791ddf073a9ca5ddf08268b7a66e1a67f359a1b8d97a636d37e914dbb0f40b', - 'src/recursive_integrity_toolkit/config.py': 'cbb96ab25130e4fe45a94cc03299aaa27015a496e38e7dcc6bec1c5d2f7e2573'} - -PHASE4_STEP7_CONFIG_INHERITED_AST_SHA256 = '92fb35fe0895f799c0b90c20d80f0262c92da8afb2b96e17154a870837ed71d5' - -PHASE4_STEP7_CONFIG_INHERITED_NODES = 40 - -def phase4_step7_runtime_boundary(path: Path, relative: str) -> None: - """Inspect fixed reviewed orchestration AST and unchanged inherited config.""" - if relative not in PHASE4_STEP7_RUNTIME_AST_SHA256: - raise ValueError("Step 7 runtime is limited to CLI and additive configuration") - try: - tree = ast.parse(path.read_text(encoding="utf-8")) - except (OSError, UnicodeError, SyntaxError) as error: - raise ValueError("Step 7 runtime cannot be inspected") from error - doc = ast.get_docstring(tree) or "" - if "Owner IDs:" not in doc or "Current phase status:" not in doc or "PR-016" not in doc: - raise ValueError("Step 7 ownership documentation is missing") - body = tree.body[1:] - if _phase4_step4_ast_digest(body) != PHASE4_STEP7_RUNTIME_AST_SHA256[relative]: - raise ValueError("Step 7 runtime differs from the reviewed orchestration") - if relative.endswith("/config.py") and _phase4_step4_ast_digest(body[:PHASE4_STEP7_CONFIG_INHERITED_NODES]) != PHASE4_STEP7_CONFIG_INHERITED_AST_SHA256: - raise ValueError("Step 7 changed inherited configuration behavior") - - -def phase4_step7_cli_boundary(root: Path) -> None: - for relative in sorted(PHASE4_STEP7_RUNTIME_AST_SHA256): - phase4_step7_runtime_boundary(root / relative, relative) - phase4_step6_output_boundary(root) - - -def phase4_step7_main(step: int = 7) -> int: - import runpy - if type(step) is not int or step != 7: - raise ValueError("Only authorized Phase 4 Step 7 traceability is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), run_name="phase4_step7_traceability_control") - control["audit_phase4_step7"](step=step) - phase4_step7_cli_boundary(ROOT) - print("Phase 4 Step 7: local audit/validate orchestration and frozen calculation/report owners: PASS") - return 0 - - -PHASE4_STEP8_RUNTIME_AST_SHA256 = {'src/recursive_integrity_toolkit/cli.py': 'c4bd7981153974857c03ebe79b7daca6cc68ab324980b8434e6ef8dd3d7fd9c8', - 'src/recursive_integrity_toolkit/config.py': 'df7f8d96997cf0bcae5c53d78a130f07ca847c3bbc609e8e6d5c507d926507fa'} - -PHASE4_STEP8_CONFIG_INHERITED_AST_SHA256 = 'cbb96ab25130e4fe45a94cc03299aaa27015a496e38e7dcc6bec1c5d2f7e2573' - -PHASE4_STEP8_CONFIG_INHERITED_NODES = 42 - -def phase4_step8_runtime_boundary(path: Path, relative: str) -> None: - """Inspect fixed reviewed orchestration AST and unchanged inherited config.""" - if relative not in PHASE4_STEP8_RUNTIME_AST_SHA256: - raise ValueError("Step 8 runtime is limited to CLI and additive configuration") - try: - tree = ast.parse(path.read_text(encoding="utf-8")) - except (OSError, UnicodeError, SyntaxError) as error: - raise ValueError("Step 8 runtime cannot be inspected") from error - doc = ast.get_docstring(tree) or "" - if "Owner IDs:" not in doc or "Current phase status:" not in doc or "PR-016" not in doc: - raise ValueError("Step 8 ownership documentation is missing") - body = tree.body[1:] - if _phase4_step4_ast_digest(body) != PHASE4_STEP8_RUNTIME_AST_SHA256[relative]: - raise ValueError("Step 8 runtime differs from the reviewed orchestration") - if relative.endswith("/config.py") and _phase4_step4_ast_digest(body[:PHASE4_STEP8_CONFIG_INHERITED_NODES]) != PHASE4_STEP8_CONFIG_INHERITED_AST_SHA256: - raise ValueError("Step 8 changed inherited configuration behavior") - - -def phase4_step8_cli_boundary(root: Path) -> None: - for relative in sorted(PHASE4_STEP8_RUNTIME_AST_SHA256): - phase4_step8_runtime_boundary(root / relative, relative) - phase4_step6_output_boundary(root) - - -def phase4_step8_main(step: int = 8) -> int: - import runpy - if type(step) is not int or step != 8: - raise ValueError("Only authorized Phase 4 Step 8 traceability is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), run_name="phase4_step8_traceability_control") - control["audit_phase4_step8"](step=step) - phase4_step8_cli_boundary(ROOT) - print("Phase 4 Step 8: explicit pair and packaged Hero orchestration and frozen calculation/report owners: PASS") - return 0 - - -def phase4_current_main(step: int = 11) -> int: - import runpy - if type(step) is not int or step != 11: - raise ValueError("Only authorized Phase 4 Step 11 traceability is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), run_name="phase4_current_traceability_control") - control["audit_phase4_current"](step=step) - # The accepted Step 10 product bytes remain frozen, including its assembly repair. - # Only the package version literal changes in the forty-module runtime. - print("Phase 4 Step 11: final dev3 candidate; 39 runtime modules and all schemas frozen; exact version literals: PASS") + module = _module_name(node, relative) + imports = [module, *(module + "." + alias.name for alias in node.names)] + for module in imports: + if module.split(".")[0] in FORBIDDEN_IMPORTS: + raise ValueError(f"Prohibited network/execution/dependency import in {relative}: {module}") + if relative in INPUT_ONLY and module.startswith(tuple("recursive_integrity_toolkit." + layer for layer in ("metrics", "lineage", "reports"))): + raise ValueError(f"Input-only module imports an analytical/report owner: {relative}") + if relative in RENDERERS and module.startswith(tuple("recursive_integrity_toolkit." + layer for layer in ("io", "metrics", "lineage", "representations"))): + raise ValueError(f"Renderer imports a calculation or input owner: {relative}") + if module.split(".")[0] == "pyarrow" and relative != "io/loaders.py": + raise ValueError(f"Optional Parquet dependency escaped its loader: {relative}") + if module.split(".")[0] == "numpy" and relative != "metrics/resampling.py": + raise ValueError(f"Numerical dependency escaped the explicit sampler: {relative}") + if isinstance(node, ast.Call): + name = node.func.id if isinstance(node.func, ast.Name) else node.func.attr if isinstance(node.func, ast.Attribute) else "" + regex_compile = (isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "re" and name == "compile") + if name in DYNAMIC_CALLS and not regex_compile: + raise ValueError(f"Dynamic execution is prohibited: {relative}") + if relative in RENDERERS and name in {"open", "read_text", "read_bytes", "write_text", "write_bytes"}: + raise ValueError(f"Renderer performs file IO: {relative}") + return {"owned_modules_checked": len(paths), "local_only_and_layer_boundaries": "PASS"} + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + if any(arg in {"--phase", "--step"} or arg.startswith(("--phase=", "--step=")) for arg in argv): + raise SystemExit("Historical phase dispatch has been retired. Run this current check without --phase/--step; prior gates are recoverable from Git history.") + argparse.ArgumentParser(description=__doc__).parse_args(argv) + print(json.dumps(audit(), indent=2)) return 0 -def cli_main(argv: list[str] | None = None) -> int: - """Select the active phase without changing historical checker semantics.""" - import argparse - - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--phase", type=int, choices=(3, 4)) - parser.add_argument("--step", type=int) - args = parser.parse_args(argv) - if args.phase is None and args.step is None: - return main() - if args.phase == 3 and args.step == PHASE3_ACTIVE_STEP: - return main() - if args.phase == 4 and args.step == 1: - return phase4_main(step=args.step) - if args.phase == 4 and args.step == 2: - return phase4_step2_main(step=args.step) - if args.phase == 4 and args.step == 3: - return phase4_step3_main(step=args.step) - if args.phase == 4 and args.step == 4: - return phase4_step4_main(step=args.step) - if args.phase == 4 and args.step == 5: - return phase4_step5_main(step=args.step) - if args.phase == 4 and args.step == 6: - return phase4_step6_main(step=args.step) - if args.phase == 4 and args.step == 7: - return phase4_step7_main(step=args.step) - if args.phase == 4 and args.step == 8: - return phase4_step8_main(step=args.step) - if args.phase == 4 and args.step == 11: - return phase4_current_main(step=args.step) - parser.error("Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5 or --phase 4 --step 6 or --phase 4 --step 7 or --phase 4 --step 8 or --phase 4 --step 11") - return 2 - - if __name__ == "__main__": - raise SystemExit(cli_main()) + raise SystemExit(main()) diff --git a/scripts/release_check.py b/scripts/release_check.py index 1935888..dbd3025 100644 --- a/scripts/release_check.py +++ b/scripts/release_check.py @@ -1,20 +1,20 @@ -"""Audit explicit phase boundaries and retain tested intermediate artifacts. +"""Current source and candidate checks for Recursive Integrity Toolkit. -Maintainer tooling only. Phase 2 authority and restoration guards stay frozen. -Phase 3 Step 11 verifies the final development milestone and its retained evidence. -Only the approved dev2 version literals change in the frozen runtime/metadata. -Publication, merge and later-phase implementation remain unauthorized. +Step 1 preserves accepted product bytes. Historical dispatch, source-body +migrations and phase registries are recoverable from the accepted Git commit. +Installed checks below retain their existing product, privacy and package cases. """ from __future__ import annotations import argparse -import ast +from functools import lru_cache import hashlib import importlib.metadata +import importlib.util +import io import json import os -from pathlib import Path -import re +from pathlib import Path, PurePosixPath import subprocess import sys import tarfile @@ -24,476 +24,78 @@ import zipfile ROOT = Path(__file__).resolve().parents[1] -STEP9_BASELINE = "e1790d60b88370651eb59dc31fbb2343e2daec1f" -PHASE2_FINAL = "78554993febb01609cb90814cc24cce2012bf7d7" -PHASE2_TREE = "706b99e27c2d284a7435ae03e2eb2376214bfa46" -PHASE2_TEST_TREE = "d787d9b06c41a91f8587f6b57a8af710419d114f" -PLAN_SHA256 = "e1c9a6776e3cd2511d37a6bbbdb4a9d5a33b66ff6b00bc00bf3e08a01e6576f1" -ACTIVE_PHASE, ACTIVE_STEP = 3, 11 -PHASE3_COMPLETE = True # Accepted implementation below; final handoff still requires all roles on HEAD. -PHASE3_IMPLEMENTATION_ACCEPTANCE = "9dc321c6997a231209653a0f86d0476824e96e3a" -PHASE3_IMPLEMENTATION_TREE = "a1086ca95075aff3ea8a83afae548806b2079dc9" -STEP10_ALLOWED = { - "README.md", "CHANGELOG.md", "docs/architecture.md", "docs/data_schema.md", - "docs/privacy.md", ".github/workflows/ci.yml", ".github/workflows/security.yml", - ".github/workflows/release.yml", ".github/workflows/golden.yml", - "tests/integration/test_ci_workflows.py", "tests/integration/test_license_notices.py", - "tests/integration/test_no_algorithms.py", "tests/integration/test_prohibited_structure.py", - "tests/integration/test_repository_structure.py", "scripts/check_spec_consistency.py", - "scripts/check_traceability.py", "scripts/release_check.py", "pyproject.toml", - "PHASE_2_COMPLETION.md", "PHASE_2_VALIDATION_REPORT.md", "PHASE_2_ARCHITECTURE_COMPLIANCE_REPORT.md", -} -PHASE3_G = { - "scripts/check_traceability.py", "scripts/check_spec_consistency.py", "scripts/release_check.py", - "tests/conftest.py", "tests/integration/test_no_algorithms.py", "tests/integration/test_ci_workflows.py", - "tests/integration/test_repository_structure.py", "tests/integration/test_owner_ids.py", - "tests/integration/test_package_import.py", "tests/integration/test_package_install.py", - "tests/integration/test_optional_dependency.py", "tests/integration/test_no_network.py", - "tests/unit/test_PR016_determinism.py", ".github/workflows/ci.yml", ".github/workflows/security.yml", - ".github/workflows/golden.yml", ".github/workflows/release.yml", "docs/architecture.md", - "docs/theory_traceability.md", "PHASE_3_DECISIONS.md", "PHASE_3_BASELINE.json", -} -STEP1_NEW = {"PHASE_3_PLAN.md", "PHASE_3_DECISIONS.md", "PHASE_3_BASELINE.json", - "tests/unit/test_phase3_contracts.py", "tests/golden/phase3_math_cases.json", "tests/golden/phase3_math_cases.md"} -STEP1_ALLOWED = PHASE3_G | STEP1_NEW | { - "src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py"} -STEP1_FINAL = "20487e356239d5b5642b6fff595b381d0faa2a4e" -STEP1_TREE = "7d9867fb7543bfd50a9b01bb48530a3411121d31" -STEP2_NEW = {"tests/unit/test_T1_representation.py", - "tests/fixtures/minimal_valid/phase3_field_records.jsonl", - "tests/fixtures/invalid_schema/phase3_representation_cases.json"} -STEP2_EXCEPTION = "tests/unit/test_phase3_contracts.py" -STEP2_ALLOWED = PHASE3_G | STEP2_NEW | { - "src/recursive_integrity_toolkit/representations/base.py", - "src/recursive_integrity_toolkit/representations/field.py", - "src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py", - "docs/data_schema.md", STEP2_EXCEPTION, -} -STEP2_FINAL = "9e5d4c4f38834c42a5b7c19de4e67f471ab9bbca" -STEP2_TREE = "096e038145bced4b01f2c0059659082a72501112" -STEP3_NEW = {"tests/fixtures/minimal_valid/phase3_exact_content.jsonl"} -STEP3_ALLOWED = PHASE3_G | STEP3_NEW | { - "src/recursive_integrity_toolkit/representations/content_hash.py", - "src/recursive_integrity_toolkit/metrics/duplicates.py", - "src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py", - "tests/unit/test_PR006_duplicates.py", "tests/unit/test_T1_representation.py", - "docs/data_schema.md", "docs/privacy.md", -} -STEP3_FINAL = "d2651ca755c82af2e2fec3963bcb6fe270a23be6" -STEP3_TREE = "7871e6253d770d1561d9f732dd5c0b0f842a4138" -STEP4_NEW = {"tests/fixtures/weighted/phase3_distribution_cases.json"} -STEP4_ALLOWED = PHASE3_G | STEP4_NEW | { - "src/recursive_integrity_toolkit/metrics/diversity.py", - "src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py", - "tests/unit/test_T1_support.py", "tests/unit/test_T1_diversity.py", - "tests/unit/test_phase3_contracts.py", -} -STEP4_FINAL = "b943a9712ed1e5b64e09c2c394077fdeed78114f" -STEP4_TREE = "e8f503b44cee0d3db059c03da6599fe44503f03b" -STEP5_NEW = {"tests/fixtures/provenance_partial/phase3_composition.json", - "tests/fixtures/provenance_unknown/phase3_grounding_crossed.json", - "tests/fixtures/weighted/phase3_source_weights.json"} -STEP5_ALLOWED = PHASE3_G | STEP5_NEW | { - "src/recursive_integrity_toolkit/metrics/provenance.py", - "src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py", - "tests/unit/test_PR004_coverage.py", "tests/unit/test_PR005_source_shares.py", "tests/unit/test_T3_provenance.py", -} -STEP5_FINAL = "4611f897d054b352733cc90543eef6f6d71cd811" -STEP5_TREE = "faffd5717dc1bf99e0081a122a338c65a68a13a7" -STEP6_NEW = {"tests/fixtures/provenance_unknown/phase3_direct_bounds.json"} -STEP6_ALLOWED = PHASE3_G | STEP6_NEW | { - "src/recursive_integrity_toolkit/metrics/bounds.py", - "src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py", - "tests/unit/test_T3_bounds.py", "tests/unit/test_T3_provenance.py", -} -STEP6_FINAL = "c7d5ceed2f154bd2ccffbe49961dbf1b65b9df63" -STEP6_TREE = "bc1878e10a9fd056afd196e85126af7ddb9555e5" -STEP7_NEW = {"tests/fixtures/minimal_valid/phase3_tail_cases.json"} -STEP7_ALLOWED = PHASE3_G | STEP7_NEW | { - "src/recursive_integrity_toolkit/metrics/tail.py", - "src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py", - "tests/unit/test_T2_tail.py", "tests/unit/test_phase3_contracts.py", -} -STEP7_FINAL = "d3d95bde50762e60b4dd03cb2bd988618c9a10fe" -STEP7_TREE = "5cb81772542ebfa87c634572068dbf9ee49f41d0" -STEP8_NEW = {"tests/fixtures/resampling/phase3_closed_cases.json"} -STEP8_ALLOWED = PHASE3_G | STEP8_NEW | { - "src/recursive_integrity_toolkit/metrics/resampling.py", - "src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py", - "tests/unit/test_T1_resampling.py", "tests/unit/test_T5_reopening.py", "tests/unit/test_phase3_contracts.py", -} -STEP8_FINAL = "a43cba0e81fd89c3b20a64737bfa89cd2bac4de9" -STEP8_TREE = "f7ecf2326770b362b51ba67357ec450e311a086a" -STEP9_NEW = {"tests/unit/test_T1_compatibility.py", - "tests/fixtures/representation_compatible/phase3_pair.json", - "tests/fixtures/representation_incompatible/phase3_pair.json"} -STEP9_ALLOWED = PHASE3_G | STEP9_NEW | { - "src/recursive_integrity_toolkit/representations/compatibility.py", - "src/recursive_integrity_toolkit/representations/field.py", - "src/recursive_integrity_toolkit/metrics/diversity.py", - "src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py", - "tests/unit/test_T1_support.py", "tests/unit/test_T1_diversity.py", "tests/unit/test_T1_representation.py", -} -STEP9_FINAL = "03fe1e7e99c8170e6be7714649ab0d926e01b6d6" -STEP9_TREE = "765d28e6ff4715fa6d59f44760f1a937489d7120" -PHASE3_STEP10_NEW = {'tests/golden/test_phase3_math.py', 'tests/integration/test_phase3_metric_pipeline.py'} -PHASE3_STEP10_ALLOWED = PHASE3_G | PHASE3_STEP10_NEW | {'tests/golden/phase3_math_cases.md', 'tests/performance/test_hero_runtime.py', 'tests/golden/phase3_math_cases.json', 'tests/performance/test_metadata_100k.py'} -STEP10_FINAL = "150a2a105e01883672ef0c2300b41b0de3be352e" -STEP10_TREE = "b609cb75913fc6352729086819a24795caae1dd8" -STEP10_CONTROL_SHA256 = "8c48a575005b6b0ed78bba8dae21703b1edd6a42bd4a548ae57b083d4f5cc9bc" -PHASE3_STEP11_NEW = {"PHASE_3_COMPLETION.md", "PHASE_3_VALIDATION_REPORT.md", - "PHASE_3_ARCHITECTURE_COMPLIANCE_REPORT.md"} -STEP11_TEST_EXCEPTIONS = {"tests/integration/test_phase3_metric_pipeline.py", - "tests/unit/test_phase3_contracts.py"} -PHASE3_STEP11_ALLOWED = PHASE3_G | PHASE3_STEP11_NEW | STEP11_TEST_EXCEPTIONS | { - "README.md", "CHANGELOG.md", "docs/data_schema.md", "docs/privacy.md", "docs/release_process.md", - "pyproject.toml", "src/recursive_integrity_toolkit/__init__.py", - "tests/integration/test_cli_validation.py", "tests/integration/test_license_notices.py", - "tests/integration/test_prohibited_structure.py", -} -STEP10_RUNTIME_SHA256 = '11b427fa9add50bcd7b2230170ed84d2e4cb4985ade9ee8f9829da0460f12f35' -STEP10_FROZEN_FILES = { - "PHASE_3_PLAN.md": "e1c9a6776e3cd2511d37a6bbbdb4a9d5a33b66ff6b00bc00bf3e08a01e6576f1", - "examples/hero/EXPECTED_OUTPUTS.md": "64d5e696c5e6067b7c34fe85b4d75760588c77fbf19bdd61ee18120385ce8c67", - "examples/hero/config.json": "2e069eb9c0105bf05c145ca15eec47834eb66f4408e8cdde9ff4935f5b920241", - "examples/hero/provenance.csv": "fc4ed3921d087e37996228dd8a37c4c51a0a3c3c91ded3e62a21e1b10f407438", - "examples/hero/records_v1.csv": "b62c9aee057c61f641009b949acb110e003acbea6a399cc607a78ee6e6f0543d", - "examples/hero/records_v2.csv": "d691605fd6bb37e785945ae34c8c18ea8409201f7d78449d39df08df7770599b", - "examples/hero/version_order.json": "3e33ca8b2095f35f55d22265071bce00119c9282b72bb02cee3275c646898387", - "pyproject.toml": "bf3954bc42f3bd9c8275bd7c6858b32266ef014a8c17148b68453a97f3625033", - "schemas/config.schema.json": "cc73411e59d77ded0f9a2a493a507c71901e0a30c7dbca548763a237241927d1", - "schemas/normalized_manifest.schema.json": "13598968bd3ed9ec017759ac4e71a118965ee8895c8c6ba503d0e225fdfad766", - "schemas/report.schema.json": "17130701fc683b3813f649bec45332fe923cd7259a76e6e8e261910207afc23b", - "schemas/schema_mapping.schema.json": "31ef2691dbb11c7861caf15853d7bd0a6dcae25fd10bf26a7b4b1aaa968df85d", - "schemas/version_order.schema.json": "94b10a5842b9f432703789d3388ec3c3d5d07e9224df4266ab29010c0390836d", - "tests/golden/phase3_math_cases.json": "b494d50a1a9bd1009c060c998e4cb3c7433d0e8949f73a8d627e538fa38884b2", - "tests/golden/phase3_math_cases.md": "5f3603ad6ad8f2b1274547f2c239461e7ee8d3b7d552b04d0dd868b862af4044" -} -CUMULATIVE_ALLOWED = STEP1_ALLOWED | STEP2_ALLOWED | STEP3_ALLOWED | STEP4_ALLOWED | STEP5_ALLOWED | STEP6_ALLOWED | STEP7_ALLOWED | STEP8_ALLOWED | STEP9_ALLOWED | PHASE3_STEP10_ALLOWED | PHASE3_STEP11_ALLOWED -CUMULATIVE_NEW = STEP1_NEW | STEP2_NEW | STEP3_NEW | STEP4_NEW | STEP5_NEW | STEP6_NEW | STEP7_NEW | STEP8_NEW | STEP9_NEW | PHASE3_STEP10_NEW | PHASE3_STEP11_NEW -COMPLETION = {"PHASE_2_COMPLETION.md", "PHASE_2_VALIDATION_REPORT.md", "PHASE_2_ARCHITECTURE_COMPLIANCE_REPORT.md"} -PROHIBITED = {"server", "webapp", "cloud", "telemetry", "plugins", "agents", "llm", "auth", "database", "policy_enforcement"} +ACCEPTED_COMMIT = "1db3b1a460c233242ff37fbe45b8fac74d6101ef" +PROTECTED_PREFIXES = ("src/", "schemas/", "examples/hero/") PARQUET_CASES = {"test_PR002_parquet_real_roundtrip", "test_PR002_parquet_real_row_limit", "test_PR002_parquet_real_invalid_file"} -RESTORATION_PATH = "VALIDATION_PLAN.md" -RESTORATION_BEFORE_SHA256 = "564abf56c91141d0cd8ca09024f1cf6dda63bf4ba12acf6563bb7cbc158a84f7" -RESTORATION_AFTER_SHA256 = "16f5fe539da2b3cff1c3e0a2854208bd7fbc1e4c5b332684265b06a03a90cf2f" +HERO_NAMES = {"records_v1.csv", "records_v2.csv", "provenance.csv", "config.json", "version_order.json", "EXPECTED_OUTPUTS.md"} +RESOURCES = {f"src/recursive_integrity_toolkit/data/hero/{name}": f"examples/hero/{name}" for name in HERO_NAMES} +RESOURCES["src/recursive_integrity_toolkit/data/report.schema.json"] = "schemas/report.schema.json" -def verify_approved_restoration(path: str, status: str, before: bytes, after: bytes) -> dict: - """Historical guard; no new restoration permission is created.""" - if path != RESTORATION_PATH or status != "M": - raise ValueError("Restoration exception does not authorize this path or change kind") - if (type(before) is not bytes or type(after) is not bytes - or hashlib.sha256(before).hexdigest() != RESTORATION_BEFORE_SHA256 - or hashlib.sha256(after).hexdigest() != RESTORATION_AFTER_SHA256): - raise ValueError("Restoration exception requires the exact approved byte transition") - return {"path": path, "before_sha256": RESTORATION_BEFORE_SHA256, - "after_sha256": RESTORATION_AFTER_SHA256, "authorization": "Theory Owner exact restoration"} +def git(*arguments: str) -> bytes: + return subprocess.check_output(["git", "-C", str(ROOT), *arguments]) -def git(*args: str) -> bytes: - return subprocess.check_output(["git", "-C", str(ROOT), *args]) - - -def verify_phase3_control(control: dict, step: int = ACTIVE_STEP) -> None: - expected = {"active_phase": 3, "active_step": ACTIVE_STEP, "baseline_commit": PHASE2_FINAL, - "baseline_tree": PHASE2_TREE, "baseline_test_tree": PHASE2_TEST_TREE, - "approved_plan_sha256": PLAN_SHA256, "baseline_core_tests": 1159, "baseline_parquet_tests": 1162, - "permitted_paths": sorted(PHASE3_STEP11_ALLOWED), "approved_decisions": [f"P3-D{i:02d}" for i in range(1, 11)], - "phase_complete": PHASE3_COMPLETE, "next_step_authorized": False, - "previous_step_commit": STEP10_FINAL, "previous_step_tree": STEP10_TREE, - "new_files_permitted": sorted(PHASE3_STEP11_NEW), "main_merge_authorized": False, - "publication_authorized": False, - "implementation_acceptance_commit": PHASE3_IMPLEMENTATION_ACCEPTANCE, - "implementation_acceptance_tree": PHASE3_IMPLEMENTATION_TREE} - if type(control) is not dict or type(step) is not int or step != ACTIVE_STEP: - raise ValueError("Unsupported active phase or step") - for key, value in expected.items(): - if type(control.get(key)) is not type(value) or control[key] != value: - raise ValueError(f"Frozen Phase 3 control mismatch: {key}") - - -def verify_phase3_changes(changes: list[tuple[str, str]], *, incremental: bool = False) -> None: - allowed = PHASE3_STEP11_ALLOWED if incremental else CUMULATIVE_ALLOWED - new_files = PHASE3_STEP11_NEW if incremental else CUMULATIVE_NEW - for status, path in changes: - if path not in allowed or status not in ("M", "A"): - raise ValueError(f"Unauthorized active-stage change: {status} {path}") - if status == "A" and path not in new_files: - raise ValueError(f"Unapproved new file: {path}") - - -def verify_prior_step_changes(changes: list[tuple[str, str]], *, step: int) -> None: - """Retain historical negative tests against their original permission sets.""" - boundaries = {1: (STEP1_ALLOWED, STEP1_NEW), 2: (STEP2_ALLOWED, STEP2_NEW), - 3: (STEP3_ALLOWED, STEP3_NEW), 4: (STEP4_ALLOWED, STEP4_NEW), 5: (STEP5_ALLOWED, STEP5_NEW), 6: (STEP6_ALLOWED, STEP6_NEW), 7: (STEP7_ALLOWED, STEP7_NEW), 8: (STEP8_ALLOWED, STEP8_NEW), 9: (STEP9_ALLOWED, STEP9_NEW), 10: (PHASE3_STEP10_ALLOWED, PHASE3_STEP10_NEW)} - if type(step) is not int or step not in boundaries: - raise ValueError("Unsupported historical step") - allowed, new_files = boundaries[step] - for status, path in changes: - if path not in allowed or status not in ("M", "A") or (status == "A" and path not in new_files): - raise ValueError("Change was not authorized in the historical step") - - -def verify_step2_contract_test_migration(before: bytes, after: bytes) -> None: - """Only the explicitly authorized forged-stage expression may change.""" - old = b'if key in ("active_phase","active_step"): control[key]=2' - new = b'if key in ("active_phase","active_step"): control[key]+=1' - if (type(before) is not bytes or type(after) is not bytes - or before.count(old) != 1 or after != before.replace(old, new, 1)): - raise ValueError("Step 2 contract-test exception exceeded its exact approved edit") - - -def verify_step10_control(control: dict) -> None: - """Verify the immutable historical control independently of the active stage.""" - if (type(control) is not dict - or hashlib.sha256(json.dumps(control, sort_keys=True, separators=(",", ":")).encode()).hexdigest() - != STEP10_CONTROL_SHA256): - raise ValueError("Historical Step 10 control changed") +@lru_cache(maxsize=1) +def _accepted_files() -> dict[str, bytes]: + """Read the existing accepted Git authority, with no new file registry.""" + raw = git("archive", "--format=zip", ACCEPTED_COMMIT) + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + return {member.filename: archive.read(member) for member in archive.infolist() if not member.is_dir()} -def verify_step11_contract_test_migration(before: bytes, after: bytes) -> None: - """Only the two explicitly approved historical-stage bindings may change.""" - replacements = ( - (b'elif key=="phase_complete": control[key]=True', - b'elif key=="phase_complete": control[key]=not control[key]'), - (b'def test_phase3_final_completion_records_not_created():', - b'def test_phase3_final_completion_records_not_created(phase3_step10_snapshot):'), - (b'assert not (ROOT/name).exists()', - b'assert not (phase3_step10_snapshot/name).exists()'), - ) - if type(before) is not bytes or type(after) is not bytes: - raise ValueError("Test migration requires source bytes") - expected = before - for old, new in replacements: - if expected.count(old) != 1: - raise ValueError("Historical contract test binding missing or duplicated") - expected = expected.replace(old, new, 1) - if after != expected: - raise ValueError("Step 11 contract-test exception exceeded its exact approved edits") +def _regular_file(root: Path, relative: str) -> Path: + path = root / relative + inside = [root.joinpath(*Path(relative).parts[:index]) + for index in range(len(Path(relative).parts) + 1)] + if not path.is_file() or any(part.is_symlink() for part in inside): + raise ValueError(f"Missing or aliased protected file: {relative}") + return path -def verify_step11_pipeline_test_migration(before: bytes, after: bytes) -> None: - """Freeze every inherited test byte except the three approved stage bindings. +def verify_source_scope(root: Path = ROOT) -> dict: + """Step 1 is administrative: reject any product mutation, even outside Git's index. - Fresh Step 11 tests may only be appended. No inherited numerical assertion, - parameter, fixture identity or test name is removed or weakened. + Later authorized implementation updates this single current boundary. It does + not append another historical dispatcher or grant authority through metadata. """ - replacements = ( - (b'gate["verify_phase3_changes"]([("M",path)],incremental=True)', - b'gate["verify_prior_step_changes"]([("M",path)],step=10)'), - (b'def test_phase3_step10_scope_is_fixed_not_self_authorized(repo_root):', - b'def test_phase3_step10_scope_is_fixed_not_self_authorized(repo_root,phase3_step10_snapshot):'), - (b'control = json.loads((repo_root/"PHASE_3_BASELINE.json").read_bytes())', - b'control = json.loads((phase3_step10_snapshot/"PHASE_3_BASELINE.json").read_bytes())'), - (b'gate["verify_phase3_control"](control,10)', b'gate["verify_step10_control"](control)'), - (b'gate["verify_phase3_control"](altered,10)', b'gate["verify_step10_control"](altered)'), - (b'gate["verify_step10_snapshot"]()["runtime_modules_unchanged"]', - b'gate["verify_step10_snapshot"](phase3_step10_snapshot)["runtime_modules_unchanged"]'), - (b'def test_phase3_step10_snapshot_rejects_byte_changes(repo_root,tmp_path,path):', - b'def test_phase3_step10_snapshot_rejects_byte_changes(repo_root,tmp_path,path,phase3_step10_snapshot):'), - (b'shutil.copytree(repo_root/"src/recursive_integrity_toolkit",', - b'shutil.copytree(phase3_step10_snapshot/"src/recursive_integrity_toolkit",'), - (b'target.write_bytes((repo_root/name).read_bytes())', - b'target.write_bytes((phase3_step10_snapshot/name).read_bytes())'), - ) - if type(before) is not bytes or type(after) is not bytes: - raise ValueError("Test migration requires source bytes") - expected = before - for old, new in replacements: - if old not in expected: - raise ValueError("Historical pipeline test binding missing") - expected = expected.replace(old, new) - if not after.startswith(expected): - raise ValueError("Step 11 changed an unapproved inherited pipeline test") - additions = ast.parse(after[len(expected):]) - if not additions.body or any(not isinstance(node, ast.FunctionDef) - or not node.name.startswith("test_phase3_step11_") for node in additions.body): - raise ValueError("Only new Step 11 test functions may be appended") - - -def verify_step11_snapshot(root: Path = ROOT) -> dict: - """Require accepted bytes after undoing exactly the two approved version edits.""" - digest = hashlib.sha256() - paths = sorted((root / "src/recursive_integrity_toolkit").rglob("*.py")) - for path in paths: - raw = path.read_bytes() - if path.relative_to(root).as_posix() == "src/recursive_integrity_toolkit/__init__.py": - current = b'__version__ = "0.1.0.dev2"' - if raw.count(current) != 1: - raise ValueError("Expected the single approved root version literal") - raw = raw.replace(current, b'__version__ = "0.1.0.dev1"', 1) - digest.update(path.relative_to(root).as_posix().encode() + b"\0" + raw + b"\0") - if len(paths) != 40 or digest.hexdigest() != STEP10_RUNTIME_SHA256: - raise ValueError("Step 11 changed runtime beyond the approved version literal") - for name, expected in STEP10_FROZEN_FILES.items(): - raw = (root / name).read_bytes() - if name == "pyproject.toml": - current = b'version = "0.1.0.dev2"' - if raw.count(current) != 1: - raise ValueError("Expected the single approved package version literal") - raw = raw.replace(current, b'version = "0.1.0.dev1"', 1) - if hashlib.sha256(raw).hexdigest() != expected: - raise ValueError(f"Step 11 frozen input, dependency or oracle changed: {name}") - return {"package_modules": 40, "approved_version_literals": 2, - "all_other_runtime_bytes_unchanged": True, "frozen_input_files": len(STEP10_FROZEN_FILES)} - - -def verify_step10_snapshot(root: Path = ROOT) -> dict: - """Require all forty accepted runtime modules and frozen inputs byte-for-byte.""" - digest = hashlib.sha256() - paths = sorted((root / "src/recursive_integrity_toolkit").rglob("*.py")) - for path in paths: - digest.update(path.relative_to(root).as_posix().encode() + b"\0" + path.read_bytes() + b"\0") - if len(paths) != 40 or digest.hexdigest() != STEP10_RUNTIME_SHA256: - raise ValueError("Step 10 cannot change the accepted runtime implementation") - for name, expected in STEP10_FROZEN_FILES.items(): - if hashlib.sha256((root / name).read_bytes()).hexdigest() != expected: - raise ValueError(f"Step 10 frozen input or oracle changed: {name}") - return {"runtime_modules_unchanged": 40, "frozen_input_files": len(STEP10_FROZEN_FILES)} - - -def audit(phase: int = 2, step: int = ACTIVE_STEP) -> dict: - required = {"LICENSE", "NOTICE", "THIRD_PARTY_NOTICES.md", "THEORY_SOURCES.md"} | COMPLETION - if phase == 3: - required |= CUMULATIVE_NEW - elif phase != 2: - raise ValueError("Unsupported audit phase") - for name in required: - if not (ROOT / name).is_file(): - raise ValueError(f"Missing required file: {name}") - project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"] - if (project["name"] != "recursive-integrity-toolkit" or project["requires-python"] != ">=3.11" - or project["license"] != "Apache-2.0" or project["version"] != "0.1.0.dev2" - or project["dependencies"] != ["numpy>=2.0", "pandas>=2.2"]): - raise ValueError("Unapproved package identity, version or dependency change") - tree = ast.parse((ROOT / "src/recursive_integrity_toolkit/__init__.py").read_text(encoding="utf-8")) - versions = [ast.literal_eval(n.value) for n in tree.body if isinstance(n, ast.Assign) - and any(isinstance(t, ast.Name) and t.id == "__version__" for t in n.targets)] - if versions != [project["version"]]: - raise ValueError("Package version mismatch") - entries = re.findall(r"\| `([^`]+\.md)` \| `([0-9a-f]{64})` \|", (ROOT / "PHASE_0_APPROVAL.md").read_text(encoding="utf-8")) - if len(entries) != 16 or any(hashlib.sha256((ROOT / n).read_bytes()).hexdigest() != h for n, h in entries): - raise ValueError("Approved Phase 0 hash mismatch") - if "Apache License" not in (ROOT / "LICENSE").read_text(encoding="utf-8"): - raise ValueError("License text missing") - notices = (ROOT / "THIRD_PARTY_NOTICES.md").read_text(encoding="utf-8") - if any(name not in notices for name in ("NumPy", "pandas", "PyArrow", "pytest", "pip-audit")): - raise ValueError("Dependency notices incomplete") - if len(list((ROOT / "src/recursive_integrity_toolkit").rglob("*.py"))) != 40: - raise ValueError("Expected forty package modules") - if any(p.is_dir() and p.name in PROHIBITED for p in ROOT.rglob("*")): - raise ValueError("Prohibited directory") - if phase == 3: - control = json.loads((ROOT / "PHASE_3_BASELINE.json").read_text(encoding="utf-8")) - verify_phase3_control(control, step) - verify_step11_snapshot() - if hashlib.sha256((ROOT / "PHASE_3_PLAN.md").read_bytes()).hexdigest() != PLAN_SHA256: - raise ValueError("Approved plan bytes changed") - if control["phase0_sha256"] != dict(entries): - raise ValueError("Authority hashes differ from original approval") - result = {"phase": phase, "active_step": step if phase == 3 else None, "package_modules": 40, - "phase0_hashes_verified": 16, "license_notices": "PASS", "runtime_dependencies": "UNCHANGED", - "phase_complete": PHASE3_COMPLETE if phase == 3 else None, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_step10_diff() -> dict: - """Historical gate, restricted to its original final checkout.""" - if git("rev-parse", "HEAD").decode().strip() != PHASE2_FINAL: - raise ValueError("Historical Phase 2 gate requires its frozen checkout; choose --phase 3") - changes, restorations = [], [] - for line in git("diff", "--name-status", "--no-renames", STEP9_BASELINE, "HEAD").decode().splitlines(): - status, path = line.split("\t", 1) - if path == RESTORATION_PATH: - restorations.append(verify_approved_restoration(path, status, git("show", f"{STEP9_BASELINE}:{path}"), git("show", f"HEAD:{path}"))) - elif path not in STEP10_ALLOWED or status not in ("M", "A"): - raise ValueError("Unapproved historical change") - if status == "A" and path not in COMPLETION: - raise ValueError("Unapproved historical new file") - changes.append((status, path)) - if git("diff", "--name-only", STEP9_BASELINE, "HEAD", "--", "src", "schemas", "examples/hero", "pyproject.toml"): - raise ValueError("Historical protected files changed") - return {"changes": changes, "approved_exact_restorations": restorations} + expected = {name: raw for name, raw in _accepted_files().items() + if name.startswith(PROTECTED_PREFIXES) or name == "pyproject.toml"} + for name, raw in expected.items(): + if _regular_file(root, name).read_bytes() != raw: + raise ValueError(f"Unauthorized product mutation during Step 1: {name}") + for prefix in PROTECTED_PREFIXES: + paths = list((root / prefix).rglob("*")) + if (root / prefix).is_symlink() or any(path.is_symlink() for path in paths): + raise ValueError(f"Aliased protected content: {prefix}") + actual = {path.relative_to(root).as_posix() for path in paths + if path.is_file() and "__pycache__" not in path.parts} + if actual != {name for name in expected if name.startswith(prefix)}: + raise ValueError(f"Unauthorized product file-set mutation during Step 1: {prefix}") + return {"protected_product_files": len(expected), "product_bytes": "unchanged"} + + +def verify_frozen_specifications(root: Path = ROOT) -> dict: + """Use the already approved specification hashes from the accepted commit.""" + hashes = json.loads(_accepted_files()["PHASE_4_BASELINE.json"])["phase0_sha256"] + for name, expected in hashes.items(): + if hashlib.sha256(_regular_file(root, name).read_bytes()).hexdigest() != expected: + raise ValueError(f"Frozen specification changed: {name}") + return {"frozen_specifications": len(hashes)} + + +def verify_resources(root: Path = ROOT) -> dict: + for destination, source in RESOURCES.items(): + if _regular_file(root, destination).read_bytes() != _regular_file(root, source).read_bytes(): + raise ValueError(f"Packaged resource differs from canonical source: {destination}") + resource_root = root / "src/recursive_integrity_toolkit/data" + if {path.relative_to(root).as_posix() for path in resource_root.rglob("*") if path.is_file()} != set(RESOURCES): + raise ValueError("Packaged resource inventory differs") + return {"canonical_resource_copies": len(RESOURCES)} -def audit_phase3_diff(step: int = ACTIVE_STEP) -> dict: - audit(3, step) - if git("rev-parse", PHASE3_IMPLEMENTATION_ACCEPTANCE + "^{tree}").decode().strip() != PHASE3_IMPLEMENTATION_TREE: - raise ValueError("Accepted Phase 3 implementation tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE3_IMPLEMENTATION_ACCEPTANCE, "HEAD"], check=True) - if git("rev-parse", PHASE2_FINAL + "^{tree}").decode().strip() != PHASE2_TREE: - raise ValueError("Baseline tree mismatch") - if git("rev-parse", PHASE2_FINAL + ":tests").decode().strip() != PHASE2_TEST_TREE: - raise ValueError("Baseline test tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE2_FINAL, "HEAD"], check=True) - changes = [tuple(line.split("\t", 1)) for line in git("diff", "--name-status", "--no-renames", PHASE2_FINAL, "HEAD").decode().splitlines()] - verify_phase3_changes(changes) - if git("rev-parse", STEP1_FINAL + "^{tree}").decode().strip() != STEP1_TREE: - raise ValueError("Previous accepted Step 1 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP1_FINAL, "HEAD"], check=True) - if git("rev-parse", STEP2_FINAL + "^{tree}").decode().strip() != STEP2_TREE: - raise ValueError("Previous accepted Step 2 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP2_FINAL, "HEAD"], check=True) - if git("rev-parse", STEP3_FINAL + "^{tree}").decode().strip() != STEP3_TREE: - raise ValueError("Previous accepted Step 3 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP3_FINAL, "HEAD"], check=True) - if git("rev-parse", STEP4_FINAL + "^{tree}").decode().strip() != STEP4_TREE: - raise ValueError("Previous accepted Step 4 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP4_FINAL, "HEAD"], check=True) - if git("rev-parse", STEP5_FINAL + "^{tree}").decode().strip() != STEP5_TREE: - raise ValueError("Previous accepted Step 5 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP5_FINAL, "HEAD"], check=True) - if git("rev-parse", STEP6_FINAL + "^{tree}").decode().strip() != STEP6_TREE: - raise ValueError("Previous accepted Step 6 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP6_FINAL, "HEAD"], check=True) - if git("rev-parse", STEP7_FINAL + "^{tree}").decode().strip() != STEP7_TREE: - raise ValueError("Previous accepted Step 7 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP7_FINAL, "HEAD"], check=True) - if git("rev-parse", STEP8_FINAL + "^{tree}").decode().strip() != STEP8_TREE: - raise ValueError("Previous accepted Step 8 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP8_FINAL, "HEAD"], check=True) - if git("rev-parse", STEP9_FINAL + "^{tree}").decode().strip() != STEP9_TREE: - raise ValueError("Accepted Step 9 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP9_FINAL, "HEAD"], check=True) - if git("diff", "--name-only", STEP9_FINAL, STEP10_FINAL, "--", "src", "schemas", "examples/hero", "pyproject.toml"): - raise ValueError("Step 10 changed a protected implementation or input") - if git("rev-parse", STEP10_FINAL + "^{tree}").decode().strip() != STEP10_TREE: - raise ValueError("Accepted Step 10 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP10_FINAL, "HEAD"], check=True) - historical = [tuple(line.split("\t", 1)) for line in - git("diff", "--name-status", "--no-renames", STEP9_FINAL, STEP10_FINAL).decode().splitlines()] - verify_prior_step_changes(historical, step=10) - verify_step11_contract_test_migration(git("show", f"{STEP10_FINAL}:tests/unit/test_phase3_contracts.py"), - (ROOT / "tests/unit/test_phase3_contracts.py").read_bytes()) - verify_step11_pipeline_test_migration(git("show", f"{STEP10_FINAL}:tests/integration/test_phase3_metric_pipeline.py"), - (ROOT / "tests/integration/test_phase3_metric_pipeline.py").read_bytes()) - # Step 9 may add pair functions but cannot alter any preexisting single-scope formula. - diversity_path = "src/recursive_integrity_toolkit/metrics/diversity.py" - before = ast.parse(git("show", f"{STEP8_FINAL}:{diversity_path}")) - current = ast.parse((ROOT / diversity_path).read_bytes()) - definitions = {n.name: ast.dump(n, include_attributes=False) for n in current.body - if isinstance(n, (ast.ClassDef, ast.FunctionDef))} - for node in before.body: - if isinstance(node, (ast.ClassDef, ast.FunctionDef)) and definitions.get(node.name) != ast.dump(node, include_attributes=False): - raise ValueError("Step 9 altered an inherited single-scope calculation") - incremental = [tuple(line.split("\t", 1)) for line in - git("diff", "--name-status", "--no-renames", STEP10_FINAL, "HEAD").decode().splitlines()] - verify_phase3_changes(incremental, incremental=True) - verify_step2_contract_test_migration(git("show", f"{STEP1_FINAL}:{STEP2_EXCEPTION}"), - git("show", f"{STEP2_FINAL}:{STEP2_EXCEPTION}")) - if git("diff", "--name-only", "HEAD", "--"): - raise ValueError("Tracked checkout differs from tested commit") - for path in ("src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py"): - before, current = ast.parse(git("show", f"{PHASE2_FINAL}:{path}")), ast.parse((ROOT / path).read_bytes()) - definitions = {n.name: ast.dump(n, include_attributes=False) for n in current.body if isinstance(n, (ast.ClassDef, ast.FunctionDef))} - for node in before.body: - if isinstance(node, (ast.ClassDef, ast.FunctionDef)) and definitions.get(node.name) != ast.dump(node, include_attributes=False): - raise ValueError(f"Inherited contract changed: {path}:{node.name}") - result = {"baseline": PHASE2_FINAL, "head": git("rev-parse", "HEAD").decode().strip(), - "changes": changes, "incremental_changes": incremental, "previous_step": STEP10_FINAL, - "old_runtime_definitions": "UNCHANGED", "active_step": step} +def audit(root: Path = ROOT) -> dict: + result = {**verify_source_scope(root), **verify_frozen_specifications(root), **verify_resources(root)} print(json.dumps(result, indent=2)) return result @@ -516,149 +118,8 @@ def verify_junit(path: Path, *, require_parquet: bool = False, minimum: int = 1) return result -def _collect(root: Path) -> tuple[list[str], str]: - env = dict(os.environ, PYTHONPATH=str(root / "src"), PYTHONDONTWRITEBYTECODE="1") - result = subprocess.run([sys.executable, "-m", "pytest", "-p", "no:cacheprovider", "--collect-only", "-q"], - cwd=root, env=env, capture_output=True, text=True, encoding="utf-8", check=True) - nodes = sorted(line.strip() for line in result.stdout.splitlines() if line.startswith("tests/") and "::" in line) - if len(nodes) != len(set(nodes)): - raise ValueError("Duplicate test node identity") - return nodes, result.stdout - - -def baseline_evidence(output: Path) -> dict: - output.mkdir(parents=True, exist_ok=True) - files = {} - with tempfile.TemporaryDirectory(prefix="rit-p3-baseline-") as temp: - root = Path(temp) - for entry in git("ls-tree", "-rz", PHASE2_FINAL).split(b"\0"): - if not entry: - continue - info, path_bytes = entry.split(b"\t", 1) - mode, kind, blob = info.decode().split() - if kind != "blob": - raise ValueError("Unexpected baseline object kind") - path = path_bytes.decode(); raw = git("show", f"{PHASE2_FINAL}:{path}") - target = root / path; target.parent.mkdir(parents=True, exist_ok=True); target.write_bytes(raw) - files[path] = {"mode": mode, "git_blob": blob, "sha256": hashlib.sha256(raw).hexdigest()} - old, old_log = _collect(root) - new, new_log = _collect(ROOT) - import io - with tempfile.TemporaryDirectory(prefix="rit-p3-step1-tests-") as temp: - prior_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP1_FINAL))) as archive: - archive.extractall(prior_root) - prior, prior_log = _collect(prior_root) - expected_prior = 1273 if os.environ.get("RIT_TEST_PARQUET") == "1" else 1270 - if len(prior) != expected_prior or set(prior) - set(new): - raise ValueError("Accepted Step 1 test identities were lost") - (output / "step1-collection.log").write_text(prior_log, encoding="utf-8") - with tempfile.TemporaryDirectory(prefix="rit-p3-step2-tests-") as temp: - prior2_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP2_FINAL))) as archive: - archive.extractall(prior2_root) - prior2, prior2_log = _collect(prior2_root) - expected_prior2 = 1432 if os.environ.get("RIT_TEST_PARQUET") == "1" else 1429 - if len(prior2) != expected_prior2 or set(prior2) - set(new): - raise ValueError("Accepted Step 2 test identities were lost") - (output / "step2-collection.log").write_text(prior2_log, encoding="utf-8") - with tempfile.TemporaryDirectory(prefix="rit-p3-step3-tests-") as temp: - prior3_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP3_FINAL))) as archive: - archive.extractall(prior3_root) - prior3, prior3_log = _collect(prior3_root) - expected_prior3 = 1578 if os.environ.get("RIT_TEST_PARQUET") == "1" else 1575 - if len(prior3) != expected_prior3 or set(prior3) - set(new): - raise ValueError("Accepted Step 3 test identities were lost") - (output / "step3-collection.log").write_text(prior3_log, encoding="utf-8") - with tempfile.TemporaryDirectory(prefix="rit-p3-step4-tests-") as temp: - prior4_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP4_FINAL))) as archive: - archive.extractall(prior4_root) - prior4, prior4_log = _collect(prior4_root) - expected_prior4 = 1717 if os.environ.get("RIT_TEST_PARQUET") == "1" else 1714 - if len(prior4) != expected_prior4 or set(prior4) - set(new): - raise ValueError("Accepted Step 4 test identities were lost") - (output / "step4-collection.log").write_text(prior4_log, encoding="utf-8") - with tempfile.TemporaryDirectory(prefix="rit-p3-step5-tests-") as temp: - prior5_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP5_FINAL))) as archive: - archive.extractall(prior5_root) - prior5, prior5_log = _collect(prior5_root) - expected_prior5 = 1891 if os.environ.get("RIT_TEST_PARQUET") == "1" else 1888 - if len(prior5) != expected_prior5 or set(prior5) - set(new): - raise ValueError("Accepted Step 5 test identities were lost") - (output / "step5-collection.log").write_text(prior5_log, encoding="utf-8") - with tempfile.TemporaryDirectory(prefix="rit-p3-step6-tests-") as temp: - prior6_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP6_FINAL))) as archive: - archive.extractall(prior6_root) - prior6, prior6_log = _collect(prior6_root) - expected_prior6 = 1990 if os.environ.get("RIT_TEST_PARQUET") == "1" else 1987 - if len(prior6) != expected_prior6 or set(prior6) - set(new): - raise ValueError("Accepted Step 6 test identities were lost") - (output / "step6-collection.log").write_text(prior6_log, encoding="utf-8") - with tempfile.TemporaryDirectory(prefix="rit-p3-step7-tests-") as temp: - prior7_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP7_FINAL))) as archive: - archive.extractall(prior7_root) - prior7, prior7_log = _collect(prior7_root) - expected_prior7 = 2120 if os.environ.get("RIT_TEST_PARQUET") == "1" else 2117 - if len(prior7) != expected_prior7 or set(prior7) - set(new): - raise ValueError("Accepted Step 7 test identities were lost") - (output / "step7-collection.log").write_text(prior7_log, encoding="utf-8") - with tempfile.TemporaryDirectory(prefix="rit-p3-step8-tests-") as temp: - prior8_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP8_FINAL))) as archive: - archive.extractall(prior8_root) - prior8, prior8_log = _collect(prior8_root) - expected_prior8 = 2271 if os.environ.get("RIT_TEST_PARQUET") == "1" else 2268 - if len(prior8) != expected_prior8 or set(prior8) - set(new): - raise ValueError("Accepted Step 8 test identities were lost") - (output / "step8-collection.log").write_text(prior8_log, encoding="utf-8") - with tempfile.TemporaryDirectory(prefix="rit-p3-step9-tests-") as temp: - prior9_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP9_FINAL))) as archive: - archive.extractall(prior9_root) - prior9, prior9_log = _collect(prior9_root) - expected_prior9 = 2407 if os.environ.get("RIT_TEST_PARQUET") == "1" else 2404 - if len(prior9) != expected_prior9 or set(prior9) - set(new): - raise ValueError("Accepted Step 9 test identities were lost") - (output / "step9-collection.log").write_text(prior9_log, encoding="utf-8") - with tempfile.TemporaryDirectory(prefix="rit-p3-step10-tests-") as temp: - prior10_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP10_FINAL))) as archive: - archive.extractall(prior10_root) - prior10, prior10_log = _collect(prior10_root) - expected_prior10 = 2475 if os.environ.get("RIT_TEST_PARQUET") == "1" else 2472 - if len(prior10) != expected_prior10 or set(prior10) - set(new): - raise ValueError("Accepted Step 10 test identities were lost") - (output / "step10-collection.log").write_text(prior10_log, encoding="utf-8") - expected = 1162 if os.environ.get("RIT_TEST_PARQUET") == "1" else 1159 - if len(files) != 197 or len(old) != expected or set(old) - set(new): - raise ValueError(f"Frozen file/test identities do not reconcile: {len(files)} files, {len(old)} baseline tests, {len(set(old)-set(new))} missing") - payload = {"baseline_commit": PHASE2_FINAL, "files": files, "baseline_nodeids": old, "current_nodeids": new, - "previous_step_commit": STEP10_FINAL, "previous_step_nodeids": prior10, - "step9_commit": STEP9_FINAL, "step9_nodeids": prior9, - "step8_commit": STEP8_FINAL, "step8_nodeids": prior8, - "step7_commit": STEP7_FINAL, "step7_nodeids": prior7, - "step6_commit": STEP6_FINAL, "step6_nodeids": prior6, - "step5_commit": STEP5_FINAL, "step5_nodeids": prior5, - "step4_commit": STEP4_FINAL, "step4_nodeids": prior4, - "step3_commit": STEP3_FINAL, "step3_nodeids": prior3, - "step2_commit": STEP2_FINAL, "step2_nodeids": prior2, - "step1_commit": STEP1_FINAL, "step1_nodeids": prior, - "missing_nodeids": [], "nodeids_sha256": hashlib.sha256(("\n".join(old)+"\n").encode()).hexdigest()} - (output / "baseline-identities.json").write_text(json.dumps(payload, indent=2)+"\n", encoding="utf-8") - (output / "baseline-collection.log").write_text(old_log, encoding="utf-8") - (output / "current-collection.log").write_text(new_log, encoding="utf-8") - summary = {"baseline_files": len(files), "inherited_tests": len(old), "current_tests": len(new), "missing_tests": 0, "previous_step_tests": len(prior10), "step9_tests": len(prior9), "step8_tests": len(prior8), "step7_tests": len(prior7), "step6_tests": len(prior6), "step5_tests": len(prior5), "step4_tests": len(prior4), "step3_tests": len(prior3), "step2_tests": len(prior2), "step1_tests": len(prior), - "nodeids_sha256": payload["nodeids_sha256"]} - print(json.dumps(summary, indent=2)) - return summary - - -def verify_distributions(directory: Path, expected_version: str = "0.1.0.dev2") -> tuple[Path, Path]: +def verify_distributions(directory: Path, expected_version: str | None = None) -> tuple[Path, Path]: + expected_version = expected_version or tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"]["version"] wheels, sdists = list(directory.glob("*.whl")), list(directory.glob("*.tar.gz")) if len(wheels) != 1 or len(sdists) != 1: raise ValueError("Require one wheel and one sdist") @@ -668,8 +129,10 @@ def verify_distributions(directory: Path, expected_version: str = "0.1.0.dev2") if wheel.read(path) != raw: raise ValueError(f"Wheel code mismatch: {path}") names = wheel.namelist() + if len(names) != len(set(names)): + raise ValueError("Duplicate wheel entries") if {n for n in names if n.endswith(".py")} != set(expected): - raise ValueError("Wheel module set differs from the forty source modules") + raise ValueError("Wheel module set differs from source") metadata = [n for n in names if n.endswith(".dist-info/METADATA")] if len(metadata) != 1 or f"Version: {expected_version}\n" not in wheel.read(metadata[0]).decode(): raise ValueError("Wheel metadata mismatch") @@ -677,14 +140,17 @@ def verify_distributions(directory: Path, expected_version: str = "0.1.0.dev2") if not any(n.endswith("/"+notice) for n in names): raise ValueError(f"Wheel lacks {notice}") with tarfile.open(sdists[0], "r:gz") as archive: - members = {m.name: m for m in archive.getmembers()} + entries = archive.getmembers() + members = {m.name: m for m in entries} + if len(members) != len(entries): + raise ValueError("Duplicate sdist entries") roots = {n.split("/", 1)[0] for n in members} if len(roots) != 1: raise ValueError("Source archive requires one root") prefix = next(iter(roots)) if {n.removeprefix(prefix + "/src/") for n in members if n.startswith(prefix + "/src/recursive_integrity_toolkit/") and n.endswith(".py")} != set(expected): - raise ValueError("Sdist module set differs from the forty source modules") + raise ValueError("Sdist module set differs from source") for notice in ("LICENSE", "NOTICE"): if f"{prefix}/{notice}" not in members: raise ValueError(f"Sdist lacks {notice}") @@ -692,7 +158,25 @@ def verify_distributions(directory: Path, expected_version: str = "0.1.0.dev2") stream = archive.extractfile(members[f"{prefix}/src/{path}"]) if stream is None or stream.read() != raw: raise ValueError(f"Sdist code mismatch: {path}") - print("wheel/sdist: 40 packaged modules match source") + with zipfile.ZipFile(wheels[0]) as archive: + actual = {name for name in archive.namelist() if name.startswith("recursive_integrity_toolkit/data/") and not name.endswith("/")} + if actual != {path.removeprefix("src/") for path in RESOURCES}: + raise ValueError("Wheel resource inventory differs") + for destination, source in RESOURCES.items(): + if archive.read(destination.removeprefix("src/")) != (ROOT / source).read_bytes(): + raise ValueError("Wheel resource bytes differ") + with tarfile.open(sdists[0], "r:gz") as archive: + prefix = next(iter({m.name.split("/")[0] for m in archive.getmembers()})) + actual = {m.name.removeprefix(prefix + "/") for m in archive.getmembers() + if m.name.startswith(prefix + "/src/recursive_integrity_toolkit/data/") and m.isfile()} + if actual != set(RESOURCES): + raise ValueError("Sdist resource inventory differs") + for destination, source in RESOURCES.items(): + member = archive.getmember(prefix + "/" + destination) + stream = archive.extractfile(member) + if not member.isfile() or stream is None or stream.read() != (ROOT / source).read_bytes(): + raise ValueError("Sdist resource bytes differ") + print(f"wheel/sdist: {len(expected)} source modules and canonical resources match") return wheels[0], sdists[0] @@ -719,7 +203,7 @@ def blocked(*args, **kwargs): assert not Path(package.__file__).resolve().is_relative_to(Path(sys.argv[1])/'src') assert package.__version__==importlib.metadata.version('recursive-integrity-toolkit') names=[package.__name__]+[m.name for m in pkgutil.walk_packages(package.__path__,package.__name__+'.')] -assert len(names)==40 +assert names for name in names: importlib.import_module(name) from recursive_integrity_toolkit.models import AuditBundle,InputSource,FileRole,CapabilityKey,CapabilityStatus,NumericalPolicy from recursive_integrity_toolkit.io.validation import validate_bundle @@ -740,7 +224,7 @@ def blocked(*args, **kwargs): assert result.observability.capabilities[CapabilityKey.MODEL_LONGITUDINAL].status is CapabilityStatus.UNAVAILABLE assert before=={n:hashlib.sha256((hero/n).read_bytes()).hexdigest() for n in files} assert not {'numpy','pandas','pyarrow'} & set(sys.modules) -print('installed: 40 imports, Step 2 field assignment, input-only Hero Level 4, no dependencies/network: PASS') +print('installed: all package imports, field assignment, input-only Hero Level 4, no dependencies/network: PASS') ''' subprocess.run([str(python), "-I", "-c", program, str(ROOT)], cwd=work, check=True) for arg in ("--help", "version"): @@ -749,7 +233,6 @@ def blocked(*args, **kwargs): subprocess.run([str(bindir / (name+".exe" if os.name == "nt" else name)), arg], cwd=work, check=True) - def smoke_installed_duplicates(wheel: Path) -> None: """A separate installed calculation check with approved dependencies present.""" with tempfile.TemporaryDirectory(prefix="rit-p3-installed-duplicates-") as temp: @@ -860,1053 +343,7 @@ def blocked(*args, **kwargs): subprocess.run([str(python), "-I", "-c", program, str(work)], cwd=work, check=True) -def verify_step10_evidence(junit: Path, output: Path | None = None) -> dict: - """Require twenty executed frozen cases and seven measured operation records. - - These are test-run observations, not a public analytical report or speed SLA. - General JUnit validation rejects failures and skips before evidence extraction. - """ - from math import isfinite - verify_junit(junit) - root = ET.parse(junit).getroot() - cases = list(root.iter("testcase")) - expected_ids = {"test_phase3_frozen_case[" + case["case_id"] + "]" - for case in json.loads((ROOT / "tests/golden/phase3_math_cases.json").read_bytes())["cases"]} - actual = {case.get("name") for case in cases if case.get("classname", "").endswith("test_phase3_math")} - if len(expected_ids) != 20 or not expected_ids <= actual: - raise ValueError("All twenty frozen mathematical cases must actually execute") - required = {"hero_input_plus_calculations", "metadata_100k_setup", "metadata_100k_field_representation", - "metadata_100k_support_diversity", "metadata_100k_provenance_join", - "metadata_100k_composition", "metadata_100k_exact_duplicates"} - observations = [] - seen = set() - for case in cases: - for prop in case.findall("./properties/property"): - if prop.get("name") != "phase3_performance": - continue - data = json.loads(prop.get("value", "")) - name = data.get("name") - if name not in required or name in seen: - raise ValueError("Missing, duplicated or unexpected performance observation") - for key in ("elapsed_seconds", "traced_current_bytes", "traced_peak_bytes"): - value = data.get(key) - if type(value) not in (int, float) or not isfinite(value) or value < 0: - raise ValueError("Performance evidence requires finite nonnegative measurements") - if data["traced_peak_bytes"] < data["traced_current_bytes"]: - raise ValueError("Invalid traced-memory evidence") - if data.get("whole_product_report_target_certified") is not False: - raise ValueError("A calculation observation cannot certify the final report target") - if data.get("record_count") != (16 if name == "hero_input_plus_calculations" else 100000): - raise ValueError("Performance evidence used an incorrect dataset size") - for key in ("memory_method", "timing_method", "python", "platform"): - if type(data.get(key)) is not str or not data[key]: - raise ValueError("Performance environment and measurement methods must be disclosed") - seen.add(name) - observations.append(data) - if seen != required: - raise ValueError("Required Step 10 performance measurements were not all executed") - result = {"frozen_cases_executed": 20, "performance_observations": sorted(observations, key=lambda d: d["name"]), - "whole_product_report_target_certified": False} - if output is not None: - output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") - return result - - -def candidate(output: Path, step: int = ACTIVE_STEP) -> None: - result = {"audit": audit_phase3_diff(step), "phase_complete": PHASE3_COMPLETE, - "acceptance_requires_all_four_workflow_roles": True, "next_step_authorized": False} - output.mkdir(parents=True, exist_ok=True) - result["core"] = verify_junit(output / "core.xml", minimum=2472) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=2475) - result["core_integration"] = verify_step10_evidence(output / "core.xml", output / "phase3-core-observations.json") - result["parquet_integration"] = verify_step10_evidence(output / "parquet.xml", output / "phase3-parquet-observations.json") - result["identities"] = baseline_evidence(output) - verify_distributions(output / "dist") - archive = output / "recursive-integrity-toolkit-phase3.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {n for n in git("ls-files", "-z").decode().split("\0") if n} - with zipfile.ZipFile(archive) as zipped: - actual = {n.removeprefix("recursive-integrity-toolkit/") for n in zipped.namelist() if not n.endswith("/")} - if actual != tracked: - raise ValueError("Candidate archive file set mismatch") - for path in actual: - if any(p in {".git", "__pycache__", ".pytest_cache", ".venv", "venv", "build", "dist"} for p in Path(path).parts) or path.endswith((".pyc", ".pyo")): - raise ValueError("Generated file in source archive") - if zipped.read("recursive-integrity-toolkit/"+path) != (ROOT / path).read_bytes(): - raise ValueError("Candidate archive byte mismatch") - (output / "phase3_repository_files.sha256").write_text( - "".join(f"{hashlib.sha256((ROOT / name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - result["source_archive"] = archive.name - result["tracked_files"] = len(tracked) - result["python"] = sys.version; result["platform"] = sys.platform; result["versions"] = {} - for name in ("numpy", "pandas", "pyarrow", "pytest", "build", "setuptools", "twine"): - try: - result["versions"][name] = importlib.metadata.version(name) - except importlib.metadata.PackageNotFoundError: - result["versions"][name] = None - (output / "phase3_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - files = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase3_artifacts.sha256") - (output / "phase3_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in files), encoding="utf-8") - print(f"Step 11 source artifact: {len(tracked)} tracked files verified. Final acceptance also requires all workflow roles on this commit.") - - -# Phase 4 gates are additive. Historical Phase 2/3 functions and constants above -# keep their original contracts; the active Phase 4 dispatcher is separate. -from functools import lru_cache -import io -import importlib.util -from types import MappingProxyType - -PHASE4_FINAL = "e3ffb8c0a88bfe31f669f9662d9b5213da628b3a" -PHASE4_TREE = "e2a25f8cfdc66c3317809c479f80fdae162e6ba9" -PHASE4_TEST_TREE = "6ab22cb9197a8f094f455b29a07a851062bb26c9" -PHASE4_PLAN_SHA256 = "5a6d65696720a426630e876d001937b2837d24774fcb6c2bd43b8e09f5ae93f0" -PHASE4_STEP1_NEW = { - "PHASE_4_PLAN.md", "PHASE_4_BASELINE.json", "PHASE_4_DECISIONS.md", - "tests/unit/test_phase4_contracts.py", "tests/integration/test_phase4_gates.py", -} -PHASE4_G = { - "PHASE_4_BASELINE.json", "PHASE_4_DECISIONS.md", "scripts/check_traceability.py", - "scripts/check_spec_consistency.py", "scripts/release_check.py", "tests/conftest.py", - "tests/unit/test_phase4_contracts.py", "tests/integration/test_phase4_gates.py", - "tests/integration/test_no_algorithms.py", "tests/integration/test_ci_workflows.py", - "tests/integration/test_repository_structure.py", "tests/integration/test_owner_ids.py", - "tests/integration/test_package_import.py", "tests/integration/test_package_install.py", - "tests/integration/test_optional_dependency.py", "tests/integration/test_no_network.py", - "tests/integration/test_schema_json.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_prohibited_structure.py", "tests/integration/test_license_notices.py", - ".github/workflows/ci.yml", ".github/workflows/security.yml", - ".github/workflows/golden.yml", ".github/workflows/release.yml", - "docs/architecture.md", "docs/theory_traceability.md", -} -PHASE4_MIGRATION_PATHS = { - "tests/unit/test_PR012_evidence_classes.py", "tests/unit/test_PR013_report_schema.py", - "tests/unit/test_PR014_unavailable.py", "tests/unit/test_PR015_redaction.py", - "tests/unit/test_PR016_determinism.py", "tests/unit/test_PR018_language.py", - "tests/integration/test_no_algorithms.py", "tests/integration/test_phase3_metric_pipeline.py", - "tests/integration/test_cli_validation.py", "tests/integration/test_ci_workflows.py", -} -PHASE4_STEP1_ALLOWED = PHASE4_G | PHASE4_STEP1_NEW | PHASE4_MIGRATION_PATHS -PHASE4_FORBIDDEN_OUTPUTS = { - "PHASE_4_COMPLETION.md", "PHASE_4_VALIDATION_REPORT.md", - "PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md", "report.json", "report.md", -} -PHASE4_APPROVAL = "批准，开始 **Phase 4 Step 1**" -PHASE4_MIGRATIONS = {'tests/integration/test_ci_workflows.py': [{'new': 'def ' - 'test_workflows_preserve_phase_boundary(repo_root: ' - 'Path,phase3_final_snapshot) -> None:\n' - ' repo_root = phase3_final_snapshot\n', - 'node': 'test_workflows_preserve_phase_boundary', - 'old': 'def ' - 'test_workflows_preserve_phase_boundary(repo_root: ' - 'Path) -> None:\n'}, - {'new': 'def ' - 'test_phase2_delivery_builds_both_formats_and_tests_installed_wheel(repo_root,phase3_final_snapshot):\n' - ' repo_root = phase3_final_snapshot\n', - 'node': 'test_phase2_delivery_builds_both_formats_and_tests_installed_wheel', - 'old': 'def ' - 'test_phase2_delivery_builds_both_formats_and_tests_installed_wheel(repo_root):\n'}, - {'new': 'def ' - 'test_phase2_hero_workflow_preserves_scaffold_only_golden_boundary(repo_root,phase3_final_snapshot):\n' - ' repo_root = phase3_final_snapshot\n', - 'node': 'test_phase2_hero_workflow_preserves_scaffold_only_golden_boundary', - 'old': 'def ' - 'test_phase2_hero_workflow_preserves_scaffold_only_golden_boundary(repo_root):\n'}], - 'tests/integration/test_cli_validation.py': [{'new': 'def ' - 'test_cli_help_runs(subprocess_env,phase3_final_subprocess_env) ' - '-> None:\n' - ' subprocess_env = ' - 'phase3_final_subprocess_env\n', - 'node': 'test_cli_help_runs', - 'old': 'def test_cli_help_runs(subprocess_env) -> ' - 'None:\n'}], - 'tests/integration/test_no_algorithms.py': [{'new': 'def ' - 'test_only_step8_authorized_modules_gain_behavior(package_root,phase3_final_package_root) ' - '-> None:\n' - ' package_root = phase3_final_package_root\n', - 'node': 'test_only_step8_authorized_modules_gain_behavior', - 'old': 'def ' - 'test_only_step8_authorized_modules_gain_behavior(package_root) ' - '-> None:\n'}, - {'new': 'def ' - 'test_protected_phase3_plus_modules_remain_placeholders(package_root,phase3_final_package_root) ' - '-> None:\n' - ' package_root = phase3_final_package_root\n', - 'node': 'test_protected_phase3_plus_modules_remain_placeholders', - 'old': 'def ' - 'test_protected_phase3_plus_modules_remain_placeholders(package_root) ' - '-> None:\n'}, - {'new': 'def ' - 'test_PR003_traceability_script_enforces_step8_scope(repo_root,phase3_final_snapshot) ' - '-> None:\n' - ' repo_root = phase3_final_snapshot\n', - 'node': 'test_PR003_traceability_script_enforces_step8_scope', - 'old': 'def ' - 'test_PR003_traceability_script_enforces_step8_scope(repo_root) ' - '-> None:\n'}], - 'tests/integration/test_phase3_metric_pipeline.py': [{'new': 'def ' - 'test_phase3_later_implementations_remain_empty(package_root,module,phase3_final_package_root):\n' - ' package_root = ' - 'phase3_final_package_root\n', - 'node': 'test_phase3_later_implementations_remain_empty', - 'old': 'def ' - 'test_phase3_later_implementations_remain_empty(package_root,module):\n'}, - {'new': 'def ' - 'test_phase3_step11_rejects_changes_beyond_version_literals(repo_root,tmp_path,path,phase3_final_snapshot):\n' - ' repo_root = phase3_final_snapshot\n', - 'node': 'test_phase3_step11_rejects_changes_beyond_version_literals', - 'old': 'def ' - 'test_phase3_step11_rejects_changes_beyond_version_literals(repo_root,tmp_path,path):\n'}, - {'new': 'def ' - 'test_phase3_step11_test_exceptions_are_exact(repo_root,phase3_step10_snapshot,phase3_final_snapshot):\n' - ' repo_root = phase3_final_snapshot\n', - 'node': 'test_phase3_step11_test_exceptions_are_exact', - 'old': 'def ' - 'test_phase3_step11_test_exceptions_are_exact(repo_root,phase3_step10_snapshot):\n'}], - 'tests/unit/test_PR012_evidence_classes.py': [{'new': 'def ' - 'test_PR012_evidence_classes_owner_and_placeholder(owner_checker, ' - 'placeholder_checker,phase3_final_owner_checker,phase3_final_placeholder_checker):\n' - ' owner_checker = phase3_final_owner_checker\n' - ' placeholder_checker = ' - 'phase3_final_placeholder_checker\n', - 'node': 'test_PR012_evidence_classes_owner_and_placeholder', - 'old': 'def ' - 'test_PR012_evidence_classes_owner_and_placeholder(owner_checker, ' - 'placeholder_checker):\n'}], - 'tests/unit/test_PR013_report_schema.py': [{'new': 'def ' - 'test_PR013_report_schema_owner_and_placeholder(owner_checker, ' - 'placeholder_checker,phase3_final_owner_checker,phase3_final_placeholder_checker):\n' - ' owner_checker = phase3_final_owner_checker\n' - ' placeholder_checker = ' - 'phase3_final_placeholder_checker\n', - 'node': 'test_PR013_report_schema_owner_and_placeholder', - 'old': 'def ' - 'test_PR013_report_schema_owner_and_placeholder(owner_checker, ' - 'placeholder_checker):\n'}], - 'tests/unit/test_PR014_unavailable.py': [{'new': 'def ' - 'test_PR014_unavailable_owner_and_placeholder(owner_checker, ' - 'placeholder_checker,phase3_final_owner_checker,phase3_final_placeholder_checker):\n' - ' owner_checker = phase3_final_owner_checker\n' - ' placeholder_checker = ' - 'phase3_final_placeholder_checker\n', - 'node': 'test_PR014_unavailable_owner_and_placeholder', - 'old': 'def ' - 'test_PR014_unavailable_owner_and_placeholder(owner_checker, ' - 'placeholder_checker):\n'}], - 'tests/unit/test_PR015_redaction.py': [{'new': 'def ' - 'test_PR015_redaction_owner_and_placeholder(owner_checker, ' - 'placeholder_checker,phase3_final_owner_checker,phase3_final_placeholder_checker):\n' - ' owner_checker = phase3_final_owner_checker\n' - ' placeholder_checker = ' - 'phase3_final_placeholder_checker\n', - 'node': 'test_PR015_redaction_owner_and_placeholder', - 'old': 'def ' - 'test_PR015_redaction_owner_and_placeholder(owner_checker, ' - 'placeholder_checker):\n'}], - 'tests/unit/test_PR016_determinism.py': [{'new': 'def ' - 'test_PR016_ordering_owner_and_no_later_behavior(owner_checker, ' - 'package_root, ' - 'placeholder_checker,phase3_final_placeholder_checker):\n' - ' placeholder_checker = ' - 'phase3_final_placeholder_checker\n', - 'node': 'test_PR016_ordering_owner_and_no_later_behavior', - 'old': 'def ' - 'test_PR016_ordering_owner_and_no_later_behavior(owner_checker, ' - 'package_root, placeholder_checker):\n'}], - 'tests/unit/test_PR018_language.py': [{'new': 'def ' - 'test_PR018_language_owner_and_placeholder(owner_checker, ' - 'placeholder_checker,phase3_final_owner_checker,phase3_final_placeholder_checker):\n' - ' owner_checker = phase3_final_owner_checker\n' - ' placeholder_checker = ' - 'phase3_final_placeholder_checker\n', - 'node': 'test_PR018_language_owner_and_placeholder', - 'old': 'def ' - 'test_PR018_language_owner_and_placeholder(owner_checker, ' - 'placeholder_checker):\n'}]} - - -@lru_cache(maxsize=1) -def _phase4_baseline_files(): - """Read immutable Git objects once, independently of the editable manifest.""" - for suffix, expected in (("^{commit}", PHASE4_FINAL), ("^{tree}", PHASE4_TREE), - (":tests", PHASE4_TEST_TREE)): - if git("rev-parse", PHASE4_FINAL + suffix).decode().strip() != expected: - raise ValueError("Pinned Phase 3 baseline identity mismatch") - blobs = {} - for entry in git("ls-tree", "-rz", PHASE4_FINAL).split(b"\0"): - if not entry: - continue - info, name = entry.split(b"\t", 1) - mode, kind, oid = info.split() - path = name.decode("utf-8") - if mode not in (b"100644", b"100755") or kind != b"blob": - raise ValueError("Unexpected baseline file type") - if Path(path).is_absolute() or ".." in Path(path).parts or path in blobs: - raise ValueError("Unsafe baseline path") - blobs[path] = oid.decode("ascii") - result = {} - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_FINAL))) as archive: - names = [n for n in archive.namelist() if not n.endswith("/")] - if len(names) != len(set(names)) or set(names) != set(blobs) or len(names) != 222: - raise ValueError("Pinned baseline archive file set mismatch") - for name in names: - raw = archive.read(name) - oid = hashlib.sha1(b"blob " + str(len(raw)).encode() + b"\0" + raw).hexdigest() - if oid != blobs[name]: - raise ValueError(f"Pinned baseline blob mismatch: {name}") - result[name] = raw - return MappingProxyType(result) - - -def phase4_expected_control() -> dict: - """Independent exact authorization, protected-byte and migration registry.""" - files = _phase4_baseline_files() - authority = re.findall(r"\| `([^`]+\.md)` \| `([0-9a-f]{64})` \|", - files["PHASE_0_APPROVAL.md"].decode()) - if len(authority) != 16: - raise ValueError("Expected sixteen original authority hashes") - return { - "control_version": "1.0", "active_phase": 4, "active_step": 1, - "baseline_commit": PHASE4_FINAL, "baseline_tree": PHASE4_TREE, - "baseline_test_tree": PHASE4_TEST_TREE, "baseline_branch": "phase3-metrics", - "work_branch": "phase4-reports-cli", - "main_at_authorization": "cfe1bd0941c1125498ac3d9d9ebf3adafa2c2fcb", - "approved_plan_sha256": PHASE4_PLAN_SHA256, "approval_date": "2026-09-18", - "approval_basis": PHASE4_APPROVAL, - "approved_decisions": [f"P4-D{i:02d}" for i in range(1, 12)], - "baseline_core_tests": 2489, "baseline_parquet_tests": 2492, - "baseline_tracked_files": 222, "baseline_package_modules": 40, - "baseline_placeholders": 16, "phase0_sha256": dict(authority), - "baseline_files_sha256": {p: hashlib.sha256(raw).hexdigest() for p, raw in sorted(files.items())}, - "permitted_paths": sorted(PHASE4_STEP1_ALLOWED), - "new_files_permitted": sorted(PHASE4_STEP1_NEW), - "historical_migration_nodes": {p: [r["node"] for r in rows] - for p, rows in sorted(PHASE4_MIGRATIONS.items())}, - "runtime_changes_authorized": False, "schema_changes_authorized": False, - "phase_complete": False, "next_step_authorized": False, - "main_merge_authorized": False, "publication_authorized": False, - } - - -def verify_phase4_control(control: dict, step: int = 1) -> None: - if type(step) is not int or step != 1 or type(control) is not dict: - raise ValueError("Unsupported Phase 4 stage/control") - try: - actual = json.dumps(control, sort_keys=True, ensure_ascii=True, allow_nan=False) - expected = json.dumps(phase4_expected_control(), sort_keys=True, ensure_ascii=True, allow_nan=False) - except (TypeError, ValueError) as error: - raise ValueError("Invalid Phase 4 control") from error - if actual != expected: - raise ValueError("Phase 4 control differs from the fixed approved Step 1 contract") - - -def verify_phase4_changes(changes: list[tuple[str, str]]) -> None: - seen = set() - for status, path in changes: - if path in seen or path not in PHASE4_STEP1_ALLOWED: - raise ValueError(f"Unapproved or repeated Phase 4 path: {path}") - seen.add(path) - required_status = "A" if path in PHASE4_STEP1_NEW else "M" - if status != required_status: - raise ValueError(f"Unapproved Phase 4 change kind: {status} {path}") - - -def verify_phase4_test_migration(path: str, before: bytes, after: bytes) -> None: - """Permit exact source bindings, retaining original assertions/parameters.""" - if path not in PHASE4_MIGRATIONS or before != _phase4_baseline_files()[path]: - raise ValueError("Migration requires the named pinned Phase 3 source") - expected = before - for row in PHASE4_MIGRATIONS[path]: - old, new = row["old"].encode(), row["new"].encode() - if expected.count(old) != 1: - raise ValueError("Historical binding is not unique") - expected = expected.replace(old, new, 1) - if not after.startswith(expected): - raise ValueError(f"Inherited test changed beyond its approved binding: {path}") - _phase4_append_only(expected, after, path) - - -def _phase4_preserve_function_header(node, path): - """Reject import-time effects in newly permitted function definitions.""" - decorators = [ast.unparse(value) for value in node.decorator_list] - expected = [] - if path == "tests/conftest.py" and node.name.startswith("phase3_final_"): - expected = ["pytest.fixture(scope='session')"] - elif path == "scripts/release_check.py" and node.name == "_phase4_baseline_files": - expected = ["lru_cache(maxsize=1)"] - if decorators != expected: - raise ValueError(f"Unapproved added function decorator: {path}:{node.name}") - for value in [*node.args.defaults, *node.args.kw_defaults]: - if value is None or isinstance(value, ast.Name) and value.id == "ROOT": - continue - try: - ast.literal_eval(value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError(f"Nonliteral added function default: {path}:{node.name}") from error - arguments = [*node.args.posonlyargs, *node.args.args, *node.args.kwonlyargs] - arguments.extend(value for value in (node.args.vararg, node.args.kwarg) if value is not None) - annotations = [value.annotation for value in arguments] + [node.returns] - allowed_nodes = (ast.Name, ast.Constant, ast.Subscript, ast.Tuple, ast.List, - ast.BinOp, ast.BitOr, ast.Load) - allowed_names = {"Path", "list", "str", "bytes", "dict", "int", "bool", "tuple", "set"} - for value in annotations: - if value is not None and any( - not isinstance(child, allowed_nodes) - or isinstance(child, ast.Name) and child.id not in allowed_names - for child in ast.walk(value)): - raise ValueError(f"Unapproved added function annotation: {path}:{node.name}") - - -def _phase4_append_only(before: bytes, after: bytes, path: str) -> None: - if not after.startswith(before): - raise ValueError(f"Inherited source prefix changed: {path}") - # Appended tests/helpers cannot shadow inherited globals or redefine nodes. - old_tree, new_tree = ast.parse(before), ast.parse(after) - def bindings(tree): - names = [] - for node in tree.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - names.append(node.name) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names.extend(n.id for target in targets for n in ast.walk(target) if isinstance(n, ast.Name)) - return names - names = bindings(new_tree) - if len(names) != len(set(names)) or any(names.count(n) != 1 for n in bindings(old_tree)): - raise ValueError(f"Appended source redefines an inherited binding: {path}") - for node in new_tree.body[len(old_tree.body):]: - if not isinstance(node, ast.FunctionDef): - raise ValueError(f"Appended test source executes or rebinds at module scope: {path}") - prefix = "phase3_final_" if path == "tests/conftest.py" else "test_phase4_" - if not node.name.startswith(prefix): - raise ValueError(f"Appended function is outside the Step 1 test/fixture scope: {path}") - _phase4_preserve_function_header(node, path) - - -def _phase4_preserve_tooling(before: bytes, after: bytes, path: str) -> None: - """All inherited maintainer definitions/constants remain exact.""" - old_text, new_text = before.decode(), after.decode() - old, new = ast.parse(old_text), ast.parse(new_text) - def entry_guard(node): - return isinstance(node, ast.If) and isinstance(node.test, ast.Compare) and "__name__" in ast.unparse(node.test) - historical = [n for n in old.body if not entry_guard(n)] - guards = [n for n in new.body if entry_guard(n)] - expected_guard = ast.parse('if __name__ == "__main__":\n raise SystemExit(cli_main())').body[0] - if len(guards) != 1 or ast.dump(guards[0]) != ast.dump(expected_guard): - raise ValueError(f"Unapproved maintainer entrypoint: {path}") - cursor = 0 - allowed_imports = {"from functools import lru_cache", "import io", "import importlib.util", "from types import MappingProxyType"} - for node in new.body: - if entry_guard(node): - continue - segment = ast.get_source_segment(new_text, node) - if cursor < len(historical) and segment == ast.get_source_segment(old_text, historical[cursor]): - cursor += 1 - continue - if isinstance(node, ast.FunctionDef) and (node.name == "cli_main" or node.name.startswith( - ("phase4_", "_phase4_", "verify_phase4_", "audit_phase4"))): - _phase4_preserve_function_header(node, path) - continue - if isinstance(node, ast.Assign) and all(isinstance(t, ast.Name) and t.id.startswith("PHASE4_") for t in node.targets): - approved_union = ast.parse("PHASE4_G | PHASE4_STEP1_NEW | PHASE4_MIGRATION_PATHS", mode="eval").body - if ([target.id for target in node.targets] == ["PHASE4_STEP1_ALLOWED"] - and ast.dump(node.value) == ast.dump(approved_union)): - continue - try: - ast.literal_eval(node.value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError(f"Nonliteral added maintainer constant: {path}") from error - continue - if isinstance(node, (ast.Import, ast.ImportFrom)) and ast.unparse(node) in allowed_imports: - continue - raise ValueError(f"Unapproved executable maintainer addition: {path}") - if cursor != len(historical): - raise ValueError(f"Historical maintainer statement changed: {path}") - def named(tree): - result = {} - for node in tree.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - keys = [node.name] - elif isinstance(node, ast.Assign): - keys = [] - for target in node.targets: - if isinstance(target, ast.Name): - keys.append(target.id) - elif isinstance(target, (ast.Tuple, ast.List)): - keys.extend(n.id for n in ast.walk(target) if isinstance(n, ast.Name)) - else: - continue - for key in keys: - if key in result: - raise ValueError(f"Duplicate maintainer binding: {key}") - result[key] = node - return result - current = named(new) - for key, node in named(old).items(): - if key not in current or ast.get_source_segment(old_text, node) != ast.get_source_segment(new_text, current[key]): - raise ValueError(f"Historical maintainer contract changed: {path}:{key}") - - -def verify_phase4_snapshot(root: Path = ROOT) -> dict: - files = _phase4_baseline_files() - for path, raw in files.items(): - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError(f"Missing or aliased inherited file: {path}") - current = target.read_bytes() - if path not in PHASE4_STEP1_ALLOWED and current != raw: - raise ValueError(f"Protected Phase 3 bytes changed: {path}") - if path in PHASE4_MIGRATIONS: - verify_phase4_test_migration(path, raw, current) - elif path.startswith("tests/") and path in PHASE4_STEP1_ALLOWED: - _phase4_append_only(raw, current, path) - elif path.startswith("scripts/") and path in PHASE4_STEP1_ALLOWED: - _phase4_preserve_tooling(raw, current, path) - actual_modules = {p.relative_to(root).as_posix() for p in (root / "src/recursive_integrity_toolkit").rglob("*.py")} - expected_modules = {p for p in files if p.startswith("src/") and p.endswith(".py")} - if actual_modules != expected_modules or len(actual_modules) != 40: - raise ValueError("Step 1 cannot change the runtime module set") - if {p.name for p in (root / "schemas").iterdir()} != {Path(p).name for p in files if p.startswith("schemas/")}: - raise ValueError("Step 1 cannot change the schema set") - for path in PHASE4_STEP1_NEW: - if not (root / path).is_file() or (root / path).is_symlink(): - raise ValueError(f"Missing or unsafe Step 1 control/test file: {path}") - if hashlib.sha256((root / "PHASE_4_PLAN.md").read_bytes()).hexdigest() != PHASE4_PLAN_SHA256: - raise ValueError("Approved Phase 4 plan bytes changed") - verify_phase4_control(json.loads((root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))) - decisions = (root / "PHASE_4_DECISIONS.md").read_text(encoding="utf-8") - if PHASE4_APPROVAL not in decisions or any(f"P4-D{i:02d}" not in decisions for i in range(1, 12)): - raise ValueError("Actual Phase 4 approval record missing") - if any((root / name).exists() for name in PHASE4_FORBIDDEN_OUTPUTS): - raise ValueError("Step 1 cannot create Phase 4 completion records or audit reports") - return {"package_modules_unchanged": 40, "schemas_unchanged": 5, - "hero_files_unchanged": 6, "historical_migrated_nodes": 16, - "phase_complete": False, "runtime_behavior_added": False} - - -def audit_phase4(step: int = 1) -> dict: - verify_phase4_control(json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")), step) - result = {"phase": 4, "active_step": step, **verify_phase4_snapshot(), - "phase0_hashes_verified": 16, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_phase4_diff(step: int = 1) -> dict: - if type(step) is not int or step != 1: - raise ValueError("Unsupported Phase 4 step") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE4_FINAL, "HEAD"], check=True) - raw = git("diff", "--name-status", "--no-renames", "-z", PHASE4_FINAL, "--").split(b"\0") - raw = [part.decode("utf-8") for part in raw if part] - if len(raw) % 2: - raise ValueError("Malformed Git difference records") - changes = list(zip(raw[::2], raw[1::2])) - changes.extend(("A", p.decode()) for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") if p) - verify_phase4_changes(changes) - result = {"baseline_commit": PHASE4_FINAL, "changed_files": len(changes), - "changes": changes, "step": 1, "scope": "PASS"} - print(json.dumps(result, indent=2)) - return result - - -def phase4_baseline_evidence(output: Path) -> dict: - """Reconcile final Phase 3 identities plus all previously required identities.""" - output.mkdir(parents=True, exist_ok=True) - inherited = baseline_evidence(output / "phase3-inherited") - with tempfile.TemporaryDirectory(prefix="rit-p4-baseline-") as temp: - baseline = Path(temp) - for name, raw in _phase4_baseline_files().items(): - target = baseline / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(raw) - old, old_log = _collect(baseline) - current, current_log = _collect(ROOT) - expected = 2492 if os.environ.get("RIT_TEST_PARQUET") == "1" else 2489 - if len(old) != expected or set(old) - set(current): - raise ValueError("Accepted final Phase 3 test identities were lost") - result = {"baseline_commit": PHASE4_FINAL, "baseline_test_tree": PHASE4_TEST_TREE, - "baseline_nodeids": old, "current_nodeids": current, "missing_nodeids": [], - "baseline_tests": len(old), "current_tests": len(current), "inherited": inherited, - "nodeids_sha256": hashlib.sha256(("\n".join(old)+"\n").encode()).hexdigest()} - (output / "phase4_test_identity_manifest.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase3-final-collection.log").write_text(old_log, encoding="utf-8") - (output / "phase4-current-collection.log").write_text(current_log, encoding="utf-8") - print(json.dumps({k: result[k] for k in ("baseline_tests", "current_tests", "missing_nodeids")}, indent=2)) - return result - - -def phase4_candidate(output: Path, step: int = 1) -> None: - """Archive a tested intermediate Step 1; never certify phase completion.""" - if os.environ.get("RIT_TEST_PARQUET") != "1": - raise ValueError("Step 1 candidate evidence requires RIT_TEST_PARQUET=1 and real PyArrow") - if importlib.util.find_spec("pyarrow") is None: - raise ValueError("Step 1 candidate evidence requires real PyArrow") - output.mkdir(parents=True, exist_ok=True) - result = audit_phase4(step) - result["diff"] = audit_phase4_diff(step) - if git("status", "--porcelain").strip(): - raise ValueError("Candidate archive requires committed, clean source") - result["core"] = verify_junit(output / "core.xml", minimum=2489) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=2492) - result["core_math_measurements"] = verify_step10_evidence(output / "core.xml", output / "phase4-core-observations.json") - result["parquet_math_measurements"] = verify_step10_evidence(output / "parquet.xml", output / "phase4-parquet-observations.json") - identity = phase4_baseline_evidence(output) - result["test_identity"] = {k: identity[k] for k in ("baseline_tests", "current_tests", "nodeids_sha256")} - expected = set() - for node in identity["current_nodeids"]: - base, bracket, parameter = node.partition("[") - owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), - name + bracket + parameter)) - for name, parquet in (("core.xml", False), ("parquet.xml", True)): - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(output / name).getroot().iter("testcase")} - target = expected if parquet else {c for c in expected if c[1] not in PARQUET_CASES} - if cases != target: - raise ValueError(f"Candidate JUnit does not execute the entire current suite: {name}") - verify_distributions(output / "dist") - archive = output / "recursive-integrity-toolkit-phase4-step1-candidate.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {p for p in git("ls-files", "-z").decode().split("\0") if p} - with zipfile.ZipFile(archive) as zipped: - names = {p.removeprefix("recursive-integrity-toolkit/") for p in zipped.namelist() if not p.endswith("/")} - if names != tracked: - raise ValueError("Step 1 source archive file set mismatch") - for name in names: - if zipped.read("recursive-integrity-toolkit/"+name) != (ROOT / name).read_bytes(): - raise ValueError(f"Step 1 source archive byte mismatch: {name}") - result.update({"commit": git("rev-parse", "HEAD").decode().strip(), - "tree": git("rev-parse", "HEAD^{tree}").decode().strip(), - "source_archive": archive.name, "tracked_files": len(tracked), - "python": sys.version, "platform": sys.platform, - "versions": {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "pytest")}}) - (output / "phase4_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4_repository_files.sha256").write_text("".join(f"{hashlib.sha256((ROOT/name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - paths = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase4_artifacts.sha256") - (output / "phase4_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in paths), encoding="utf-8") - print(f"Phase 4 Step 1 candidate: {len(tracked)} source files verified; Phase 4 remains incomplete.") - - -def phase4_cli_main() -> int: - parser = argparse.ArgumentParser(description="Phase 4 Step 1 maintainer gate") - parser.add_argument("--phase", type=int, choices=(4,), required=True) - parser.add_argument("--step", type=int, choices=(1,), required=True) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) - parser.add_argument("--require-parquet", action="store_true") - parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - raise ValueError("Phase 4 Step 1 cannot certify a final phase delivery") - if args.junit is not None: - verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - phase4_baseline_evidence(args.baseline_evidence) - elif args.dist is not None: - audit_phase4(args.step) - wheel, _ = verify_distributions(args.dist) - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - elif args.smoke_wheel is not None: - smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - phase4_candidate(args.candidate, args.step) - else: - audit_phase4(args.step) - if args.diff: - audit_phase4_diff(args.step) - return 0 - - -# Phase 4 Step 2 is additive. Step 1 gates above retain their original semantics. -PHASE4_STEP1_FINAL = "a7f3c46d6ca05de36bfcb60f496ebb2d5ab4a37c" -PHASE4_STEP1_LOCAL = "5aa9cd6d06bfffec9cc280a52277aa9152a8627e" -PHASE4_STEP1_TREE = "7f8ef492568456bf5d46fb1b7e2631cb9b94e3fd" -PHASE4_STEP1_TEST_TREE = "2ce77e331a0fc377387735bb55dffd3be630b59c" -PHASE4_STEP2_APPROVAL = "批准，开始 **Phase 4 Step 2**" -PHASE4_STEP2_APPROVAL_DATE = "2026-09-19" -PHASE4_STEP2_REPORT_SCHEMA_SHA256 = "34b7edb8e8672fca3f3d4f6fe647b4d8967acb6f0435544ba449c1c44f7bc1f6" -PHASE4_STEP2_ALLOWED = { - "PHASE_4_BASELINE.json", "PHASE_4_DECISIONS.md", "scripts/check_traceability.py", - "scripts/check_spec_consistency.py", "scripts/release_check.py", "tests/conftest.py", - "tests/unit/test_phase4_contracts.py", "tests/integration/test_phase4_gates.py", - "tests/integration/test_no_algorithms.py", "tests/integration/test_ci_workflows.py", - "tests/integration/test_repository_structure.py", "tests/integration/test_owner_ids.py", - "tests/integration/test_package_import.py", "tests/integration/test_package_install.py", - "tests/integration/test_optional_dependency.py", "tests/integration/test_no_network.py", - "tests/integration/test_schema_json.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_prohibited_structure.py", "tests/integration/test_license_notices.py", - ".github/workflows/ci.yml", ".github/workflows/security.yml", - ".github/workflows/golden.yml", ".github/workflows/release.yml", - "docs/architecture.md", "docs/theory_traceability.md", - "src/recursive_integrity_toolkit/result.py", "schemas/report.schema.json", - "docs/report_schema.md", "tests/unit/test_PR012_evidence_classes.py", - "tests/unit/test_PR013_report_schema.py", -} -PHASE4_STEP2_NEW = () -PHASE4_STEP2_MIGRATIONS = {'tests/integration/test_phase4_gates.py': [{'new': 'def phase4_mutation_tree(repo_root, ' - 'tmp_path_factory, phase4_gate_tools, ' - 'phase4_step1_snapshot):', - 'node': 'phase4_mutation_tree', - 'old': 'def phase4_mutation_tree(repo_root, ' - 'tmp_path_factory, phase4_gate_tools):'}, - {'new': 'shutil.copyfile(phase4_step1_snapshot / ' - 'relative, destination)', - 'node': 'phase4_mutation_tree', - 'old': 'shutil.copyfile(repo_root / relative, ' - 'destination)'}, - {'new': 'def ' - 'test_phase4_final_snapshot_preserves_all_current_runtime_bytes(repo_root, ' - 'phase3_final_snapshot, ' - 'phase4_step1_snapshot):\n' - ' repo_root = phase4_step1_snapshot', - 'node': 'test_phase4_final_snapshot_preserves_all_current_runtime_bytes', - 'old': 'def ' - 'test_phase4_final_snapshot_preserves_all_current_runtime_bytes(repo_root, ' - 'phase3_final_snapshot):'}, - {'new': 'def ' - 'test_phase4_active_workflows_preserve_full_matrix_and_use_current_dispatch(repo_root, ' - 'phase4_step1_snapshot):\n' - ' repo_root = phase4_step1_snapshot', - 'node': 'test_phase4_active_workflows_preserve_full_matrix_and_use_current_dispatch', - 'old': 'def ' - 'test_phase4_active_workflows_preserve_full_matrix_and_use_current_dispatch(repo_root):'}], - 'tests/unit/test_phase4_contracts.py': [{'new': 'def phase4_control(phase4_step1_snapshot):\n' - ' return json.loads((phase4_step1_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))', - 'node': 'phase4_control', - 'old': 'def phase4_control():\n' - ' return json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))'}, - {'new': 'def ' - 'test_phase4_registered_migrations_accept_only_the_approved_inventory(phase4_tools, ' - 'phase3_final_snapshot, phase4_step1_snapshot):', - 'node': 'test_phase4_registered_migrations_accept_only_the_approved_inventory', - 'old': 'def ' - 'test_phase4_registered_migrations_accept_only_the_approved_inventory(phase4_tools, ' - 'phase3_final_snapshot):'}, - {'new': 'path, (phase3_final_snapshot / ' - 'path).read_bytes(), (phase4_step1_snapshot / ' - 'path).read_bytes(),', - 'node': 'test_phase4_registered_migrations_accept_only_the_approved_inventory', - 'old': 'path, (phase3_final_snapshot / ' - 'path).read_bytes(), (ROOT / ' - 'path).read_bytes(),'}]} - - -def _phase4_step1_files(): - """Read the exact accepted Step 1 Git tree and verify every blob identity.""" - for suffix, expected in (("^{commit}", PHASE4_STEP1_FINAL), ("^{tree}", PHASE4_STEP1_TREE), - (":tests", PHASE4_STEP1_TEST_TREE)): - if git("rev-parse", PHASE4_STEP1_FINAL + suffix).decode().strip() != expected: - raise ValueError("Pinned Phase 4 Step 1 identity mismatch") - objects = {} - for entry in git("ls-tree", "-rz", PHASE4_STEP1_FINAL).split(b"\0"): - if not entry: - continue - metadata, raw_path = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - path = raw_path.decode("utf-8") - if (mode not in (b"100644", b"100755") or kind != b"blob" or path in objects - or path.startswith("/") or ".." in path.split("/") or ".git" in path.split("/")): - raise ValueError("Unsafe pinned Step 1 Git object") - objects[path] = oid.decode("ascii") - if len(objects) != 227: - raise ValueError("Pinned Step 1 must contain exactly 227 files") - files = {} - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_STEP1_FINAL))) as archive: - names = [item.filename for item in archive.infolist() if not item.is_dir()] - if len(names) != len(objects) or set(names) != set(objects): - raise ValueError("Pinned Step 1 archive identity mismatch") - for name in names: - raw = archive.read(name) - oid = hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - if oid != objects[name]: - raise ValueError(f"Pinned Step 1 blob mismatch: {name}") - files[name] = raw - return MappingProxyType(files) - - -def phase4_step2_expected_control() -> dict: - """Actual Step 2 approval is independent of the mutable control document.""" - prior = _phase4_step1_files() - result = phase4_expected_control() - result.update({ - "control_version": "1.1", "active_step": 2, - "approval_date": PHASE4_STEP2_APPROVAL_DATE, "approval_basis": PHASE4_STEP2_APPROVAL, - "previous_step_commit": PHASE4_STEP1_FINAL, - "previous_step_local_commit": PHASE4_STEP1_LOCAL, - "previous_step_tree": PHASE4_STEP1_TREE, - "previous_step_test_tree": PHASE4_STEP1_TEST_TREE, - "previous_step_core_tests": 2584, "previous_step_parquet_tests": 2587, - "previous_step_files_sha256": {p: hashlib.sha256(raw).hexdigest() for p, raw in sorted(prior.items())}, - "permitted_paths": sorted(PHASE4_STEP2_ALLOWED), "new_files_permitted": [], - "runtime_changes_authorized": True, "schema_changes_authorized": True, - "runtime_paths_authorized": ["src/recursive_integrity_toolkit/result.py"], - "schema_paths_authorized": ["schemas/report.schema.json"], - "step1_historical_binding_nodes": {p: sorted({r["node"] for r in rows}) - for p, rows in sorted(PHASE4_STEP2_MIGRATIONS.items())}, - }) - return result - - -def verify_phase4_step2_control(control: dict, step: int = 2) -> None: - if type(step) is not int or step != 2 or type(control) is not dict: - raise ValueError("Unsupported Phase 4 Step 2 stage/control") - try: - actual = json.dumps(control, sort_keys=True, ensure_ascii=True, allow_nan=False) - expected = json.dumps(phase4_step2_expected_control(), sort_keys=True, ensure_ascii=True, allow_nan=False) - except (TypeError, ValueError) as error: - raise ValueError("Invalid Phase 4 Step 2 control") from error - if actual != expected: - raise ValueError("Phase 4 control differs from the independently approved Step 2 contract") - - -def verify_phase4_step2_changes(changes: list[tuple[str, str]]) -> None: - seen = set() - for status, path in changes: - if path in seen or path not in PHASE4_STEP2_ALLOWED or status != "M": - raise ValueError(f"Unapproved Phase 4 Step 2 path/operation: {status} {path}") - seen.add(path) - - -def _phase4_step2_header(node, path): - """Allow explicit test parametrization/fixtures, never definition-time effects.""" - for decorator in node.decorator_list: - if not isinstance(decorator, ast.Call) or ast.unparse(decorator.func) not in { - "pytest.fixture", "pytest.mark.parametrize"}: - raise ValueError(f"Unapproved Step 2 test decorator: {path}:{node.name}") - if ast.unparse(decorator.func) == "pytest.fixture" and not node.name.startswith("phase4_step"): - raise ValueError("A Step 2 fixture must have an explicit scoped name") - if any(keyword.arg is None for keyword in decorator.keywords): - raise ValueError("Decorator expansion is outside the Step 2 contract") - for value in [*decorator.args, *(keyword.value for keyword in decorator.keywords)]: - try: - ast.literal_eval(value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError("Step 2 decorator arguments must be literal") from error - clone = ast.parse(ast.unparse(node)).body[0] - clone.decorator_list = [] - _phase4_preserve_function_header(clone, path) - - -def _phase4_step2_append_only(before: bytes, after: bytes, path: str) -> None: - if not after.startswith(before): - raise ValueError(f"Inherited Step 1 prefix changed: {path}") - old, new = ast.parse(before), ast.parse(after) - bindings = set() - for node in old.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - bindings.add(node.name) - elif isinstance(node, (ast.Import, ast.ImportFrom)): - bindings.update(alias.asname or alias.name.split(".")[0] for alias in node.names) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - bindings.update(child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)) - for node in new.body[len(old.body):]: - if not isinstance(node, ast.FunctionDef) or node.name in bindings: - raise ValueError(f"Step 2 addition executes, rebinds or shadows inherited source: {path}") - if not node.name.startswith(("test_phase4_step2_", "phase4_step2_", "phase4_step1_")): - raise ValueError(f"Step 2 added test/helper is not explicitly scoped: {path}:{node.name}") - if path == "tests/conftest.py" and node.name != "phase4_step1_snapshot": - raise ValueError("Only the pinned Step 1 shared fixture is authorized") - bindings.add(node.name) - _phase4_step2_header(node, path) - - -def verify_phase4_step2_test_migration(path: str, before: bytes, after: bytes) -> None: - prior = _phase4_step1_files() - if path not in prior or path not in PHASE4_STEP2_ALLOWED or before != prior[path] or not path.startswith("tests/"): - raise ValueError("Step 2 migration requires the exact named Step 1 source") - _phase4_step2_check_test_migration(path, before, after) - - -def _phase4_step2_check_test_migration(path: str, before: bytes, after: bytes) -> None: - """Check a source already read from the verified Step 1 immutable mapping.""" - expected = before - for row in PHASE4_STEP2_MIGRATIONS.get(path, []): - old, new = row["old"].encode(), row["new"].encode() - if expected.count(old) != 1: - raise ValueError("Step 1 historical binding is not unique") - expected = expected.replace(old, new, 1) - _phase4_step2_append_only(expected, after, path) - - -def _phase4_step2_preserve_tooling(before: bytes, after: bytes, path: str) -> None: - """Normalize only the exact new dispatch branch, then preserve all old code.""" - if path == "scripts/release_check.py": - old = ' return phase4_cli_main() if explicit_phase4 else main()' - new = (' explicit_step2 = "--step=2" in argv or any(a == "--step" and b == "2" for a, b in zip(argv, argv[1:]))\n' - ' if explicit_phase4 and explicit_step2:\n' - ' return phase4_step2_cli_main()\n' - ' return phase4_cli_main() if explicit_phase4 else main()') - replacements = [(old, new)] - else: - old = ' if args.phase == 4 and args.step == 1:\n return phase4_main(step=args.step)\n' - new = old + ' if args.phase == 4 and args.step == 2:\n return phase4_step2_main(step=args.step)\n' - replacements = [(old, new), - ('Choose explicit --phase 3 --step 11 or --phase 4 --step 1', - 'Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2')] - expected = before - for old, new in replacements: - if expected.count(old.encode()) != 1: - raise ValueError(f"Historical dispatcher identity mismatch: {path}") - expected = expected.replace(old.encode(), new.encode(), 1) - _phase4_preserve_tooling(expected, after, path) - - -def verify_phase4_step2_snapshot(root: Path = ROOT) -> dict: - prior = _phase4_step1_files() - for path, raw in prior.items(): - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError(f"Missing or aliased inherited Step 1 file: {path}") - current = target.read_bytes() - if path not in PHASE4_STEP2_ALLOWED and current != raw: - raise ValueError(f"Protected Step 1 bytes changed: {path}") - if path.startswith("tests/") and path in PHASE4_STEP2_ALLOWED: - _phase4_step2_check_test_migration(path, raw, current) - elif path.startswith("scripts/") and path in PHASE4_STEP2_ALLOWED: - _phase4_step2_preserve_tooling(raw, current, path) - elif path in {"docs/architecture.md", "docs/theory_traceability.md", "PHASE_4_DECISIONS.md"}: - if not current.startswith(raw): - raise ValueError(f"Step 1 historical documentation prefix changed: {path}") - actual_modules = {p.relative_to(root).as_posix() for p in (root / "src/recursive_integrity_toolkit").rglob("*.py")} - expected_modules = {p for p in prior if p.startswith("src/") and p.endswith(".py")} - if actual_modules != expected_modules or len(actual_modules) != 40: - raise ValueError("Step 2 cannot change the runtime module set") - if {p.name for p in (root / "schemas").iterdir()} != {Path(p).name for p in prior if p.startswith("schemas/")}: - raise ValueError("Step 2 cannot change the schema set") - if hashlib.sha256((root / "PHASE_4_PLAN.md").read_bytes()).hexdigest() != PHASE4_PLAN_SHA256: - raise ValueError("Approved Phase 4 plan changed") - verify_phase4_step2_control(json.loads((root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))) - decisions = (root / "PHASE_4_DECISIONS.md").read_text(encoding="utf-8") - if PHASE4_STEP2_APPROVAL not in decisions or PHASE4_STEP1_FINAL not in decisions: - raise ValueError("Actual Step 2 authorization or Step 1 evidence anchor missing") - if any((root / name).exists() for name in PHASE4_FORBIDDEN_OUTPUTS): - raise ValueError("Step 2 cannot create Phase 4 completion records or audit outputs") - import runpy - checker = runpy.run_path(str(root / "scripts/check_traceability.py"), run_name="phase4_step2_contract_boundary") - checker["phase4_step2_result_boundary"](root / "src/recursive_integrity_toolkit/result.py") - schema_bytes = (root / "schemas/report.schema.json").read_bytes() - if hashlib.sha256(schema_bytes).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Step 2 report schema differs from its independently reviewed bytes") - schema = json.loads(schema_bytes) - if (schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema" - or schema.get("type") != "object" or schema.get("additionalProperties") is not False): - raise ValueError("Step 2 report schema dialect/closed root mismatch") - return {"package_modules": 40, "frozen_runtime_modules": 39, "frozen_schemas": 4, - "hero_files_unchanged": 6, "historical_phase3_migrated_nodes": 16, - "phase_complete": False, "result_contracts_enabled": True, - "adapters_enabled": False, "cli_analysis_enabled": False} - - -def audit_phase4_step2(step: int = 2) -> dict: - verify_phase4_step2_control(json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")), step) - result = {"phase": 4, "active_step": step, **verify_phase4_step2_snapshot(), - "phase0_hashes_verified": 16, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_phase4_step2_diff(step: int = 2) -> dict: - if type(step) is not int or step != 2: - raise ValueError("Unsupported Phase 4 Step 2 stage") - _phase4_step1_files() - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE4_STEP1_FINAL, "HEAD"], check=True) - raw = git("diff", "--name-status", "--no-renames", "-z", PHASE4_STEP1_FINAL, "--").split(b"\0") - raw = [part.decode("utf-8") for part in raw if part] - if len(raw) % 2: - raise ValueError("Malformed Step 2 Git difference records") - changes = list(zip(raw[::2], raw[1::2])) - changes.extend(("A", p.decode()) for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") if p) - verify_phase4_step2_changes(changes) - result = {"previous_step_commit": PHASE4_STEP1_FINAL, "changed_files": len(changes), - "changes": changes, "step": 2, "scope": "PASS"} - print(json.dumps(result, indent=2)) - return result - - -def phase4_step2_baseline_evidence(output: Path) -> dict: - """Reconcile all inherited identities, including the accepted Step 1 suite.""" - output.mkdir(parents=True, exist_ok=True) - inherited = phase4_baseline_evidence(output / "phase4-step1-inherited") - with tempfile.TemporaryDirectory(prefix="rit-p4-step1-identities-") as temp: - baseline = Path(temp) - for name, raw in _phase4_step1_files().items(): - target = baseline / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(raw) - old, old_log = _collect(baseline) - current, current_log = _collect(ROOT) - expected = 2587 if os.environ.get("RIT_TEST_PARQUET") == "1" else 2584 - if len(old) != expected or set(old) - set(current): - raise ValueError("Accepted Phase 4 Step 1 test identities were lost") - result = {"baseline_commit": PHASE4_STEP1_FINAL, "baseline_test_tree": PHASE4_STEP1_TEST_TREE, - "baseline_nodeids": old, "current_nodeids": current, "missing_nodeids": [], - "baseline_tests": len(old), "current_tests": len(current), "inherited": inherited, - "nodeids_sha256": hashlib.sha256(("\n".join(old)+"\n").encode()).hexdigest()} - (output / "phase4_step2_test_identity_manifest.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4-step1-collection.log").write_text(old_log, encoding="utf-8") - (output / "phase4-step2-collection.log").write_text(current_log, encoding="utf-8") - print(json.dumps({k: result[k] for k in ("baseline_tests", "current_tests", "missing_nodeids")}, indent=2)) - return result - - -def phase4_step2_candidate(output: Path, step: int = 2) -> None: - """Build tested Step 2 intermediate evidence without declaring Phase 4 complete.""" - if os.environ.get("RIT_TEST_PARQUET") != "1" or importlib.util.find_spec("pyarrow") is None: - raise ValueError("Step 2 candidate evidence requires RIT_TEST_PARQUET=1 and real PyArrow") - output.mkdir(parents=True, exist_ok=True) - result = audit_phase4_step2(step) - result["diff"] = audit_phase4_step2_diff(step) - if git("status", "--porcelain").strip(): - raise ValueError("Step 2 candidate archive requires committed, clean source") - result["core"] = verify_junit(output / "core.xml", minimum=2584) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=2587) - result["core_math_measurements"] = verify_step10_evidence(output / "core.xml", output / "phase4-step2-core-observations.json") - result["parquet_math_measurements"] = verify_step10_evidence(output / "parquet.xml", output / "phase4-step2-parquet-observations.json") - identity = phase4_step2_baseline_evidence(output) - result["test_identity"] = {k: identity[k] for k in ("baseline_tests", "current_tests", "nodeids_sha256")} - expected = set() - for node in identity["current_nodeids"]: - base, bracket, parameter = node.partition("[") - owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) - for name, parquet in (("core.xml", False), ("parquet.xml", True)): - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(output / name).getroot().iter("testcase")} - target = expected if parquet else {c for c in expected if c[1] not in PARQUET_CASES} - if cases != target: - raise ValueError(f"Step 2 JUnit does not execute the entire current suite: {name}") - wheel, _ = verify_distributions(output / "dist") - phase4_step2_installed_contract_smoke(wheel) - archive = output / "recursive-integrity-toolkit-phase4-step2-candidate.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {p for p in git("ls-files", "-z").decode().split("\0") if p} - with zipfile.ZipFile(archive) as zipped: - names = {p.removeprefix("recursive-integrity-toolkit/") for p in zipped.namelist() if not p.endswith("/")} - if names != tracked or len(tracked) != 227: - raise ValueError("Step 2 source archive file set mismatch") - for name in names: - if zipped.read("recursive-integrity-toolkit/"+name) != (ROOT / name).read_bytes(): - raise ValueError(f"Step 2 source archive byte mismatch: {name}") - result.update({"commit": git("rev-parse", "HEAD").decode().strip(), - "tree": git("rev-parse", "HEAD^{tree}").decode().strip(), - "source_archive": archive.name, "tracked_files": len(tracked), - "python": sys.version, "platform": sys.platform, - "versions": {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "pytest")}}) - (output / "phase4_step2_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4_step2_repository_files.sha256").write_text("".join(f"{hashlib.sha256((ROOT/name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - paths = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase4_step2_artifacts.sha256") - (output / "phase4_step2_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in paths), encoding="utf-8") - print(f"Phase 4 Step 2 candidate: {len(tracked)} source files verified; Phase 4 remains incomplete.") - - -def phase4_step2_installed_contract_smoke(wheel: Path) -> None: +def smoke_installed_contract(wheel: Path) -> None: """Exercise installed canonical contracts with no optional or analytical imports.""" with tempfile.TemporaryDirectory(prefix="rit-p4-step2-installed-contract-") as temp: work = Path(temp) @@ -2028,570 +465,7 @@ def reject_schema_files(event, args): subprocess.run([str(python), "-I", "-c", program, str(work), str(ROOT)], cwd=work, check=True) -def phase4_step2_cli_main() -> int: - parser = argparse.ArgumentParser(description="Phase 4 Step 2 maintainer gate") - parser.add_argument("--phase", type=int, choices=(4,), required=True) - parser.add_argument("--step", type=int, choices=(2,), required=True) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) - parser.add_argument("--require-parquet", action="store_true") - parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - raise ValueError("Phase 4 Step 2 cannot certify a final phase delivery") - if args.junit is not None: - verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - phase4_step2_baseline_evidence(args.baseline_evidence) - elif args.dist is not None: - audit_phase4_step2(args.step) - wheel, _ = verify_distributions(args.dist) - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - elif args.smoke_wheel is not None: - smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - phase4_step2_candidate(args.candidate, args.step) - else: - audit_phase4_step2(args.step) - if args.diff: - audit_phase4_step2_diff(args.step) - return 0 - - -# Phase 4 Step 3 adds pure adapters. All earlier stage gates remain callable. -PHASE4_STEP2_FINAL = "fcea74e2b1858e83cdbfd1b8212d15343d63ff08" -PHASE4_STEP2_TREE = "c3ce887fd393cfc6c18364546005f2b74a63b746" -PHASE4_STEP2_TEST_TREE = "9d6ac0cbf3827ab54c7852503d99d442a3796259" -PHASE4_STEP3_APPROVAL = "开始 **Phase 4 Step 3**" -PHASE4_STEP3_APPROVAL_DATE = "2026-09-19" -PHASE4_STEP3_ALLOWED = {'.github/workflows/ci.yml', - '.github/workflows/golden.yml', - '.github/workflows/release.yml', - '.github/workflows/security.yml', - 'PHASE_4_BASELINE.json', - 'PHASE_4_DECISIONS.md', - 'docs/architecture.md', - 'docs/report_schema.md', - 'docs/theory_traceability.md', - 'scripts/check_spec_consistency.py', - 'scripts/check_traceability.py', - 'scripts/release_check.py', - 'src/recursive_integrity_toolkit/reports/assembly.py', - 'tests/conftest.py', - 'tests/integration/test_ci_workflows.py', - 'tests/integration/test_hero_structure.py', - 'tests/integration/test_license_notices.py', - 'tests/integration/test_no_algorithms.py', - 'tests/integration/test_no_network.py', - 'tests/integration/test_optional_dependency.py', - 'tests/integration/test_owner_ids.py', - 'tests/integration/test_package_import.py', - 'tests/integration/test_package_install.py', - 'tests/integration/test_partial_lineage_report.py', - 'tests/integration/test_partial_provenance_report.py', - 'tests/integration/test_phase4_gates.py', - 'tests/integration/test_prohibited_structure.py', - 'tests/integration/test_repository_structure.py', - 'tests/integration/test_schema_json.py', - 'tests/unit/test_PR012_evidence_classes.py', - 'tests/unit/test_PR014_unavailable.py', - 'tests/unit/test_PR018_language.py', - 'tests/unit/test_phase4_contracts.py'} -PHASE4_STEP3_NEW = () -PHASE4_STEP3_DOCUMENT_CORRECTIONS = { - "docs/report_schema.md": [{ - "old": "| `derived_metrics.provenance.weighted_source_type_masses` | number | `user_declared_weight_mass` | `derived_metric` | `PR-005` | `PR-005.source_weight_mass` | 2 | Null only with unavailable reasons |", - "new": "| `derived_metrics.provenance.weighted_source_type_masses` | category_mass_map | `user_declared_weight_mass` | `derived_metric` | `PR-005` | `PR-005.source_weight_mass` | 2 | Null only with unavailable reasons |", - }], -} -PHASE4_STEP3_MIGRATIONS = {'tests/integration/test_phase4_gates.py': [{'new': 'def ' - 'test_phase4_step2_current_runtime_opens_only_the_canonical_result(repo_root, ' - 'phase3_final_snapshot, ' - 'phase4_step2_snapshot):\n' - ' repo_root = phase4_step2_snapshot\n', - 'node': 'test_phase4_step2_current_runtime_opens_only_the_canonical_result', - 'old': 'def ' - 'test_phase4_step2_current_runtime_opens_only_the_canonical_result(repo_root, ' - 'phase3_final_snapshot):\n'}, - {'new': 'def ' - 'test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step2_snapshot):\n' - ' repo_root = phase4_step2_snapshot\n', - 'node': 'test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch', - 'old': 'def ' - 'test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root):\n'}, - {'new': 'def ' - 'test_phase4_step2_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step2_snapshot):', - 'node': 'test_phase4_step2_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step2_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation):'}, - {'new': ' shutil.copyfile(phase4_step2_snapshot ' - '/ relative, destination)', - 'node': 'test_phase4_step2_current_snapshot_checks_valid_tree_before_mutations', - 'old': ' shutil.copyfile(repo_root / relative, ' - 'destination)'}], - 'tests/unit/test_phase4_contracts.py': [{'new': 'def ' - 'test_phase4_step2_approved_control_keeps_independent_step1_anchors(phase4_tools, ' - 'phase4_step2_snapshot):\n' - ' control = json.loads((phase4_step2_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))', - 'node': 'test_phase4_step2_approved_control_keeps_independent_step1_anchors', - 'old': 'def ' - 'test_phase4_step2_approved_control_keeps_independent_step1_anchors(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))'}, - {'new': 'def ' - 'test_phase4_step2_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value, phase4_step2_snapshot):\n' - ' control = json.loads((phase4_step2_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))', - 'node': 'test_phase4_step2_control_rejects_forged_scope_and_stage', - 'old': 'def ' - 'test_phase4_step2_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))'}, - {'new': 'def ' - 'test_phase4_step2_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step, phase4_step2_snapshot):\n' - ' control = json.loads((phase4_step2_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))', - 'node': 'test_phase4_step2_control_rejects_unapproved_dispatch', - 'old': 'def ' - 'test_phase4_step2_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))'}, - {'new': 'def ' - 'test_phase4_step2_control_requires_explicit_approval_fields(phase4_tools, ' - 'field, phase4_step2_snapshot):\n' - ' control = json.loads((phase4_step2_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))', - 'node': 'test_phase4_step2_control_requires_explicit_approval_fields', - 'old': 'def ' - 'test_phase4_step2_control_requires_explicit_approval_fields(phase4_tools, ' - 'field):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))'}, - {'new': 'def ' - 'test_phase4_step2_control_cannot_mint_extra_permission(phase4_tools, ' - 'phase4_step2_snapshot):\n' - ' control = json.loads((phase4_step2_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))', - 'node': 'test_phase4_step2_control_cannot_mint_extra_permission', - 'old': 'def ' - 'test_phase4_step2_control_cannot_mint_extra_permission(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))'}, - {'new': 'def ' - 'test_phase4_step2_historical_migrations_are_five_explicit_step1_bindings(phase4_tools, ' - 'phase4_step1_snapshot, phase4_step2_snapshot):', - 'node': 'test_phase4_step2_historical_migrations_are_five_explicit_step1_bindings', - 'old': 'def ' - 'test_phase4_step2_historical_migrations_are_five_explicit_step1_bindings(phase4_tools, ' - 'phase4_step1_snapshot):'}, - {'new': 'def ' - 'test_phase4_step2_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, ' - 'phase4_step1_snapshot, mutation, ' - 'phase4_step2_snapshot):', - 'node': 'test_phase4_step2_historical_guard_rejects_assertion_and_binding_weakening', - 'old': 'def ' - 'test_phase4_step2_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, ' - 'phase4_step1_snapshot, mutation):'}, - {'new': ' path, (phase4_step1_snapshot / ' - 'path).read_bytes(), (phase4_step2_snapshot / ' - 'path).read_bytes(),', - 'node': 'test_phase4_step2_historical_migrations_are_five_explicit_step1_bindings', - 'old': ' path, (phase4_step1_snapshot / ' - 'path).read_bytes(), (ROOT / path).read_bytes(),'}, - {'new': ' before = (phase4_step1_snapshot / ' - 'path).read_bytes()\n' - ' after = (phase4_step2_snapshot / ' - 'path).read_bytes()', - 'node': 'test_phase4_step2_historical_guard_rejects_assertion_and_binding_weakening', - 'old': ' before = (phase4_step1_snapshot / ' - 'path).read_bytes()\n' - ' after = (ROOT / path).read_bytes()'}]} - - -def _phase4_step2_files(): - """Read the exact accepted Step 2 Git tree and verify every blob identity.""" - for suffix, expected in (("^{commit}", PHASE4_STEP2_FINAL), ("^{tree}", PHASE4_STEP2_TREE), - (":tests", PHASE4_STEP2_TEST_TREE)): - if git("rev-parse", PHASE4_STEP2_FINAL + suffix).decode().strip() != expected: - raise ValueError("Pinned Phase 4 Step 2 identity mismatch") - objects = {} - for entry in git("ls-tree", "-rz", PHASE4_STEP2_FINAL).split(b"\0"): - if not entry: - continue - metadata, raw_path = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - path = raw_path.decode("utf-8") - if (mode not in (b"100644", b"100755") or kind != b"blob" or path in objects - or path.startswith("/") or ".." in path.split("/") or ".git" in path.split("/")): - raise ValueError("Unsafe pinned Step 2 Git object") - objects[path] = oid.decode("ascii") - if len(objects) != 227: - raise ValueError("Pinned Step 2 must contain exactly 227 files") - files = {} - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_STEP2_FINAL))) as archive: - names = [item.filename for item in archive.infolist() if not item.is_dir()] - if len(names) != len(objects) or set(names) != set(objects): - raise ValueError("Pinned Step 2 archive identity mismatch") - for name in names: - raw = archive.read(name) - oid = hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - if oid != objects[name]: - raise ValueError(f"Pinned Step 2 blob mismatch: {name}") - files[name] = raw - return MappingProxyType(files) - - -def phase4_step3_expected_control() -> dict: - """Actual Step 3 approval is independent of the mutable control document.""" - prior = _phase4_step2_files() - result = phase4_step2_expected_control() - result.pop("previous_step_local_commit") - result.update({ - "control_version": "1.2", "active_step": 3, - "approval_date": PHASE4_STEP3_APPROVAL_DATE, "approval_basis": PHASE4_STEP3_APPROVAL, - "previous_step_commit": PHASE4_STEP2_FINAL, - "previous_step_tree": PHASE4_STEP2_TREE, - "previous_step_test_tree": PHASE4_STEP2_TEST_TREE, - "previous_step_core_tests": 2728, "previous_step_parquet_tests": 2731, - "previous_step_files_sha256": {p: hashlib.sha256(raw).hexdigest() for p, raw in sorted(prior.items())}, - "permitted_paths": sorted(PHASE4_STEP3_ALLOWED), "new_files_permitted": [], - "runtime_changes_authorized": True, "schema_changes_authorized": False, - "runtime_paths_authorized": ["src/recursive_integrity_toolkit/reports/assembly.py"], - "schema_paths_authorized": [], - "step2_historical_binding_nodes": {p: sorted({r["node"] for r in rows}) - for p, rows in sorted(PHASE4_STEP3_MIGRATIONS.items())}, - }) - return result - - -def verify_phase4_step3_control(control: dict, step: int = 3) -> None: - if type(step) is not int or step != 3 or type(control) is not dict: - raise ValueError("Unsupported Phase 4 Step 3 stage/control") - try: - actual = json.dumps(control, sort_keys=True, ensure_ascii=True, allow_nan=False) - expected = json.dumps(phase4_step3_expected_control(), sort_keys=True, ensure_ascii=True, allow_nan=False) - except (TypeError, ValueError) as error: - raise ValueError("Invalid Phase 4 Step 3 control") from error - if actual != expected: - raise ValueError("Phase 4 control differs from the independently approved Step 3 contract") - - -def verify_phase4_step3_changes(changes: list[tuple[str, str]]) -> None: - seen = set() - for status, path in changes: - if path in seen or path not in PHASE4_STEP3_ALLOWED or status != "M": - raise ValueError(f"Unapproved Phase 4 Step 3 path/operation: {status} {path}") - seen.add(path) - - -def _phase4_step3_header(node, path): - """Allow explicit test parametrization/fixtures, never definition-time effects.""" - for decorator in node.decorator_list: - if not isinstance(decorator, ast.Call) or ast.unparse(decorator.func) not in { - "pytest.fixture", "pytest.mark.parametrize"}: - raise ValueError(f"Unapproved Step 3 test decorator: {path}:{node.name}") - if ast.unparse(decorator.func) == "pytest.fixture" and not node.name.startswith("phase4_step"): - raise ValueError("A Step 3 fixture must have an explicit scoped name") - if any(keyword.arg is None for keyword in decorator.keywords): - raise ValueError("Decorator expansion is outside the Step 3 contract") - for value in [*decorator.args, *(keyword.value for keyword in decorator.keywords)]: - try: - ast.literal_eval(value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError("Step 3 decorator arguments must be literal") from error - clone = ast.parse(ast.unparse(node)).body[0] - clone.decorator_list = [] - _phase4_preserve_function_header(clone, path) - - -def _phase4_step3_append_only(before: bytes, after: bytes, path: str) -> None: - if not after.startswith(before): - raise ValueError(f"Inherited Step 2 prefix changed: {path}") - old, new = ast.parse(before), ast.parse(after) - bindings = set() - for node in old.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - bindings.add(node.name) - elif isinstance(node, (ast.Import, ast.ImportFrom)): - bindings.update(alias.asname or alias.name.split(".")[0] for alias in node.names) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - bindings.update(child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)) - for node in new.body[len(old.body):]: - if not isinstance(node, ast.FunctionDef) or node.name in bindings: - raise ValueError(f"Step 3 addition executes, rebinds or shadows inherited source: {path}") - allowed_snapshot = path == "tests/conftest.py" and node.name == "phase4_step2_snapshot" - if not allowed_snapshot and not node.name.startswith(("test_phase4_step3_", "phase4_step3_")): - raise ValueError(f"Step 3 added test/helper is not explicitly scoped: {path}:{node.name}") - if path == "tests/conftest.py" and node.name != "phase4_step2_snapshot": - raise ValueError("Only the pinned Step 2 shared fixture is authorized") - bindings.add(node.name) - _phase4_step3_header(node, path) - - -def verify_phase4_step3_test_migration(path: str, before: bytes, after: bytes) -> None: - prior = _phase4_step2_files() - if path not in prior or path not in PHASE4_STEP3_ALLOWED or before != prior[path] or not path.startswith("tests/"): - raise ValueError("Step 3 migration requires the exact named Step 2 source") - _phase4_step3_check_test_migration(path, before, after) - - -def _phase4_step3_check_test_migration(path: str, before: bytes, after: bytes) -> None: - """Check a source already read from the verified Step 2 immutable mapping.""" - expected = before - for row in PHASE4_STEP3_MIGRATIONS.get(path, []): - old, new = row["old"].encode(), row["new"].encode() - if expected.count(old) != 1: - raise ValueError("Step 2 historical binding is not unique") - expected = expected.replace(old, new, 1) - _phase4_step3_append_only(expected, after, path) - - -def _phase4_step3_preserve_tooling(before: bytes, after: bytes, path: str) -> None: - """Permit the exact Step 3 dispatch addition and preserve every older statement.""" - if path == "scripts/release_check.py": - old = ('def cli_main() -> int:\n' - ' argv = sys.argv[1:]\n' - ' explicit_phase4 = "--phase=4" in argv or any(a == "--phase" and b == "4" for a, b in zip(argv, argv[1:]))\n' - ' explicit_step2 = "--step=2" in argv or any(a == "--step" and b == "2" for a, b in zip(argv, argv[1:]))\n' - ' if explicit_phase4 and explicit_step2:\n' - ' return phase4_step2_cli_main()\n' - ' return phase4_cli_main() if explicit_phase4 else main()') - new = old.replace(' return phase4_cli_main() if explicit_phase4 else main()', - ' explicit_step3 = "--step=3" in argv or any(a == "--step" and b == "3" for a, b in zip(argv, argv[1:]))\n' - ' if explicit_phase4 and explicit_step3:\n' - ' return phase4_step3_cli_main()\n' - ' return phase4_cli_main() if explicit_phase4 else main()') - replacements = [(old, new)] - else: - old = ' if args.phase == 4 and args.step == 2:\n return phase4_step2_main(step=args.step)\n' - new = old + ' if args.phase == 4 and args.step == 3:\n return phase4_step3_main(step=args.step)\n' - replacements = [(old, new), - ('Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2', - 'Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3')] - expected = before - for old, new in replacements: - if expected.count(old.encode()) != 1: - raise ValueError(f"Historical dispatcher identity mismatch: {path}") - expected = expected.replace(old.encode(), new.encode(), 1) - _phase4_preserve_tooling(expected, after, path) - - -def verify_phase4_step3_snapshot(root: Path = ROOT) -> dict: - prior = _phase4_step2_files() - for path, raw in prior.items(): - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError(f"Missing or aliased inherited Step 2 file: {path}") - current = target.read_bytes() - if path not in PHASE4_STEP3_ALLOWED and current != raw: - raise ValueError(f"Protected Step 2 bytes changed: {path}") - if path.startswith("tests/") and path in PHASE4_STEP3_ALLOWED: - _phase4_step3_check_test_migration(path, raw, current) - elif path.startswith("scripts/") and path in PHASE4_STEP3_ALLOWED: - _phase4_step3_preserve_tooling(raw, current, path) - elif path in {"docs/architecture.md", "docs/theory_traceability.md", "docs/report_schema.md", "PHASE_4_DECISIONS.md"}: - expected = raw - for correction in PHASE4_STEP3_DOCUMENT_CORRECTIONS.get(path, []): - old, new = correction["old"].encode(), correction["new"].encode() - if expected.count(old) != 1: - raise ValueError(f"Historical documentation correction is not unique: {path}") - expected = expected.replace(old, new, 1) - if not current.startswith(expected): - raise ValueError(f"Step 2 historical documentation prefix changed: {path}") - actual_modules = {p.relative_to(root).as_posix() for p in (root / "src/recursive_integrity_toolkit").rglob("*.py")} - expected_modules = {p for p in prior if p.startswith("src/") and p.endswith(".py")} - if actual_modules != expected_modules or len(actual_modules) != 40: - raise ValueError("Step 3 cannot change the runtime module set") - if {p.name for p in (root / "schemas").iterdir()} != {Path(p).name for p in prior if p.startswith("schemas/")}: - raise ValueError("Step 3 cannot change the schema set") - if hashlib.sha256((root / "PHASE_4_PLAN.md").read_bytes()).hexdigest() != PHASE4_PLAN_SHA256: - raise ValueError("Approved Phase 4 plan changed") - verify_phase4_step3_control(json.loads((root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))) - decisions = (root / "PHASE_4_DECISIONS.md").read_text(encoding="utf-8") - if PHASE4_STEP3_APPROVAL not in decisions or PHASE4_STEP2_FINAL not in decisions: - raise ValueError("Actual Step 3 authorization or Step 2 evidence anchor missing") - if any((root / name).exists() for name in PHASE4_FORBIDDEN_OUTPUTS): - raise ValueError("Step 3 cannot create Phase 4 completion records or audit outputs") - import runpy - checker = runpy.run_path(str(root / "scripts/check_traceability.py"), run_name="phase4_step3_assembly_boundary") - checker["phase4_step2_result_boundary"](root / "src/recursive_integrity_toolkit/result.py") - checker["phase4_step3_assembly_boundary"](root / "src/recursive_integrity_toolkit/reports/assembly.py") - schema_bytes = (root / "schemas/report.schema.json").read_bytes() - if hashlib.sha256(schema_bytes).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Step 3 report schema differs from its independently reviewed bytes") - schema = json.loads(schema_bytes) - if (schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema" - or schema.get("type") != "object" or schema.get("additionalProperties") is not False): - raise ValueError("Step 3 report schema dialect/closed root mismatch") - return {"package_modules": 40, "frozen_runtime_modules": 39, "frozen_schemas": 5, - "hero_files_unchanged": 6, "historical_phase3_migrated_nodes": 16, - "phase_complete": False, "result_contracts_enabled": True, - "adapters_enabled": True, "cli_analysis_enabled": False} - - -def audit_phase4_step3(step: int = 3) -> dict: - verify_phase4_step3_control(json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")), step) - result = {"phase": 4, "active_step": step, **verify_phase4_step3_snapshot(), - "phase0_hashes_verified": 16, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_phase4_step3_diff(step: int = 3) -> dict: - if type(step) is not int or step != 3: - raise ValueError("Unsupported Phase 4 Step 3 stage") - _phase4_step2_files() - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE4_STEP2_FINAL, "HEAD"], check=True) - raw = git("diff", "--name-status", "--no-renames", "-z", PHASE4_STEP2_FINAL, "--").split(b"\0") - raw = [part.decode("utf-8") for part in raw if part] - if len(raw) % 2: - raise ValueError("Malformed Step 3 Git difference records") - changes = list(zip(raw[::2], raw[1::2])) - changes.extend(("A", p.decode()) for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") if p) - verify_phase4_step3_changes(changes) - result = {"previous_step_commit": PHASE4_STEP2_FINAL, "changed_files": len(changes), - "changes": changes, "step": 3, "scope": "PASS"} - print(json.dumps(result, indent=2)) - return result - - -def phase4_step3_baseline_evidence(output: Path) -> dict: - """Reconcile all inherited identities, including the accepted Step 2 suite.""" - output.mkdir(parents=True, exist_ok=True) - inherited = phase4_step2_baseline_evidence(output / "phase4-step2-inherited") - with tempfile.TemporaryDirectory(prefix="rit-p4-step2-identities-") as temp: - baseline = Path(temp) - for name, raw in _phase4_step2_files().items(): - target = baseline / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(raw) - old, old_log = _collect(baseline) - current, current_log = _collect(ROOT) - expected = 2731 if os.environ.get("RIT_TEST_PARQUET") == "1" else 2728 - if len(old) != expected or set(old) - set(current): - raise ValueError("Accepted Phase 4 Step 2 test identities were lost") - result = {"baseline_commit": PHASE4_STEP2_FINAL, "baseline_test_tree": PHASE4_STEP2_TEST_TREE, - "baseline_nodeids": old, "current_nodeids": current, "missing_nodeids": [], - "baseline_tests": len(old), "current_tests": len(current), "inherited": inherited, - "nodeids_sha256": hashlib.sha256(("\n".join(old)+"\n").encode()).hexdigest()} - (output / "phase4_step3_test_identity_manifest.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4-step2-collection.log").write_text(old_log, encoding="utf-8") - (output / "phase4-step3-collection.log").write_text(current_log, encoding="utf-8") - print(json.dumps({k: result[k] for k in ("baseline_tests", "current_tests", "missing_nodeids")}, indent=2)) - return result - - -def phase4_step3_candidate(output: Path, step: int = 3) -> None: - """Build tested Step 3 intermediate evidence without declaring Phase 4 complete.""" - if os.environ.get("RIT_TEST_PARQUET") != "1" or importlib.util.find_spec("pyarrow") is None: - raise ValueError("Step 3 candidate evidence requires RIT_TEST_PARQUET=1 and real PyArrow") - output.mkdir(parents=True, exist_ok=True) - result = audit_phase4_step3(step) - result["diff"] = audit_phase4_step3_diff(step) - if git("status", "--porcelain").strip(): - raise ValueError("Step 3 candidate archive requires committed, clean source") - result["core"] = verify_junit(output / "core.xml", minimum=2728) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=2731) - result["core_math_measurements"] = verify_step10_evidence(output / "core.xml", output / "phase4-step3-core-observations.json") - result["parquet_math_measurements"] = verify_step10_evidence(output / "parquet.xml", output / "phase4-step3-parquet-observations.json") - identity = phase4_step3_baseline_evidence(output) - result["test_identity"] = {k: identity[k] for k in ("baseline_tests", "current_tests", "nodeids_sha256")} - expected = set() - for node in identity["current_nodeids"]: - base, bracket, parameter = node.partition("[") - owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) - for name, parquet in (("core.xml", False), ("parquet.xml", True)): - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(output / name).getroot().iter("testcase")} - target = expected if parquet else {c for c in expected if c[1] not in PARQUET_CASES} - if cases != target: - raise ValueError(f"Step 3 JUnit does not execute the entire current suite: {name}") - wheel, _ = verify_distributions(output / "dist") - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - archive = output / "recursive-integrity-toolkit-phase4-step3-candidate.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {p for p in git("ls-files", "-z").decode().split("\0") if p} - with zipfile.ZipFile(archive) as zipped: - names = {p.removeprefix("recursive-integrity-toolkit/") for p in zipped.namelist() if not p.endswith("/")} - if names != tracked or len(tracked) != 227: - raise ValueError("Step 3 source archive file set mismatch") - for name in names: - if zipped.read("recursive-integrity-toolkit/"+name) != (ROOT / name).read_bytes(): - raise ValueError(f"Step 3 source archive byte mismatch: {name}") - result.update({"commit": git("rev-parse", "HEAD").decode().strip(), - "tree": git("rev-parse", "HEAD^{tree}").decode().strip(), - "source_archive": archive.name, "tracked_files": len(tracked), - "python": sys.version, "platform": sys.platform, - "versions": {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "pytest")}}) - (output / "phase4_step3_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4_step3_repository_files.sha256").write_text("".join(f"{hashlib.sha256((ROOT/name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - paths = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase4_step3_artifacts.sha256") - (output / "phase4_step3_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in paths), encoding="utf-8") - print(f"Phase 4 Step 3 candidate: {len(tracked)} source files verified; Phase 4 remains incomplete.") - - -def phase4_step3_cli_main() -> int: - parser = argparse.ArgumentParser(description="Phase 4 Step 3 maintainer gate") - parser.add_argument("--phase", type=int, choices=(4,), required=True) - parser.add_argument("--step", type=int, choices=(3,), required=True) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) - parser.add_argument("--require-parquet", action="store_true") - parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - raise ValueError("Phase 4 Step 3 cannot certify a final phase delivery") - if args.junit is not None: - verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - phase4_step3_baseline_evidence(args.baseline_evidence) - elif args.dist is not None: - audit_phase4_step3(args.step) - wheel, _ = verify_distributions(args.dist) - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - elif args.smoke_wheel is not None: - smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - phase4_step3_candidate(args.candidate, args.step) - else: - audit_phase4_step3(args.step) - if args.diff: - audit_phase4_step3_diff(args.step) - return 0 - - -def phase4_step3_installed_assembly_smoke(wheel: Path) -> None: +def smoke_installed_assembly(wheel: Path) -> None: """Exercise an installed, hand-authored typed handoff with every effect blocked.""" with tempfile.TemporaryDirectory(prefix="rit-p4-step3-installed-assembly-") as temp: work = Path(temp) @@ -2773,607 +647,7 @@ def deny_effects(event, arguments): subprocess.run([str(python), "-I", "-c", program, str(work), str(ROOT)], cwd=work, check=True) -# Phase 4 Step 4 adds explicit privacy views without changing inherited gates. -PHASE4_STEP3_FINAL = "974545c456e535ba1e1c5b6bf4ae0ce25bc04b57" -PHASE4_STEP3_TREE = "15498f3f00d0ce048c3cba2156eb7a44f096ae88" -PHASE4_STEP3_TEST_TREE = "17b87858f77ee4413c98cfd76f16f52170246741" -PHASE4_STEP4_APPROVAL = "继续" -PHASE4_STEP4_APPROVAL_DATE = "2026-09-19" -PHASE4_STEP4_NEW = () -PHASE4_STEP4_MIGRATIONS = {'tests/integration/test_phase4_gates.py': [{'new': 'def ' - 'test_phase4_step3_current_runtime_opens_only_assembly_and_freezes_schema(repo_root, ' - 'phase4_step2_snapshot, ' - 'phase4_step3_snapshot):\n' - ' repo_root = ' - 'phase4_step3_snapshot\n', - 'node': 'test_phase4_step3_current_runtime_opens_only_assembly_and_freezes_schema', - 'old': 'def ' - 'test_phase4_step3_current_runtime_opens_only_assembly_and_freezes_schema(repo_root, ' - 'phase4_step2_snapshot):\n'}, - {'new': 'def ' - 'test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step3_snapshot):\n' - ' repo_root = ' - 'phase4_step3_snapshot\n', - 'node': 'test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch', - 'old': 'def ' - 'test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root):\n'}, - {'new': 'def ' - 'test_phase4_step3_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, ' - 'phase4_gate_tools, ' - 'mutation, ' - 'phase4_step3_snapshot):\n', - 'node': 'test_phase4_step3_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step3_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, ' - 'phase4_gate_tools, ' - 'mutation):\n'}, - {'new': ' ' - 'shutil.copyfile(phase4_step3_snapshot ' - '/ relative, ' - 'destination)\n', - 'node': 'test_phase4_step3_current_snapshot_checks_valid_tree_before_mutations', - 'old': ' ' - 'shutil.copyfile(repo_root ' - '/ relative, ' - 'destination)\n'}], - 'tests/unit/test_phase4_contracts.py': [{'new': 'def ' - 'test_phase4_step3_approved_control_keeps_independent_step2_anchors(phase4_tools, ' - 'phase4_step3_snapshot):\n' - ' control = ' - 'json.loads((phase4_step3_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step3_approved_control_keeps_independent_step2_anchors', - 'old': 'def ' - 'test_phase4_step3_approved_control_keeps_independent_step2_anchors(phase4_tools):\n' - ' control = ' - 'json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step3_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value, ' - 'phase4_step3_snapshot):\n' - ' control = ' - 'json.loads((phase4_step3_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step3_control_rejects_forged_scope_and_stage', - 'old': 'def ' - 'test_phase4_step3_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value):\n' - ' control = ' - 'json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step3_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step, ' - 'phase4_step3_snapshot):\n' - ' control = ' - 'json.loads((phase4_step3_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step3_control_rejects_unapproved_dispatch', - 'old': 'def ' - 'test_phase4_step3_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step):\n' - ' control = ' - 'json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step3_control_requires_explicit_approval_fields(phase4_tools, ' - 'field, ' - 'phase4_step3_snapshot):\n' - ' control = ' - 'json.loads((phase4_step3_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step3_control_requires_explicit_approval_fields', - 'old': 'def ' - 'test_phase4_step3_control_requires_explicit_approval_fields(phase4_tools, ' - 'field):\n' - ' control = ' - 'json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step3_control_cannot_mint_extra_permission(phase4_tools, ' - 'phase4_step3_snapshot):\n' - ' control = ' - 'json.loads((phase4_step3_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step3_control_cannot_mint_extra_permission', - 'old': 'def ' - 'test_phase4_step3_control_cannot_mint_extra_permission(phase4_tools):\n' - ' control = ' - 'json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step3_historical_migrations_preserve_ten_step2_gate_nodes(phase4_tools, ' - 'phase4_step2_snapshot, ' - 'phase4_step3_snapshot):\n', - 'node': 'test_phase4_step3_historical_migrations_preserve_ten_step2_gate_nodes', - 'old': 'def ' - 'test_phase4_step3_historical_migrations_preserve_ten_step2_gate_nodes(phase4_tools, ' - 'phase4_step2_snapshot):\n'}, - {'new': ' path, ' - '(phase4_step2_snapshot / ' - 'path).read_bytes(), ' - '(phase4_step3_snapshot / ' - 'path).read_bytes(),\n', - 'node': 'test_phase4_step3_historical_migrations_preserve_ten_step2_gate_nodes', - 'old': ' path, ' - '(phase4_step2_snapshot / ' - 'path).read_bytes(), (ROOT / ' - 'path).read_bytes(),\n'}, - {'new': 'def ' - 'test_phase4_step3_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, ' - 'phase4_step2_snapshot, ' - 'mutation, ' - 'phase4_step3_snapshot):\n', - 'node': 'test_phase4_step3_historical_guard_rejects_assertion_and_binding_weakening', - 'old': 'def ' - 'test_phase4_step3_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, ' - 'phase4_step2_snapshot, ' - 'mutation):\n'}, - {'new': ' after = ' - '(phase4_step3_snapshot / ' - 'path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step3_test_migration"]\n', - 'node': 'test_phase4_step3_historical_guard_rejects_assertion_and_binding_weakening', - 'old': ' after = (ROOT / ' - 'path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step3_test_migration"]\n'}]} -PHASE4_STEP4_ALLOWED = {'.github/workflows/ci.yml', - '.github/workflows/golden.yml', - '.github/workflows/release.yml', - '.github/workflows/security.yml', - 'PHASE_4_BASELINE.json', - 'PHASE_4_DECISIONS.md', - 'docs/architecture.md', - 'docs/privacy.md', - 'docs/report_schema.md', - 'docs/theory_traceability.md', - 'scripts/check_spec_consistency.py', - 'scripts/check_traceability.py', - 'scripts/release_check.py', - 'src/recursive_integrity_toolkit/config.py', - 'src/recursive_integrity_toolkit/reports/assembly.py', - 'src/recursive_integrity_toolkit/result.py', - 'src/recursive_integrity_toolkit/utils/hashing.py', - 'src/recursive_integrity_toolkit/utils/logging.py', - 'tests/conftest.py', - 'tests/integration/test_ci_workflows.py', - 'tests/integration/test_hero_structure.py', - 'tests/integration/test_license_notices.py', - 'tests/integration/test_no_algorithms.py', - 'tests/integration/test_no_network.py', - 'tests/integration/test_optional_dependency.py', - 'tests/integration/test_owner_ids.py', - 'tests/integration/test_package_import.py', - 'tests/integration/test_package_install.py', - 'tests/integration/test_phase4_gates.py', - 'tests/integration/test_prohibited_structure.py', - 'tests/integration/test_repository_structure.py', - 'tests/integration/test_schema_json.py', - 'tests/unit/test_PR015_redaction.py', - 'tests/unit/test_PR016_determinism.py', - 'tests/unit/test_PR018_language.py', - 'tests/unit/test_phase4_contracts.py'} - - -def _phase4_step3_files(): - """Read the exact accepted Step 3 Git tree and verify every blob identity.""" - for suffix, expected in (("^{commit}", PHASE4_STEP3_FINAL), ("^{tree}", PHASE4_STEP3_TREE), - (":tests", PHASE4_STEP3_TEST_TREE)): - if git("rev-parse", PHASE4_STEP3_FINAL + suffix).decode().strip() != expected: - raise ValueError("Pinned Phase 4 Step 3 identity mismatch") - objects = {} - for entry in git("ls-tree", "-rz", PHASE4_STEP3_FINAL).split(b"\0"): - if not entry: - continue - metadata, raw_path = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - path = raw_path.decode("utf-8") - if (mode not in (b"100644", b"100755") or kind != b"blob" or path in objects - or path.startswith("/") or ".." in path.split("/") or ".git" in path.split("/")): - raise ValueError("Unsafe pinned Step 3 Git object") - objects[path] = oid.decode("ascii") - if len(objects) != 227: - raise ValueError("Pinned Step 3 must contain exactly 227 files") - files = {} - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_STEP3_FINAL))) as archive: - names = [item.filename for item in archive.infolist() if not item.is_dir()] - if len(names) != len(objects) or set(names) != set(objects): - raise ValueError("Pinned Step 3 archive identity mismatch") - for name in names: - raw = archive.read(name) - oid = hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - if oid != objects[name]: - raise ValueError(f"Pinned Step 3 blob mismatch: {name}") - files[name] = raw - return MappingProxyType(files) - - -def phase4_step4_expected_control() -> dict: - """Actual Step 4 approval is independent of the mutable control document.""" - prior = _phase4_step3_files() - result = phase4_step3_expected_control() - result.update({ - "control_version": "1.3", "active_step": 4, - "approval_date": PHASE4_STEP4_APPROVAL_DATE, "approval_basis": PHASE4_STEP4_APPROVAL, - "previous_step_commit": PHASE4_STEP3_FINAL, - "previous_step_tree": PHASE4_STEP3_TREE, - "previous_step_test_tree": PHASE4_STEP3_TEST_TREE, - "previous_step_core_tests": 2876, "previous_step_parquet_tests": 2879, - "previous_step_files_sha256": {p: hashlib.sha256(raw).hexdigest() for p, raw in sorted(prior.items())}, - "permitted_paths": sorted(PHASE4_STEP4_ALLOWED), "new_files_permitted": [], - "runtime_changes_authorized": True, "schema_changes_authorized": False, - "runtime_paths_authorized": sorted(p for p in PHASE4_STEP4_ALLOWED if p.startswith("src/")), - "schema_paths_authorized": [], - "step3_historical_binding_nodes": {p: sorted({r["node"] for r in rows}) - for p, rows in sorted(PHASE4_STEP4_MIGRATIONS.items())}, - }) - return result - - -def verify_phase4_step4_control(control: dict, step: int = 4) -> None: - if type(step) is not int or step != 4 or type(control) is not dict: - raise ValueError("Unsupported Phase 4 Step 4 stage/control") - try: - actual = json.dumps(control, sort_keys=True, ensure_ascii=True, allow_nan=False) - expected = json.dumps(phase4_step4_expected_control(), sort_keys=True, ensure_ascii=True, allow_nan=False) - except (TypeError, ValueError) as error: - raise ValueError("Invalid Phase 4 Step 4 control") from error - if actual != expected: - raise ValueError("Phase 4 control differs from the independently approved Step 4 contract") - - -def verify_phase4_step4_changes(changes: list[tuple[str, str]]) -> None: - seen = set() - for status, path in changes: - if path in seen or path not in PHASE4_STEP4_ALLOWED or status != "M": - raise ValueError(f"Unapproved Phase 4 Step 4 path/operation: {status} {path}") - seen.add(path) - - -def _phase4_step4_header(node, path): - """Allow explicit test parametrization/fixtures, never definition-time effects.""" - for decorator in node.decorator_list: - if not isinstance(decorator, ast.Call) or ast.unparse(decorator.func) not in { - "pytest.fixture", "pytest.mark.parametrize"}: - raise ValueError(f"Unapproved Step 4 test decorator: {path}:{node.name}") - if ast.unparse(decorator.func) == "pytest.fixture" and not node.name.startswith("phase4_step"): - raise ValueError("A Step 4 fixture must have an explicit scoped name") - if any(keyword.arg is None for keyword in decorator.keywords): - raise ValueError("Decorator expansion is outside the Step 4 contract") - for value in [*decorator.args, *(keyword.value for keyword in decorator.keywords)]: - try: - ast.literal_eval(value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError("Step 4 decorator arguments must be literal") from error - clone = ast.parse(ast.unparse(node)).body[0] - clone.decorator_list = [] - _phase4_preserve_function_header(clone, path) - - -def _phase4_step4_append_only(before: bytes, after: bytes, path: str) -> None: - if not after.startswith(before): - raise ValueError(f"Inherited Step 3 prefix changed: {path}") - old, new = ast.parse(before), ast.parse(after) - bindings = set() - for node in old.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - bindings.add(node.name) - elif isinstance(node, (ast.Import, ast.ImportFrom)): - bindings.update(alias.asname or alias.name.split(".")[0] for alias in node.names) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - bindings.update(child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)) - for node in new.body[len(old.body):]: - if not isinstance(node, ast.FunctionDef) or node.name in bindings: - raise ValueError(f"Step 4 addition executes, rebinds or shadows inherited source: {path}") - allowed_snapshot = path == "tests/conftest.py" and node.name == "phase4_step3_snapshot" - if not allowed_snapshot and not node.name.startswith(("test_phase4_step4_", "phase4_step4_")): - raise ValueError(f"Step 4 added test/helper is not explicitly scoped: {path}:{node.name}") - if path == "tests/conftest.py" and node.name != "phase4_step3_snapshot": - raise ValueError("Only the pinned Step 3 shared fixture is authorized") - bindings.add(node.name) - _phase4_step4_header(node, path) - - -def verify_phase4_step4_test_migration(path: str, before: bytes, after: bytes) -> None: - prior = _phase4_step3_files() - if path not in prior or path not in PHASE4_STEP4_ALLOWED or before != prior[path] or not path.startswith("tests/"): - raise ValueError("Step 4 migration requires the exact named Step 3 source") - _phase4_step4_check_test_migration(path, before, after) - - -def _phase4_step4_check_test_migration(path: str, before: bytes, after: bytes) -> None: - """Check a source already read from the verified Step 3 immutable mapping.""" - expected = before - for row in PHASE4_STEP4_MIGRATIONS.get(path, []): - old, new = row["old"].encode(), row["new"].encode() - if expected.count(old) != 1: - raise ValueError("Step 3 historical binding is not unique") - expected = expected.replace(old, new, 1) - _phase4_step4_append_only(expected, after, path) - - -def _phase4_step4_preserve_tooling(before: bytes, after: bytes, path: str) -> None: - """Preserve exact inherited statements using one source-line split per tree.""" - if path == "scripts/release_check.py": - old = ' return phase4_cli_main() if explicit_phase4 else main()' - new = (' explicit_step4 = "--step=4" in argv or any(a == "--step" and b == "4" for a, b in zip(argv, argv[1:]))\n' - ' if explicit_phase4 and explicit_step4:\n' - ' return phase4_step4_cli_main()\n' + old) - # Earlier preserved dispatch functions contain this source too; bind cli_main only. - parsed = ast.parse(before) - entry = next(node for node in parsed.body if isinstance(node, ast.FunctionDef) and node.name == "cli_main") - old_entry = ast.get_source_segment(before.decode(), entry) - if old_entry.count(old) != 1: - raise ValueError("Step 3 release dispatcher identity mismatch") - replacements = [(old_entry, old_entry.replace(old, new, 1))] - else: - old = ' if args.phase == 4 and args.step == 3:\n return phase4_step3_main(step=args.step)\n' - new = old + ' if args.phase == 4 and args.step == 4:\n return phase4_step4_main(step=args.step)\n' - replacements = [(old, new), - ('Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3', - 'Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4')] - expected = before - for old, new in replacements: - if expected.count(old.encode()) != 1: - raise ValueError(f"Historical dispatcher identity mismatch: {path}") - expected = expected.replace(old.encode(), new.encode(), 1) - trees = [ast.parse(expected), ast.parse(after)] - lines = [expected.splitlines(keepends=True), after.splitlines(keepends=True)] - def segment(index, node): - if node.lineno == node.end_lineno: - return lines[index][node.lineno - 1][node.col_offset:node.end_col_offset] - return (lines[index][node.lineno - 1][node.col_offset:] - + b"".join(lines[index][node.lineno:node.end_lineno - 1]) - + lines[index][node.end_lineno - 1][:node.end_col_offset]) - def entry_guard(node): - return isinstance(node, ast.If) and isinstance(node.test, ast.Compare) and "__name__" in ast.unparse(node.test) - guards = [n for n in trees[1].body if entry_guard(n)] - expected_guard = ast.parse('if __name__ == "__main__":\n raise SystemExit(cli_main())').body[0] - if len(guards) != 1 or ast.dump(guards[0]) != ast.dump(expected_guard): - raise ValueError(f"Unapproved Step 4 maintainer entrypoint: {path}") - historical = [n for n in trees[0].body if not entry_guard(n)] - current = [n for n in trees[1].body if not entry_guard(n)] - old_entries = [(segment(0, n), ast.dump(n)) for n in historical] - cursor = 0 - for node in current: - if cursor < len(old_entries) and (segment(1, node), ast.dump(node)) == old_entries[cursor]: - cursor += 1 - continue - if isinstance(node, ast.FunctionDef) and (node.name == "_phase4_step3_files" or node.name.startswith( - ("phase4_step4_", "_phase4_step4_", "verify_phase4_step4_", "audit_phase4_step4"))): - _phase4_preserve_function_header(node, path) - continue - if isinstance(node, ast.Assign) and all(isinstance(target, ast.Name) and target.id.startswith( - ("PHASE4_STEP4_", "PHASE4_STEP3_")) for target in node.targets): - try: - ast.literal_eval(node.value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError(f"Nonliteral added Step 4 maintainer constant: {path}") from error - continue - raise ValueError(f"Unapproved Step 4 maintainer addition: {path}") - if cursor != len(historical): - raise ValueError(f"Inherited Step 3 maintainer statement changed: {path}") - def binding_counts(tree): - counts = {} - for node in tree.body: - names = [] - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - names = [node.name] - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names = [child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)] - elif isinstance(node, (ast.Import, ast.ImportFrom)): - names = [alias.asname or alias.name.split(".")[0] for alias in node.names] - for name in names: - counts[name] = counts.get(name, 0) + 1 - return counts - inherited_counts = binding_counts(trees[0]) - for name, count in binding_counts(trees[1]).items(): - if count > inherited_counts.get(name, 1): - raise ValueError(f"Duplicate Step 4 maintainer binding: {path}:{name}") - - -def verify_phase4_step4_snapshot(root: Path = ROOT) -> dict: - prior = _phase4_step3_files() - for path, raw in prior.items(): - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError(f"Missing or aliased inherited Step 3 file: {path}") - current = target.read_bytes() - if path not in PHASE4_STEP4_ALLOWED and current != raw: - raise ValueError(f"Protected Step 3 bytes changed: {path}") - if path.startswith("tests/") and path in PHASE4_STEP4_ALLOWED: - _phase4_step4_check_test_migration(path, raw, current) - elif path.startswith("scripts/") and path in PHASE4_STEP4_ALLOWED: - _phase4_step4_preserve_tooling(raw, current, path) - elif path in {"docs/architecture.md", "docs/theory_traceability.md", "docs/report_schema.md", "docs/privacy.md", "PHASE_4_DECISIONS.md"}: - if not current.startswith(raw): - raise ValueError(f"Step 3 historical documentation prefix changed: {path}") - actual_modules = {p.relative_to(root).as_posix() for p in (root / "src/recursive_integrity_toolkit").rglob("*.py")} - expected_modules = {p for p in prior if p.startswith("src/") and p.endswith(".py")} - if actual_modules != expected_modules or len(actual_modules) != 40: - raise ValueError("Step 4 cannot change the runtime module set") - if {p.name for p in (root / "schemas").iterdir()} != {Path(p).name for p in prior if p.startswith("schemas/")}: - raise ValueError("Step 4 cannot change the schema set") - if hashlib.sha256((root / "PHASE_4_PLAN.md").read_bytes()).hexdigest() != PHASE4_PLAN_SHA256: - raise ValueError("Approved Phase 4 plan changed") - verify_phase4_step4_control(json.loads((root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))) - decisions = (root / "PHASE_4_DECISIONS.md").read_text(encoding="utf-8") - if PHASE4_STEP4_APPROVAL not in decisions or PHASE4_STEP3_FINAL not in decisions: - raise ValueError("Actual Step 4 authorization or Step 3 evidence anchor missing") - if any((root / name).exists() for name in PHASE4_FORBIDDEN_OUTPUTS): - raise ValueError("Step 4 cannot create Phase 4 completion records or audit outputs") - import runpy - checker = runpy.run_path(str(root / "scripts/check_traceability.py"), run_name="phase4_step4_assembly_boundary") - checker["phase4_step4_privacy_boundary"](root) - schema_bytes = (root / "schemas/report.schema.json").read_bytes() - if hashlib.sha256(schema_bytes).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Step 4 report schema differs from its independently reviewed bytes") - schema = json.loads(schema_bytes) - if (schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema" - or schema.get("type") != "object" or schema.get("additionalProperties") is not False): - raise ValueError("Step 4 report schema dialect/closed root mismatch") - return {"package_modules": 40, "frozen_runtime_modules": 35, "frozen_schemas": 5, - "hero_files_unchanged": 6, "historical_phase3_migrated_nodes": 16, - "phase_complete": False, "result_contracts_enabled": True, - "adapters_enabled": True, "privacy_views_enabled": True, "cli_analysis_enabled": False} - - -def audit_phase4_step4(step: int = 4) -> dict: - verify_phase4_step4_control(json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")), step) - result = {"phase": 4, "active_step": step, **verify_phase4_step4_snapshot(), - "phase0_hashes_verified": 16, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_phase4_step4_diff(step: int = 4) -> dict: - if type(step) is not int or step != 4: - raise ValueError("Unsupported Phase 4 Step 4 stage") - _phase4_step3_files() - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE4_STEP3_FINAL, "HEAD"], check=True) - raw = git("diff", "--name-status", "--no-renames", "-z", PHASE4_STEP3_FINAL, "--").split(b"\0") - raw = [part.decode("utf-8") for part in raw if part] - if len(raw) % 2: - raise ValueError("Malformed Step 4 Git difference records") - changes = list(zip(raw[::2], raw[1::2])) - changes.extend(("A", p.decode()) for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") if p) - verify_phase4_step4_changes(changes) - result = {"previous_step_commit": PHASE4_STEP3_FINAL, "changed_files": len(changes), - "changes": changes, "step": 4, "scope": "PASS"} - print(json.dumps(result, indent=2)) - return result - - -def phase4_step4_baseline_evidence(output: Path) -> dict: - """Reconcile all inherited identities, including the accepted Step 3 suite.""" - output.mkdir(parents=True, exist_ok=True) - inherited = phase4_step3_baseline_evidence(output / "phase4-step3-inherited") - with tempfile.TemporaryDirectory(prefix="rit-p4-step3-identities-") as temp: - baseline = Path(temp) - for name, raw in _phase4_step3_files().items(): - target = baseline / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(raw) - old, old_log = _collect(baseline) - current, current_log = _collect(ROOT) - expected = 2879 if os.environ.get("RIT_TEST_PARQUET") == "1" else 2876 - if len(old) != expected or set(old) - set(current): - raise ValueError("Accepted Phase 4 Step 3 test identities were lost") - result = {"baseline_commit": PHASE4_STEP3_FINAL, "baseline_test_tree": PHASE4_STEP3_TEST_TREE, - "baseline_nodeids": old, "current_nodeids": current, "missing_nodeids": [], - "baseline_tests": len(old), "current_tests": len(current), "inherited": inherited, - "nodeids_sha256": hashlib.sha256(("\n".join(old)+"\n").encode()).hexdigest()} - (output / "phase4_step4_test_identity_manifest.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4-step3-collection.log").write_text(old_log, encoding="utf-8") - (output / "phase4-step4-collection.log").write_text(current_log, encoding="utf-8") - print(json.dumps({k: result[k] for k in ("baseline_tests", "current_tests", "missing_nodeids")}, indent=2)) - return result - - -def phase4_step4_candidate(output: Path, step: int = 4) -> None: - """Build tested Step 4 intermediate evidence without declaring Phase 4 complete.""" - if os.environ.get("RIT_TEST_PARQUET") != "1" or importlib.util.find_spec("pyarrow") is None: - raise ValueError("Step 4 candidate evidence requires RIT_TEST_PARQUET=1 and real PyArrow") - output.mkdir(parents=True, exist_ok=True) - result = audit_phase4_step4(step) - result["diff"] = audit_phase4_step4_diff(step) - if git("status", "--porcelain").strip(): - raise ValueError("Step 4 candidate archive requires committed, clean source") - result["core"] = verify_junit(output / "core.xml", minimum=2876) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=2879) - result["core_math_measurements"] = verify_step10_evidence(output / "core.xml", output / "phase4-step4-core-observations.json") - result["parquet_math_measurements"] = verify_step10_evidence(output / "parquet.xml", output / "phase4-step4-parquet-observations.json") - identity = phase4_step4_baseline_evidence(output) - result["test_identity"] = {k: identity[k] for k in ("baseline_tests", "current_tests", "nodeids_sha256")} - expected = set() - for node in identity["current_nodeids"]: - base, bracket, parameter = node.partition("[") - owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) - for name, parquet in (("core.xml", False), ("parquet.xml", True)): - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(output / name).getroot().iter("testcase")} - target = expected if parquet else {c for c in expected if c[1] not in PARQUET_CASES} - if cases != target: - raise ValueError(f"Step 4 JUnit does not execute the entire current suite: {name}") - wheel, _ = verify_distributions(output / "dist") - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - archive = output / "recursive-integrity-toolkit-phase4-step4-candidate.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {p for p in git("ls-files", "-z").decode().split("\0") if p} - with zipfile.ZipFile(archive) as zipped: - names = {p.removeprefix("recursive-integrity-toolkit/") for p in zipped.namelist() if not p.endswith("/")} - if names != tracked or len(tracked) != 227: - raise ValueError("Step 4 source archive file set mismatch") - for name in names: - if zipped.read("recursive-integrity-toolkit/"+name) != (ROOT / name).read_bytes(): - raise ValueError(f"Step 4 source archive byte mismatch: {name}") - result.update({"commit": git("rev-parse", "HEAD").decode().strip(), - "tree": git("rev-parse", "HEAD^{tree}").decode().strip(), - "source_archive": archive.name, "tracked_files": len(tracked), - "python": sys.version, "platform": sys.platform, - "versions": {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "pytest")}}) - (output / "phase4_step4_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4_step4_repository_files.sha256").write_text("".join(f"{hashlib.sha256((ROOT/name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - paths = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase4_step4_artifacts.sha256") - (output / "phase4_step4_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in paths), encoding="utf-8") - print(f"Phase 4 Step 4 candidate: {len(tracked)} source files verified; Phase 4 remains incomplete.") - - -def phase4_step4_cli_main() -> int: - parser = argparse.ArgumentParser(description="Phase 4 Step 4 maintainer gate") - parser.add_argument("--phase", type=int, choices=(4,), required=True) - parser.add_argument("--step", type=int, choices=(4,), required=True) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) - parser.add_argument("--require-parquet", action="store_true") - parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - raise ValueError("Phase 4 Step 4 cannot certify a final phase delivery") - if args.junit is not None: - verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - phase4_step4_baseline_evidence(args.baseline_evidence) - elif args.dist is not None: - audit_phase4_step4(args.step) - wheel, _ = verify_distributions(args.dist) - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - elif args.smoke_wheel is not None: - smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - phase4_step4_candidate(args.candidate, args.step) - else: - audit_phase4_step4(args.step) - if args.diff: - audit_phase4_step4_diff(args.step) - return 0 - - -def phase4_step4_installed_privacy_smoke(wheel: Path) -> None: +def smoke_installed_privacy(wheel: Path) -> None: """Exercise installed privacy, config and diagnostics after accepted calculation.""" with tempfile.TemporaryDirectory(prefix="rit-p4-step4-installed-privacy-") as temp: work = Path(temp) @@ -3636,591 +910,7 @@ def deny_effects(event, arguments): subprocess.run([str(python), "-I", "-c", program, str(work), str(ROOT)], cwd=work, check=True) -# Phase 4 Step 5 adds pure renderers while retaining every inherited gate. - - -PHASE4_STEP4_FINAL = "49454a9b162cb8d35e38d1cb1ae32cb208d3a01c" - - -PHASE4_STEP4_TREE = "79264d845e08f73ee68e160dea04f7786f18fd0a" - - -PHASE4_STEP4_TEST_TREE = "4aa4ca0b22f8b8c25afe27feeaba713d6f0830bf" - - -PHASE4_STEP5_APPROVAL = "很好，Phase 4 Step 5 继续" - - -PHASE4_STEP5_APPROVAL_DATE = "2026-09-20" - - -PHASE4_STEP5_NEW = () - - -PHASE4_STEP5_MIGRATIONS = {'tests/integration/test_phase4_gates.py': [{'new': 'def ' - 'test_phase4_step4_current_runtime_opens_only_privacy_metadata_modules_and_freezes_schema(repo_root, ' - 'phase4_step3_snapshot, phase4_step4_snapshot):\n' - ' repo_root = phase4_step4_snapshot\n', - 'node': 'test_phase4_step4_current_runtime_opens_only_privacy_metadata_modules_and_freezes_schema', - 'old': 'def ' - 'test_phase4_step4_current_runtime_opens_only_privacy_metadata_modules_and_freezes_schema(repo_root, ' - 'phase4_step3_snapshot):\n'}, - {'new': 'def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\n' - ' repo_root = phase4_step4_snapshot\n', - 'node': 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch', - 'old': 'def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root):\n'}, - {'new': 'def ' - 'test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step4_snapshot):\n', - 'node': 'test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation):\n'}, - {'new': ' shutil.copyfile(phase4_step4_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step4_snapshot"]\n', - 'node': 'test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations', - 'old': ' shutil.copyfile(repo_root / relative, ' - 'destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step4_snapshot"]\n'}], - 'tests/unit/test_phase4_contracts.py': [{'new': 'def ' - 'test_phase4_step4_approved_control_keeps_independent_step3_anchors(phase4_tools, ' - 'phase4_step4_snapshot):\n' - ' control = json.loads((phase4_step4_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step4_approved_control_keeps_independent_step3_anchors', - 'old': 'def ' - 'test_phase4_step4_approved_control_keeps_independent_step3_anchors(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step4_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value, phase4_step4_snapshot):\n' - ' control = json.loads((phase4_step4_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step4_control_rejects_forged_scope_and_stage', - 'old': 'def ' - 'test_phase4_step4_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step4_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step, phase4_step4_snapshot):\n' - ' control = json.loads((phase4_step4_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step4_control_rejects_unapproved_dispatch', - 'old': 'def ' - 'test_phase4_step4_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step4_control_requires_explicit_approval_fields(phase4_tools, ' - 'field, phase4_step4_snapshot):\n' - ' control = json.loads((phase4_step4_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step4_control_requires_explicit_approval_fields', - 'old': 'def ' - 'test_phase4_step4_control_requires_explicit_approval_fields(phase4_tools, ' - 'field):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step4_control_cannot_mint_extra_permission(phase4_tools, ' - 'phase4_step4_snapshot):\n' - ' control = json.loads((phase4_step4_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step4_control_cannot_mint_extra_permission', - 'old': 'def ' - 'test_phase4_step4_control_cannot_mint_extra_permission(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step4_historical_migrations_preserve_ten_step3_gate_nodes(phase4_tools, ' - 'phase4_step3_snapshot, phase4_step4_snapshot):\n', - 'node': 'test_phase4_step4_historical_migrations_preserve_ten_step3_gate_nodes', - 'old': 'def ' - 'test_phase4_step4_historical_migrations_preserve_ten_step3_gate_nodes(phase4_tools, ' - 'phase4_step3_snapshot):\n'}, - {'new': ' path, (phase4_step3_snapshot / ' - 'path).read_bytes(), (phase4_step4_snapshot / ' - 'path).read_bytes(),\n', - 'node': 'test_phase4_step4_historical_migrations_preserve_ten_step3_gate_nodes', - 'old': ' path, (phase4_step3_snapshot / ' - 'path).read_bytes(), (ROOT / path).read_bytes(),\n'}, - {'new': 'def ' - 'test_phase4_step4_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, ' - 'phase4_step3_snapshot, mutation, phase4_step4_snapshot):\n', - 'node': 'test_phase4_step4_historical_guard_rejects_assertion_and_binding_weakening', - 'old': 'def ' - 'test_phase4_step4_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, ' - 'phase4_step3_snapshot, mutation):\n'}, - {'new': ' after = (phase4_step4_snapshot / path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step4_test_migration"]\n', - 'node': 'test_phase4_step4_historical_guard_rejects_assertion_and_binding_weakening', - 'old': ' after = (ROOT / path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step4_test_migration"]\n'}]} - - -PHASE4_STEP5_ALLOWED = {'.github/workflows/ci.yml', - '.github/workflows/golden.yml', - '.github/workflows/release.yml', - '.github/workflows/security.yml', - 'PHASE_4_BASELINE.json', - 'PHASE_4_DECISIONS.md', - 'docs/architecture.md', - 'docs/report_schema.md', - 'docs/theory_traceability.md', - 'scripts/check_spec_consistency.py', - 'scripts/check_traceability.py', - 'scripts/release_check.py', - 'src/recursive_integrity_toolkit/reports/json_report.py', - 'src/recursive_integrity_toolkit/reports/markdown_report.py', - 'tests/conftest.py', - 'tests/integration/test_ci_workflows.py', - 'tests/integration/test_hero_structure.py', - 'tests/integration/test_license_notices.py', - 'tests/integration/test_no_algorithms.py', - 'tests/integration/test_no_network.py', - 'tests/integration/test_optional_dependency.py', - 'tests/integration/test_owner_ids.py', - 'tests/integration/test_package_import.py', - 'tests/integration/test_package_install.py', - 'tests/integration/test_phase4_gates.py', - 'tests/integration/test_prohibited_structure.py', - 'tests/integration/test_repository_structure.py', - 'tests/integration/test_schema_json.py', - 'tests/unit/test_PR013_report_schema.py', - 'tests/unit/test_PR016_determinism.py', - 'tests/unit/test_PR018_language.py', - 'tests/unit/test_phase4_contracts.py'} - - -def _phase4_step4_files(): - """Read the exact accepted Step 4 Git tree and verify every blob identity.""" - for suffix, expected in (("^{commit}", PHASE4_STEP4_FINAL), ("^{tree}", PHASE4_STEP4_TREE), - (":tests", PHASE4_STEP4_TEST_TREE)): - if git("rev-parse", PHASE4_STEP4_FINAL + suffix).decode().strip() != expected: - raise ValueError("Pinned Phase 4 Step 4 identity mismatch") - objects = {} - for entry in git("ls-tree", "-rz", PHASE4_STEP4_FINAL).split(b"\0"): - if not entry: - continue - metadata, raw_path = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - path = raw_path.decode("utf-8") - if (mode not in (b"100644", b"100755") or kind != b"blob" or path in objects - or path.startswith("/") or ".." in path.split("/") or ".git" in path.split("/")): - raise ValueError("Unsafe pinned Step 4 Git object") - objects[path] = oid.decode("ascii") - if len(objects) != 227: - raise ValueError("Pinned Step 4 must contain exactly 227 files") - files = {} - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_STEP4_FINAL))) as archive: - names = [item.filename for item in archive.infolist() if not item.is_dir()] - if len(names) != len(objects) or set(names) != set(objects): - raise ValueError("Pinned Step 4 archive identity mismatch") - for name in names: - raw = archive.read(name) - oid = hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - if oid != objects[name]: - raise ValueError(f"Pinned Step 4 blob mismatch: {name}") - files[name] = raw - return MappingProxyType(files) - - -def phase4_step5_expected_control() -> dict: - """Actual Step 5 approval is independent of the mutable control document.""" - prior = _phase4_step4_files() - result = phase4_step4_expected_control() - result.update({ - "control_version": "1.4", "active_step": 5, - "approval_date": PHASE4_STEP5_APPROVAL_DATE, "approval_basis": PHASE4_STEP5_APPROVAL, - "previous_step_commit": PHASE4_STEP4_FINAL, - "previous_step_tree": PHASE4_STEP4_TREE, - "previous_step_test_tree": PHASE4_STEP4_TEST_TREE, - "previous_step_core_tests": 3071, "previous_step_parquet_tests": 3074, - "previous_step_files_sha256": {p: hashlib.sha256(raw).hexdigest() for p, raw in sorted(prior.items())}, - "permitted_paths": sorted(PHASE4_STEP5_ALLOWED), "new_files_permitted": [], - "runtime_changes_authorized": True, "schema_changes_authorized": False, - "runtime_paths_authorized": sorted(p for p in PHASE4_STEP5_ALLOWED if p.startswith("src/")), - "schema_paths_authorized": [], - "step4_historical_binding_nodes": {p: sorted({r["node"] for r in rows}) - for p, rows in sorted(PHASE4_STEP5_MIGRATIONS.items())}, - }) - return result - - -def verify_phase4_step5_control(control: dict, step: int = 5) -> None: - if type(step) is not int or step != 5 or type(control) is not dict: - raise ValueError("Unsupported Phase 4 Step 5 stage/control") - try: - actual = json.dumps(control, sort_keys=True, ensure_ascii=True, allow_nan=False) - expected = json.dumps(phase4_step5_expected_control(), sort_keys=True, ensure_ascii=True, allow_nan=False) - except (TypeError, ValueError) as error: - raise ValueError("Invalid Phase 4 Step 5 control") from error - if actual != expected: - raise ValueError("Phase 4 control differs from the independently approved Step 5 contract") - - -def verify_phase4_step5_changes(changes: list[tuple[str, str]]) -> None: - seen = set() - for status, path in changes: - if path in seen or path not in PHASE4_STEP5_ALLOWED or status != "M": - raise ValueError(f"Unapproved Phase 4 Step 5 path/operation: {status} {path}") - seen.add(path) - - -def _phase4_step5_header(node, path): - """Allow explicit test parametrization/fixtures, never definition-time effects.""" - for decorator in node.decorator_list: - if not isinstance(decorator, ast.Call) or ast.unparse(decorator.func) not in { - "pytest.fixture", "pytest.mark.parametrize"}: - raise ValueError(f"Unapproved Step 5 test decorator: {path}:{node.name}") - if ast.unparse(decorator.func) == "pytest.fixture" and not node.name.startswith("phase4_step"): - raise ValueError("A Step 5 fixture must have an explicit scoped name") - if any(keyword.arg is None for keyword in decorator.keywords): - raise ValueError("Decorator expansion is outside the Step 5 contract") - for value in [*decorator.args, *(keyword.value for keyword in decorator.keywords)]: - try: - ast.literal_eval(value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError("Step 5 decorator arguments must be literal") from error - clone = ast.parse(ast.unparse(node)).body[0] - clone.decorator_list = [] - _phase4_preserve_function_header(clone, path) - - -def _phase4_step5_append_only(before: bytes, after: bytes, path: str) -> None: - if not after.startswith(before): - raise ValueError(f"Inherited Step 4 prefix changed: {path}") - old, new = ast.parse(before), ast.parse(after) - bindings = set() - for node in old.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - bindings.add(node.name) - elif isinstance(node, (ast.Import, ast.ImportFrom)): - bindings.update(alias.asname or alias.name.split(".")[0] for alias in node.names) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - bindings.update(child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)) - for node in new.body[len(old.body):]: - if not isinstance(node, ast.FunctionDef) or node.name in bindings: - raise ValueError(f"Step 5 addition executes, rebinds or shadows inherited source: {path}") - allowed_snapshot = path == "tests/conftest.py" and node.name == "phase4_step4_snapshot" - if not allowed_snapshot and not node.name.startswith(("test_phase4_step5_", "phase4_step5_")): - raise ValueError(f"Step 5 added test/helper is not explicitly scoped: {path}:{node.name}") - if path == "tests/conftest.py" and node.name != "phase4_step4_snapshot": - raise ValueError("Only the pinned Step 4 shared fixture is authorized") - bindings.add(node.name) - _phase4_step5_header(node, path) - - -def verify_phase4_step5_test_migration(path: str, before: bytes, after: bytes) -> None: - prior = _phase4_step4_files() - if path not in prior or path not in PHASE4_STEP5_ALLOWED or before != prior[path] or not path.startswith("tests/"): - raise ValueError("Step 5 migration requires the exact named Step 4 source") - _phase4_step5_check_test_migration(path, before, after) - - -def _phase4_step5_check_test_migration(path: str, before: bytes, after: bytes) -> None: - """Check a source already read from the verified Step 4 immutable mapping.""" - expected = before - for row in PHASE4_STEP5_MIGRATIONS.get(path, []): - old, new = row["old"].encode(), row["new"].encode() - if expected.count(old) != 1: - raise ValueError("Step 4 historical binding is not unique") - expected = expected.replace(old, new, 1) - _phase4_step5_append_only(expected, after, path) - - -def _phase4_step5_preserve_tooling(before: bytes, after: bytes, path: str) -> None: - """Preserve exact inherited statements using one source-line split per tree.""" - if path == "scripts/release_check.py": - old = ' return phase4_cli_main() if explicit_phase4 else main()' - new = (' explicit_step5 = "--step=5" in argv or any(a == "--step" and b == "5" for a, b in zip(argv, argv[1:]))\n' - ' if explicit_phase4 and explicit_step5:\n' - ' return phase4_step5_cli_main()\n' + old) - # Earlier preserved dispatch functions contain this source too; bind cli_main only. - parsed = ast.parse(before) - entry = next(node for node in parsed.body if isinstance(node, ast.FunctionDef) and node.name == "cli_main") - old_entry = ast.get_source_segment(before.decode(), entry) - if old_entry.count(old) != 1: - raise ValueError("Step 4 release dispatcher identity mismatch") - replacements = [(old_entry, old_entry.replace(old, new, 1))] - else: - old = ' if args.phase == 4 and args.step == 4:\n return phase4_step4_main(step=args.step)\n' - new = old + ' if args.phase == 4 and args.step == 5:\n return phase4_step5_main(step=args.step)\n' - replacements = [(old, new), - ('Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4', - 'Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5')] - expected = before - for old, new in replacements: - if expected.count(old.encode()) != 1: - raise ValueError(f"Historical dispatcher identity mismatch: {path}") - expected = expected.replace(old.encode(), new.encode(), 1) - trees = [ast.parse(expected), ast.parse(after)] - lines = [expected.splitlines(keepends=True), after.splitlines(keepends=True)] - def segment(index, node): - if node.lineno == node.end_lineno: - return lines[index][node.lineno - 1][node.col_offset:node.end_col_offset] - return (lines[index][node.lineno - 1][node.col_offset:] - + b"".join(lines[index][node.lineno:node.end_lineno - 1]) - + lines[index][node.end_lineno - 1][:node.end_col_offset]) - def entry_guard(node): - return isinstance(node, ast.If) and isinstance(node.test, ast.Compare) and "__name__" in ast.unparse(node.test) - guards = [n for n in trees[1].body if entry_guard(n)] - expected_guard = ast.parse('if __name__ == "__main__":\n raise SystemExit(cli_main())').body[0] - if len(guards) != 1 or ast.dump(guards[0]) != ast.dump(expected_guard): - raise ValueError(f"Unapproved Step 5 maintainer entrypoint: {path}") - historical = [n for n in trees[0].body if not entry_guard(n)] - current = [n for n in trees[1].body if not entry_guard(n)] - old_entries = [(segment(0, n), ast.dump(n)) for n in historical] - cursor = 0 - for node in current: - if cursor < len(old_entries) and (segment(1, node), ast.dump(node)) == old_entries[cursor]: - cursor += 1 - continue - if isinstance(node, ast.FunctionDef) and (node.name == "_phase4_step4_files" or node.name.startswith( - ("phase4_step5_", "_phase4_step5_", "verify_phase4_step5_", "audit_phase4_step5"))): - _phase4_preserve_function_header(node, path) - continue - if isinstance(node, ast.Assign) and all(isinstance(target, ast.Name) and target.id.startswith( - ("PHASE4_STEP5_", "PHASE4_STEP4_")) for target in node.targets): - try: - ast.literal_eval(node.value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError(f"Nonliteral added Step 5 maintainer constant: {path}") from error - continue - raise ValueError(f"Unapproved Step 5 maintainer addition: {path}") - if cursor != len(historical): - raise ValueError(f"Inherited Step 4 maintainer statement changed: {path}") - def binding_counts(tree): - counts = {} - for node in tree.body: - names = [] - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - names = [node.name] - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names = [child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)] - elif isinstance(node, (ast.Import, ast.ImportFrom)): - names = [alias.asname or alias.name.split(".")[0] for alias in node.names] - for name in names: - counts[name] = counts.get(name, 0) + 1 - return counts - inherited_counts = binding_counts(trees[0]) - for name, count in binding_counts(trees[1]).items(): - if count > inherited_counts.get(name, 1): - raise ValueError(f"Duplicate Step 5 maintainer binding: {path}:{name}") - - -def verify_phase4_step5_snapshot(root: Path = ROOT) -> dict: - prior = _phase4_step4_files() - for path, raw in prior.items(): - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError(f"Missing or aliased inherited Step 4 file: {path}") - current = target.read_bytes() - if path not in PHASE4_STEP5_ALLOWED and current != raw: - raise ValueError(f"Protected Step 4 bytes changed: {path}") - if path.startswith("tests/") and path in PHASE4_STEP5_ALLOWED: - _phase4_step5_check_test_migration(path, raw, current) - elif path.startswith("scripts/") and path in PHASE4_STEP5_ALLOWED: - _phase4_step5_preserve_tooling(raw, current, path) - elif path in {"docs/architecture.md", "docs/theory_traceability.md", "docs/report_schema.md", "PHASE_4_DECISIONS.md"}: - if not current.startswith(raw): - raise ValueError(f"Step 4 historical documentation prefix changed: {path}") - actual_modules = {p.relative_to(root).as_posix() for p in (root / "src/recursive_integrity_toolkit").rglob("*.py")} - expected_modules = {p for p in prior if p.startswith("src/") and p.endswith(".py")} - if actual_modules != expected_modules or len(actual_modules) != 40: - raise ValueError("Step 5 cannot change the runtime module set") - if {p.name for p in (root / "schemas").iterdir()} != {Path(p).name for p in prior if p.startswith("schemas/")}: - raise ValueError("Step 5 cannot change the schema set") - if hashlib.sha256((root / "PHASE_4_PLAN.md").read_bytes()).hexdigest() != PHASE4_PLAN_SHA256: - raise ValueError("Approved Phase 4 plan changed") - verify_phase4_step5_control(json.loads((root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))) - decisions = (root / "PHASE_4_DECISIONS.md").read_text(encoding="utf-8") - if PHASE4_STEP5_APPROVAL not in decisions or PHASE4_STEP4_FINAL not in decisions: - raise ValueError("Actual Step 5 authorization or Step 4 evidence anchor missing") - if any((root / name).exists() for name in PHASE4_FORBIDDEN_OUTPUTS): - raise ValueError("Step 5 cannot create Phase 4 completion records or audit outputs") - import runpy - checker = runpy.run_path(str(root / "scripts/check_traceability.py"), run_name="phase4_step5_renderer_boundary") - checker["phase4_step5_renderers_boundary"](root) - schema_bytes = (root / "schemas/report.schema.json").read_bytes() - if hashlib.sha256(schema_bytes).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Step 5 report schema differs from its independently reviewed bytes") - schema = json.loads(schema_bytes) - if (schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema" - or schema.get("type") != "object" or schema.get("additionalProperties") is not False): - raise ValueError("Step 5 report schema dialect/closed root mismatch") - return {"package_modules": 40, "frozen_runtime_modules": 38, "frozen_schemas": 5, - "hero_files_unchanged": 6, "historical_phase3_migrated_nodes": 16, - "phase_complete": False, "result_contracts_enabled": True, - "adapters_enabled": True, "privacy_views_enabled": True, "renderers_enabled": True, "cli_analysis_enabled": False} - - -def audit_phase4_step5(step: int = 5) -> dict: - verify_phase4_step5_control(json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")), step) - result = {"phase": 4, "active_step": step, **verify_phase4_step5_snapshot(), - "phase0_hashes_verified": 16, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_phase4_step5_diff(step: int = 5) -> dict: - if type(step) is not int or step != 5: - raise ValueError("Unsupported Phase 4 Step 5 stage") - _phase4_step4_files() - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE4_STEP4_FINAL, "HEAD"], check=True) - raw = git("diff", "--name-status", "--no-renames", "-z", PHASE4_STEP4_FINAL, "--").split(b"\0") - raw = [part.decode("utf-8") for part in raw if part] - if len(raw) % 2: - raise ValueError("Malformed Step 5 Git difference records") - changes = list(zip(raw[::2], raw[1::2])) - changes.extend(("A", p.decode()) for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") if p) - verify_phase4_step5_changes(changes) - result = {"previous_step_commit": PHASE4_STEP4_FINAL, "changed_files": len(changes), - "changes": changes, "step": 5, "scope": "PASS"} - print(json.dumps(result, indent=2)) - return result - - -def phase4_step5_baseline_evidence(output: Path) -> dict: - """Reconcile all inherited identities, including the accepted Step 4 suite.""" - output.mkdir(parents=True, exist_ok=True) - inherited = phase4_step4_baseline_evidence(output / "phase4-step4-inherited") - with tempfile.TemporaryDirectory(prefix="rit-p4-step4-identities-") as temp: - baseline = Path(temp) - for name, raw in _phase4_step4_files().items(): - target = baseline / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(raw) - old, old_log = _collect(baseline) - current, current_log = _collect(ROOT) - expected = 3074 if os.environ.get("RIT_TEST_PARQUET") == "1" else 3071 - if len(old) != expected or set(old) - set(current): - raise ValueError("Accepted Phase 4 Step 4 test identities were lost") - result = {"baseline_commit": PHASE4_STEP4_FINAL, "baseline_test_tree": PHASE4_STEP4_TEST_TREE, - "baseline_nodeids": old, "current_nodeids": current, "missing_nodeids": [], - "baseline_tests": len(old), "current_tests": len(current), "inherited": inherited, - "nodeids_sha256": hashlib.sha256(("\n".join(old)+"\n").encode()).hexdigest()} - (output / "phase4_step5_test_identity_manifest.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4-step4-collection.log").write_text(old_log, encoding="utf-8") - (output / "phase4-step5-collection.log").write_text(current_log, encoding="utf-8") - print(json.dumps({k: result[k] for k in ("baseline_tests", "current_tests", "missing_nodeids")}, indent=2)) - return result - - -def phase4_step5_candidate(output: Path, step: int = 5) -> None: - """Build tested Step 5 intermediate evidence without declaring Phase 4 complete.""" - if os.environ.get("RIT_TEST_PARQUET") != "1" or importlib.util.find_spec("pyarrow") is None: - raise ValueError("Step 5 candidate evidence requires RIT_TEST_PARQUET=1 and real PyArrow") - output.mkdir(parents=True, exist_ok=True) - result = audit_phase4_step5(step) - result["diff"] = audit_phase4_step5_diff(step) - if git("status", "--porcelain").strip(): - raise ValueError("Step 5 candidate archive requires committed, clean source") - result["core"] = verify_junit(output / "core.xml", minimum=3071) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=3074) - result["core_math_measurements"] = verify_step10_evidence(output / "core.xml", output / "phase4-step5-core-observations.json") - result["parquet_math_measurements"] = verify_step10_evidence(output / "parquet.xml", output / "phase4-step5-parquet-observations.json") - identity = phase4_step5_baseline_evidence(output) - result["test_identity"] = {k: identity[k] for k in ("baseline_tests", "current_tests", "nodeids_sha256")} - expected = set() - for node in identity["current_nodeids"]: - base, bracket, parameter = node.partition("[") - owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) - for name, parquet in (("core.xml", False), ("parquet.xml", True)): - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(output / name).getroot().iter("testcase")} - target = expected if parquet else {c for c in expected if c[1] not in PARQUET_CASES} - if cases != target: - raise ValueError(f"Step 5 JUnit does not execute the entire current suite: {name}") - wheel, _ = verify_distributions(output / "dist") - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - archive = output / "recursive-integrity-toolkit-phase4-step5-candidate.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {p for p in git("ls-files", "-z").decode().split("\0") if p} - with zipfile.ZipFile(archive) as zipped: - entries = [item.filename for item in zipped.infolist() if not item.is_dir()] - prefix = "recursive-integrity-toolkit/" - if (len(entries) != len(set(entries)) or any(not name.startswith(prefix) for name in entries)): - raise ValueError("Step 5 source archive has duplicate or unprefixed entries") - names = {name.removeprefix(prefix) for name in entries} - if names != tracked or len(tracked) != 227 or len(entries) != 227: - raise ValueError("Step 5 source archive file set mismatch") - for name in names: - if zipped.read("recursive-integrity-toolkit/"+name) != (ROOT / name).read_bytes(): - raise ValueError(f"Step 5 source archive byte mismatch: {name}") - result.update({"commit": git("rev-parse", "HEAD").decode().strip(), - "tree": git("rev-parse", "HEAD^{tree}").decode().strip(), - "source_archive": archive.name, "tracked_files": len(tracked), - "python": sys.version, "platform": sys.platform, - "versions": {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "pytest")}}) - (output / "phase4_step5_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4_step5_repository_files.sha256").write_text("".join(f"{hashlib.sha256((ROOT/name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - paths = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase4_step5_artifacts.sha256") - (output / "phase4_step5_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in paths), encoding="utf-8") - print(f"Phase 4 Step 5 candidate: {len(tracked)} source files verified; Phase 4 remains incomplete.") - - -def phase4_step5_cli_main() -> int: - parser = argparse.ArgumentParser(description="Phase 4 Step 5 maintainer gate") - parser.add_argument("--phase", type=int, choices=(4,), required=True) - parser.add_argument("--step", type=int, choices=(5,), required=True) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) - parser.add_argument("--require-parquet", action="store_true") - parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - raise ValueError("Phase 4 Step 5 cannot certify a final phase delivery") - if args.junit is not None: - verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - phase4_step5_baseline_evidence(args.baseline_evidence) - elif args.dist is not None: - audit_phase4_step5(args.step) - wheel, _ = verify_distributions(args.dist) - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - elif args.smoke_wheel is not None: - smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - phase4_step5_candidate(args.candidate, args.step) - else: - audit_phase4_step5(args.step) - if args.diff: - audit_phase4_step5_diff(args.step) - return 0 - - -def phase4_step5_installed_renderers_smoke(wheel: Path) -> None: +def smoke_installed_renderers(wheel: Path) -> None: """Exercise the installed pure renderers against prepared protected evidence.""" with tempfile.TemporaryDirectory(prefix="rit-p4-step5-installed-renderers-") as temp: work = Path(temp) @@ -4378,2363 +1068,7 @@ def deny_effects(event, arguments): subprocess.run([str(python), "-I", "-c", program, str(work), str(ROOT)], cwd=work, check=True) -# Phase 4 Step 6: safe local publication; no CLI or analytical expansion. - - -PHASE4_STEP5_FINAL = 'c1667179e895a798bba2349960162542efa5ef87' - - -PHASE4_STEP5_TREE = 'd83f9fc8e0b569f2d796bd2a83b87422fe350fba' - - -PHASE4_STEP5_TEST_TREE = '974f8e65904c24ed4e226996eaf571aab6123965' - - -PHASE4_STEP6_APPROVAL = '批准开始Phase 4 **Step 6**' - - -PHASE4_STEP6_APPROVAL_DATE = '2026-09-20' - - -PHASE4_STEP6_NEW = ('tests/unit/test_phase4_output_safety.py',) - - -PHASE4_STEP6_MIGRATIONS = {'tests/integration/test_phase4_gates.py': [{'new': 'def ' - 'test_phase4_step5_current_runtime_opens_only_renderers_and_freezes_schema(repo_root, ' - 'phase4_step4_snapshot, phase4_step5_snapshot):\n' - ' repo_root = phase4_step5_snapshot\n' - ' package = "src/recursive_integrity_toolkit"\n' - ' current = {path.relative_to(repo_root).as_posix(): ' - 'hashlib.sha256(path.read_bytes()).hexdigest()\n' - ' for path in (repo_root / ' - 'package).rglob("*.py")}\n' - ' frozen = ' - '{path.relative_to(phase4_step4_snapshot).as_posix(): ' - 'hashlib.sha256(path.read_bytes()).hexdigest()\n' - ' for path in (phase4_step4_snapshot / ' - 'package).rglob("*.py")}\n' - ' assert len(current) == len(frozen) == 40\n' - ' assert current.keys() == frozen.keys()\n' - ' opened = {package + "/" + relative for relative in ' - '(\n' - ' "reports/json_report.py", ' - '"reports/markdown_report.py",\n' - ' )}\n' - ' assert {path for path in current if current[path] ' - '!= frozen[path]} == opened\n' - ' assert {path: digest for path, digest in ' - 'current.items() if path not in opened} == {\n' - ' path: digest for path, digest in frozen.items() ' - 'if path not in opened\n' - ' }\n' - ' schemas = {path.name: path.read_bytes() for path in ' - '(repo_root / "schemas").glob("*.json")}\n' - ' assert len(schemas) == 5\n' - ' assert schemas == {path.name: path.read_bytes() for ' - 'path in (phase4_step4_snapshot / ' - '"schemas").glob("*.json")}', - 'node': 'test_phase4_step5_current_runtime_opens_only_renderers_and_freezes_schema', - 'old': 'def ' - 'test_phase4_step5_current_runtime_opens_only_renderers_and_freezes_schema(repo_root, ' - 'phase4_step4_snapshot):\n' - ' package = "src/recursive_integrity_toolkit"\n' - ' current = {path.relative_to(repo_root).as_posix(): ' - 'hashlib.sha256(path.read_bytes()).hexdigest()\n' - ' for path in (repo_root / ' - 'package).rglob("*.py")}\n' - ' frozen = ' - '{path.relative_to(phase4_step4_snapshot).as_posix(): ' - 'hashlib.sha256(path.read_bytes()).hexdigest()\n' - ' for path in (phase4_step4_snapshot / ' - 'package).rglob("*.py")}\n' - ' assert len(current) == len(frozen) == 40\n' - ' assert current.keys() == frozen.keys()\n' - ' opened = {package + "/" + relative for relative in ' - '(\n' - ' "reports/json_report.py", ' - '"reports/markdown_report.py",\n' - ' )}\n' - ' assert {path for path in current if current[path] ' - '!= frozen[path]} == opened\n' - ' assert {path: digest for path, digest in ' - 'current.items() if path not in opened} == {\n' - ' path: digest for path, digest in frozen.items() ' - 'if path not in opened\n' - ' }\n' - ' schemas = {path.name: path.read_bytes() for path in ' - '(repo_root / "schemas").glob("*.json")}\n' - ' assert len(schemas) == 5\n' - ' assert schemas == {path.name: path.read_bytes() for ' - 'path in (phase4_step4_snapshot / ' - '"schemas").glob("*.json")}'}, - {'new': 'def ' - 'test_phase4_step5_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step5_snapshot):\n' - ' repo_root = phase4_step5_snapshot\n' - ' names = {"ci.yml", "golden.yml", "security.yml", ' - '"release.yml"}\n' - ' root = repo_root / ".github/workflows"\n' - ' expected_timeouts = {"ci.yml": [60, 20], ' - '"golden.yml": [15], "security.yml": [20], ' - '"release.yml": [25]}\n' - ' assert {path.name for path in root.glob("*.yml")} ' - '== names\n' - ' for name in sorted(names):\n' - ' text = (root / ' - 'name).read_text(encoding="utf-8")\n' - ' assert [int(line.split(":", 1)[1]) for line in ' - 'text.splitlines()\n' - ' if ' - 'line.strip().startswith("timeout-minutes:")] == ' - 'expected_timeouts[name]\n' - ' assert "Phase 4 Step 5" in text, name\n' - ' assert "--phase 4 --step 5" in text, name\n' - ' for older in ("--phase 4 --step 4", "--phase 4 ' - '--step 3", "--phase 4 --step 2", "--phase 4 --step 1", ' - '"--phase 3 --step 11"):\n' - ' assert older not in text, (name, older)\n' - ' assert "permissions:\\n contents: read" in ' - 'text\n' - ' assert "persist-credentials: false" in text\n' - ' assert "timeout-minutes:" in text and "set -euo ' - 'pipefail" in text\n' - ' assert "actions/upload-artifact@v4" in text and ' - '"if-no-files-found: error" in text\n' - ' for line in text.splitlines():\n' - ' if "python scripts/release_check.py" in ' - 'line:\n' - ' assert "--phase 4 --step 5" in line, ' - '(name, line)\n' - ' for forbidden in ("continue-on-error:", "|| ' - 'true", "contents: write", "id-token: write", "twine ' - 'upload", "git push"):\n' - ' assert forbidden not in text, (name, ' - 'forbidden)\n' - ' ci = (root / "ci.yml").read_text(encoding="utf-8")\n' - " for required in ('os: [ubuntu-latest, " - 'windows-latest]\', \'python-version: ["3.11", ' - '"3.12"]\',\n' - " 'dependencies: [current, " - 'minimum]\', \'"numpy==2.0.0" "pandas==2.2.2"\',\n' - ' \'RIT_TEST_PARQUET: "0"\', ' - '\'RIT_TEST_PARQUET: "1"\', \'--require-parquet\',\n' - " '--baseline-evidence', 'python -m " - "pip check',\n" - " '--minimum-tests 3071', " - "'--minimum-tests 3074',\n" - ' "find_spec(\'pyarrow\') is None", ' - "'import pyarrow'):\n" - ' assert required in ci\n' - ' assert ci.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2\n' - ' assert " -k " not in ci\n' - ' golden = (root / ' - '"golden.yml").read_text(encoding="utf-8")\n' - ' for required in ' - '("tests/golden/test_phase3_math.py", ' - '"tests/integration/test_hero_structure.py",\n' - ' ' - '"tests/integration/test_phase3_metric_pipeline.py"):\n' - ' assert required in golden\n' - ' security = (root / ' - '"security.yml").read_text(encoding="utf-8")\n' - ' for required in ' - "('tests/integration/test_no_network.py', " - "'tests/integration/test_optional_dependency.py',\n" - ' ' - "'tests/unit/test_PR012_evidence_classes.py', " - "'tests/unit/test_PR013_report_schema.py',\n" - ' ' - "'tests/unit/test_PR014_unavailable.py', " - "'tests/unit/test_PR015_redaction.py',\n" - ' ' - "'tests/unit/test_PR016_determinism.py', " - "'tests/unit/test_PR018_language.py',\n" - ' ' - "'tests/integration/test_partial_provenance_report.py',\n" - ' ' - "'tests/integration/test_partial_lineage_report.py',\n" - ' ' - "'tests/unit/test_phase4_contracts.py', " - "'tests/integration/test_phase4_gates.py'):\n" - ' assert required in security\n' - ' release = (root / ' - '"release.yml").read_text(encoding="utf-8")\n' - ' assert "python -m build" in release and "python -m ' - 'twine check --strict" in release\n' - ' assert "--dist" in release and "--candidate" in ' - 'release\n' - ' assert "--delivery" not in release\n' - ' assert ' - '"recursive-integrity-toolkit-phase4-step5-candidate" in ' - 'release\n' - ' assert "rit-phase4-step4" not in release\n' - ' assert "--minimum-tests 3071" in release and ' - '"--minimum-tests 3074" in release\n' - ' assert release.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2', - 'node': 'test_phase4_step5_current_workflows_preserve_matrix_and_use_active_dispatch', - 'old': 'def ' - 'test_phase4_step5_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root):\n' - ' names = {"ci.yml", "golden.yml", "security.yml", ' - '"release.yml"}\n' - ' root = repo_root / ".github/workflows"\n' - ' expected_timeouts = {"ci.yml": [60, 20], ' - '"golden.yml": [15], "security.yml": [20], ' - '"release.yml": [25]}\n' - ' assert {path.name for path in root.glob("*.yml")} ' - '== names\n' - ' for name in sorted(names):\n' - ' text = (root / ' - 'name).read_text(encoding="utf-8")\n' - ' assert [int(line.split(":", 1)[1]) for line in ' - 'text.splitlines()\n' - ' if ' - 'line.strip().startswith("timeout-minutes:")] == ' - 'expected_timeouts[name]\n' - ' assert "Phase 4 Step 5" in text, name\n' - ' assert "--phase 4 --step 5" in text, name\n' - ' for older in ("--phase 4 --step 4", "--phase 4 ' - '--step 3", "--phase 4 --step 2", "--phase 4 --step 1", ' - '"--phase 3 --step 11"):\n' - ' assert older not in text, (name, older)\n' - ' assert "permissions:\\n contents: read" in ' - 'text\n' - ' assert "persist-credentials: false" in text\n' - ' assert "timeout-minutes:" in text and "set -euo ' - 'pipefail" in text\n' - ' assert "actions/upload-artifact@v4" in text and ' - '"if-no-files-found: error" in text\n' - ' for line in text.splitlines():\n' - ' if "python scripts/release_check.py" in ' - 'line:\n' - ' assert "--phase 4 --step 5" in line, ' - '(name, line)\n' - ' for forbidden in ("continue-on-error:", "|| ' - 'true", "contents: write", "id-token: write", "twine ' - 'upload", "git push"):\n' - ' assert forbidden not in text, (name, ' - 'forbidden)\n' - ' ci = (root / "ci.yml").read_text(encoding="utf-8")\n' - " for required in ('os: [ubuntu-latest, " - 'windows-latest]\', \'python-version: ["3.11", ' - '"3.12"]\',\n' - " 'dependencies: [current, " - 'minimum]\', \'"numpy==2.0.0" "pandas==2.2.2"\',\n' - ' \'RIT_TEST_PARQUET: "0"\', ' - '\'RIT_TEST_PARQUET: "1"\', \'--require-parquet\',\n' - " '--baseline-evidence', 'python -m " - "pip check',\n" - " '--minimum-tests 3071', " - "'--minimum-tests 3074',\n" - ' "find_spec(\'pyarrow\') is None", ' - "'import pyarrow'):\n" - ' assert required in ci\n' - ' assert ci.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2\n' - ' assert " -k " not in ci\n' - ' golden = (root / ' - '"golden.yml").read_text(encoding="utf-8")\n' - ' for required in ' - '("tests/golden/test_phase3_math.py", ' - '"tests/integration/test_hero_structure.py",\n' - ' ' - '"tests/integration/test_phase3_metric_pipeline.py"):\n' - ' assert required in golden\n' - ' security = (root / ' - '"security.yml").read_text(encoding="utf-8")\n' - ' for required in ' - "('tests/integration/test_no_network.py', " - "'tests/integration/test_optional_dependency.py',\n" - ' ' - "'tests/unit/test_PR012_evidence_classes.py', " - "'tests/unit/test_PR013_report_schema.py',\n" - ' ' - "'tests/unit/test_PR014_unavailable.py', " - "'tests/unit/test_PR015_redaction.py',\n" - ' ' - "'tests/unit/test_PR016_determinism.py', " - "'tests/unit/test_PR018_language.py',\n" - ' ' - "'tests/integration/test_partial_provenance_report.py',\n" - ' ' - "'tests/integration/test_partial_lineage_report.py',\n" - ' ' - "'tests/unit/test_phase4_contracts.py', " - "'tests/integration/test_phase4_gates.py'):\n" - ' assert required in security\n' - ' release = (root / ' - '"release.yml").read_text(encoding="utf-8")\n' - ' assert "python -m build" in release and "python -m ' - 'twine check --strict" in release\n' - ' assert "--dist" in release and "--candidate" in ' - 'release\n' - ' assert "--delivery" not in release\n' - ' assert ' - '"recursive-integrity-toolkit-phase4-step5-candidate" in ' - 'release\n' - ' assert "rit-phase4-step4" not in release\n' - ' assert "--minimum-tests 3071" in release and ' - '"--minimum-tests 3074" in release\n' - ' assert release.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2'}, - {'new': 'def ' - 'test_phase4_step5_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step5_snapshot):\n' - ' import json\n' - '\n' - ' paths = ' - '{path.relative_to(phase4_step5_snapshot).as_posix()\n' - ' for path in ' - 'phase4_step5_snapshot.rglob("*") if path.is_file()}\n' - ' assert len(paths) == 227\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step5_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step5_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 38\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["privacy_views_enabled"] is True\n' - ' assert baseline["renderers_enabled"] is True\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 4\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n repo_root = ' - 'phase4_step4_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n"),\n' - ' "historical_document": ' - '("docs/report_schema.md", b"# Report Schema\\n",\n' - ' b"# Unauthorized ' - 'historical document\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "html": ' - '"src/recursive_integrity_toolkit/reports/html_report.py",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "config": ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' "hashing": ' - '"src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' "logging": ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 5 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 6\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)', - 'node': 'test_phase4_step5_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step5_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation):\n' - ' import json\n' - '\n' - ' tracked = subprocess.check_output(["git", "-C", ' - 'str(repo_root), "ls-files", "-z"])\n' - ' paths = set(tracked.decode().split("\\0")) - {""}\n' - ' assert len(paths) == 227\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(repo_root / relative, ' - 'destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step5_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 38\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["privacy_views_enabled"] is True\n' - ' assert baseline["renderers_enabled"] is True\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 4\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n repo_root = ' - 'phase4_step4_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n"),\n' - ' "historical_document": ' - '("docs/report_schema.md", b"# Report Schema\\n",\n' - ' b"# Unauthorized ' - 'historical document\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "html": ' - '"src/recursive_integrity_toolkit/reports/html_report.py",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "config": ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' "hashing": ' - '"src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' "logging": ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 5 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 6\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)'}, - {'new': 'def phase4_mutation_tree(repo_root, tmp_path_factory, ' - 'phase4_gate_tools, phase4_step1_snapshot):\n' - ' """One source copy; each individual mutation is ' - 'restored in a finally block."""\n' - ' root = ' - 'tmp_path_factory.mktemp("phase4-active-mutations")\n' - ' paths = ' - '{path.relative_to(phase4_step1_snapshot).as_posix()\n' - ' for path in ' - 'phase4_step1_snapshot.rglob("*") if path.is_file()}\n' - ' ' - 'paths.update(phase4_gate_tools["PHASE4_STEP1_NEW"])\n' - ' for relative in sorted(paths):\n' - ' destination = root / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step1_snapshot / ' - 'relative, destination)\n' - ' return root', - 'node': 'phase4_mutation_tree', - 'old': 'def phase4_mutation_tree(repo_root, tmp_path_factory, ' - 'phase4_gate_tools, phase4_step1_snapshot):\n' - ' """One source copy; each individual mutation is ' - 'restored in a finally block."""\n' - ' root = ' - 'tmp_path_factory.mktemp("phase4-active-mutations")\n' - ' tracked = subprocess.check_output(["git", "-C", ' - 'str(repo_root), "ls-files", "-z"])\n' - ' paths = set(tracked.decode().split("\\0")) - {""}\n' - ' ' - 'paths.update(phase4_gate_tools["PHASE4_STEP1_NEW"])\n' - ' for relative in sorted(paths):\n' - ' destination = root / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step1_snapshot / ' - 'relative, destination)\n' - ' return root'}, - {'new': 'def ' - 'test_phase4_step2_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step2_snapshot):\n' - ' import json\n' - '\n' - ' paths = ' - '{path.relative_to(phase4_step2_snapshot).as_posix()\n' - ' for path in ' - 'phase4_step2_snapshot.rglob("*") if path.is_file()}\n' - ' assert len(paths) == 227\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step2_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step2_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["frozen_runtime_modules"] == 39\n' - ' assert baseline["frozen_schemas"] == 4\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is False\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'phase4_control["active_step"] == 1\\n\', b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_control(control: dict, step: int = 1) -> ' - 'None:",\n' - ' b"def ' - 'verify_phase4_control(control: dict, step: int = 1) -> ' - 'None:\\n return"),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/config.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md",\n' - ' "premature_report": "report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 2 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 3\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/cli.py")\n' - ' else:\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/config.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' elif mutation.startswith("schema_"):\n' - ' path = tmp_path / "schemas/report.schema.json"\n' - ' schema = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "schema_open_root":\n' - ' schema["additionalProperties"] = True\n' - ' else:\n' - ' schema["$schema"] = ' - '"http://json-schema.org/draft-07/schema#"\n' - ' path.write_text(json.dumps(schema), ' - 'encoding="utf-8")\n' - ' elif mutation == "model_computation_import":\n' - ' path = tmp_path / ' - '"src/recursive_integrity_toolkit/result.py"\n' - ' path.write_bytes(path.read_bytes() + b"\\nfrom ' - '.metrics.diversity import ' - 'effective_state_diversity\\n")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)', - 'node': 'test_phase4_step2_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step2_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step2_snapshot):\n' - ' import json\n' - '\n' - ' tracked = subprocess.check_output(["git", "-C", ' - 'str(repo_root), "ls-files", "-z"])\n' - ' paths = set(tracked.decode().split("\\0")) - {""}\n' - ' assert len(paths) == 227\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step2_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step2_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["frozen_runtime_modules"] == 39\n' - ' assert baseline["frozen_schemas"] == 4\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is False\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'phase4_control["active_step"] == 1\\n\', b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_control(control: dict, step: int = 1) -> ' - 'None:",\n' - ' b"def ' - 'verify_phase4_control(control: dict, step: int = 1) -> ' - 'None:\\n return"),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/config.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md",\n' - ' "premature_report": "report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 2 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 3\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/cli.py")\n' - ' else:\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/config.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' elif mutation.startswith("schema_"):\n' - ' path = tmp_path / "schemas/report.schema.json"\n' - ' schema = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "schema_open_root":\n' - ' schema["additionalProperties"] = True\n' - ' else:\n' - ' schema["$schema"] = ' - '"http://json-schema.org/draft-07/schema#"\n' - ' path.write_text(json.dumps(schema), ' - 'encoding="utf-8")\n' - ' elif mutation == "model_computation_import":\n' - ' path = tmp_path / ' - '"src/recursive_integrity_toolkit/result.py"\n' - ' path.write_bytes(path.read_bytes() + b"\\nfrom ' - '.metrics.diversity import ' - 'effective_state_diversity\\n")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)'}, - {'new': 'def ' - 'test_phase4_step3_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step3_snapshot):\n' - ' import json\n' - '\n' - ' paths = ' - '{path.relative_to(phase4_step3_snapshot).as_posix()\n' - ' for path in ' - 'phase4_step3_snapshot.rglob("*") if path.is_file()}\n' - ' assert len(paths) == 227\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step3_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step3_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 39\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 2\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step2_control(control: dict, step: int = ' - '2) -> None:",\n' - ' b"def ' - 'verify_phase4_step2_control(control: dict, step: int = ' - '2) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step2_snapshot):\\n repo_root = ' - 'phase4_step2_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step2_snapshot):\\n"),\n' - ' "historical_document": ' - '("docs/report_schema.md", b"# Report Schema\\n", b"# ' - 'Modified Report Schema\\n"),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "frozen_schema": "schemas/config.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "renderer": ' - '"src/recursive_integrity_toolkit/reports/json_report.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md",\n' - ' "premature_report": "report.json",\n' - ' }\n' - ' assembly_injections = {\n' - ' "assembly_metric_import": b"\\nfrom ' - '..metrics.diversity import ' - 'effective_state_diversity\\n",\n' - ' "assembly_calculation_import": b"\\nfrom ' - '..metrics import coverage\\n",\n' - ' "assembly_numeric_import": b"\\nimport ' - 'numpy\\n",\n' - ' "assembly_network_import": b"\\nimport ' - 'socket\\n",\n' - ' "assembly_file_io": ' - 'b"\\nopen(\'unapproved.txt\', ' - '\'w\').write(\'data\')\\n",\n' - ' "assembly_dynamic_execution": ' - 'b"\\nexec(\'forged = True\')\\n",\n' - ' "assembly_new_formula": b"\\ndef ' - 'invented_metric(values):\\n return sum(value * value ' - 'for value in values)\\n",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 3 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 4\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/result.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' elif mutation == "schema_open_root":\n' - ' path = tmp_path / "schemas/report.schema.json"\n' - ' schema = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' schema["additionalProperties"] = True\n' - ' path.write_text(json.dumps(schema), ' - 'encoding="utf-8")\n' - ' elif mutation in assembly_injections:\n' - ' path = tmp_path / ' - '"src/recursive_integrity_toolkit/reports/assembly.py"\n' - ' path.write_bytes(path.read_bytes() + ' - 'assembly_injections[mutation])\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)', - 'node': 'test_phase4_step3_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step3_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step3_snapshot):\n' - ' import json\n' - '\n' - ' tracked = subprocess.check_output(["git", "-C", ' - 'str(repo_root), "ls-files", "-z"])\n' - ' paths = set(tracked.decode().split("\\0")) - {""}\n' - ' assert len(paths) == 227\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step3_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step3_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 39\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 2\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step2_control(control: dict, step: int = ' - '2) -> None:",\n' - ' b"def ' - 'verify_phase4_step2_control(control: dict, step: int = ' - '2) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step2_snapshot):\\n repo_root = ' - 'phase4_step2_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step2_snapshot):\\n"),\n' - ' "historical_document": ' - '("docs/report_schema.md", b"# Report Schema\\n", b"# ' - 'Modified Report Schema\\n"),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "frozen_schema": "schemas/config.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "renderer": ' - '"src/recursive_integrity_toolkit/reports/json_report.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md",\n' - ' "premature_report": "report.json",\n' - ' }\n' - ' assembly_injections = {\n' - ' "assembly_metric_import": b"\\nfrom ' - '..metrics.diversity import ' - 'effective_state_diversity\\n",\n' - ' "assembly_calculation_import": b"\\nfrom ' - '..metrics import coverage\\n",\n' - ' "assembly_numeric_import": b"\\nimport ' - 'numpy\\n",\n' - ' "assembly_network_import": b"\\nimport ' - 'socket\\n",\n' - ' "assembly_file_io": ' - 'b"\\nopen(\'unapproved.txt\', ' - '\'w\').write(\'data\')\\n",\n' - ' "assembly_dynamic_execution": ' - 'b"\\nexec(\'forged = True\')\\n",\n' - ' "assembly_new_formula": b"\\ndef ' - 'invented_metric(values):\\n return sum(value * value ' - 'for value in values)\\n",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 3 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 4\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/result.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' elif mutation == "schema_open_root":\n' - ' path = tmp_path / "schemas/report.schema.json"\n' - ' schema = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' schema["additionalProperties"] = True\n' - ' path.write_text(json.dumps(schema), ' - 'encoding="utf-8")\n' - ' elif mutation in assembly_injections:\n' - ' path = tmp_path / ' - '"src/recursive_integrity_toolkit/reports/assembly.py"\n' - ' path.write_bytes(path.read_bytes() + ' - 'assembly_injections[mutation])\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)'}, - {'new': 'def ' - 'test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step4_snapshot):\n' - ' import json\n' - '\n' - ' paths = ' - '{path.relative_to(phase4_step4_snapshot).as_posix()\n' - ' for path in ' - 'phase4_step4_snapshot.rglob("*") if path.is_file()}\n' - ' assert len(paths) == 227\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step4_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step4_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 35\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["privacy_views_enabled"] is True\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 3\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step3_control(control: dict, step: int = ' - '3) -> None:",\n' - ' b"def ' - 'verify_phase4_step3_control(control: dict, step: int = ' - '3) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step3_snapshot):\\n repo_root = ' - 'phase4_step3_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step3_snapshot):\\n"),\n' - ' "historical_document": ("docs/privacy.md", b"# ' - 'Privacy and Local Input Boundaries\\n",\n' - ' b"# Unauthorized ' - 'Privacy and Local Input Boundaries\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "renderer": ' - '"src/recursive_integrity_toolkit/reports/json_report.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 4 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 5\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/json_report.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)', - 'node': 'test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step4_snapshot):\n' - ' import json\n' - '\n' - ' tracked = subprocess.check_output(["git", "-C", ' - 'str(repo_root), "ls-files", "-z"])\n' - ' paths = set(tracked.decode().split("\\0")) - {""}\n' - ' assert len(paths) == 227\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step4_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step4_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 35\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["privacy_views_enabled"] is True\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 3\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step3_control(control: dict, step: int = ' - '3) -> None:",\n' - ' b"def ' - 'verify_phase4_step3_control(control: dict, step: int = ' - '3) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step3_snapshot):\\n repo_root = ' - 'phase4_step3_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step3_snapshot):\\n"),\n' - ' "historical_document": ("docs/privacy.md", b"# ' - 'Privacy and Local Input Boundaries\\n",\n' - ' b"# Unauthorized ' - 'Privacy and Local Input Boundaries\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "renderer": ' - '"src/recursive_integrity_toolkit/reports/json_report.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 4 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 5\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/json_report.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)'}, - {'new': 'def ' - 'test_phase4_step4_privacy_ast_checks_current_source_before_rejecting_scope_injection(repo_root, ' - 'tmp_path, relative, injection, phase4_step5_snapshot):\n' - ' checker = runpy.run_path(str(repo_root / ' - '"scripts/check_traceability.py"), ' - 'run_name="phase4_step4_ast_tests")\n' - ' verify = checker["phase4_step4_runtime_boundary"]\n' - ' repository_path = ' - '"src/recursive_integrity_toolkit/" + relative\n' - ' current = (phase4_step5_snapshot if relative == ' - '"utils/logging.py" else repo_root) / repository_path\n' - ' verify(current, repository_path)\n' - ' copy = tmp_path / "source.py"\n' - ' copy.write_bytes(current.read_bytes() + injection)\n' - ' with pytest.raises(ValueError):\n' - ' verify(copy, repository_path)', - 'node': 'test_phase4_step4_privacy_ast_checks_current_source_before_rejecting_scope_injection', - 'old': 'def ' - 'test_phase4_step4_privacy_ast_checks_current_source_before_rejecting_scope_injection(repo_root, ' - 'tmp_path, relative, injection):\n' - ' checker = runpy.run_path(str(repo_root / ' - '"scripts/check_traceability.py"), ' - 'run_name="phase4_step4_ast_tests")\n' - ' verify = checker["phase4_step4_runtime_boundary"]\n' - ' repository_path = ' - '"src/recursive_integrity_toolkit/" + relative\n' - ' current = repo_root / repository_path\n' - ' verify(current, repository_path)\n' - ' copy = tmp_path / "source.py"\n' - ' copy.write_bytes(current.read_bytes() + injection)\n' - ' with pytest.raises(ValueError):\n' - ' verify(copy, repository_path)'}], - 'tests/unit/test_phase4_contracts.py': [{'new': 'def ' - 'test_phase4_step5_approved_control_keeps_independent_step4_anchors(phase4_tools, ' - 'phase4_step5_snapshot):\n' - ' control = json.loads((phase4_step5_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' assert phase4_tools["PHASE4_STEP4_FINAL"] == ' - '"49454a9b162cb8d35e38d1cb1ae32cb208d3a01c"\n' - ' assert phase4_tools["PHASE4_STEP4_TREE"] == ' - '"79264d845e08f73ee68e160dea04f7786f18fd0a"\n' - ' assert phase4_tools["PHASE4_STEP4_TEST_TREE"] == ' - '"4aa4ca0b22f8b8c25afe27feeaba713d6f0830bf"\n' - ' assert control["active_phase"] == 4 and ' - 'control["active_step"] == 5\n' - ' assert control["approval_basis"] == "很好，Phase 4 Step 5 ' - '继续"\n' - ' assert control["baseline_commit"] == BASELINE\n' - ' assert control["previous_step_commit"] == ' - '"49454a9b162cb8d35e38d1cb1ae32cb208d3a01c"\n' - ' assert control["previous_step_tree"] == ' - '"79264d845e08f73ee68e160dea04f7786f18fd0a"\n' - ' assert control["previous_step_test_tree"] == ' - '"4aa4ca0b22f8b8c25afe27feeaba713d6f0830bf"\n' - ' assert control["previous_step_core_tests"] == 3071\n' - ' assert control["previous_step_parquet_tests"] == 3074\n' - ' assert len(control["previous_step_files_sha256"]) == ' - '227\n' - ' assert set(control["runtime_paths_authorized"]) == {\n' - ' ' - '"src/recursive_integrity_toolkit/reports/json_report.py",\n' - ' ' - '"src/recursive_integrity_toolkit/reports/markdown_report.py",\n' - ' }\n' - ' assert control["schema_changes_authorized"] is False\n' - ' assert control["schema_paths_authorized"] == []\n' - ' assert control["new_files_permitted"] == []\n' - ' for name in ("phase_complete", "next_step_authorized", ' - '"main_merge_authorized", "publication_authorized"):\n' - ' assert control[name] is False\n' - ' phase4_tools["verify_phase4_step5_control"](control, ' - 'step=5)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step4_control"](control, ' - 'step=4)', - 'node': 'test_phase4_step5_approved_control_keeps_independent_step4_anchors', - 'old': 'def ' - 'test_phase4_step5_approved_control_keeps_independent_step4_anchors(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' assert phase4_tools["PHASE4_STEP4_FINAL"] == ' - '"49454a9b162cb8d35e38d1cb1ae32cb208d3a01c"\n' - ' assert phase4_tools["PHASE4_STEP4_TREE"] == ' - '"79264d845e08f73ee68e160dea04f7786f18fd0a"\n' - ' assert phase4_tools["PHASE4_STEP4_TEST_TREE"] == ' - '"4aa4ca0b22f8b8c25afe27feeaba713d6f0830bf"\n' - ' assert control["active_phase"] == 4 and ' - 'control["active_step"] == 5\n' - ' assert control["approval_basis"] == "很好，Phase 4 Step 5 ' - '继续"\n' - ' assert control["baseline_commit"] == BASELINE\n' - ' assert control["previous_step_commit"] == ' - '"49454a9b162cb8d35e38d1cb1ae32cb208d3a01c"\n' - ' assert control["previous_step_tree"] == ' - '"79264d845e08f73ee68e160dea04f7786f18fd0a"\n' - ' assert control["previous_step_test_tree"] == ' - '"4aa4ca0b22f8b8c25afe27feeaba713d6f0830bf"\n' - ' assert control["previous_step_core_tests"] == 3071\n' - ' assert control["previous_step_parquet_tests"] == 3074\n' - ' assert len(control["previous_step_files_sha256"]) == ' - '227\n' - ' assert set(control["runtime_paths_authorized"]) == {\n' - ' ' - '"src/recursive_integrity_toolkit/reports/json_report.py",\n' - ' ' - '"src/recursive_integrity_toolkit/reports/markdown_report.py",\n' - ' }\n' - ' assert control["schema_changes_authorized"] is False\n' - ' assert control["schema_paths_authorized"] == []\n' - ' assert control["new_files_permitted"] == []\n' - ' for name in ("phase_complete", "next_step_authorized", ' - '"main_merge_authorized", "publication_authorized"):\n' - ' assert control[name] is False\n' - ' phase4_tools["verify_phase4_step5_control"](control, ' - 'step=5)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step4_control"](control, ' - 'step=4)'}, - {'new': 'def ' - 'test_phase4_step5_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value, phase4_step5_snapshot):\n' - ' control = json.loads((phase4_step5_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' verify = phase4_tools["verify_phase4_step5_control"]\n' - ' verify(control, step=5)\n' - ' control[field] = value\n' - ' with pytest.raises(ValueError):\n' - ' verify(control, step=5)', - 'node': 'test_phase4_step5_control_rejects_forged_scope_and_stage', - 'old': 'def ' - 'test_phase4_step5_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' verify = phase4_tools["verify_phase4_step5_control"]\n' - ' verify(control, step=5)\n' - ' control[field] = value\n' - ' with pytest.raises(ValueError):\n' - ' verify(control, step=5)'}, - {'new': 'def ' - 'test_phase4_step5_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step, phase4_step5_snapshot):\n' - ' control = json.loads((phase4_step5_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step5_control"](control, ' - 'step=5)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step5_control"](control, ' - 'step=step)', - 'node': 'test_phase4_step5_control_rejects_unapproved_dispatch', - 'old': 'def ' - 'test_phase4_step5_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step5_control"](control, ' - 'step=5)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step5_control"](control, ' - 'step=step)'}, - {'new': 'def ' - 'test_phase4_step5_control_requires_explicit_approval_fields(phase4_tools, ' - 'field, phase4_step5_snapshot):\n' - ' control = json.loads((phase4_step5_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step5_control"](control)\n' - ' del control[field]\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step5_control"](control)', - 'node': 'test_phase4_step5_control_requires_explicit_approval_fields', - 'old': 'def ' - 'test_phase4_step5_control_requires_explicit_approval_fields(phase4_tools, ' - 'field):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step5_control"](control)\n' - ' del control[field]\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step5_control"](control)'}, - {'new': 'def ' - 'test_phase4_step5_control_cannot_mint_extra_permission(phase4_tools, ' - 'phase4_step5_snapshot):\n' - ' control = json.loads((phase4_step5_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step5_control"](control)\n' - ' control["approved_scope_expansion"] = {"step": 6, ' - '"publication": True}\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step5_control"](control)', - 'node': 'test_phase4_step5_control_cannot_mint_extra_permission', - 'old': 'def ' - 'test_phase4_step5_control_cannot_mint_extra_permission(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step5_control"](control)\n' - ' control["approved_scope_expansion"] = {"step": 6, ' - '"publication": True}\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step5_control"](control)'}, - {'new': 'def ' - 'test_phase4_step5_historical_migrations_preserve_ten_step4_gate_nodes(phase4_tools, ' - 'phase4_step4_snapshot, phase4_step5_snapshot):\n' - ' registry = phase4_tools["PHASE4_STEP5_MIGRATIONS"]\n' - ' expected = {\n' - ' "tests/unit/test_phase4_contracts.py": {\n' - ' ' - '"test_phase4_step4_approved_control_keeps_independent_step3_anchors",\n' - ' ' - '"test_phase4_step4_control_rejects_forged_scope_and_stage",\n' - ' ' - '"test_phase4_step4_control_rejects_unapproved_dispatch",\n' - ' ' - '"test_phase4_step4_control_requires_explicit_approval_fields",\n' - ' ' - '"test_phase4_step4_control_cannot_mint_extra_permission",\n' - ' ' - '"test_phase4_step4_historical_migrations_preserve_ten_step3_gate_nodes",\n' - ' ' - '"test_phase4_step4_historical_guard_rejects_assertion_and_binding_weakening",\n' - ' },\n' - ' "tests/integration/test_phase4_gates.py": {\n' - ' ' - '"test_phase4_step4_current_runtime_opens_only_privacy_metadata_modules_and_freezes_schema",\n' - ' ' - '"test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch",\n' - ' ' - '"test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations",\n' - ' },\n' - ' }\n' - ' assert set(registry) == set(expected)\n' - ' assert sum(len(rows) for rows in registry.values()) == ' - '13\n' - ' for path, nodes in expected.items():\n' - ' assert {row["node"] for row in registry[path]} == ' - 'nodes\n' - ' ' - 'phase4_tools["verify_phase4_step5_test_migration"](\n' - ' path, (phase4_step4_snapshot / ' - 'path).read_bytes(), (phase4_step5_snapshot / ' - 'path).read_bytes(),\n' - ' )', - 'node': 'test_phase4_step5_historical_migrations_preserve_ten_step4_gate_nodes', - 'old': 'def ' - 'test_phase4_step5_historical_migrations_preserve_ten_step4_gate_nodes(phase4_tools, ' - 'phase4_step4_snapshot):\n' - ' registry = phase4_tools["PHASE4_STEP5_MIGRATIONS"]\n' - ' expected = {\n' - ' "tests/unit/test_phase4_contracts.py": {\n' - ' ' - '"test_phase4_step4_approved_control_keeps_independent_step3_anchors",\n' - ' ' - '"test_phase4_step4_control_rejects_forged_scope_and_stage",\n' - ' ' - '"test_phase4_step4_control_rejects_unapproved_dispatch",\n' - ' ' - '"test_phase4_step4_control_requires_explicit_approval_fields",\n' - ' ' - '"test_phase4_step4_control_cannot_mint_extra_permission",\n' - ' ' - '"test_phase4_step4_historical_migrations_preserve_ten_step3_gate_nodes",\n' - ' ' - '"test_phase4_step4_historical_guard_rejects_assertion_and_binding_weakening",\n' - ' },\n' - ' "tests/integration/test_phase4_gates.py": {\n' - ' ' - '"test_phase4_step4_current_runtime_opens_only_privacy_metadata_modules_and_freezes_schema",\n' - ' ' - '"test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch",\n' - ' ' - '"test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations",\n' - ' },\n' - ' }\n' - ' assert set(registry) == set(expected)\n' - ' assert sum(len(rows) for rows in registry.values()) == ' - '13\n' - ' for path, nodes in expected.items():\n' - ' assert {row["node"] for row in registry[path]} == ' - 'nodes\n' - ' ' - 'phase4_tools["verify_phase4_step5_test_migration"](\n' - ' path, (phase4_step4_snapshot / ' - 'path).read_bytes(), (ROOT / path).read_bytes(),\n' - ' )'}, - {'new': 'def ' - 'test_phase4_step5_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, ' - 'phase4_step4_snapshot, mutation, phase4_step5_snapshot):\n' - ' path = "tests/unit/test_phase4_contracts.py"\n' - ' before = (phase4_step4_snapshot / path).read_bytes()\n' - ' after = (phase4_step5_snapshot / path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step5_test_migration"]\n' - ' verify(path, before, after)\n' - ' if mutation == "old_assertion":\n' - ' original = b\' assert control["active_phase"] ' - '== 4 and control["active_step"] == 4\\n\'\n' - ' assert after.count(original) == 1\n' - ' after = after.replace(original, b" assert ' - 'True\\n", 1)\n' - ' elif mutation == "wrong_snapshot":\n' - " original = (b'def " - 'test_phase4_step4_control_cannot_mint_extra_permission(phase4_tools, ' - "phase4_step4_snapshot):\\n'\n" - " b' control = " - 'json.loads((phase4_step4_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\\n\')\n' - ' assert after.count(original) == 1\n' - ' after = after.replace(original, ' - "original.replace(b'(phase4_step4_snapshot /', b'(ROOT /'), " - '1)\n' - ' elif mutation == "rebind_node":\n' - ' after += b"\\ndef ' - 'test_phase4_step4_fixed_boundary_opens_only_approved_existing_paths():\\n ' - 'assert True\\n"\n' - ' else:\n' - ' after += b"\\ndef ' - 'test_phase4_step5_mutator(value=globals().clear()):\\n ' - 'pass\\n"\n' - ' with pytest.raises(ValueError):\n' - ' verify(path, before, after)', - 'node': 'test_phase4_step5_historical_guard_rejects_assertion_and_binding_weakening', - 'old': 'def ' - 'test_phase4_step5_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, ' - 'phase4_step4_snapshot, mutation):\n' - ' path = "tests/unit/test_phase4_contracts.py"\n' - ' before = (phase4_step4_snapshot / path).read_bytes()\n' - ' after = (ROOT / path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step5_test_migration"]\n' - ' verify(path, before, after)\n' - ' if mutation == "old_assertion":\n' - ' original = b\' assert control["active_phase"] ' - '== 4 and control["active_step"] == 4\\n\'\n' - ' assert after.count(original) == 1\n' - ' after = after.replace(original, b" assert ' - 'True\\n", 1)\n' - ' elif mutation == "wrong_snapshot":\n' - " original = (b'def " - 'test_phase4_step4_control_cannot_mint_extra_permission(phase4_tools, ' - "phase4_step4_snapshot):\\n'\n" - " b' control = " - 'json.loads((phase4_step4_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\\n\')\n' - ' assert after.count(original) == 1\n' - ' after = after.replace(original, ' - "original.replace(b'(phase4_step4_snapshot /', b'(ROOT /'), " - '1)\n' - ' elif mutation == "rebind_node":\n' - ' after += b"\\ndef ' - 'test_phase4_step4_fixed_boundary_opens_only_approved_existing_paths():\\n ' - 'assert True\\n"\n' - ' else:\n' - ' after += b"\\ndef ' - 'test_phase4_step5_mutator(value=globals().clear()):\\n ' - 'pass\\n"\n' - ' with pytest.raises(ValueError):\n' - ' verify(path, before, after)'}]} - - -PHASE4_STEP6_ALLOWED = {'.github/workflows/ci.yml', - '.github/workflows/golden.yml', - '.github/workflows/release.yml', - '.github/workflows/security.yml', - 'PHASE_4_BASELINE.json', - 'PHASE_4_DECISIONS.md', - 'docs/architecture.md', - 'docs/cli.md', - 'docs/privacy.md', - 'docs/theory_traceability.md', - 'scripts/check_spec_consistency.py', - 'scripts/check_traceability.py', - 'scripts/release_check.py', - 'src/recursive_integrity_toolkit/utils/logging.py', - 'src/recursive_integrity_toolkit/utils/paths.py', - 'tests/conftest.py', - 'tests/integration/test_ci_workflows.py', - 'tests/integration/test_hero_structure.py', - 'tests/integration/test_license_notices.py', - 'tests/integration/test_no_algorithms.py', - 'tests/integration/test_no_network.py', - 'tests/integration/test_optional_dependency.py', - 'tests/integration/test_owner_ids.py', - 'tests/integration/test_package_import.py', - 'tests/integration/test_package_install.py', - 'tests/integration/test_phase4_gates.py', - 'tests/integration/test_prohibited_structure.py', - 'tests/integration/test_repository_structure.py', - 'tests/integration/test_schema_json.py', - 'tests/unit/test_phase4_contracts.py', - 'tests/unit/test_phase4_output_safety.py'} - - -def _phase4_step5_files(): - """Read the exact accepted Step 5 Git tree and verify every blob identity.""" - for suffix, expected in (("^{commit}", PHASE4_STEP5_FINAL), ("^{tree}", PHASE4_STEP5_TREE), - (":tests", PHASE4_STEP5_TEST_TREE)): - if git("rev-parse", PHASE4_STEP5_FINAL + suffix).decode().strip() != expected: - raise ValueError("Pinned Phase 4 Step 5 identity mismatch") - objects = {} - for entry in git("ls-tree", "-rz", PHASE4_STEP5_FINAL).split(b"\0"): - if not entry: - continue - metadata, raw_path = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - path = raw_path.decode("utf-8") - if (mode not in (b"100644", b"100755") or kind != b"blob" or path in objects - or path.startswith("/") or ".." in path.split("/") or ".git" in path.split("/")): - raise ValueError("Unsafe pinned Step 5 Git object") - objects[path] = oid.decode("ascii") - if len(objects) != 227: - raise ValueError("Pinned Step 5 must contain exactly 227 files") - files = {} - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_STEP5_FINAL))) as archive: - names = [item.filename for item in archive.infolist() if not item.is_dir()] - if len(names) != len(objects) or set(names) != set(objects): - raise ValueError("Pinned Step 5 archive identity mismatch") - for name in names: - raw = archive.read(name) - oid = hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - if oid != objects[name]: - raise ValueError(f"Pinned Step 5 blob mismatch: {name}") - files[name] = raw - return MappingProxyType(files) - - -def phase4_step6_expected_control() -> dict: - """Actual Step 6 approval is independent of the mutable control document.""" - prior = _phase4_step5_files() - result = phase4_step5_expected_control() - result.update({ - "control_version": "1.5", "active_step": 6, - "approval_date": PHASE4_STEP6_APPROVAL_DATE, "approval_basis": PHASE4_STEP6_APPROVAL, - "previous_step_commit": PHASE4_STEP5_FINAL, - "previous_step_tree": PHASE4_STEP5_TREE, - "previous_step_test_tree": PHASE4_STEP5_TEST_TREE, - "previous_step_core_tests": 3215, "previous_step_parquet_tests": 3218, - "previous_step_files_sha256": {p: hashlib.sha256(raw).hexdigest() for p, raw in sorted(prior.items())}, - "permitted_paths": sorted(PHASE4_STEP6_ALLOWED), "new_files_permitted": list(PHASE4_STEP6_NEW), - "runtime_changes_authorized": True, "schema_changes_authorized": False, - "runtime_paths_authorized": sorted(p for p in PHASE4_STEP6_ALLOWED if p.startswith("src/")), - "schema_paths_authorized": [], - "step5_historical_binding_nodes": {p: sorted({r["node"] for r in rows}) - for p, rows in sorted(PHASE4_STEP6_MIGRATIONS.items())}, - }) - return result - - -def verify_phase4_step6_control(control: dict, step: int = 6) -> None: - if type(step) is not int or step != 6 or type(control) is not dict: - raise ValueError("Unsupported Phase 4 Step 6 stage/control") - try: - actual = json.dumps(control, sort_keys=True, ensure_ascii=True, allow_nan=False) - expected = json.dumps(phase4_step6_expected_control(), sort_keys=True, ensure_ascii=True, allow_nan=False) - except (TypeError, ValueError) as error: - raise ValueError("Invalid Phase 4 Step 6 control") from error - if actual != expected: - raise ValueError("Phase 4 control differs from the independently approved Step 6 contract") - - -def verify_phase4_step6_changes(changes: list[tuple[str, str]]) -> None: - seen = set() - for status, path in changes: - if path in seen or path not in PHASE4_STEP6_ALLOWED or status != ("A" if path in PHASE4_STEP6_NEW else "M"): - raise ValueError(f"Unapproved Phase 4 Step 6 path/operation: {status} {path}") - seen.add(path) - - -def _phase4_step6_header(node, path): - """Allow explicit test parametrization/fixtures, never definition-time effects.""" - for decorator in node.decorator_list: - if not isinstance(decorator, ast.Call) or ast.unparse(decorator.func) not in { - "pytest.fixture", "pytest.mark.parametrize"}: - raise ValueError(f"Unapproved Step 6 test decorator: {path}:{node.name}") - if ast.unparse(decorator.func) == "pytest.fixture" and not node.name.startswith("phase4_step"): - raise ValueError("A Step 6 fixture must have an explicit scoped name") - if any(keyword.arg is None for keyword in decorator.keywords): - raise ValueError("Decorator expansion is outside the Step 6 contract") - for value in [*decorator.args, *(keyword.value for keyword in decorator.keywords)]: - try: - ast.literal_eval(value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError("Step 6 decorator arguments must be literal") from error - clone = ast.parse(ast.unparse(node)).body[0] - clone.decorator_list = [] - _phase4_preserve_function_header(clone, path) - - -def _phase4_step6_append_only(before: bytes, after: bytes, path: str) -> None: - if not after.startswith(before): - raise ValueError(f"Inherited Step 5 prefix changed: {path}") - old, new = ast.parse(before), ast.parse(after) - bindings = set() - for node in old.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - bindings.add(node.name) - elif isinstance(node, (ast.Import, ast.ImportFrom)): - bindings.update(alias.asname or alias.name.split(".")[0] for alias in node.names) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - bindings.update(child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)) - for node in new.body[len(old.body):]: - if not isinstance(node, ast.FunctionDef) or node.name in bindings: - raise ValueError(f"Step 6 addition executes, rebinds or shadows inherited source: {path}") - allowed_snapshot = path == "tests/conftest.py" and node.name == "phase4_step5_snapshot" - if not allowed_snapshot and not node.name.startswith(("test_phase4_step6_", "phase4_step6_")): - raise ValueError(f"Step 6 added test/helper is not explicitly scoped: {path}:{node.name}") - if path == "tests/conftest.py" and node.name != "phase4_step5_snapshot": - raise ValueError("Only the pinned Step 5 shared fixture is authorized") - bindings.add(node.name) - _phase4_step6_header(node, path) - - -def verify_phase4_step6_test_migration(path: str, before: bytes, after: bytes) -> None: - prior = _phase4_step5_files() - if path not in prior or path not in PHASE4_STEP6_ALLOWED or before != prior[path] or not path.startswith("tests/"): - raise ValueError("Step 6 migration requires the exact named Step 5 source") - _phase4_step6_check_test_migration(path, before, after) - - -def _phase4_step6_check_test_migration(path: str, before: bytes, after: bytes) -> None: - """Check a source already read from the verified Step 5 immutable mapping.""" - expected = before - for row in PHASE4_STEP6_MIGRATIONS.get(path, []): - old, new = row["old"].encode(), row["new"].encode() - if expected.count(old) != 1: - raise ValueError("Step 5 historical binding is not unique") - expected = expected.replace(old, new, 1) - _phase4_step6_append_only(expected, after, path) - - -def _phase4_step6_preserve_tooling(before: bytes, after: bytes, path: str) -> None: - """Preserve exact inherited statements using one source-line split per tree.""" - if path == "scripts/release_check.py": - old = ' return phase4_cli_main() if explicit_phase4 else main()' - new = (' explicit_step6 = "--step=6" in argv or any(a == "--step" and b == "6" for a, b in zip(argv, argv[1:]))\n' - ' if explicit_phase4 and explicit_step6:\n' - ' return phase4_step6_cli_main()\n' + old) - # Earlier preserved dispatch functions contain this source too; bind cli_main only. - parsed = ast.parse(before) - entry = next(node for node in parsed.body if isinstance(node, ast.FunctionDef) and node.name == "cli_main") - old_entry = ast.get_source_segment(before.decode(), entry) - if old_entry.count(old) != 1: - raise ValueError("Step 5 release dispatcher identity mismatch") - replacements = [(old_entry, old_entry.replace(old, new, 1))] - else: - old = ' if args.phase == 4 and args.step == 5:\n return phase4_step5_main(step=args.step)\n' - new = old + ' if args.phase == 4 and args.step == 6:\n return phase4_step6_main(step=args.step)\n' - replacements = [(old, new), - ('Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5', - 'Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5 or --phase 4 --step 6')] - expected = before - for old, new in replacements: - if expected.count(old.encode()) != 1: - raise ValueError(f"Historical dispatcher identity mismatch: {path}") - expected = expected.replace(old.encode(), new.encode(), 1) - trees = [ast.parse(expected), ast.parse(after)] - lines = [expected.splitlines(keepends=True), after.splitlines(keepends=True)] - def segment(index, node): - if node.lineno == node.end_lineno: - return lines[index][node.lineno - 1][node.col_offset:node.end_col_offset] - return (lines[index][node.lineno - 1][node.col_offset:] - + b"".join(lines[index][node.lineno:node.end_lineno - 1]) - + lines[index][node.end_lineno - 1][:node.end_col_offset]) - def entry_guard(node): - return isinstance(node, ast.If) and isinstance(node.test, ast.Compare) and "__name__" in ast.unparse(node.test) - guards = [n for n in trees[1].body if entry_guard(n)] - expected_guard = ast.parse('if __name__ == "__main__":\n raise SystemExit(cli_main())').body[0] - if len(guards) != 1 or ast.dump(guards[0]) != ast.dump(expected_guard): - raise ValueError(f"Unapproved Step 6 maintainer entrypoint: {path}") - historical = [n for n in trees[0].body if not entry_guard(n)] - current = [n for n in trees[1].body if not entry_guard(n)] - old_entries = [(segment(0, n), ast.dump(n)) for n in historical] - cursor = 0 - for node in current: - if cursor < len(old_entries) and (segment(1, node), ast.dump(node)) == old_entries[cursor]: - cursor += 1 - continue - if isinstance(node, ast.FunctionDef) and (node.name == "_phase4_step5_files" or node.name.startswith( - ("phase4_step6_", "_phase4_step6_", "verify_phase4_step6_", "audit_phase4_step6"))): - _phase4_preserve_function_header(node, path) - continue - if isinstance(node, ast.Assign) and all(isinstance(target, ast.Name) and target.id.startswith( - ("PHASE4_STEP6_", "PHASE4_STEP5_")) for target in node.targets): - try: - ast.literal_eval(node.value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError(f"Nonliteral added Step 6 maintainer constant: {path}") from error - continue - raise ValueError(f"Unapproved Step 6 maintainer addition: {path}") - if cursor != len(historical): - raise ValueError(f"Inherited Step 5 maintainer statement changed: {path}") - def binding_counts(tree): - counts = {} - for node in tree.body: - names = [] - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - names = [node.name] - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names = [child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)] - elif isinstance(node, (ast.Import, ast.ImportFrom)): - names = [alias.asname or alias.name.split(".")[0] for alias in node.names] - for name in names: - counts[name] = counts.get(name, 0) + 1 - return counts - inherited_counts = binding_counts(trees[0]) - for name, count in binding_counts(trees[1]).items(): - if count > inherited_counts.get(name, 1): - raise ValueError(f"Duplicate Step 6 maintainer binding: {path}:{name}") - - -def verify_phase4_step6_snapshot(root: Path = ROOT) -> dict: - prior = _phase4_step5_files() - for path, raw in prior.items(): - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError(f"Missing or aliased inherited Step 5 file: {path}") - current = target.read_bytes() - if path not in PHASE4_STEP6_ALLOWED and current != raw: - raise ValueError(f"Protected Step 5 bytes changed: {path}") - if path.startswith("tests/") and path in PHASE4_STEP6_ALLOWED: - _phase4_step6_check_test_migration(path, raw, current) - elif path.startswith("scripts/") and path in PHASE4_STEP6_ALLOWED: - _phase4_step6_preserve_tooling(raw, current, path) - elif path in {"docs/architecture.md", "docs/theory_traceability.md", "docs/privacy.md", "docs/cli.md", "PHASE_4_DECISIONS.md"}: - if not current.startswith(raw): - raise ValueError(f"Step 5 historical documentation prefix changed: {path}") - for path in PHASE4_STEP6_NEW: - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError("Missing Step 6 output safety tests") - _phase4_step6_new_test(target.read_bytes(), path) - for path in sorted(p for p in PHASE4_STEP6_ALLOWED if p.startswith("src/")): - _phase4_step6_preserve_runtime(prior[path], (root / path).read_bytes(), path) - actual_modules = {p.relative_to(root).as_posix() for p in (root / "src/recursive_integrity_toolkit").rglob("*.py")} - expected_modules = {p for p in prior if p.startswith("src/") and p.endswith(".py")} - if actual_modules != expected_modules or len(actual_modules) != 40: - raise ValueError("Step 6 cannot change the runtime module set") - if {p.name for p in (root / "schemas").iterdir()} != {Path(p).name for p in prior if p.startswith("schemas/")}: - raise ValueError("Step 6 cannot change the schema set") - if hashlib.sha256((root / "PHASE_4_PLAN.md").read_bytes()).hexdigest() != PHASE4_PLAN_SHA256: - raise ValueError("Approved Phase 4 plan changed") - verify_phase4_step6_control(json.loads((root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))) - decisions = (root / "PHASE_4_DECISIONS.md").read_text(encoding="utf-8") - if PHASE4_STEP6_APPROVAL not in decisions or PHASE4_STEP5_FINAL not in decisions: - raise ValueError("Actual Step 6 authorization or Step 5 evidence anchor missing") - if any((root / name).exists() for name in PHASE4_FORBIDDEN_OUTPUTS): - raise ValueError("Step 6 cannot create Phase 4 completion records or audit outputs") - import runpy - checker = runpy.run_path(str(root / "scripts/check_traceability.py"), run_name="phase4_step6_renderer_boundary") - checker["phase4_step6_output_boundary"](root) - schema_bytes = (root / "schemas/report.schema.json").read_bytes() - if hashlib.sha256(schema_bytes).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Step 6 report schema differs from its independently reviewed bytes") - schema = json.loads(schema_bytes) - if (schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema" - or schema.get("type") != "object" or schema.get("additionalProperties") is not False): - raise ValueError("Step 6 report schema dialect/closed root mismatch") - return {"package_modules": 40, "frozen_runtime_modules": 38, "frozen_schemas": 5, - "hero_files_unchanged": 6, "historical_phase3_migrated_nodes": 16, - "phase_complete": False, "result_contracts_enabled": True, - "adapters_enabled": True, "privacy_views_enabled": True, "renderers_enabled": True, "output_publication_enabled": True, "cli_analysis_enabled": False} - - -def audit_phase4_step6(step: int = 6) -> dict: - verify_phase4_step6_control(json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")), step) - result = {"phase": 4, "active_step": step, **verify_phase4_step6_snapshot(), - "phase0_hashes_verified": 16, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_phase4_step6_diff(step: int = 6) -> dict: - if type(step) is not int or step != 6: - raise ValueError("Unsupported Phase 4 Step 6 stage") - _phase4_step5_files() - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE4_STEP5_FINAL, "HEAD"], check=True) - raw = git("diff", "--name-status", "--no-renames", "-z", PHASE4_STEP5_FINAL, "--").split(b"\0") - raw = [part.decode("utf-8") for part in raw if part] - if len(raw) % 2: - raise ValueError("Malformed Step 6 Git difference records") - changes = list(zip(raw[::2], raw[1::2])) - changes.extend(("A", p.decode()) for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") if p) - verify_phase4_step6_changes(changes) - result = {"previous_step_commit": PHASE4_STEP5_FINAL, "changed_files": len(changes), - "changes": changes, "step": 6, "scope": "PASS"} - print(json.dumps(result, indent=2)) - return result - - -def phase4_step6_baseline_evidence(output: Path) -> dict: - """Reconcile all inherited identities, including the accepted Step 5 suite.""" - output.mkdir(parents=True, exist_ok=True) - inherited = phase4_step5_baseline_evidence(output / "phase4-step5-inherited") - with tempfile.TemporaryDirectory(prefix="rit-p4-step5-identities-") as temp: - baseline = Path(temp) - for name, raw in _phase4_step5_files().items(): - target = baseline / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(raw) - old, old_log = _collect(baseline) - current, current_log = _collect(ROOT) - expected = 3218 if os.environ.get("RIT_TEST_PARQUET") == "1" else 3215 - if len(old) != expected or set(old) - set(current): - raise ValueError("Accepted Phase 4 Step 5 test identities were lost") - result = {"baseline_commit": PHASE4_STEP5_FINAL, "baseline_test_tree": PHASE4_STEP5_TEST_TREE, - "baseline_nodeids": old, "current_nodeids": current, "missing_nodeids": [], - "baseline_tests": len(old), "current_tests": len(current), "inherited": inherited, - "nodeids_sha256": hashlib.sha256(("\n".join(old)+"\n").encode()).hexdigest()} - (output / "phase4_step6_test_identity_manifest.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4-step5-collection.log").write_text(old_log, encoding="utf-8") - (output / "phase4-step6-collection.log").write_text(current_log, encoding="utf-8") - print(json.dumps({k: result[k] for k in ("baseline_tests", "current_tests", "missing_nodeids")}, indent=2)) - return result - - -def phase4_step6_candidate(output: Path, step: int = 6) -> None: - """Build tested Step 6 intermediate evidence without declaring Phase 4 complete.""" - if os.environ.get("RIT_TEST_PARQUET") != "1" or importlib.util.find_spec("pyarrow") is None: - raise ValueError("Step 6 candidate evidence requires RIT_TEST_PARQUET=1 and real PyArrow") - output.mkdir(parents=True, exist_ok=True) - result = audit_phase4_step6(step) - result["diff"] = audit_phase4_step6_diff(step) - if git("status", "--porcelain").strip(): - raise ValueError("Step 6 candidate archive requires committed, clean source") - result["core"] = verify_junit(output / "core.xml", minimum=3215) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=3218) - result["core_math_measurements"] = verify_step10_evidence(output / "core.xml", output / "phase4-step6-core-observations.json") - result["parquet_math_measurements"] = verify_step10_evidence(output / "parquet.xml", output / "phase4-step6-parquet-observations.json") - identity = phase4_step6_baseline_evidence(output) - result["test_identity"] = {k: identity[k] for k in ("baseline_tests", "current_tests", "nodeids_sha256")} - expected = set() - for node in identity["current_nodeids"]: - base, bracket, parameter = node.partition("[") - owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) - for name, parquet in (("core.xml", False), ("parquet.xml", True)): - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(output / name).getroot().iter("testcase")} - target = expected if parquet else {c for c in expected if c[1] not in PARQUET_CASES} - if cases != target: - raise ValueError(f"Step 6 JUnit does not execute the entire current suite: {name}") - wheel, _ = verify_distributions(output / "dist") - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - phase4_step6_installed_publication_smoke(wheel) - archive = output / "recursive-integrity-toolkit-phase4-step6-candidate.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {p for p in git("ls-files", "-z").decode().split("\0") if p} - with zipfile.ZipFile(archive) as zipped: - entries = [item.filename for item in zipped.infolist() if not item.is_dir()] - prefix = "recursive-integrity-toolkit/" - if (len(entries) != len(set(entries)) or any(not name.startswith(prefix) for name in entries)): - raise ValueError("Step 6 source archive has duplicate or unprefixed entries") - names = {name.removeprefix(prefix) for name in entries} - if names != tracked or len(tracked) != 228 or len(entries) != 228: - raise ValueError("Step 6 source archive file set mismatch") - for name in names: - if zipped.read("recursive-integrity-toolkit/"+name) != (ROOT / name).read_bytes(): - raise ValueError(f"Step 6 source archive byte mismatch: {name}") - result.update({"commit": git("rev-parse", "HEAD").decode().strip(), - "tree": git("rev-parse", "HEAD^{tree}").decode().strip(), - "source_archive": archive.name, "tracked_files": len(tracked), - "python": sys.version, "platform": sys.platform, - "versions": {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "pytest")}}) - (output / "phase4_step6_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4_step6_repository_files.sha256").write_text("".join(f"{hashlib.sha256((ROOT/name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - paths = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase4_step6_artifacts.sha256") - (output / "phase4_step6_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in paths), encoding="utf-8") - print(f"Phase 4 Step 6 candidate: {len(tracked)} source files verified; Phase 4 remains incomplete.") - - -def phase4_step6_cli_main() -> int: - parser = argparse.ArgumentParser(description="Phase 4 Step 6 maintainer gate") - parser.add_argument("--phase", type=int, choices=(4,), required=True) - parser.add_argument("--step", type=int, choices=(6,), required=True) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) - parser.add_argument("--require-parquet", action="store_true") - parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - raise ValueError("Phase 4 Step 6 cannot certify a final phase delivery") - if args.junit is not None: - verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - phase4_step6_baseline_evidence(args.baseline_evidence) - elif args.dist is not None: - audit_phase4_step6(args.step) - wheel, _ = verify_distributions(args.dist) - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - phase4_step6_installed_publication_smoke(wheel) - elif args.smoke_wheel is not None: - smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - phase4_step6_candidate(args.candidate, args.step) - else: - audit_phase4_step6(args.step) - if args.diff: - audit_phase4_step6_diff(args.step) - return 0 - - -def _phase4_step6_new_test(raw: bytes, path: str) -> None: - """Only literal test declarations and explicitly listed safe imports.""" - allowed = {"__future__", "errno", "io", "json", "os", "pathlib", "types", "pytest", - "recursive_integrity_toolkit.utils"} - for index, node in enumerate(ast.parse(raw).body): - if index == 0 and isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str): - continue - if isinstance(node, (ast.Import, ast.ImportFrom)): - modules = [node.module] if isinstance(node, ast.ImportFrom) else [alias.name for alias in node.names] - if any(module not in allowed for module in modules): - raise ValueError("Unapproved Step 6 test import") - continue - if not isinstance(node, ast.FunctionDef) or not node.name.startswith(("test_phase4_step6_", "phase4_step6_")): - raise ValueError("Step 6 output tests gained definition-time execution") - _phase4_step6_header(node, path) - - -def _phase4_step6_preserve_runtime(before: bytes, after: bytes, path: str) -> None: - """Allow ownership-header maintenance and byte-exact additive helpers only.""" - try: - old, new = ast.parse(before), ast.parse(after) - except (SyntaxError, UnicodeError, ValueError) as error: - raise ValueError("Output helper source cannot be inspected") from error - if not ast.get_docstring(old) or not ast.get_docstring(new): - raise ValueError("Output helper ownership documentation is required") - old_tail = b"".join(before.splitlines(keepends=True)[old.body[0].end_lineno:]) - new_tail = b"".join(after.splitlines(keepends=True)[new.body[0].end_lineno:]) - if not new_tail.startswith(old_tail): - raise ValueError(f"Inherited input/diagnostic bytes changed: {path}") - - - -def phase4_step6_installed_publication_smoke(wheel: Path) -> None: +def smoke_installed_publication(wheel: Path) -> None: """Exercise installed output safety outside the source checkout.""" with tempfile.TemporaryDirectory(prefix="rit-p4-step6-installed-") as temporary: work = Path(temporary) @@ -6746,1531 +1080,7 @@ def phase4_step6_installed_publication_smoke(wheel: Path) -> None: subprocess.run([str(python), "-I", "-c", program, str(work), str(ROOT)], cwd=work, check=True) - -# Phase 4 Step 7: single-version CLI orchestration, no later-stage expansion. - - -PHASE4_STEP6_FINAL = 'aa2355f2359c3af4fc05345c792f9903b0d5858c' - - -PHASE4_STEP6_TREE = 'd9b6910079f49a2c739f86eceec05040918a41a5' - - -PHASE4_STEP6_TEST_TREE = '8d3a03c61da807d1fef8d26fc4af0d6f72e4546e' - - -PHASE4_STEP7_APPROVAL = 'Phase 4 Step 7 继续' - - -PHASE4_STEP7_APPROVAL_DATE = '2026-09-21' - - -PHASE4_STEP7_NEW = ('tests/integration/test_phase4_cli.py',) - - -PHASE4_STEP7_ALLOWED = {'.github/workflows/ci.yml', - '.github/workflows/golden.yml', - '.github/workflows/release.yml', - '.github/workflows/security.yml', - 'PHASE_4_BASELINE.json', - 'PHASE_4_DECISIONS.md', - 'docs/architecture.md', - 'docs/cli.md', - 'docs/theory_traceability.md', - 'scripts/check_spec_consistency.py', - 'scripts/check_traceability.py', - 'scripts/release_check.py', - 'src/recursive_integrity_toolkit/cli.py', - 'src/recursive_integrity_toolkit/config.py', - 'tests/conftest.py', - 'tests/integration/test_ci_workflows.py', - 'tests/integration/test_cli_validation.py', - 'tests/integration/test_hero_structure.py', - 'tests/integration/test_license_notices.py', - 'tests/integration/test_no_algorithms.py', - 'tests/integration/test_no_network.py', - 'tests/integration/test_optional_dependency.py', - 'tests/integration/test_owner_ids.py', - 'tests/integration/test_package_import.py', - 'tests/integration/test_package_install.py', - 'tests/integration/test_phase4_cli.py', - 'tests/integration/test_phase4_gates.py', - 'tests/integration/test_prohibited_structure.py', - 'tests/integration/test_repository_structure.py', - 'tests/integration/test_schema_json.py', - 'tests/unit/test_phase4_contracts.py'} - - -PHASE4_STEP7_MIGRATIONS = {'tests/integration/test_phase4_gates.py': [{'new': 'def ' - 'test_phase4_step6_current_runtime_opens_only_output_helpers_and_freezes_schema(repo_root, ' - 'phase4_step5_snapshot, phase4_step6_snapshot):\n' - ' repo_root = phase4_step6_snapshot\n' - ' package = "src/recursive_integrity_toolkit"\n' - ' current = {path.relative_to(repo_root).as_posix(): ' - 'hashlib.sha256(path.read_bytes()).hexdigest()\n' - ' for path in (repo_root / ' - 'package).rglob("*.py")}\n' - ' frozen = ' - '{path.relative_to(phase4_step5_snapshot).as_posix(): ' - 'hashlib.sha256(path.read_bytes()).hexdigest()\n' - ' for path in (phase4_step5_snapshot / ' - 'package).rglob("*.py")}\n' - ' assert len(current) == len(frozen) == 40\n' - ' assert current.keys() == frozen.keys()\n' - ' opened = {package + "/" + relative for relative in ' - '(\n' - ' "utils/paths.py", "utils/logging.py",\n' - ' )}\n' - ' assert {path for path in current if current[path] ' - '!= frozen[path]} == opened\n' - ' assert {path: digest for path, digest in ' - 'current.items() if path not in opened} == {\n' - ' path: digest for path, digest in frozen.items() ' - 'if path not in opened\n' - ' }\n' - ' schemas = {path.name: path.read_bytes() for path in ' - '(repo_root / "schemas").glob("*.json")}\n' - ' assert len(schemas) == 5\n' - ' assert schemas == {path.name: path.read_bytes() for ' - 'path in (phase4_step5_snapshot / ' - '"schemas").glob("*.json")}', - 'node': 'test_phase4_step6_current_runtime_opens_only_output_helpers_and_freezes_schema', - 'old': 'def ' - 'test_phase4_step6_current_runtime_opens_only_output_helpers_and_freezes_schema(repo_root, ' - 'phase4_step5_snapshot):\n' - ' package = "src/recursive_integrity_toolkit"\n' - ' current = {path.relative_to(repo_root).as_posix(): ' - 'hashlib.sha256(path.read_bytes()).hexdigest()\n' - ' for path in (repo_root / ' - 'package).rglob("*.py")}\n' - ' frozen = ' - '{path.relative_to(phase4_step5_snapshot).as_posix(): ' - 'hashlib.sha256(path.read_bytes()).hexdigest()\n' - ' for path in (phase4_step5_snapshot / ' - 'package).rglob("*.py")}\n' - ' assert len(current) == len(frozen) == 40\n' - ' assert current.keys() == frozen.keys()\n' - ' opened = {package + "/" + relative for relative in ' - '(\n' - ' "utils/paths.py", "utils/logging.py",\n' - ' )}\n' - ' assert {path for path in current if current[path] ' - '!= frozen[path]} == opened\n' - ' assert {path: digest for path, digest in ' - 'current.items() if path not in opened} == {\n' - ' path: digest for path, digest in frozen.items() ' - 'if path not in opened\n' - ' }\n' - ' schemas = {path.name: path.read_bytes() for path in ' - '(repo_root / "schemas").glob("*.json")}\n' - ' assert len(schemas) == 5\n' - ' assert schemas == {path.name: path.read_bytes() for ' - 'path in (phase4_step5_snapshot / ' - '"schemas").glob("*.json")}'}, - {'new': 'def ' - 'test_phase4_step6_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step6_snapshot):\n' - ' repo_root = phase4_step6_snapshot\n' - ' names = {"ci.yml", "golden.yml", "security.yml", ' - '"release.yml"}\n' - ' root = repo_root / ".github/workflows"\n' - ' expected_timeouts = {"ci.yml": [60, 20], ' - '"golden.yml": [15], "security.yml": [20], ' - '"release.yml": [25]}\n' - ' assert {path.name for path in root.glob("*.yml")} ' - '== names\n' - ' for name in sorted(names):\n' - ' text = (root / ' - 'name).read_text(encoding="utf-8")\n' - ' assert [int(line.split(":", 1)[1]) for line in ' - 'text.splitlines()\n' - ' if ' - 'line.strip().startswith("timeout-minutes:")] == ' - 'expected_timeouts[name]\n' - ' assert "Phase 4 Step 6" in text, name\n' - ' assert "--phase 4 --step 6" in text, name\n' - ' for older in ("--phase 4 --step 5", "--phase 4 ' - '--step 4", "--phase 4 --step 3", "--phase 4 --step 2", ' - '"--phase 4 --step 1", "--phase 3 --step 11"):\n' - ' assert older not in text, (name, older)\n' - ' assert "permissions:\\n contents: read" in ' - 'text\n' - ' assert "persist-credentials: false" in text\n' - ' assert "timeout-minutes:" in text and "set -euo ' - 'pipefail" in text\n' - ' assert "actions/upload-artifact@v4" in text and ' - '"if-no-files-found: error" in text\n' - ' for line in text.splitlines():\n' - ' if "python scripts/release_check.py" in ' - 'line:\n' - ' assert "--phase 4 --step 6" in line, ' - '(name, line)\n' - ' for forbidden in ("continue-on-error:", "|| ' - 'true", "contents: write", "id-token: write", "twine ' - 'upload", "git push"):\n' - ' assert forbidden not in text, (name, ' - 'forbidden)\n' - ' ci = (root / "ci.yml").read_text(encoding="utf-8")\n' - " for required in ('os: [ubuntu-latest, " - 'windows-latest]\', \'python-version: ["3.11", ' - '"3.12"]\',\n' - " 'dependencies: [current, " - 'minimum]\', \'"numpy==2.0.0" "pandas==2.2.2"\',\n' - ' \'RIT_TEST_PARQUET: "0"\', ' - '\'RIT_TEST_PARQUET: "1"\', \'--require-parquet\',\n' - " '--baseline-evidence', 'python -m " - "pip check',\n" - " '--minimum-tests 3215', " - "'--minimum-tests 3218',\n" - ' "find_spec(\'pyarrow\') is None", ' - "'import pyarrow'):\n" - ' assert required in ci\n' - ' assert ci.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2\n' - ' assert " -k " not in ci\n' - ' golden = (root / ' - '"golden.yml").read_text(encoding="utf-8")\n' - ' for required in ' - '("tests/golden/test_phase3_math.py", ' - '"tests/integration/test_hero_structure.py",\n' - ' ' - '"tests/integration/test_phase3_metric_pipeline.py"):\n' - ' assert required in golden\n' - ' security = (root / ' - '"security.yml").read_text(encoding="utf-8")\n' - ' for required in ' - "('tests/integration/test_no_network.py', " - "'tests/integration/test_optional_dependency.py',\n" - ' ' - "'tests/unit/test_PR012_evidence_classes.py', " - "'tests/unit/test_PR013_report_schema.py',\n" - ' ' - "'tests/unit/test_PR014_unavailable.py', " - "'tests/unit/test_PR015_redaction.py',\n" - ' ' - "'tests/unit/test_PR016_determinism.py', " - "'tests/unit/test_PR018_language.py',\n" - ' ' - "'tests/integration/test_partial_provenance_report.py',\n" - ' ' - "'tests/integration/test_partial_lineage_report.py',\n" - ' ' - "'tests/unit/test_phase4_output_safety.py', " - "'tests/unit/test_phase4_contracts.py', " - "'tests/integration/test_phase4_gates.py'):\n" - ' assert required in security\n' - ' release = (root / ' - '"release.yml").read_text(encoding="utf-8")\n' - ' assert "python -m build" in release and "python -m ' - 'twine check --strict" in release\n' - ' assert "--dist" in release and "--candidate" in ' - 'release\n' - ' assert "--delivery" not in release\n' - ' assert ' - '"recursive-integrity-toolkit-phase4-step6-candidate" in ' - 'release\n' - ' assert "rit-phase4-step4" not in release\n' - ' assert "--minimum-tests 3215" in release and ' - '"--minimum-tests 3218" in release\n' - ' assert release.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2', - 'node': 'test_phase4_step6_current_workflows_preserve_matrix_and_use_active_dispatch', - 'old': 'def ' - 'test_phase4_step6_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root):\n' - ' names = {"ci.yml", "golden.yml", "security.yml", ' - '"release.yml"}\n' - ' root = repo_root / ".github/workflows"\n' - ' expected_timeouts = {"ci.yml": [60, 20], ' - '"golden.yml": [15], "security.yml": [20], ' - '"release.yml": [25]}\n' - ' assert {path.name for path in root.glob("*.yml")} ' - '== names\n' - ' for name in sorted(names):\n' - ' text = (root / ' - 'name).read_text(encoding="utf-8")\n' - ' assert [int(line.split(":", 1)[1]) for line in ' - 'text.splitlines()\n' - ' if ' - 'line.strip().startswith("timeout-minutes:")] == ' - 'expected_timeouts[name]\n' - ' assert "Phase 4 Step 6" in text, name\n' - ' assert "--phase 4 --step 6" in text, name\n' - ' for older in ("--phase 4 --step 5", "--phase 4 ' - '--step 4", "--phase 4 --step 3", "--phase 4 --step 2", ' - '"--phase 4 --step 1", "--phase 3 --step 11"):\n' - ' assert older not in text, (name, older)\n' - ' assert "permissions:\\n contents: read" in ' - 'text\n' - ' assert "persist-credentials: false" in text\n' - ' assert "timeout-minutes:" in text and "set -euo ' - 'pipefail" in text\n' - ' assert "actions/upload-artifact@v4" in text and ' - '"if-no-files-found: error" in text\n' - ' for line in text.splitlines():\n' - ' if "python scripts/release_check.py" in ' - 'line:\n' - ' assert "--phase 4 --step 6" in line, ' - '(name, line)\n' - ' for forbidden in ("continue-on-error:", "|| ' - 'true", "contents: write", "id-token: write", "twine ' - 'upload", "git push"):\n' - ' assert forbidden not in text, (name, ' - 'forbidden)\n' - ' ci = (root / "ci.yml").read_text(encoding="utf-8")\n' - " for required in ('os: [ubuntu-latest, " - 'windows-latest]\', \'python-version: ["3.11", ' - '"3.12"]\',\n' - " 'dependencies: [current, " - 'minimum]\', \'"numpy==2.0.0" "pandas==2.2.2"\',\n' - ' \'RIT_TEST_PARQUET: "0"\', ' - '\'RIT_TEST_PARQUET: "1"\', \'--require-parquet\',\n' - " '--baseline-evidence', 'python -m " - "pip check',\n" - " '--minimum-tests 3215', " - "'--minimum-tests 3218',\n" - ' "find_spec(\'pyarrow\') is None", ' - "'import pyarrow'):\n" - ' assert required in ci\n' - ' assert ci.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2\n' - ' assert " -k " not in ci\n' - ' golden = (root / ' - '"golden.yml").read_text(encoding="utf-8")\n' - ' for required in ' - '("tests/golden/test_phase3_math.py", ' - '"tests/integration/test_hero_structure.py",\n' - ' ' - '"tests/integration/test_phase3_metric_pipeline.py"):\n' - ' assert required in golden\n' - ' security = (root / ' - '"security.yml").read_text(encoding="utf-8")\n' - ' for required in ' - "('tests/integration/test_no_network.py', " - "'tests/integration/test_optional_dependency.py',\n" - ' ' - "'tests/unit/test_PR012_evidence_classes.py', " - "'tests/unit/test_PR013_report_schema.py',\n" - ' ' - "'tests/unit/test_PR014_unavailable.py', " - "'tests/unit/test_PR015_redaction.py',\n" - ' ' - "'tests/unit/test_PR016_determinism.py', " - "'tests/unit/test_PR018_language.py',\n" - ' ' - "'tests/integration/test_partial_provenance_report.py',\n" - ' ' - "'tests/integration/test_partial_lineage_report.py',\n" - ' ' - "'tests/unit/test_phase4_output_safety.py', " - "'tests/unit/test_phase4_contracts.py', " - "'tests/integration/test_phase4_gates.py'):\n" - ' assert required in security\n' - ' release = (root / ' - '"release.yml").read_text(encoding="utf-8")\n' - ' assert "python -m build" in release and "python -m ' - 'twine check --strict" in release\n' - ' assert "--dist" in release and "--candidate" in ' - 'release\n' - ' assert "--delivery" not in release\n' - ' assert ' - '"recursive-integrity-toolkit-phase4-step6-candidate" in ' - 'release\n' - ' assert "rit-phase4-step4" not in release\n' - ' assert "--minimum-tests 3215" in release and ' - '"--minimum-tests 3218" in release\n' - ' assert release.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2'}, - {'new': 'def ' - 'test_phase4_step6_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step6_snapshot):\n' - ' import json\n' - '\n' - ' paths = ' - '{path.relative_to(phase4_step6_snapshot).as_posix()\n' - ' for path in ' - 'phase4_step6_snapshot.rglob("*") if path.is_file()}\n' - ' assert len(paths) == 228\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step6_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step6_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 38\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["privacy_views_enabled"] is True\n' - ' assert baseline["renderers_enabled"] is True\n' - ' assert baseline["output_publication_enabled"] is ' - 'True\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 4\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n repo_root = ' - 'phase4_step4_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n"),\n' - ' "historical_document": ("docs/cli.md", b"# ' - 'CLI\\n",\n' - ' b"# Unauthorized ' - 'historical document\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "html": ' - '"src/recursive_integrity_toolkit/reports/html_report.py",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "config": ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' "hashing": ' - '"src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' "logging": ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 6 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 7\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)', - 'node': 'test_phase4_step6_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step6_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation):\n' - ' import json\n' - '\n' - ' tracked = subprocess.check_output(["git", "-C", ' - 'str(repo_root), "ls-files", "-z"])\n' - ' paths = set(tracked.decode().split("\\0")) - {""}\n' - ' assert len(paths) == 228\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(repo_root / relative, ' - 'destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step6_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 38\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["privacy_views_enabled"] is True\n' - ' assert baseline["renderers_enabled"] is True\n' - ' assert baseline["output_publication_enabled"] is ' - 'True\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 4\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n repo_root = ' - 'phase4_step4_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n"),\n' - ' "historical_document": ("docs/cli.md", b"# ' - 'CLI\\n",\n' - ' b"# Unauthorized ' - 'historical document\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "html": ' - '"src/recursive_integrity_toolkit/reports/html_report.py",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "config": ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' "hashing": ' - '"src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' "logging": ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 6 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 7\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)'}, - {'new': 'def ' - 'test_phase4_step4_privacy_ast_checks_current_source_before_rejecting_scope_injection(repo_root, ' - 'tmp_path, relative, injection, phase4_step5_snapshot, ' - 'phase4_step6_snapshot):\n' - ' checker = runpy.run_path(str(repo_root / ' - '"scripts/check_traceability.py"), ' - 'run_name="phase4_step4_ast_tests")\n' - ' verify = checker["phase4_step4_runtime_boundary"]\n' - ' repository_path = ' - '"src/recursive_integrity_toolkit/" + relative\n' - ' current = (phase4_step5_snapshot if relative == ' - '"utils/logging.py" else phase4_step6_snapshot if ' - 'relative == "config.py" else repo_root) / ' - 'repository_path\n' - ' verify(current, repository_path)\n' - ' copy = tmp_path / "source.py"\n' - ' copy.write_bytes(current.read_bytes() + injection)\n' - ' with pytest.raises(ValueError):\n' - ' verify(copy, repository_path)', - 'node': 'test_phase4_step4_privacy_ast_checks_current_source_before_rejecting_scope_injection', - 'old': 'def ' - 'test_phase4_step4_privacy_ast_checks_current_source_before_rejecting_scope_injection(repo_root, ' - 'tmp_path, relative, injection, phase4_step5_snapshot):\n' - ' checker = runpy.run_path(str(repo_root / ' - '"scripts/check_traceability.py"), ' - 'run_name="phase4_step4_ast_tests")\n' - ' verify = checker["phase4_step4_runtime_boundary"]\n' - ' repository_path = ' - '"src/recursive_integrity_toolkit/" + relative\n' - ' current = (phase4_step5_snapshot if relative == ' - '"utils/logging.py" else repo_root) / repository_path\n' - ' verify(current, repository_path)\n' - ' copy = tmp_path / "source.py"\n' - ' copy.write_bytes(current.read_bytes() + injection)\n' - ' with pytest.raises(ValueError):\n' - ' verify(copy, repository_path)'}, - {'new': 'def ' - 'test_phase4_step1_keeps_current_help_and_future_commands_unopened(subprocess_env, ' - 'tmp_path, phase4_step6_snapshot):\n' - ' subprocess_env = dict(subprocess_env, ' - 'PYTHONPATH=str(phase4_step6_snapshot / "src"))\n' - ' before = set(tmp_path.iterdir())\n' - ' result = subprocess.run(\n' - ' [sys.executable, "-m", ' - '"recursive_integrity_toolkit", "--help"],\n' - ' cwd=tmp_path, env=subprocess_env, text=True, ' - 'capture_output=True, check=False,\n' - ' )\n' - ' assert result.returncode == 0 and "version" in ' - 'result.stdout\n' - ' assert "Analytical audit functionality is not ' - 'implemented" in " ".join(result.stdout.split())\n' - ' rejected = subprocess.run(\n' - ' [sys.executable, "-m", ' - '"recursive_integrity_toolkit", "audit"],\n' - ' cwd=tmp_path, env=subprocess_env, text=True, ' - 'capture_output=True, check=False,\n' - ' )\n' - ' assert rejected.returncode == 2\n' - ' assert set(tmp_path.iterdir()) == before', - 'node': 'test_phase4_step1_keeps_current_help_and_future_commands_unopened', - 'old': 'def ' - 'test_phase4_step1_keeps_current_help_and_future_commands_unopened(subprocess_env, ' - 'tmp_path):\n' - ' before = set(tmp_path.iterdir())\n' - ' result = subprocess.run(\n' - ' [sys.executable, "-m", ' - '"recursive_integrity_toolkit", "--help"],\n' - ' cwd=tmp_path, env=subprocess_env, text=True, ' - 'capture_output=True, check=False,\n' - ' )\n' - ' assert result.returncode == 0 and "version" in ' - 'result.stdout\n' - ' assert "Analytical audit functionality is not ' - 'implemented" in " ".join(result.stdout.split())\n' - ' rejected = subprocess.run(\n' - ' [sys.executable, "-m", ' - '"recursive_integrity_toolkit", "audit"],\n' - ' cwd=tmp_path, env=subprocess_env, text=True, ' - 'capture_output=True, check=False,\n' - ' )\n' - ' assert rejected.returncode == 2\n' - ' assert set(tmp_path.iterdir()) == before'}], - 'tests/unit/test_phase4_contracts.py': [{'new': 'def ' - 'test_phase4_step6_approved_control_keeps_independent_step5_anchors(phase4_tools, ' - 'phase4_step6_snapshot):\n' - ' control = json.loads((phase4_step6_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' assert phase4_tools["PHASE4_STEP5_FINAL"] == ' - '"c1667179e895a798bba2349960162542efa5ef87"\n' - ' assert phase4_tools["PHASE4_STEP5_TREE"] == ' - '"d83f9fc8e0b569f2d796bd2a83b87422fe350fba"\n' - ' assert phase4_tools["PHASE4_STEP5_TEST_TREE"] == ' - '"974f8e65904c24ed4e226996eaf571aab6123965"\n' - ' assert control["active_phase"] == 4 and ' - 'control["active_step"] == 6\n' - ' assert control["approval_basis"] == "批准开始Phase 4 ' - '**Step 6**"\n' - ' assert control["baseline_commit"] == BASELINE\n' - ' assert control["previous_step_commit"] == ' - '"c1667179e895a798bba2349960162542efa5ef87"\n' - ' assert control["previous_step_tree"] == ' - '"d83f9fc8e0b569f2d796bd2a83b87422fe350fba"\n' - ' assert control["previous_step_test_tree"] == ' - '"974f8e65904c24ed4e226996eaf571aab6123965"\n' - ' assert control["previous_step_core_tests"] == 3215\n' - ' assert control["previous_step_parquet_tests"] == 3218\n' - ' assert len(control["previous_step_files_sha256"]) == ' - '227\n' - ' assert set(control["runtime_paths_authorized"]) == {\n' - ' "src/recursive_integrity_toolkit/utils/paths.py",\n' - ' ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' }\n' - ' assert control["schema_changes_authorized"] is False\n' - ' assert control["schema_paths_authorized"] == []\n' - ' assert control["new_files_permitted"] == ' - '["tests/unit/test_phase4_output_safety.py"]\n' - ' for name in ("phase_complete", "next_step_authorized", ' - '"main_merge_authorized", "publication_authorized"):\n' - ' assert control[name] is False\n' - ' phase4_tools["verify_phase4_step6_control"](control, ' - 'step=6)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step4_control"](control, ' - 'step=4)', - 'node': 'test_phase4_step6_approved_control_keeps_independent_step5_anchors', - 'old': 'def ' - 'test_phase4_step6_approved_control_keeps_independent_step5_anchors(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' assert phase4_tools["PHASE4_STEP5_FINAL"] == ' - '"c1667179e895a798bba2349960162542efa5ef87"\n' - ' assert phase4_tools["PHASE4_STEP5_TREE"] == ' - '"d83f9fc8e0b569f2d796bd2a83b87422fe350fba"\n' - ' assert phase4_tools["PHASE4_STEP5_TEST_TREE"] == ' - '"974f8e65904c24ed4e226996eaf571aab6123965"\n' - ' assert control["active_phase"] == 4 and ' - 'control["active_step"] == 6\n' - ' assert control["approval_basis"] == "批准开始Phase 4 ' - '**Step 6**"\n' - ' assert control["baseline_commit"] == BASELINE\n' - ' assert control["previous_step_commit"] == ' - '"c1667179e895a798bba2349960162542efa5ef87"\n' - ' assert control["previous_step_tree"] == ' - '"d83f9fc8e0b569f2d796bd2a83b87422fe350fba"\n' - ' assert control["previous_step_test_tree"] == ' - '"974f8e65904c24ed4e226996eaf571aab6123965"\n' - ' assert control["previous_step_core_tests"] == 3215\n' - ' assert control["previous_step_parquet_tests"] == 3218\n' - ' assert len(control["previous_step_files_sha256"]) == ' - '227\n' - ' assert set(control["runtime_paths_authorized"]) == {\n' - ' "src/recursive_integrity_toolkit/utils/paths.py",\n' - ' ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' }\n' - ' assert control["schema_changes_authorized"] is False\n' - ' assert control["schema_paths_authorized"] == []\n' - ' assert control["new_files_permitted"] == ' - '["tests/unit/test_phase4_output_safety.py"]\n' - ' for name in ("phase_complete", "next_step_authorized", ' - '"main_merge_authorized", "publication_authorized"):\n' - ' assert control[name] is False\n' - ' phase4_tools["verify_phase4_step6_control"](control, ' - 'step=6)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step4_control"](control, ' - 'step=4)'}, - {'new': 'def ' - 'test_phase4_step6_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value, phase4_step6_snapshot):\n' - ' control = json.loads((phase4_step6_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' verify = phase4_tools["verify_phase4_step6_control"]\n' - ' verify(control, step=6)\n' - ' control[field] = value\n' - ' with pytest.raises(ValueError):\n' - ' verify(control, step=6)', - 'node': 'test_phase4_step6_control_rejects_forged_scope_and_stage', - 'old': 'def ' - 'test_phase4_step6_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' verify = phase4_tools["verify_phase4_step6_control"]\n' - ' verify(control, step=6)\n' - ' control[field] = value\n' - ' with pytest.raises(ValueError):\n' - ' verify(control, step=6)'}, - {'new': 'def ' - 'test_phase4_step6_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step, phase4_step6_snapshot):\n' - ' control = json.loads((phase4_step6_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step6_control"](control, ' - 'step=6)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step6_control"](control, ' - 'step=step)', - 'node': 'test_phase4_step6_control_rejects_unapproved_dispatch', - 'old': 'def ' - 'test_phase4_step6_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step6_control"](control, ' - 'step=6)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step6_control"](control, ' - 'step=step)'}, - {'new': 'def ' - 'test_phase4_step6_control_requires_explicit_approval_fields(phase4_tools, ' - 'field, phase4_step6_snapshot):\n' - ' control = json.loads((phase4_step6_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step6_control"](control)\n' - ' del control[field]\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step6_control"](control)', - 'node': 'test_phase4_step6_control_requires_explicit_approval_fields', - 'old': 'def ' - 'test_phase4_step6_control_requires_explicit_approval_fields(phase4_tools, ' - 'field):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step6_control"](control)\n' - ' del control[field]\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step6_control"](control)'}, - {'new': 'def ' - 'test_phase4_step6_control_cannot_mint_extra_permission(phase4_tools, ' - 'phase4_step6_snapshot):\n' - ' control = json.loads((phase4_step6_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step6_control"](control)\n' - ' control["approved_scope_expansion"] = {"step": 6, ' - '"publication": True}\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step6_control"](control)', - 'node': 'test_phase4_step6_control_cannot_mint_extra_permission', - 'old': 'def ' - 'test_phase4_step6_control_cannot_mint_extra_permission(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step6_control"](control)\n' - ' control["approved_scope_expansion"] = {"step": 6, ' - '"publication": True}\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step6_control"](control)'}, - {'new': 'def ' - 'test_phase4_step6_historical_bindings_preserve_assertions_and_headers(phase4_tools, ' - 'phase4_step5_snapshot, phase4_step6_snapshot):\n' - ' import ast\n' - '\n' - ' registry = phase4_tools["PHASE4_STEP6_MIGRATIONS"]\n' - ' assert set(registry) == ' - '{"tests/unit/test_phase4_contracts.py", ' - '"tests/integration/test_phase4_gates.py"}\n' - ' assert sorted(len(rows) for rows in registry.values()) ' - '== [7, 8]\n' - ' expected_inventory = {"phase4_mutation_tree"} | {\n' - ' "test_phase4_step" + str(step) + ' - '"_current_snapshot_checks_valid_tree_before_mutations"\n' - ' for step in (2, 3, 4, 5)\n' - ' }\n' - ' inventory_seen = set()\n' - ' for path, rows in registry.items():\n' - ' ' - 'phase4_tools["verify_phase4_step6_test_migration"](\n' - ' path, (phase4_step5_snapshot / ' - 'path).read_bytes(), (phase4_step6_snapshot / ' - 'path).read_bytes())\n' - ' for row in rows:\n' - ' trees = [ast.parse(row[key]) for key in ' - '("old", "new")]\n' - ' assertions = [[ast.dump(n) for n in ' - 'ast.walk(tree) if isinstance(n, ast.Assert)] for tree in ' - 'trees]\n' - ' assert assertions[0] == assertions[1]\n' - ' assert trees[0].body[0].name == ' - 'trees[1].body[0].name == row["node"]\n' - ' if row["node"] in expected_inventory:\n' - ' inventory_seen.add(row["node"])\n' - ' assert \'"ls-files"\' in row["old"] and ' - '\'rglob("*")\' in row["new"]\n' - ' assert inventory_seen == expected_inventory', - 'node': 'test_phase4_step6_historical_bindings_preserve_assertions_and_headers', - 'old': 'def ' - 'test_phase4_step6_historical_bindings_preserve_assertions_and_headers(phase4_tools, ' - 'phase4_step5_snapshot):\n' - ' import ast\n' - '\n' - ' registry = phase4_tools["PHASE4_STEP6_MIGRATIONS"]\n' - ' assert set(registry) == ' - '{"tests/unit/test_phase4_contracts.py", ' - '"tests/integration/test_phase4_gates.py"}\n' - ' assert sorted(len(rows) for rows in registry.values()) ' - '== [7, 8]\n' - ' expected_inventory = {"phase4_mutation_tree"} | {\n' - ' "test_phase4_step" + str(step) + ' - '"_current_snapshot_checks_valid_tree_before_mutations"\n' - ' for step in (2, 3, 4, 5)\n' - ' }\n' - ' inventory_seen = set()\n' - ' for path, rows in registry.items():\n' - ' ' - 'phase4_tools["verify_phase4_step6_test_migration"](\n' - ' path, (phase4_step5_snapshot / ' - 'path).read_bytes(), (ROOT / path).read_bytes())\n' - ' for row in rows:\n' - ' trees = [ast.parse(row[key]) for key in ' - '("old", "new")]\n' - ' assertions = [[ast.dump(n) for n in ' - 'ast.walk(tree) if isinstance(n, ast.Assert)] for tree in ' - 'trees]\n' - ' assert assertions[0] == assertions[1]\n' - ' assert trees[0].body[0].name == ' - 'trees[1].body[0].name == row["node"]\n' - ' if row["node"] in expected_inventory:\n' - ' inventory_seen.add(row["node"])\n' - ' assert \'"ls-files"\' in row["old"] and ' - '\'rglob("*")\' in row["new"]\n' - ' assert inventory_seen == expected_inventory'}, - {'new': 'def ' - 'test_phase4_step6_historical_guard_rejects_weakening(phase4_tools, ' - 'phase4_step5_snapshot, mutation, phase4_step6_snapshot):\n' - ' path = "tests/integration/test_phase4_gates.py" if ' - 'mutation in ("inventory", "logging") else ' - '"tests/unit/test_phase4_contracts.py"\n' - ' before = (phase4_step5_snapshot / path).read_bytes()\n' - ' after = (phase4_step6_snapshot / path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step6_test_migration"]\n' - ' verify(path, before, after)\n' - ' if mutation == "assertion":\n' - ' needle = b\' assert control["active_phase"] == ' - '4 and control["active_step"] == 5\\n\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b" assert ' - 'True\\n", 1)\n' - ' elif mutation == "binding":\n' - " needle = b'json.loads((phase4_step5_snapshot / " - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\'\n' - ' assert needle in after\n' - ' after = after.replace(needle, ' - 'needle.replace(b"phase4_step5_snapshot", b"ROOT"), 1)\n' - ' elif mutation == "inventory":\n' - " needle = b'for path in " - 'phase4_step1_snapshot.rglob("*") if path.is_file()\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, ' - 'needle.replace(b"phase4_step1_snapshot", b"repo_root"), ' - '1)\n' - ' elif mutation == "logging":\n' - " needle = b'(phase4_step5_snapshot if relative == " - '"utils/logging.py" else repo_root)\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b"repo_root", 1)\n' - ' elif mutation == "shadow":\n' - ' after += b"\\ndef ' - 'test_phase4_step5_fixed_boundary_opens_only_approved_existing_paths():\\n ' - 'assert True\\n"\n' - ' else:\n' - ' after += b"\\ndef ' - 'test_phase4_step6_bad(value=globals().clear()):\\n ' - 'pass\\n"\n' - ' with pytest.raises(ValueError):\n' - ' verify(path, before, after)', - 'node': 'test_phase4_step6_historical_guard_rejects_weakening', - 'old': 'def ' - 'test_phase4_step6_historical_guard_rejects_weakening(phase4_tools, ' - 'phase4_step5_snapshot, mutation):\n' - ' path = "tests/integration/test_phase4_gates.py" if ' - 'mutation in ("inventory", "logging") else ' - '"tests/unit/test_phase4_contracts.py"\n' - ' before = (phase4_step5_snapshot / path).read_bytes()\n' - ' after = (ROOT / path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step6_test_migration"]\n' - ' verify(path, before, after)\n' - ' if mutation == "assertion":\n' - ' needle = b\' assert control["active_phase"] == ' - '4 and control["active_step"] == 5\\n\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b" assert ' - 'True\\n", 1)\n' - ' elif mutation == "binding":\n' - " needle = b'json.loads((phase4_step5_snapshot / " - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\'\n' - ' assert needle in after\n' - ' after = after.replace(needle, ' - 'needle.replace(b"phase4_step5_snapshot", b"ROOT"), 1)\n' - ' elif mutation == "inventory":\n' - " needle = b'for path in " - 'phase4_step1_snapshot.rglob("*") if path.is_file()\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, ' - 'needle.replace(b"phase4_step1_snapshot", b"repo_root"), ' - '1)\n' - ' elif mutation == "logging":\n' - " needle = b'(phase4_step5_snapshot if relative == " - '"utils/logging.py" else repo_root)\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b"repo_root", 1)\n' - ' elif mutation == "shadow":\n' - ' after += b"\\ndef ' - 'test_phase4_step5_fixed_boundary_opens_only_approved_existing_paths():\\n ' - 'assert True\\n"\n' - ' else:\n' - ' after += b"\\ndef ' - 'test_phase4_step6_bad(value=globals().clear()):\\n ' - 'pass\\n"\n' - ' with pytest.raises(ValueError):\n' - ' verify(path, before, after)'}]} - - -def _phase4_step6_files(): - """Read the exact accepted Step 6 Git tree and verify every blob identity.""" - for suffix, expected in (("^{commit}", PHASE4_STEP6_FINAL), ("^{tree}", PHASE4_STEP6_TREE), - (":tests", PHASE4_STEP6_TEST_TREE)): - if git("rev-parse", PHASE4_STEP6_FINAL + suffix).decode().strip() != expected: - raise ValueError("Pinned Phase 4 Step 6 identity mismatch") - objects = {} - for entry in git("ls-tree", "-rz", PHASE4_STEP6_FINAL).split(b"\0"): - if not entry: - continue - metadata, raw_path = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - path = raw_path.decode("utf-8") - if (mode not in (b"100644", b"100755") or kind != b"blob" or path in objects - or path.startswith("/") or ".." in path.split("/") or ".git" in path.split("/")): - raise ValueError("Unsafe pinned Step 6 Git object") - objects[path] = oid.decode("ascii") - if len(objects) != 228: - raise ValueError("Pinned Step 6 must contain exactly 228 files") - files = {} - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_STEP6_FINAL))) as archive: - names = [item.filename for item in archive.infolist() if not item.is_dir()] - if len(names) != len(objects) or set(names) != set(objects): - raise ValueError("Pinned Step 6 archive identity mismatch") - for name in names: - raw = archive.read(name) - oid = hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - if oid != objects[name]: - raise ValueError(f"Pinned Step 6 blob mismatch: {name}") - files[name] = raw - return MappingProxyType(files) - - -def phase4_step7_expected_control() -> dict: - """Actual Step 7 approval is independent of the mutable control document.""" - prior = _phase4_step6_files() - result = phase4_step6_expected_control() - result.update({ - "control_version": "1.6", "active_step": 7, - "approval_date": PHASE4_STEP7_APPROVAL_DATE, "approval_basis": PHASE4_STEP7_APPROVAL, - "previous_step_commit": PHASE4_STEP6_FINAL, - "previous_step_tree": PHASE4_STEP6_TREE, - "previous_step_test_tree": PHASE4_STEP6_TEST_TREE, - "previous_step_core_tests": 3420, "previous_step_parquet_tests": 3423, - "previous_step_files_sha256": {p: hashlib.sha256(raw).hexdigest() for p, raw in sorted(prior.items())}, - "permitted_paths": sorted(PHASE4_STEP7_ALLOWED), "new_files_permitted": list(PHASE4_STEP7_NEW), - "runtime_changes_authorized": True, "schema_changes_authorized": False, - "runtime_paths_authorized": sorted(p for p in PHASE4_STEP7_ALLOWED if p.startswith("src/")), - "schema_paths_authorized": [], - "step6_historical_binding_nodes": {p: sorted({r["node"] for r in rows}) - for p, rows in sorted(PHASE4_STEP7_MIGRATIONS.items())}, - }) - return result - - -def verify_phase4_step7_control(control: dict, step: int = 7) -> None: - if type(step) is not int or step != 7 or type(control) is not dict: - raise ValueError("Unsupported Phase 4 Step 7 stage/control") - try: - actual = json.dumps(control, sort_keys=True, ensure_ascii=True, allow_nan=False) - expected = json.dumps(phase4_step7_expected_control(), sort_keys=True, ensure_ascii=True, allow_nan=False) - except (TypeError, ValueError) as error: - raise ValueError("Invalid Phase 4 Step 7 control") from error - if actual != expected: - raise ValueError("Phase 4 control differs from the independently approved Step 7 contract") - - -def verify_phase4_step7_changes(changes: list[tuple[str, str]]) -> None: - seen = set() - for status, path in changes: - if path in seen or path not in PHASE4_STEP7_ALLOWED or status != ("A" if path in PHASE4_STEP7_NEW else "M"): - raise ValueError(f"Unapproved Phase 4 Step 7 path/operation: {status} {path}") - seen.add(path) - - -def _phase4_step7_header(node, path): - """Allow explicit test parametrization/fixtures, never definition-time effects.""" - for decorator in node.decorator_list: - if not isinstance(decorator, ast.Call) or ast.unparse(decorator.func) not in { - "pytest.fixture", "pytest.mark.parametrize"}: - raise ValueError(f"Unapproved Step 7 test decorator: {path}:{node.name}") - if ast.unparse(decorator.func) == "pytest.fixture" and not node.name.startswith("phase4_step"): - raise ValueError("A Step 7 fixture must have an explicit scoped name") - if any(keyword.arg is None for keyword in decorator.keywords): - raise ValueError("Decorator expansion is outside the Step 7 contract") - for value in [*decorator.args, *(keyword.value for keyword in decorator.keywords)]: - try: - ast.literal_eval(value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError("Step 7 decorator arguments must be literal") from error - clone = ast.parse(ast.unparse(node)).body[0] - clone.decorator_list = [] - _phase4_preserve_function_header(clone, path) - - -def _phase4_step7_append_only(before: bytes, after: bytes, path: str) -> None: - if not after.startswith(before): - raise ValueError(f"Inherited Step 6 prefix changed: {path}") - old, new = ast.parse(before), ast.parse(after) - bindings = set() - for node in old.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - bindings.add(node.name) - elif isinstance(node, (ast.Import, ast.ImportFrom)): - bindings.update(alias.asname or alias.name.split(".")[0] for alias in node.names) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - bindings.update(child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)) - for node in new.body[len(old.body):]: - if not isinstance(node, ast.FunctionDef) or node.name in bindings: - raise ValueError(f"Step 7 addition executes, rebinds or shadows inherited source: {path}") - allowed_snapshot = path == "tests/conftest.py" and node.name == "phase4_step6_snapshot" - if not allowed_snapshot and not node.name.startswith(("test_phase4_step7_", "phase4_step7_")): - raise ValueError(f"Step 7 added test/helper is not explicitly scoped: {path}:{node.name}") - if path == "tests/conftest.py" and node.name != "phase4_step6_snapshot": - raise ValueError("Only the pinned Step 6 shared fixture is authorized") - bindings.add(node.name) - _phase4_step7_header(node, path) - - -def verify_phase4_step7_test_migration(path: str, before: bytes, after: bytes) -> None: - prior = _phase4_step6_files() - if path not in prior or path not in PHASE4_STEP7_ALLOWED or before != prior[path] or not path.startswith("tests/"): - raise ValueError("Step 7 migration requires the exact named Step 6 source") - _phase4_step7_check_test_migration(path, before, after) - - -def _phase4_step7_check_test_migration(path: str, before: bytes, after: bytes) -> None: - """Check a source already read from the verified Step 6 immutable mapping.""" - expected = before - for row in PHASE4_STEP7_MIGRATIONS.get(path, []): - old, new = row["old"].encode(), row["new"].encode() - if expected.count(old) != 1: - raise ValueError("Step 6 historical binding is not unique") - expected = expected.replace(old, new, 1) - _phase4_step7_append_only(expected, after, path) - - -def _phase4_step7_preserve_tooling(before: bytes, after: bytes, path: str) -> None: - """Preserve exact inherited statements using one source-line split per tree.""" - if path == "scripts/release_check.py": - old = ' return phase4_cli_main() if explicit_phase4 else main()' - new = (' explicit_step7 = "--step=7" in argv or any(a == "--step" and b == "7" for a, b in zip(argv, argv[1:]))\n' - ' if explicit_phase4 and explicit_step7:\n' - ' return phase4_step7_cli_main()\n' + old) - # Earlier preserved dispatch functions contain this source too; bind cli_main only. - parsed = ast.parse(before) - entry = next(node for node in parsed.body if isinstance(node, ast.FunctionDef) and node.name == "cli_main") - old_entry = ast.get_source_segment(before.decode(), entry) - if old_entry.count(old) != 1: - raise ValueError("Step 6 release dispatcher identity mismatch") - replacements = [(old_entry, old_entry.replace(old, new, 1))] - else: - old = ' if args.phase == 4 and args.step == 6:\n return phase4_step6_main(step=args.step)\n' - new = old + ' if args.phase == 4 and args.step == 7:\n return phase4_step7_main(step=args.step)\n' - replacements = [(old, new), - ('Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5 or --phase 4 --step 6', - 'Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5 or --phase 4 --step 6 or --phase 4 --step 7')] - expected = before - for old, new in replacements: - if expected.count(old.encode()) != 1: - raise ValueError(f"Historical dispatcher identity mismatch: {path}") - expected = expected.replace(old.encode(), new.encode(), 1) - trees = [ast.parse(expected), ast.parse(after)] - lines = [expected.splitlines(keepends=True), after.splitlines(keepends=True)] - def segment(index, node): - if node.lineno == node.end_lineno: - return lines[index][node.lineno - 1][node.col_offset:node.end_col_offset] - return (lines[index][node.lineno - 1][node.col_offset:] - + b"".join(lines[index][node.lineno:node.end_lineno - 1]) - + lines[index][node.end_lineno - 1][:node.end_col_offset]) - def entry_guard(node): - return isinstance(node, ast.If) and isinstance(node.test, ast.Compare) and "__name__" in ast.unparse(node.test) - guards = [n for n in trees[1].body if entry_guard(n)] - expected_guard = ast.parse('if __name__ == "__main__":\n raise SystemExit(cli_main())').body[0] - if len(guards) != 1 or ast.dump(guards[0]) != ast.dump(expected_guard): - raise ValueError(f"Unapproved Step 7 maintainer entrypoint: {path}") - historical = [n for n in trees[0].body if not entry_guard(n)] - current = [n for n in trees[1].body if not entry_guard(n)] - old_entries = [(segment(0, n), ast.dump(n)) for n in historical] - cursor = 0 - for node in current: - if cursor < len(old_entries) and (segment(1, node), ast.dump(node)) == old_entries[cursor]: - cursor += 1 - continue - if isinstance(node, ast.FunctionDef) and (node.name == "_phase4_step6_files" or node.name.startswith( - ("phase4_step7_", "_phase4_step7_", "verify_phase4_step7_", "audit_phase4_step7"))): - _phase4_preserve_function_header(node, path) - continue - if isinstance(node, ast.Assign) and all(isinstance(target, ast.Name) and target.id.startswith( - ("PHASE4_STEP7_", "PHASE4_STEP6_")) for target in node.targets): - try: - ast.literal_eval(node.value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError(f"Nonliteral added Step 7 maintainer constant: {path}") from error - continue - raise ValueError(f"Unapproved Step 7 maintainer addition: {path}") - if cursor != len(historical): - raise ValueError(f"Inherited Step 6 maintainer statement changed: {path}") - def binding_counts(tree): - counts = {} - for node in tree.body: - names = [] - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - names = [node.name] - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names = [child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)] - elif isinstance(node, (ast.Import, ast.ImportFrom)): - names = [alias.asname or alias.name.split(".")[0] for alias in node.names] - for name in names: - counts[name] = counts.get(name, 0) + 1 - return counts - inherited_counts = binding_counts(trees[0]) - for name, count in binding_counts(trees[1]).items(): - if count > inherited_counts.get(name, 1): - raise ValueError(f"Duplicate Step 7 maintainer binding: {path}:{name}") - - -def verify_phase4_step7_snapshot(root: Path = ROOT) -> dict: - prior = _phase4_step6_files() - for path, raw in prior.items(): - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError(f"Missing or aliased inherited Step 6 file: {path}") - current = target.read_bytes() - if path not in PHASE4_STEP7_ALLOWED and current != raw: - raise ValueError(f"Protected Step 6 bytes changed: {path}") - if path.startswith("tests/") and path in PHASE4_STEP7_ALLOWED: - _phase4_step7_check_test_migration(path, raw, current) - elif path.startswith("scripts/") and path in PHASE4_STEP7_ALLOWED: - _phase4_step7_preserve_tooling(raw, current, path) - elif path in {"docs/architecture.md", "docs/theory_traceability.md", "PHASE_4_DECISIONS.md"}: - if not current.startswith(raw): - raise ValueError(f"Step 6 historical documentation prefix changed: {path}") - if not (root / "docs/cli.md").read_bytes().endswith(prior["docs/cli.md"]): - raise ValueError("Step 7 CLI documentation must retain the historical notes verbatim") - for path in PHASE4_STEP7_NEW: - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError("Missing Step 7 output safety tests") - _phase4_step7_new_test(target.read_bytes(), path) - for path in ("src/recursive_integrity_toolkit/config.py",): - _phase4_step7_preserve_runtime(prior[path], (root / path).read_bytes(), path) - actual_modules = {p.relative_to(root).as_posix() for p in (root / "src/recursive_integrity_toolkit").rglob("*.py")} - expected_modules = {p for p in prior if p.startswith("src/") and p.endswith(".py")} - if actual_modules != expected_modules or len(actual_modules) != 40: - raise ValueError("Step 7 cannot change the runtime module set") - if {p.name for p in (root / "schemas").iterdir()} != {Path(p).name for p in prior if p.startswith("schemas/")}: - raise ValueError("Step 7 cannot change the schema set") - if hashlib.sha256((root / "PHASE_4_PLAN.md").read_bytes()).hexdigest() != PHASE4_PLAN_SHA256: - raise ValueError("Approved Phase 4 plan changed") - verify_phase4_step7_control(json.loads((root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))) - decisions = (root / "PHASE_4_DECISIONS.md").read_text(encoding="utf-8") - if PHASE4_STEP7_APPROVAL not in decisions or PHASE4_STEP6_FINAL not in decisions: - raise ValueError("Actual Step 7 authorization or Step 6 evidence anchor missing") - if any((root / name).exists() for name in PHASE4_FORBIDDEN_OUTPUTS): - raise ValueError("Step 7 cannot create Phase 4 completion records or audit outputs") - import runpy - checker = runpy.run_path(str(root / "scripts/check_traceability.py"), run_name="phase4_step7_renderer_boundary") - checker["phase4_step7_cli_boundary"](root) - schema_bytes = (root / "schemas/report.schema.json").read_bytes() - if hashlib.sha256(schema_bytes).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Step 7 report schema differs from its independently reviewed bytes") - schema = json.loads(schema_bytes) - if (schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema" - or schema.get("type") != "object" or schema.get("additionalProperties") is not False): - raise ValueError("Step 7 report schema dialect/closed root mismatch") - return {"package_modules": 40, "frozen_runtime_modules": 38, "frozen_schemas": 5, - "hero_files_unchanged": 6, "historical_phase3_migrated_nodes": 16, - "phase_complete": False, "result_contracts_enabled": True, - "adapters_enabled": True, "privacy_views_enabled": True, "renderers_enabled": True, "output_publication_enabled": True, "cli_analysis_enabled": True, "comparison_enabled": False, "example_enabled": False} - - -def audit_phase4_step7(step: int = 7) -> dict: - verify_phase4_step7_control(json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")), step) - result = {"phase": 4, "active_step": step, **verify_phase4_step7_snapshot(), - "phase0_hashes_verified": 16, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_phase4_step7_diff(step: int = 7) -> dict: - if type(step) is not int or step != 7: - raise ValueError("Unsupported Phase 4 Step 7 stage") - _phase4_step6_files() - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE4_STEP6_FINAL, "HEAD"], check=True) - raw = git("diff", "--name-status", "--no-renames", "-z", PHASE4_STEP6_FINAL, "--").split(b"\0") - raw = [part.decode("utf-8") for part in raw if part] - if len(raw) % 2: - raise ValueError("Malformed Step 7 Git difference records") - changes = list(zip(raw[::2], raw[1::2])) - changes.extend(("A", p.decode()) for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") if p) - verify_phase4_step7_changes(changes) - result = {"previous_step_commit": PHASE4_STEP6_FINAL, "changed_files": len(changes), - "changes": changes, "step": 7, "scope": "PASS"} - print(json.dumps(result, indent=2)) - return result - - -def phase4_step7_baseline_evidence(output: Path) -> dict: - """Reconcile all inherited identities, including the accepted Step 6 suite.""" - output.mkdir(parents=True, exist_ok=True) - inherited = phase4_step6_baseline_evidence(output / "phase4-step6-inherited") - with tempfile.TemporaryDirectory(prefix="rit-p4-step6-identities-") as temp: - baseline = Path(temp) - for name, raw in _phase4_step6_files().items(): - target = baseline / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(raw) - old, old_log = _collect(baseline) - current, current_log = _collect(ROOT) - expected = 3423 if os.environ.get("RIT_TEST_PARQUET") == "1" else 3420 - if len(old) != expected or set(old) - set(current): - raise ValueError("Accepted Phase 4 Step 6 test identities were lost") - result = {"baseline_commit": PHASE4_STEP6_FINAL, "baseline_test_tree": PHASE4_STEP6_TEST_TREE, - "baseline_nodeids": old, "current_nodeids": current, "missing_nodeids": [], - "baseline_tests": len(old), "current_tests": len(current), "inherited": inherited, - "nodeids_sha256": hashlib.sha256(("\n".join(old)+"\n").encode()).hexdigest()} - (output / "phase4_step7_test_identity_manifest.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4-step6-collection.log").write_text(old_log, encoding="utf-8") - (output / "phase4-step7-collection.log").write_text(current_log, encoding="utf-8") - print(json.dumps({k: result[k] for k in ("baseline_tests", "current_tests", "missing_nodeids")}, indent=2)) - return result - - -def phase4_step7_candidate(output: Path, step: int = 7) -> None: - """Build tested Step 7 intermediate evidence without declaring Phase 4 complete.""" - if os.environ.get("RIT_TEST_PARQUET") != "1" or importlib.util.find_spec("pyarrow") is None: - raise ValueError("Step 7 candidate evidence requires RIT_TEST_PARQUET=1 and real PyArrow") - output.mkdir(parents=True, exist_ok=True) - result = audit_phase4_step7(step) - result["diff"] = audit_phase4_step7_diff(step) - if git("status", "--porcelain").strip(): - raise ValueError("Step 7 candidate archive requires committed, clean source") - result["core"] = verify_junit(output / "core.xml", minimum=3420) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=3423) - result["core_math_measurements"] = verify_step10_evidence(output / "core.xml", output / "phase4-step7-core-observations.json") - result["parquet_math_measurements"] = verify_step10_evidence(output / "parquet.xml", output / "phase4-step7-parquet-observations.json") - identity = phase4_step7_baseline_evidence(output) - result["test_identity"] = {k: identity[k] for k in ("baseline_tests", "current_tests", "nodeids_sha256")} - expected = set() - for node in identity["current_nodeids"]: - base, bracket, parameter = node.partition("[") - owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) - for name, parquet in (("core.xml", False), ("parquet.xml", True)): - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(output / name).getroot().iter("testcase")} - target = expected if parquet else {c for c in expected if c[1] not in PARQUET_CASES} - if cases != target: - raise ValueError(f"Step 7 JUnit does not execute the entire current suite: {name}") - wheel, _ = verify_distributions(output / "dist") - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - phase4_step6_installed_publication_smoke(wheel) - phase4_step7_installed_cli_smoke(wheel) - archive = output / "recursive-integrity-toolkit-phase4-step7-candidate.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {p for p in git("ls-files", "-z").decode().split("\0") if p} - with zipfile.ZipFile(archive) as zipped: - entries = [item.filename for item in zipped.infolist() if not item.is_dir()] - prefix = "recursive-integrity-toolkit/" - if (len(entries) != len(set(entries)) or any(not name.startswith(prefix) for name in entries)): - raise ValueError("Step 7 source archive has duplicate or unprefixed entries") - names = {name.removeprefix(prefix) for name in entries} - if names != tracked or len(tracked) != 229 or len(entries) != 229: - raise ValueError("Step 7 source archive file set mismatch") - for name in names: - if zipped.read("recursive-integrity-toolkit/"+name) != (ROOT / name).read_bytes(): - raise ValueError(f"Step 7 source archive byte mismatch: {name}") - result.update({"commit": git("rev-parse", "HEAD").decode().strip(), - "tree": git("rev-parse", "HEAD^{tree}").decode().strip(), - "source_archive": archive.name, "tracked_files": len(tracked), - "python": sys.version, "platform": sys.platform, - "versions": {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "pytest")}}) - (output / "phase4_step7_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4_step7_repository_files.sha256").write_text("".join(f"{hashlib.sha256((ROOT/name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - paths = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase4_step7_artifacts.sha256") - (output / "phase4_step7_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in paths), encoding="utf-8") - print(f"Phase 4 Step 7 candidate: {len(tracked)} source files verified; Phase 4 remains incomplete.") - - -def phase4_step7_cli_main() -> int: - parser = argparse.ArgumentParser(description="Phase 4 Step 7 maintainer gate") - parser.add_argument("--phase", type=int, choices=(4,), required=True) - parser.add_argument("--step", type=int, choices=(7,), required=True) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) - parser.add_argument("--require-parquet", action="store_true") - parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - raise ValueError("Phase 4 Step 7 cannot certify a final phase delivery") - if args.junit is not None: - verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - phase4_step7_baseline_evidence(args.baseline_evidence) - elif args.dist is not None: - audit_phase4_step7(args.step) - wheel, _ = verify_distributions(args.dist) - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - phase4_step6_installed_publication_smoke(wheel) - phase4_step7_installed_cli_smoke(wheel) - elif args.smoke_wheel is not None: - smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - phase4_step7_candidate(args.candidate, args.step) - else: - audit_phase4_step7(args.step) - if args.diff: - audit_phase4_step7_diff(args.step) - return 0 - - -def _phase4_step7_new_test(raw: bytes, path: str) -> None: - """Only literal test declarations and explicitly listed safe imports.""" - allowed = {"__future__", "json", "pathlib", "pytest"} - for index, node in enumerate(ast.parse(raw).body): - if index == 0 and isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str): - continue - if isinstance(node, (ast.Import, ast.ImportFrom)): - modules = [node.module] if isinstance(node, ast.ImportFrom) else [alias.name for alias in node.names] - if any(module not in allowed for module in modules): - raise ValueError("Unapproved Step 7 test import") - continue - if not isinstance(node, ast.FunctionDef) or not node.name.startswith(("test_phase4_step7_", "phase4_step7_")): - raise ValueError("Step 7 output tests gained definition-time execution") - _phase4_step7_header(node, path) - - -def _phase4_step7_preserve_runtime(before: bytes, after: bytes, path: str) -> None: - """Allow ownership-header maintenance and byte-exact additive helpers only.""" - try: - old, new = ast.parse(before), ast.parse(after) - except (SyntaxError, UnicodeError, ValueError) as error: - raise ValueError("Output helper source cannot be inspected") from error - if not ast.get_docstring(old) or not ast.get_docstring(new): - raise ValueError("Output helper ownership documentation is required") - old_tail = b"".join(before.splitlines(keepends=True)[old.body[0].end_lineno:]) - new_tail = b"".join(after.splitlines(keepends=True)[new.body[0].end_lineno:]) - if not new_tail.startswith(old_tail): - raise ValueError(f"Inherited input/diagnostic bytes changed: {path}") - - -def phase4_step7_installed_cli_smoke(wheel: Path) -> None: +def smoke_installed_cli(wheel: Path) -> None: """Run the installed CLI with caller core dependencies outside the checkout.""" with tempfile.TemporaryDirectory(prefix="rit-p4-step7-installed-") as temporary: work = Path(temporary) @@ -8280,1442 +1090,7 @@ def phase4_step7_installed_cli_smoke(wheel: Path) -> None: subprocess.run([sys.executable, "-I", "-c", program, str(target), str(work), str(ROOT)], cwd=work, check=True) - -# Phase 4 Step 8: explicit pair and exact packaged resources. - - -PHASE4_STEP7_FINAL = 'bacd33ae65e392872776c6d976401fdc3d565f63' - - -PHASE4_STEP7_TREE = '4c16914bb0f0c5281efc36624efb8b1a1d4a174e' - - -PHASE4_STEP7_TEST_TREE = '96317cb7b25236b79efc0f2d93e1e0f5f132fdb2' - - -PHASE4_STEP8_APPROVAL = 'phase 4 step 8 开始' - - -PHASE4_STEP8_APPROVAL_DATE = '2026-09-21' - - -PHASE4_STEP8_NEW = ('src/recursive_integrity_toolkit/data/hero/EXPECTED_OUTPUTS.md', 'src/recursive_integrity_toolkit/data/hero/config.json', 'src/recursive_integrity_toolkit/data/hero/provenance.csv', 'src/recursive_integrity_toolkit/data/hero/records_v1.csv', 'src/recursive_integrity_toolkit/data/hero/records_v2.csv', 'src/recursive_integrity_toolkit/data/hero/version_order.json', 'src/recursive_integrity_toolkit/data/report.schema.json') - - -PHASE4_STEP8_ALLOWED = {'.github/workflows/ci.yml', - '.github/workflows/golden.yml', - '.github/workflows/release.yml', - '.github/workflows/security.yml', - 'PHASE_4_BASELINE.json', - 'PHASE_4_DECISIONS.md', - 'docs/architecture.md', - 'docs/cli.md', - 'docs/theory_traceability.md', - 'pyproject.toml', - 'scripts/check_spec_consistency.py', - 'scripts/check_traceability.py', - 'scripts/release_check.py', - 'src/recursive_integrity_toolkit/cli.py', - 'src/recursive_integrity_toolkit/config.py', - 'src/recursive_integrity_toolkit/data/hero/EXPECTED_OUTPUTS.md', - 'src/recursive_integrity_toolkit/data/hero/config.json', - 'src/recursive_integrity_toolkit/data/hero/provenance.csv', - 'src/recursive_integrity_toolkit/data/hero/records_v1.csv', - 'src/recursive_integrity_toolkit/data/hero/records_v2.csv', - 'src/recursive_integrity_toolkit/data/hero/version_order.json', - 'src/recursive_integrity_toolkit/data/report.schema.json', - 'tests/conftest.py', - 'tests/integration/test_ci_workflows.py', - 'tests/integration/test_hero_end_to_end.py', - 'tests/integration/test_hero_structure.py', - 'tests/integration/test_license_notices.py', - 'tests/integration/test_no_algorithms.py', - 'tests/integration/test_no_network.py', - 'tests/integration/test_optional_dependency.py', - 'tests/integration/test_owner_ids.py', - 'tests/integration/test_package_import.py', - 'tests/integration/test_package_install.py', - 'tests/integration/test_phase4_cli.py', - 'tests/integration/test_phase4_gates.py', - 'tests/integration/test_prohibited_structure.py', - 'tests/integration/test_repository_structure.py', - 'tests/integration/test_schema_json.py', - 'tests/unit/test_phase4_contracts.py'} - - -PHASE4_STEP8_RESOURCES = {'src/recursive_integrity_toolkit/data/hero/EXPECTED_OUTPUTS.md': 'examples/hero/EXPECTED_OUTPUTS.md', - 'src/recursive_integrity_toolkit/data/hero/config.json': 'examples/hero/config.json', - 'src/recursive_integrity_toolkit/data/hero/provenance.csv': 'examples/hero/provenance.csv', - 'src/recursive_integrity_toolkit/data/hero/records_v1.csv': 'examples/hero/records_v1.csv', - 'src/recursive_integrity_toolkit/data/hero/records_v2.csv': 'examples/hero/records_v2.csv', - 'src/recursive_integrity_toolkit/data/hero/version_order.json': 'examples/hero/version_order.json', - 'src/recursive_integrity_toolkit/data/report.schema.json': 'schemas/report.schema.json'} - - -PHASE4_STEP8_PACKAGING = '\n[tool.setuptools.package-data]\nrecursive_integrity_toolkit = ["data/hero/*", "data/report.schema.json"]\n' - - -PHASE4_STEP8_MIGRATIONS = {'tests/integration/test_phase4_gates.py': [{'new': 'def ' - 'test_phase4_step7_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step7_snapshot):\n' - ' repo_root = phase4_step7_snapshot\n' - ' names = {"ci.yml", "golden.yml", "security.yml", ' - '"release.yml"}\n' - ' root = repo_root / ".github/workflows"\n' - ' expected_timeouts = {"ci.yml": [60, 20], ' - '"golden.yml": [15], "security.yml": [20], ' - '"release.yml": [40]}\n' - ' assert {path.name for path in root.glob("*.yml")} ' - '== names\n' - ' for name in sorted(names):\n' - ' text = (root / ' - 'name).read_text(encoding="utf-8")\n' - ' assert [int(line.split(":", 1)[1]) for line in ' - 'text.splitlines()\n' - ' if ' - 'line.strip().startswith("timeout-minutes:")] == ' - 'expected_timeouts[name]\n' - ' assert "Phase 4 Step 7" in text, name\n' - ' assert "--phase 4 --step 7" in text, name\n' - ' for older in ("--phase 4 --step 6", "--phase 4 ' - '--step 5", "--phase 4 --step 4", "--phase 4 --step 3", ' - '"--phase 4 --step 2", "--phase 4 --step 1", "--phase 3 ' - '--step 11"):\n' - ' assert older not in text, (name, older)\n' - ' assert "permissions:\\n contents: read" in ' - 'text\n' - ' assert "persist-credentials: false" in text\n' - ' assert "timeout-minutes:" in text and "set -euo ' - 'pipefail" in text\n' - ' assert "actions/upload-artifact@v4" in text and ' - '"if-no-files-found: error" in text\n' - ' for line in text.splitlines():\n' - ' if "python scripts/release_check.py" in ' - 'line:\n' - ' assert "--phase 4 --step 7" in line, ' - '(name, line)\n' - ' for forbidden in ("continue-on-error:", "|| ' - 'true", "contents: write", "id-token: write", "twine ' - 'upload", "git push"):\n' - ' assert forbidden not in text, (name, ' - 'forbidden)\n' - ' ci = (root / "ci.yml").read_text(encoding="utf-8")\n' - " for required in ('os: [ubuntu-latest, " - 'windows-latest]\', \'python-version: ["3.11", ' - '"3.12"]\',\n' - " 'dependencies: [current, " - 'minimum]\', \'"numpy==2.0.0" "pandas==2.2.2"\',\n' - ' \'RIT_TEST_PARQUET: "0"\', ' - '\'RIT_TEST_PARQUET: "1"\', \'--require-parquet\',\n' - " '--baseline-evidence', 'python -m " - "pip check',\n" - " '--minimum-tests 3420', " - "'--minimum-tests 3423',\n" - ' "find_spec(\'pyarrow\') is None", ' - "'import pyarrow'):\n" - ' assert required in ci\n' - ' assert ci.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2\n' - ' assert " -k " not in ci\n' - ' golden = (root / ' - '"golden.yml").read_text(encoding="utf-8")\n' - ' for required in ' - '("tests/golden/test_phase3_math.py", ' - '"tests/integration/test_hero_structure.py",\n' - ' ' - '"tests/integration/test_phase3_metric_pipeline.py"):\n' - ' assert required in golden\n' - ' security = (root / ' - '"security.yml").read_text(encoding="utf-8")\n' - ' for required in ' - "('tests/integration/test_no_network.py', " - "'tests/integration/test_optional_dependency.py',\n" - ' ' - "'tests/unit/test_PR012_evidence_classes.py', " - "'tests/unit/test_PR013_report_schema.py',\n" - ' ' - "'tests/unit/test_PR014_unavailable.py', " - "'tests/unit/test_PR015_redaction.py',\n" - ' ' - "'tests/unit/test_PR016_determinism.py', " - "'tests/unit/test_PR018_language.py',\n" - ' ' - "'tests/integration/test_partial_provenance_report.py',\n" - ' ' - "'tests/integration/test_partial_lineage_report.py',\n" - ' ' - "'tests/integration/test_phase4_cli.py', " - "'tests/unit/test_phase4_output_safety.py', " - "'tests/unit/test_phase4_contracts.py', " - "'tests/integration/test_phase4_gates.py'):\n" - ' assert required in security\n' - ' release = (root / ' - '"release.yml").read_text(encoding="utf-8")\n' - ' assert "python -m build" in release and "python -m ' - 'twine check --strict" in release\n' - ' assert "--dist" in release and "--candidate" in ' - 'release\n' - ' assert "--delivery" not in release\n' - ' assert ' - '"recursive-integrity-toolkit-phase4-step7-candidate" in ' - 'release\n' - ' assert "rit-phase4-step4" not in release\n' - ' assert "--minimum-tests 3420" in release and ' - '"--minimum-tests 3423" in release\n' - ' assert release.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2', - 'node': 'test_phase4_step7_current_workflows_preserve_matrix_and_use_active_dispatch', - 'old': 'def ' - 'test_phase4_step7_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root):\n' - ' names = {"ci.yml", "golden.yml", "security.yml", ' - '"release.yml"}\n' - ' root = repo_root / ".github/workflows"\n' - ' expected_timeouts = {"ci.yml": [60, 20], ' - '"golden.yml": [15], "security.yml": [20], ' - '"release.yml": [40]}\n' - ' assert {path.name for path in root.glob("*.yml")} ' - '== names\n' - ' for name in sorted(names):\n' - ' text = (root / ' - 'name).read_text(encoding="utf-8")\n' - ' assert [int(line.split(":", 1)[1]) for line in ' - 'text.splitlines()\n' - ' if ' - 'line.strip().startswith("timeout-minutes:")] == ' - 'expected_timeouts[name]\n' - ' assert "Phase 4 Step 7" in text, name\n' - ' assert "--phase 4 --step 7" in text, name\n' - ' for older in ("--phase 4 --step 6", "--phase 4 ' - '--step 5", "--phase 4 --step 4", "--phase 4 --step 3", ' - '"--phase 4 --step 2", "--phase 4 --step 1", "--phase 3 ' - '--step 11"):\n' - ' assert older not in text, (name, older)\n' - ' assert "permissions:\\n contents: read" in ' - 'text\n' - ' assert "persist-credentials: false" in text\n' - ' assert "timeout-minutes:" in text and "set -euo ' - 'pipefail" in text\n' - ' assert "actions/upload-artifact@v4" in text and ' - '"if-no-files-found: error" in text\n' - ' for line in text.splitlines():\n' - ' if "python scripts/release_check.py" in ' - 'line:\n' - ' assert "--phase 4 --step 7" in line, ' - '(name, line)\n' - ' for forbidden in ("continue-on-error:", "|| ' - 'true", "contents: write", "id-token: write", "twine ' - 'upload", "git push"):\n' - ' assert forbidden not in text, (name, ' - 'forbidden)\n' - ' ci = (root / "ci.yml").read_text(encoding="utf-8")\n' - " for required in ('os: [ubuntu-latest, " - 'windows-latest]\', \'python-version: ["3.11", ' - '"3.12"]\',\n' - " 'dependencies: [current, " - 'minimum]\', \'"numpy==2.0.0" "pandas==2.2.2"\',\n' - ' \'RIT_TEST_PARQUET: "0"\', ' - '\'RIT_TEST_PARQUET: "1"\', \'--require-parquet\',\n' - " '--baseline-evidence', 'python -m " - "pip check',\n" - " '--minimum-tests 3420', " - "'--minimum-tests 3423',\n" - ' "find_spec(\'pyarrow\') is None", ' - "'import pyarrow'):\n" - ' assert required in ci\n' - ' assert ci.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2\n' - ' assert " -k " not in ci\n' - ' golden = (root / ' - '"golden.yml").read_text(encoding="utf-8")\n' - ' for required in ' - '("tests/golden/test_phase3_math.py", ' - '"tests/integration/test_hero_structure.py",\n' - ' ' - '"tests/integration/test_phase3_metric_pipeline.py"):\n' - ' assert required in golden\n' - ' security = (root / ' - '"security.yml").read_text(encoding="utf-8")\n' - ' for required in ' - "('tests/integration/test_no_network.py', " - "'tests/integration/test_optional_dependency.py',\n" - ' ' - "'tests/unit/test_PR012_evidence_classes.py', " - "'tests/unit/test_PR013_report_schema.py',\n" - ' ' - "'tests/unit/test_PR014_unavailable.py', " - "'tests/unit/test_PR015_redaction.py',\n" - ' ' - "'tests/unit/test_PR016_determinism.py', " - "'tests/unit/test_PR018_language.py',\n" - ' ' - "'tests/integration/test_partial_provenance_report.py',\n" - ' ' - "'tests/integration/test_partial_lineage_report.py',\n" - ' ' - "'tests/integration/test_phase4_cli.py', " - "'tests/unit/test_phase4_output_safety.py', " - "'tests/unit/test_phase4_contracts.py', " - "'tests/integration/test_phase4_gates.py'):\n" - ' assert required in security\n' - ' release = (root / ' - '"release.yml").read_text(encoding="utf-8")\n' - ' assert "python -m build" in release and "python -m ' - 'twine check --strict" in release\n' - ' assert "--dist" in release and "--candidate" in ' - 'release\n' - ' assert "--delivery" not in release\n' - ' assert ' - '"recursive-integrity-toolkit-phase4-step7-candidate" in ' - 'release\n' - ' assert "rit-phase4-step4" not in release\n' - ' assert "--minimum-tests 3420" in release and ' - '"--minimum-tests 3423" in release\n' - ' assert release.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2'}, - {'new': 'def ' - 'test_phase4_step7_current_cli_ast_rejects_injection(repo_root, ' - 'tmp_path, relative, injection, phase4_step7_snapshot):\n' - ' repo_root = phase4_step7_snapshot\n' - ' checker = runpy.run_path(str(repo_root / ' - '"scripts/check_traceability.py"), ' - 'run_name="phase4_step7_ast_tests")\n' - ' verify = checker["phase4_step7_runtime_boundary"]\n' - ' path = "src/recursive_integrity_toolkit/" + ' - 'relative\n' - ' verify(repo_root / path, path)\n' - ' changed = tmp_path / "changed.py"\n' - ' changed.write_bytes((repo_root / path).read_bytes() ' - '+ injection)\n' - ' with pytest.raises(ValueError):\n' - ' verify(changed, path)', - 'node': 'test_phase4_step7_current_cli_ast_rejects_injection', - 'old': 'def ' - 'test_phase4_step7_current_cli_ast_rejects_injection(repo_root, ' - 'tmp_path, relative, injection):\n' - ' checker = runpy.run_path(str(repo_root / ' - '"scripts/check_traceability.py"), ' - 'run_name="phase4_step7_ast_tests")\n' - ' verify = checker["phase4_step7_runtime_boundary"]\n' - ' path = "src/recursive_integrity_toolkit/" + ' - 'relative\n' - ' verify(repo_root / path, path)\n' - ' changed = tmp_path / "changed.py"\n' - ' changed.write_bytes((repo_root / path).read_bytes() ' - '+ injection)\n' - ' with pytest.raises(ValueError):\n' - ' verify(changed, path)'}, - {'new': 'def ' - 'test_phase4_step7_new_cli_tests_reject_definition_effects(repo_root, ' - 'phase4_gate_tools, injection, phase4_step7_snapshot):\n' - ' repo_root = phase4_step7_snapshot\n' - ' path = "tests/integration/test_phase4_cli.py"\n' - ' raw = (repo_root / path).read_bytes()\n' - ' verify = ' - 'phase4_gate_tools["_phase4_step7_new_test"]\n' - ' verify(raw, path)\n' - ' with pytest.raises(ValueError):\n' - ' verify(raw + injection, path)', - 'node': 'test_phase4_step7_new_cli_tests_reject_definition_effects', - 'old': 'def ' - 'test_phase4_step7_new_cli_tests_reject_definition_effects(repo_root, ' - 'phase4_gate_tools, injection):\n' - ' path = "tests/integration/test_phase4_cli.py"\n' - ' raw = (repo_root / path).read_bytes()\n' - ' verify = ' - 'phase4_gate_tools["_phase4_step7_new_test"]\n' - ' verify(raw, path)\n' - ' with pytest.raises(ValueError):\n' - ' verify(raw + injection, path)'}, - {'new': 'def ' - 'test_phase4_step7_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step7_snapshot):\n' - ' import json\n' - '\n' - ' paths = ' - '{path.relative_to(phase4_step7_snapshot).as_posix()\n' - ' for path in ' - 'phase4_step7_snapshot.rglob("*") if path.is_file()}\n' - ' assert len(paths) == 229\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step7_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step7_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 38\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["privacy_views_enabled"] is True\n' - ' assert baseline["renderers_enabled"] is True\n' - ' assert baseline["output_publication_enabled"] is ' - 'True\n' - ' assert baseline["cli_analysis_enabled"] is True\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 4\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n repo_root = ' - 'phase4_step4_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n"),\n' - ' "historical_document": ("docs/cli.md", b"# ' - 'CLI\\n",\n' - ' b"# Unauthorized ' - 'historical document\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "html": ' - '"src/recursive_integrity_toolkit/reports/html_report.py",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "config": ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' "hashing": ' - '"src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' "logging": ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 7 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 8\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)', - 'node': 'test_phase4_step7_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step7_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation):\n' - ' import json\n' - '\n' - ' tracked = subprocess.check_output(["git", "-C", ' - 'str(repo_root), "ls-files", "-z"])\n' - ' paths = set(tracked.decode().split("\\0")) - {""}\n' - ' assert len(paths) == 229\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(repo_root / relative, ' - 'destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step7_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 38\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["privacy_views_enabled"] is True\n' - ' assert baseline["renderers_enabled"] is True\n' - ' assert baseline["output_publication_enabled"] is ' - 'True\n' - ' assert baseline["cli_analysis_enabled"] is True\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 4\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n repo_root = ' - 'phase4_step4_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n"),\n' - ' "historical_document": ("docs/cli.md", b"# ' - 'CLI\\n",\n' - ' b"# Unauthorized ' - 'historical document\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "html": ' - '"src/recursive_integrity_toolkit/reports/html_report.py",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "config": ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' "hashing": ' - '"src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' "logging": ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 7 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 8\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)'}], - 'tests/unit/test_phase4_contracts.py': [{'new': 'def ' - 'test_phase4_step7_approved_control_keeps_independent_step6_anchors(phase4_tools, ' - 'phase4_step7_snapshot):\n' - ' control = json.loads((phase4_step7_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' assert phase4_tools["PHASE4_STEP6_FINAL"] == ' - '"aa2355f2359c3af4fc05345c792f9903b0d5858c"\n' - ' assert phase4_tools["PHASE4_STEP6_TREE"] == ' - '"d9b6910079f49a2c739f86eceec05040918a41a5"\n' - ' assert phase4_tools["PHASE4_STEP6_TEST_TREE"] == ' - '"8d3a03c61da807d1fef8d26fc4af0d6f72e4546e"\n' - ' assert control["active_phase"] == 4 and ' - 'control["active_step"] == 7\n' - ' assert control["approval_basis"] == "Phase 4 Step 7 ' - '继续"\n' - ' assert control["baseline_commit"] == BASELINE\n' - ' assert control["previous_step_commit"] == ' - '"aa2355f2359c3af4fc05345c792f9903b0d5858c"\n' - ' assert control["previous_step_tree"] == ' - '"d9b6910079f49a2c739f86eceec05040918a41a5"\n' - ' assert control["previous_step_test_tree"] == ' - '"8d3a03c61da807d1fef8d26fc4af0d6f72e4546e"\n' - ' assert control["previous_step_core_tests"] == 3420\n' - ' assert control["previous_step_parquet_tests"] == 3423\n' - ' assert len(control["previous_step_files_sha256"]) == ' - '228\n' - ' assert set(control["runtime_paths_authorized"]) == {\n' - ' "src/recursive_integrity_toolkit/cli.py",\n' - ' "src/recursive_integrity_toolkit/config.py",\n' - ' }\n' - ' assert control["schema_changes_authorized"] is False\n' - ' assert control["schema_paths_authorized"] == []\n' - ' assert control["new_files_permitted"] == ' - '["tests/integration/test_phase4_cli.py"]\n' - ' for name in ("phase_complete", "next_step_authorized", ' - '"main_merge_authorized", "publication_authorized"):\n' - ' assert control[name] is False\n' - ' phase4_tools["verify_phase4_step7_control"](control, ' - 'step=7)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step4_control"](control, ' - 'step=4)', - 'node': 'test_phase4_step7_approved_control_keeps_independent_step6_anchors', - 'old': 'def ' - 'test_phase4_step7_approved_control_keeps_independent_step6_anchors(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' assert phase4_tools["PHASE4_STEP6_FINAL"] == ' - '"aa2355f2359c3af4fc05345c792f9903b0d5858c"\n' - ' assert phase4_tools["PHASE4_STEP6_TREE"] == ' - '"d9b6910079f49a2c739f86eceec05040918a41a5"\n' - ' assert phase4_tools["PHASE4_STEP6_TEST_TREE"] == ' - '"8d3a03c61da807d1fef8d26fc4af0d6f72e4546e"\n' - ' assert control["active_phase"] == 4 and ' - 'control["active_step"] == 7\n' - ' assert control["approval_basis"] == "Phase 4 Step 7 ' - '继续"\n' - ' assert control["baseline_commit"] == BASELINE\n' - ' assert control["previous_step_commit"] == ' - '"aa2355f2359c3af4fc05345c792f9903b0d5858c"\n' - ' assert control["previous_step_tree"] == ' - '"d9b6910079f49a2c739f86eceec05040918a41a5"\n' - ' assert control["previous_step_test_tree"] == ' - '"8d3a03c61da807d1fef8d26fc4af0d6f72e4546e"\n' - ' assert control["previous_step_core_tests"] == 3420\n' - ' assert control["previous_step_parquet_tests"] == 3423\n' - ' assert len(control["previous_step_files_sha256"]) == ' - '228\n' - ' assert set(control["runtime_paths_authorized"]) == {\n' - ' "src/recursive_integrity_toolkit/cli.py",\n' - ' "src/recursive_integrity_toolkit/config.py",\n' - ' }\n' - ' assert control["schema_changes_authorized"] is False\n' - ' assert control["schema_paths_authorized"] == []\n' - ' assert control["new_files_permitted"] == ' - '["tests/integration/test_phase4_cli.py"]\n' - ' for name in ("phase_complete", "next_step_authorized", ' - '"main_merge_authorized", "publication_authorized"):\n' - ' assert control[name] is False\n' - ' phase4_tools["verify_phase4_step7_control"](control, ' - 'step=7)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step4_control"](control, ' - 'step=4)'}, - {'new': 'def ' - 'test_phase4_step7_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value, phase4_step7_snapshot):\n' - ' control = json.loads((phase4_step7_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' verify = phase4_tools["verify_phase4_step7_control"]\n' - ' verify(control, step=7)\n' - ' control[field] = value\n' - ' with pytest.raises(ValueError):\n' - ' verify(control, step=7)', - 'node': 'test_phase4_step7_control_rejects_forged_scope_and_stage', - 'old': 'def ' - 'test_phase4_step7_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' verify = phase4_tools["verify_phase4_step7_control"]\n' - ' verify(control, step=7)\n' - ' control[field] = value\n' - ' with pytest.raises(ValueError):\n' - ' verify(control, step=7)'}, - {'new': 'def ' - 'test_phase4_step7_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step, phase4_step7_snapshot):\n' - ' control = json.loads((phase4_step7_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step7_control"](control, ' - 'step=7)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step7_control"](control, ' - 'step=step)', - 'node': 'test_phase4_step7_control_rejects_unapproved_dispatch', - 'old': 'def ' - 'test_phase4_step7_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step7_control"](control, ' - 'step=7)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step7_control"](control, ' - 'step=step)'}, - {'new': 'def ' - 'test_phase4_step7_control_requires_explicit_approval_fields(phase4_tools, ' - 'field, phase4_step7_snapshot):\n' - ' control = json.loads((phase4_step7_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step7_control"](control)\n' - ' del control[field]\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step7_control"](control)', - 'node': 'test_phase4_step7_control_requires_explicit_approval_fields', - 'old': 'def ' - 'test_phase4_step7_control_requires_explicit_approval_fields(phase4_tools, ' - 'field):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step7_control"](control)\n' - ' del control[field]\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step7_control"](control)'}, - {'new': 'def ' - 'test_phase4_step7_control_cannot_mint_extra_permission(phase4_tools, ' - 'phase4_step7_snapshot):\n' - ' control = json.loads((phase4_step7_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step7_control"](control)\n' - ' control["approved_scope_expansion"] = {"step": 7, ' - '"publication": True}\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step7_control"](control)', - 'node': 'test_phase4_step7_control_cannot_mint_extra_permission', - 'old': 'def ' - 'test_phase4_step7_control_cannot_mint_extra_permission(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step7_control"](control)\n' - ' control["approved_scope_expansion"] = {"step": 7, ' - '"publication": True}\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step7_control"](control)'}, - {'new': 'def ' - 'test_phase4_step7_historical_bindings_preserve_assertions(phase4_tools, ' - 'phase4_step6_snapshot, phase4_step7_snapshot):\n' - ' import ast\n' - ' registry = phase4_tools["PHASE4_STEP7_MIGRATIONS"]\n' - ' assert set(registry) == ' - '{"tests/unit/test_phase4_contracts.py", ' - '"tests/integration/test_phase4_gates.py"}\n' - ' assert sorted(len(rows) for rows in registry.values()) ' - '== [5, 7]\n' - ' for path, rows in registry.items():\n' - ' ' - 'phase4_tools["verify_phase4_step7_test_migration"](path, ' - '(phase4_step6_snapshot / path).read_bytes(), ' - '(phase4_step7_snapshot / path).read_bytes())\n' - ' for row in rows:\n' - ' trees = [ast.parse(row[key]) for key in ' - '("old", "new")]\n' - ' assert trees[0].body[0].name == ' - 'trees[1].body[0].name == row["node"]\n' - ' assertions = [[ast.dump(n) for n in ' - 'ast.walk(tree) if isinstance(n, ast.Assert)] for tree in ' - 'trees]\n' - ' assert assertions[0] == assertions[1]', - 'node': 'test_phase4_step7_historical_bindings_preserve_assertions', - 'old': 'def ' - 'test_phase4_step7_historical_bindings_preserve_assertions(phase4_tools, ' - 'phase4_step6_snapshot):\n' - ' import ast\n' - ' registry = phase4_tools["PHASE4_STEP7_MIGRATIONS"]\n' - ' assert set(registry) == ' - '{"tests/unit/test_phase4_contracts.py", ' - '"tests/integration/test_phase4_gates.py"}\n' - ' assert sorted(len(rows) for rows in registry.values()) ' - '== [5, 7]\n' - ' for path, rows in registry.items():\n' - ' ' - 'phase4_tools["verify_phase4_step7_test_migration"](path, ' - '(phase4_step6_snapshot / path).read_bytes(), (ROOT / ' - 'path).read_bytes())\n' - ' for row in rows:\n' - ' trees = [ast.parse(row[key]) for key in ' - '("old", "new")]\n' - ' assert trees[0].body[0].name == ' - 'trees[1].body[0].name == row["node"]\n' - ' assertions = [[ast.dump(n) for n in ' - 'ast.walk(tree) if isinstance(n, ast.Assert)] for tree in ' - 'trees]\n' - ' assert assertions[0] == assertions[1]'}, - {'new': 'def ' - 'test_phase4_step7_historical_guard_rejects_weakening(phase4_tools, ' - 'phase4_step6_snapshot, mutation, phase4_step7_snapshot):\n' - ' path = "tests/integration/test_phase4_gates.py" if ' - 'mutation == "config" else ' - '"tests/unit/test_phase4_contracts.py"\n' - ' before, after = (phase4_step6_snapshot / ' - 'path).read_bytes(), (phase4_step7_snapshot / ' - 'path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step7_test_migration"]\n' - ' verify(path, before, after)\n' - ' if mutation == "assertion":\n' - ' needle = b\' assert control["active_phase"] == ' - '4 and control["active_step"] == 6\\n\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b" assert ' - 'True\\n", 1)\n' - ' elif mutation == "binding":\n' - " needle = b'json.loads((phase4_step6_snapshot / " - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\'\n' - ' assert needle in after\n' - ' after = after.replace(needle, ' - 'needle.replace(b"phase4_step6_snapshot", b"ROOT"), 1)\n' - ' elif mutation == "config":\n' - " needle = b'else phase4_step6_snapshot if relative " - '== "config.py" else repo_root\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b"else repo_root", ' - '1)\n' - ' elif mutation == "shadow":\n' - ' after += b"\\ndef ' - 'test_phase4_step6_fixed_boundary_opens_only_approved_existing_paths():\\n ' - 'assert True\\n"\n' - ' else:\n' - ' after += b"\\ndef ' - 'test_phase4_step7_bad(value=globals().clear()):\\n ' - 'pass\\n"\n' - ' with pytest.raises(ValueError):\n' - ' verify(path, before, after)', - 'node': 'test_phase4_step7_historical_guard_rejects_weakening', - 'old': 'def ' - 'test_phase4_step7_historical_guard_rejects_weakening(phase4_tools, ' - 'phase4_step6_snapshot, mutation):\n' - ' path = "tests/integration/test_phase4_gates.py" if ' - 'mutation == "config" else ' - '"tests/unit/test_phase4_contracts.py"\n' - ' before, after = (phase4_step6_snapshot / ' - 'path).read_bytes(), (ROOT / path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step7_test_migration"]\n' - ' verify(path, before, after)\n' - ' if mutation == "assertion":\n' - ' needle = b\' assert control["active_phase"] == ' - '4 and control["active_step"] == 6\\n\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b" assert ' - 'True\\n", 1)\n' - ' elif mutation == "binding":\n' - " needle = b'json.loads((phase4_step6_snapshot / " - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\'\n' - ' assert needle in after\n' - ' after = after.replace(needle, ' - 'needle.replace(b"phase4_step6_snapshot", b"ROOT"), 1)\n' - ' elif mutation == "config":\n' - " needle = b'else phase4_step6_snapshot if relative " - '== "config.py" else repo_root\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b"else repo_root", ' - '1)\n' - ' elif mutation == "shadow":\n' - ' after += b"\\ndef ' - 'test_phase4_step6_fixed_boundary_opens_only_approved_existing_paths():\\n ' - 'assert True\\n"\n' - ' else:\n' - ' after += b"\\ndef ' - 'test_phase4_step7_bad(value=globals().clear()):\\n ' - 'pass\\n"\n' - ' with pytest.raises(ValueError):\n' - ' verify(path, before, after)'}]} - - -def _phase4_step7_files(): - """Read the exact accepted Step 7 Git tree and verify every blob identity.""" - for suffix, expected in (("^{commit}", PHASE4_STEP7_FINAL), ("^{tree}", PHASE4_STEP7_TREE), - (":tests", PHASE4_STEP7_TEST_TREE)): - if git("rev-parse", PHASE4_STEP7_FINAL + suffix).decode().strip() != expected: - raise ValueError("Pinned Phase 4 Step 7 identity mismatch") - objects = {} - for entry in git("ls-tree", "-rz", PHASE4_STEP7_FINAL).split(b"\0"): - if not entry: - continue - metadata, raw_path = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - path = raw_path.decode("utf-8") - if (mode not in (b"100644", b"100755") or kind != b"blob" or path in objects - or path.startswith("/") or ".." in path.split("/") or ".git" in path.split("/")): - raise ValueError("Unsafe pinned Step 7 Git object") - objects[path] = oid.decode("ascii") - if len(objects) != 229: - raise ValueError("Pinned Step 7 must contain exactly 229 files") - files = {} - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_STEP7_FINAL))) as archive: - names = [item.filename for item in archive.infolist() if not item.is_dir()] - if len(names) != len(objects) or set(names) != set(objects): - raise ValueError("Pinned Step 7 archive identity mismatch") - for name in names: - raw = archive.read(name) - oid = hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - if oid != objects[name]: - raise ValueError(f"Pinned Step 7 blob mismatch: {name}") - files[name] = raw - return MappingProxyType(files) - - -def phase4_step8_expected_control() -> dict: - """Actual Step 8 approval is independent of the mutable control document.""" - prior = _phase4_step7_files() - result = phase4_step7_expected_control() - result.update({ - "control_version": "1.7", "active_step": 8, - "approval_date": PHASE4_STEP8_APPROVAL_DATE, "approval_basis": PHASE4_STEP8_APPROVAL, - "previous_step_commit": PHASE4_STEP7_FINAL, - "previous_step_tree": PHASE4_STEP7_TREE, - "previous_step_test_tree": PHASE4_STEP7_TEST_TREE, - "previous_step_core_tests": 3620, "previous_step_parquet_tests": 3623, - "previous_step_files_sha256": {p: hashlib.sha256(raw).hexdigest() for p, raw in sorted(prior.items())}, - "permitted_paths": sorted(PHASE4_STEP8_ALLOWED), "new_files_permitted": list(PHASE4_STEP8_NEW), - "runtime_changes_authorized": True, "schema_changes_authorized": False, - "runtime_paths_authorized": sorted(p for p in PHASE4_STEP8_ALLOWED if p.startswith("src/") and p.endswith(".py")), - "schema_paths_authorized": [], "package_resource_copies": PHASE4_STEP8_RESOURCES, - "package_data_declaration_only": True, - "step7_historical_binding_nodes": {p: sorted({r["node"] for r in rows}) - for p, rows in sorted(PHASE4_STEP8_MIGRATIONS.items())}, - }) - return result - - -def verify_phase4_step8_control(control: dict, step: int = 8) -> None: - if type(step) is not int or step != 8 or type(control) is not dict: - raise ValueError("Unsupported Phase 4 Step 8 stage/control") - try: - actual = json.dumps(control, sort_keys=True, ensure_ascii=True, allow_nan=False) - expected = json.dumps(phase4_step8_expected_control(), sort_keys=True, ensure_ascii=True, allow_nan=False) - except (TypeError, ValueError) as error: - raise ValueError("Invalid Phase 4 Step 8 control") from error - if actual != expected: - raise ValueError("Phase 4 control differs from the independently approved Step 8 contract") - - -def verify_phase4_step8_changes(changes: list[tuple[str, str]]) -> None: - seen = set() - for status, path in changes: - if path in seen or path not in PHASE4_STEP8_ALLOWED or status != ("A" if path in PHASE4_STEP8_NEW else "M"): - raise ValueError(f"Unapproved Phase 4 Step 8 path/operation: {status} {path}") - seen.add(path) - - -def _phase4_step8_header(node, path): - """Allow explicit test parametrization/fixtures, never definition-time effects.""" - for decorator in node.decorator_list: - if not isinstance(decorator, ast.Call) or ast.unparse(decorator.func) not in { - "pytest.fixture", "pytest.mark.parametrize"}: - raise ValueError(f"Unapproved Step 8 test decorator: {path}:{node.name}") - if ast.unparse(decorator.func) == "pytest.fixture" and not node.name.startswith("phase4_step"): - raise ValueError("A Step 8 fixture must have an explicit scoped name") - if any(keyword.arg is None for keyword in decorator.keywords): - raise ValueError("Decorator expansion is outside the Step 8 contract") - for value in [*decorator.args, *(keyword.value for keyword in decorator.keywords)]: - try: - ast.literal_eval(value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError("Step 8 decorator arguments must be literal") from error - clone = ast.parse(ast.unparse(node)).body[0] - clone.decorator_list = [] - _phase4_preserve_function_header(clone, path) - - -def _phase4_step8_append_only(before: bytes, after: bytes, path: str) -> None: - if not after.startswith(before): - raise ValueError(f"Inherited Step 7 prefix changed: {path}") - old, new = ast.parse(before), ast.parse(after) - bindings = set() - for node in old.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - bindings.add(node.name) - elif isinstance(node, (ast.Import, ast.ImportFrom)): - bindings.update(alias.asname or alias.name.split(".")[0] for alias in node.names) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - bindings.update(child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)) - for node in new.body[len(old.body):]: - if not isinstance(node, ast.FunctionDef) or node.name in bindings: - raise ValueError(f"Step 8 addition executes, rebinds or shadows inherited source: {path}") - allowed_snapshot = path == "tests/conftest.py" and node.name == "phase4_step7_snapshot" - if not allowed_snapshot and not node.name.startswith(("test_phase4_step8_", "phase4_step8_")): - raise ValueError(f"Step 8 added test/helper is not explicitly scoped: {path}:{node.name}") - if path == "tests/conftest.py" and node.name != "phase4_step7_snapshot": - raise ValueError("Only the pinned Step 7 shared fixture is authorized") - bindings.add(node.name) - _phase4_step8_header(node, path) - - -def verify_phase4_step8_test_migration(path: str, before: bytes, after: bytes) -> None: - prior = _phase4_step7_files() - if path not in prior or path not in PHASE4_STEP8_ALLOWED or before != prior[path] or not path.startswith("tests/"): - raise ValueError("Step 8 migration requires the exact named Step 7 source") - _phase4_step8_check_test_migration(path, before, after) - - -def _phase4_step8_check_test_migration(path: str, before: bytes, after: bytes) -> None: - """Check a source already read from the verified Step 7 immutable mapping.""" - expected = before - for row in PHASE4_STEP8_MIGRATIONS.get(path, []): - old, new = row["old"].encode(), row["new"].encode() - if expected.count(old) != 1: - raise ValueError("Step 7 historical binding is not unique") - expected = expected.replace(old, new, 1) - _phase4_step8_append_only(expected, after, path) - - -def _phase4_step8_preserve_tooling(before: bytes, after: bytes, path: str) -> None: - """Preserve exact inherited statements using one source-line split per tree.""" - if path == "scripts/release_check.py": - old = ' return phase4_cli_main() if explicit_phase4 else main()' - new = (' explicit_step8 = "--step=8" in argv or any(a == "--step" and b == "8" for a, b in zip(argv, argv[1:]))\n' - ' if explicit_phase4 and explicit_step8:\n' - ' return phase4_step8_cli_main()\n' + old) - # Earlier preserved dispatch functions contain this source too; bind cli_main only. - parsed = ast.parse(before) - entry = next(node for node in parsed.body if isinstance(node, ast.FunctionDef) and node.name == "cli_main") - old_entry = ast.get_source_segment(before.decode(), entry) - if old_entry.count(old) != 1: - raise ValueError("Step 7 release dispatcher identity mismatch") - replacements = [(old_entry, old_entry.replace(old, new, 1))] - else: - old = ' if args.phase == 4 and args.step == 7:\n return phase4_step7_main(step=args.step)\n' - new = old + ' if args.phase == 4 and args.step == 8:\n return phase4_step8_main(step=args.step)\n' - replacements = [(old, new), - ('Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5 or --phase 4 --step 6 or --phase 4 --step 7', - 'Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5 or --phase 4 --step 6 or --phase 4 --step 7 or --phase 4 --step 8')] - expected = before - for old, new in replacements: - if expected.count(old.encode()) != 1: - raise ValueError(f"Historical dispatcher identity mismatch: {path}") - expected = expected.replace(old.encode(), new.encode(), 1) - trees = [ast.parse(expected), ast.parse(after)] - lines = [expected.splitlines(keepends=True), after.splitlines(keepends=True)] - def segment(index, node): - if node.lineno == node.end_lineno: - return lines[index][node.lineno - 1][node.col_offset:node.end_col_offset] - return (lines[index][node.lineno - 1][node.col_offset:] - + b"".join(lines[index][node.lineno:node.end_lineno - 1]) - + lines[index][node.end_lineno - 1][:node.end_col_offset]) - def entry_guard(node): - return isinstance(node, ast.If) and isinstance(node.test, ast.Compare) and "__name__" in ast.unparse(node.test) - guards = [n for n in trees[1].body if entry_guard(n)] - expected_guard = ast.parse('if __name__ == "__main__":\n raise SystemExit(cli_main())').body[0] - if len(guards) != 1 or ast.dump(guards[0]) != ast.dump(expected_guard): - raise ValueError(f"Unapproved Step 8 maintainer entrypoint: {path}") - historical = [n for n in trees[0].body if not entry_guard(n)] - current = [n for n in trees[1].body if not entry_guard(n)] - old_entries = [(segment(0, n), ast.dump(n)) for n in historical] - cursor = 0 - for node in current: - if cursor < len(old_entries) and (segment(1, node), ast.dump(node)) == old_entries[cursor]: - cursor += 1 - continue - if isinstance(node, ast.FunctionDef) and (node.name == "_phase4_step7_files" or node.name.startswith( - ("phase4_step8_", "_phase4_step8_", "verify_phase4_step8_", "audit_phase4_step8"))): - _phase4_preserve_function_header(node, path) - continue - if isinstance(node, ast.Assign) and all(isinstance(target, ast.Name) and target.id.startswith( - ("PHASE4_STEP8_", "PHASE4_STEP7_")) for target in node.targets): - try: - ast.literal_eval(node.value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError(f"Nonliteral added Step 8 maintainer constant: {path}") from error - continue - raise ValueError(f"Unapproved Step 8 maintainer addition: {path}") - if cursor != len(historical): - raise ValueError(f"Inherited Step 7 maintainer statement changed: {path}") - def binding_counts(tree): - counts = {} - for node in tree.body: - names = [] - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - names = [node.name] - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names = [child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)] - elif isinstance(node, (ast.Import, ast.ImportFrom)): - names = [alias.asname or alias.name.split(".")[0] for alias in node.names] - for name in names: - counts[name] = counts.get(name, 0) + 1 - return counts - inherited_counts = binding_counts(trees[0]) - for name, count in binding_counts(trees[1]).items(): - if count > inherited_counts.get(name, 1): - raise ValueError(f"Duplicate Step 8 maintainer binding: {path}:{name}") - - -def verify_phase4_step8_snapshot(root: Path = ROOT) -> dict: - prior = _phase4_step7_files() - for path, raw in prior.items(): - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError(f"Missing or aliased inherited Step 7 file: {path}") - current = target.read_bytes() - if path not in PHASE4_STEP8_ALLOWED and current != raw: - raise ValueError(f"Protected Step 7 bytes changed: {path}") - if path.startswith("tests/") and path in PHASE4_STEP8_ALLOWED: - _phase4_step8_check_test_migration(path, raw, current) - elif path.startswith("scripts/") and path in PHASE4_STEP8_ALLOWED: - _phase4_step8_preserve_tooling(raw, current, path) - elif path in {"docs/architecture.md", "docs/theory_traceability.md", "PHASE_4_DECISIONS.md"}: - if not current.startswith(raw): - raise ValueError(f"Step 7 historical documentation prefix changed: {path}") - if not (root / "docs/cli.md").read_bytes().endswith(prior["docs/cli.md"]): - raise ValueError("Step 8 CLI documentation must retain the historical notes verbatim") - phase4_step8_resources(root, prior) - for path in ("src/recursive_integrity_toolkit/config.py",): - _phase4_step8_preserve_runtime(prior[path], (root / path).read_bytes(), path) - actual_modules = {p.relative_to(root).as_posix() for p in (root / "src/recursive_integrity_toolkit").rglob("*.py")} - expected_modules = {p for p in prior if p.startswith("src/") and p.endswith(".py")} - if actual_modules != expected_modules or len(actual_modules) != 40: - raise ValueError("Step 8 cannot change the runtime module set") - if {p.name for p in (root / "schemas").iterdir()} != {Path(p).name for p in prior if p.startswith("schemas/")}: - raise ValueError("Step 8 cannot change the schema set") - if hashlib.sha256((root / "PHASE_4_PLAN.md").read_bytes()).hexdigest() != PHASE4_PLAN_SHA256: - raise ValueError("Approved Phase 4 plan changed") - verify_phase4_step8_control(json.loads((root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))) - decisions = (root / "PHASE_4_DECISIONS.md").read_text(encoding="utf-8") - if PHASE4_STEP8_APPROVAL not in decisions or PHASE4_STEP7_FINAL not in decisions: - raise ValueError("Actual Step 8 authorization or Step 7 evidence anchor missing") - if any((root / name).exists() for name in PHASE4_FORBIDDEN_OUTPUTS): - raise ValueError("Step 8 cannot create Phase 4 completion records or audit outputs") - import runpy - checker = runpy.run_path(str(root / "scripts/check_traceability.py"), run_name="phase4_step8_renderer_boundary") - checker["phase4_step8_cli_boundary"](root) - schema_bytes = (root / "schemas/report.schema.json").read_bytes() - if hashlib.sha256(schema_bytes).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Step 8 report schema differs from its independently reviewed bytes") - schema = json.loads(schema_bytes) - if (schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema" - or schema.get("type") != "object" or schema.get("additionalProperties") is not False): - raise ValueError("Step 8 report schema dialect/closed root mismatch") - return {"package_modules": 40, "frozen_runtime_modules": 38, "frozen_schemas": 5, - "hero_files_unchanged": 6, "historical_phase3_migrated_nodes": 16, - "phase_complete": False, "result_contracts_enabled": True, - "adapters_enabled": True, "privacy_views_enabled": True, "renderers_enabled": True, "output_publication_enabled": True, "cli_analysis_enabled": True, "comparison_enabled": True, "example_enabled": True, "packaged_resources": 7} - - -def audit_phase4_step8(step: int = 8) -> dict: - verify_phase4_step8_control(json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")), step) - result = {"phase": 4, "active_step": step, **verify_phase4_step8_snapshot(), - "phase0_hashes_verified": 16, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_phase4_step8_diff(step: int = 8) -> dict: - if type(step) is not int or step != 8: - raise ValueError("Unsupported Phase 4 Step 8 stage") - _phase4_step7_files() - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE4_STEP7_FINAL, "HEAD"], check=True) - raw = git("diff", "--name-status", "--no-renames", "-z", PHASE4_STEP7_FINAL, "--").split(b"\0") - raw = [part.decode("utf-8") for part in raw if part] - if len(raw) % 2: - raise ValueError("Malformed Step 8 Git difference records") - changes = list(zip(raw[::2], raw[1::2])) - changes.extend(("A", p.decode()) for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") if p) - verify_phase4_step8_changes(changes) - result = {"previous_step_commit": PHASE4_STEP7_FINAL, "changed_files": len(changes), - "changes": changes, "step": 8, "scope": "PASS"} - print(json.dumps(result, indent=2)) - return result - - -def phase4_step8_baseline_evidence(output: Path) -> dict: - """Reconcile all inherited identities, including the accepted Step 7 suite.""" - output.mkdir(parents=True, exist_ok=True) - inherited = phase4_step7_baseline_evidence(output / "phase4-step7-inherited") - with tempfile.TemporaryDirectory(prefix="rit-p4-step7-identities-") as temp: - baseline = Path(temp) - for name, raw in _phase4_step7_files().items(): - target = baseline / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(raw) - old, old_log = _collect(baseline) - current, current_log = _collect(ROOT) - expected = 3623 if os.environ.get("RIT_TEST_PARQUET") == "1" else 3620 - if len(old) != expected or set(old) - set(current): - raise ValueError("Accepted Phase 4 Step 7 test identities were lost") - result = {"baseline_commit": PHASE4_STEP7_FINAL, "baseline_test_tree": PHASE4_STEP7_TEST_TREE, - "baseline_nodeids": old, "current_nodeids": current, "missing_nodeids": [], - "baseline_tests": len(old), "current_tests": len(current), "inherited": inherited, - "nodeids_sha256": hashlib.sha256(("\n".join(old)+"\n").encode()).hexdigest()} - (output / "phase4_step8_test_identity_manifest.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4-step7-collection.log").write_text(old_log, encoding="utf-8") - (output / "phase4-step8-collection.log").write_text(current_log, encoding="utf-8") - print(json.dumps({k: result[k] for k in ("baseline_tests", "current_tests", "missing_nodeids")}, indent=2)) - return result - - -def phase4_step8_candidate(output: Path, step: int = 8) -> None: - """Build tested Step 8 intermediate evidence without declaring Phase 4 complete.""" - if os.environ.get("RIT_TEST_PARQUET") != "1" or importlib.util.find_spec("pyarrow") is None: - raise ValueError("Step 8 candidate evidence requires RIT_TEST_PARQUET=1 and real PyArrow") - output.mkdir(parents=True, exist_ok=True) - result = audit_phase4_step8(step) - result["diff"] = audit_phase4_step8_diff(step) - if git("status", "--porcelain").strip(): - raise ValueError("Step 8 candidate archive requires committed, clean source") - result["core"] = verify_junit(output / "core.xml", minimum=3620) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=3623) - result["core_math_measurements"] = verify_step10_evidence(output / "core.xml", output / "phase4-step8-core-observations.json") - result["parquet_math_measurements"] = verify_step10_evidence(output / "parquet.xml", output / "phase4-step8-parquet-observations.json") - identity = phase4_step8_baseline_evidence(output) - result["test_identity"] = {k: identity[k] for k in ("baseline_tests", "current_tests", "nodeids_sha256")} - expected = set() - for node in identity["current_nodeids"]: - base, bracket, parameter = node.partition("[") - owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) - for name, parquet in (("core.xml", False), ("parquet.xml", True)): - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(output / name).getroot().iter("testcase")} - target = expected if parquet else {c for c in expected if c[1] not in PARQUET_CASES} - if cases != target: - raise ValueError(f"Step 8 JUnit does not execute the entire current suite: {name}") - wheel, sdist = phase4_step8_distributions(output / "dist") - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - phase4_step6_installed_publication_smoke(wheel) - phase4_step7_installed_cli_smoke(wheel) - phase4_step8_installed_example_smoke(wheel) - phase4_step8_sdist_smoke(sdist) - archive = output / "recursive-integrity-toolkit-phase4-step8-candidate.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {p for p in git("ls-files", "-z").decode().split("\0") if p} - with zipfile.ZipFile(archive) as zipped: - entries = [item.filename for item in zipped.infolist() if not item.is_dir()] - prefix = "recursive-integrity-toolkit/" - if (len(entries) != len(set(entries)) or any(not name.startswith(prefix) for name in entries)): - raise ValueError("Step 8 source archive has duplicate or unprefixed entries") - names = {name.removeprefix(prefix) for name in entries} - if names != tracked or len(tracked) != 236 or len(entries) != 236: - raise ValueError("Step 8 source archive file set mismatch") - for name in names: - if zipped.read("recursive-integrity-toolkit/"+name) != (ROOT / name).read_bytes(): - raise ValueError(f"Step 8 source archive byte mismatch: {name}") - result.update({"commit": git("rev-parse", "HEAD").decode().strip(), - "tree": git("rev-parse", "HEAD^{tree}").decode().strip(), - "source_archive": archive.name, "tracked_files": len(tracked), - "python": sys.version, "platform": sys.platform, - "versions": {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "pytest")}}) - (output / "phase4_step8_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4_step8_repository_files.sha256").write_text("".join(f"{hashlib.sha256((ROOT/name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - paths = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase4_step8_artifacts.sha256") - (output / "phase4_step8_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in paths), encoding="utf-8") - print(f"Phase 4 Step 8 candidate: {len(tracked)} source files verified; Phase 4 remains incomplete.") - - -def phase4_step8_cli_main() -> int: - parser = argparse.ArgumentParser(description="Phase 4 Step 8 maintainer gate") - parser.add_argument("--phase", type=int, choices=(4,), required=True) - parser.add_argument("--step", type=int, choices=(8,), required=True) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) - parser.add_argument("--require-parquet", action="store_true") - parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - raise ValueError("Phase 4 Step 8 cannot certify a final phase delivery") - if args.junit is not None: - verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - phase4_step8_baseline_evidence(args.baseline_evidence) - elif args.dist is not None: - audit_phase4_step8(args.step) - wheel, sdist = phase4_step8_distributions(args.dist) - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - phase4_step6_installed_publication_smoke(wheel) - phase4_step7_installed_cli_smoke(wheel) - phase4_step8_installed_example_smoke(wheel) - phase4_step8_sdist_smoke(sdist) - elif args.smoke_wheel is not None: - smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - phase4_step8_candidate(args.candidate, args.step) - else: - audit_phase4_step8(args.step) - if args.diff: - audit_phase4_step8_diff(args.step) - return 0 - - -def _phase4_step8_preserve_runtime(before: bytes, after: bytes, path: str) -> None: - """Allow ownership-header maintenance and byte-exact additive helpers only.""" - try: - old, new = ast.parse(before), ast.parse(after) - except (SyntaxError, UnicodeError, ValueError) as error: - raise ValueError("Output helper source cannot be inspected") from error - if not ast.get_docstring(old) or not ast.get_docstring(new): - raise ValueError("Output helper ownership documentation is required") - old_tail = b"".join(before.splitlines(keepends=True)[old.body[0].end_lineno:]) - new_tail = b"".join(after.splitlines(keepends=True)[new.body[0].end_lineno:]) - if not new_tail.startswith(old_tail): - raise ValueError(f"Inherited input/diagnostic bytes changed: {path}") - - -def phase4_step8_resources(root: Path, prior: dict) -> None: - """Exact approved copies, with no build/dependency/version change.""" - for destination, source in PHASE4_STEP8_RESOURCES.items(): - path = root / destination - if not path.is_file() or path.is_symlink() or path.read_bytes() != (root / source).read_bytes(): - raise ValueError("Step 8 packaged resource differs from its canonical source") - resource_root = root / "src/recursive_integrity_toolkit/data" - if any(p.is_symlink() for p in resource_root.rglob("*")) or {p.relative_to(root).as_posix() for p in resource_root.rglob("*") if p.is_file()} != set(PHASE4_STEP8_RESOURCES): - raise ValueError("Step 8 packaged resource set differs from its exact allowlist") - before = prior["pyproject.toml"] - marker = b'include = ["recursive_integrity_toolkit*"]\n' - if before.count(marker) != 1 or (root / "pyproject.toml").read_bytes() != before.replace(marker, marker + PHASE4_STEP8_PACKAGING.encode(), 1): - raise ValueError("Step 8 packaging may add only the exact package-data declaration") - - -def phase4_step8_distributions(directory: Path, expected_version: str = "0.1.0.dev2"): - wheel, sdist = verify_distributions(directory, expected_version=expected_version) - with zipfile.ZipFile(wheel) as archive: - actual = {name for name in archive.namelist() if name.startswith("recursive_integrity_toolkit/data/") and not name.endswith("/")} - if actual != {path.removeprefix("src/") for path in PHASE4_STEP8_RESOURCES}: - raise ValueError("Wheel resource inventory differs") - for destination, source in PHASE4_STEP8_RESOURCES.items(): - if archive.read(destination.removeprefix("src/")) != (ROOT / source).read_bytes(): - raise ValueError("Wheel resource bytes differ") - with tarfile.open(sdist, "r:gz") as archive: - root = next(iter({m.name.split("/")[0] for m in archive.getmembers()})) - for destination, source in PHASE4_STEP8_RESOURCES.items(): - member = archive.getmember(root + "/" + destination) - if not member.isfile() or archive.extractfile(member).read() != (ROOT / source).read_bytes(): - raise ValueError("Sdist resource bytes differ") - print("Step 8 wheel/sdist: seven exact canonical resources: PASS") - return wheel, sdist - - - -def phase4_step8_example_program() -> str: +def installed_example_program() -> str: """Independent installed acceptance, using frozen section 8 Hero values.""" return r'''import hashlib, importlib.abc, json, socket, sys, urllib.request from pathlib import Path @@ -9776,1584 +1151,148 @@ def deny(*args,**kwargs):raise AssertionError('network forbidden') ''' -def phase4_step8_installed_example_smoke(wheel: Path) -> None: +def smoke_installed_example(wheel: Path) -> None: with tempfile.TemporaryDirectory(prefix="rit-p4-step8-wheel-") as temporary: work = Path(temporary) target = work / "installed" subprocess.run([sys.executable, "-m", "pip", "install", "--no-index", "--no-deps", "--target", str(target), str(wheel.resolve())], cwd=work, check=True) - hashes = {destination.removeprefix("src/recursive_integrity_toolkit/"): hashlib.sha256((ROOT/source).read_bytes()).hexdigest() for destination,source in PHASE4_STEP8_RESOURCES.items()} - subprocess.run([sys.executable, "-I", "-c", phase4_step8_example_program(), str(target), str(work), str(ROOT), json.dumps(hashes)], cwd=work, check=True) + hashes = {destination.removeprefix("src/recursive_integrity_toolkit/"): hashlib.sha256((ROOT/source).read_bytes()).hexdigest() for destination,source in RESOURCES.items()} + subprocess.run([sys.executable, "-I", "-c", installed_example_program(), str(target), str(work), str(ROOT), json.dumps(hashes)], cwd=work, check=True) + + +def extract_sdist(sdist: Path, destination: Path) -> Path: + """Preflight regular archive members before copying any source bytes.""" + with tarfile.open(sdist, "r:gz") as archive: + members = archive.getmembers() + names = [member.name for member in members] + if len(names) != len(set(names)): + raise ValueError("Duplicate sdist entries") + roots = {PurePosixPath(name).parts[0] for name in names if PurePosixPath(name).parts} + if len(roots) != 1: + raise ValueError("Sdist needs exactly one root") + for member in members: + parts = PurePosixPath(member.name).parts + if (not parts or member.name.startswith("/") or ".." in parts + or "\\" in member.name or any(":" in part for part in parts) + or not (member.isdir() or member.isfile())): + raise ValueError("Unsafe sdist member") + target = destination.joinpath(*parts) + if any(path.is_symlink() for path in (target, *target.parents)): + raise ValueError("Aliased sdist extraction destination") + for member in members: + if member.isdir(): + continue + target = destination / member.name + target.parent.mkdir(parents=True, exist_ok=True) + stream = archive.extractfile(member) + if stream is None: + raise ValueError("Missing regular sdist content") + target.write_bytes(stream.read()) + return destination / next(iter(roots)) -def phase4_step8_sdist_smoke(sdist: Path) -> None: +def smoke_sdist(sdist: Path) -> None: """Extract regular sdist members and audit without build/download operations.""" with tempfile.TemporaryDirectory(prefix="rit-p4-step8-sdist-") as temporary: work = Path(temporary) - extracted = work / "source" - with tarfile.open(sdist, "r:gz") as archive: - roots = {m.name.split("/")[0] for m in archive.getmembers()} - if len(roots) != 1: - raise ValueError("Sdist needs exactly one root") - for member in archive.getmembers(): - if member.isdir(): - continue - if (not member.isfile() or member.name.startswith("/") or ".." in member.name.split("/") or "\\" in member.name): - raise ValueError("Unsafe sdist member") - target = extracted / member.name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(archive.extractfile(member).read()) - source = extracted / next(iter(roots)) / "src" - hashes = {destination.removeprefix("src/recursive_integrity_toolkit/"): hashlib.sha256((ROOT/original).read_bytes()).hexdigest() for destination,original in PHASE4_STEP8_RESOURCES.items()} - subprocess.run([sys.executable, "-I", "-c", phase4_step8_example_program(), str(source), str(work), str(ROOT), json.dumps(hashes)], cwd=work, check=True) - - - -# Phase 4 Step 9: independently authored report oracles and adversarial integration. - - -PHASE4_STEP8_FINAL = '63e347a785838b64a9c32e636ea48c664458cee4' - - -PHASE4_STEP8_TREE = '3ac450b5f67cad738849de18601999fefee98f63' - - -PHASE4_STEP8_TEST_TREE = 'ddaaab24d83080b7c1a27229c15e6e4a3f5b3cd8' - - -PHASE4_STEP9_APPROVAL = 'Phase 4 **Step 9** 开始' - - -PHASE4_STEP9_APPROVAL_DATE = '2026-09-22' - - -PHASE4_STEP9_NEW = ('tests/golden/phase4_hero_redacted.json', - 'tests/golden/phase4_hero_redacted.md', - 'tests/golden/phase4_hero_report.json', - 'tests/golden/phase4_hero_report.md', - 'tests/golden/phase4_report_cases.md', - 'tests/golden/phase4_report_expected.json', - 'tests/golden/test_phase4_reports.py') - - -PHASE4_STEP9_ALLOWED = {'.github/workflows/ci.yml', - '.github/workflows/golden.yml', - '.github/workflows/release.yml', - '.github/workflows/security.yml', - 'PHASE_4_BASELINE.json', - 'PHASE_4_DECISIONS.md', - 'docs/architecture.md', - 'docs/theory_traceability.md', - 'scripts/build_golden.py', - 'scripts/check_spec_consistency.py', - 'scripts/check_traceability.py', - 'scripts/normalize_golden.py', - 'scripts/release_check.py', - 'tests/conftest.py', - 'tests/golden/README.md', - 'tests/golden/phase4_hero_redacted.json', - 'tests/golden/phase4_hero_redacted.md', - 'tests/golden/phase4_hero_report.json', - 'tests/golden/phase4_hero_report.md', - 'tests/golden/phase4_report_cases.md', - 'tests/golden/phase4_report_expected.json', - 'tests/golden/test_phase4_reports.py', - 'tests/integration/test_ci_workflows.py', - 'tests/integration/test_hero_end_to_end.py', - 'tests/integration/test_hero_structure.py', - 'tests/integration/test_license_notices.py', - 'tests/integration/test_no_algorithms.py', - 'tests/integration/test_no_network.py', - 'tests/integration/test_optional_dependency.py', - 'tests/integration/test_owner_ids.py', - 'tests/integration/test_package_import.py', - 'tests/integration/test_package_install.py', - 'tests/integration/test_partial_lineage_report.py', - 'tests/integration/test_partial_provenance_report.py', - 'tests/integration/test_phase4_cli.py', - 'tests/integration/test_phase4_gates.py', - 'tests/integration/test_prohibited_structure.py', - 'tests/integration/test_repository_structure.py', - 'tests/integration/test_schema_json.py', - 'tests/unit/test_phase4_contracts.py'} - - -PHASE4_STEP9_MIGRATIONS = {'tests/integration/test_phase4_gates.py': [{'new': 'def ' - 'test_phase4_step8_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step8_snapshot):\n' - ' repo_root = phase4_step8_snapshot\n' - ' names = {"ci.yml", "golden.yml", ' - '"security.yml", "release.yml"}\n' - ' root = repo_root / ".github/workflows"\n' - ' expected_timeouts = {"ci.yml": [60, 20], ' - '"golden.yml": [15], "security.yml": [20], ' - '"release.yml": [40]}\n' - ' assert {path.name for path in ' - 'root.glob("*.yml")} == names\n' - ' for name in sorted(names):\n' - ' text = (root / ' - 'name).read_text(encoding="utf-8")\n' - ' assert [int(line.split(":", 1)[1]) ' - 'for line in text.splitlines()\n' - ' if ' - 'line.strip().startswith("timeout-minutes:")] ' - '== expected_timeouts[name]\n' - ' assert "Phase 4 Step 8" in text, ' - 'name\n' - ' assert "--phase 4 --step 8" in text, ' - 'name\n' - ' for older in ("--phase 4 --step 7", ' - '"--phase 4 --step 6", "--phase 4 --step 5", ' - '"--phase 4 --step 4", "--phase 4 --step 3", ' - '"--phase 4 --step 2", "--phase 4 --step 1", ' - '"--phase 3 --step 11"):\n' - ' assert older not in text, (name, ' - 'older)\n' - ' assert "permissions:\\n contents: ' - 'read" in text\n' - ' assert "persist-credentials: false" ' - 'in text\n' - ' assert "timeout-minutes:" in text and ' - '"set -euo pipefail" in text\n' - ' assert "actions/upload-artifact@v4" ' - 'in text and "if-no-files-found: error" in ' - 'text\n' - ' for line in text.splitlines():\n' - ' if "python ' - 'scripts/release_check.py" in line:\n' - ' assert "--phase 4 --step 8" ' - 'in line, (name, line)\n' - ' for forbidden in ' - '("continue-on-error:", "|| true", "contents: ' - 'write", "id-token: write", "twine upload", ' - '"git push"):\n' - ' assert forbidden not in text, ' - '(name, forbidden)\n' - ' ci = (root / ' - '"ci.yml").read_text(encoding="utf-8")\n' - " for required in ('os: [ubuntu-latest, " - 'windows-latest]\', \'python-version: ["3.11", ' - '"3.12"]\',\n' - " 'dependencies: [current, " - 'minimum]\', \'"numpy==2.0.0" ' - '"pandas==2.2.2"\',\n' - " 'RIT_TEST_PARQUET: " - '"0"\', \'RIT_TEST_PARQUET: "1"\', ' - "'--require-parquet',\n" - " '--baseline-evidence', " - "'python -m pip check',\n" - " '--minimum-tests 3620', " - "'--minimum-tests 3623',\n" - ' "find_spec(\'pyarrow\') ' - 'is None", \'import pyarrow\'):\n' - ' assert required in ci\n' - ' assert ci.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2\n' - ' assert " -k " not in ci\n' - ' golden = (root / ' - '"golden.yml").read_text(encoding="utf-8")\n' - ' for required in ' - '("tests/golden/test_phase3_math.py", ' - '"tests/integration/test_hero_structure.py",\n' - ' ' - '"tests/integration/test_phase3_metric_pipeline.py", ' - '"tests/integration/test_hero_end_to_end.py"):\n' - ' assert required in golden\n' - ' security = (root / ' - '"security.yml").read_text(encoding="utf-8")\n' - ' for required in ' - "('tests/integration/test_no_network.py', " - "'tests/integration/test_optional_dependency.py',\n" - ' ' - "'tests/unit/test_PR012_evidence_classes.py', " - "'tests/unit/test_PR013_report_schema.py',\n" - ' ' - "'tests/unit/test_PR014_unavailable.py', " - "'tests/unit/test_PR015_redaction.py',\n" - ' ' - "'tests/unit/test_PR016_determinism.py', " - "'tests/unit/test_PR018_language.py',\n" - ' ' - "'tests/integration/test_partial_provenance_report.py',\n" - ' ' - "'tests/integration/test_partial_lineage_report.py',\n" - ' ' - "'tests/integration/test_phase4_cli.py', " - "'tests/unit/test_phase4_output_safety.py', " - "'tests/unit/test_phase4_contracts.py', " - "'tests/integration/test_phase4_gates.py'):\n" - ' assert required in security\n' - ' release = (root / ' - '"release.yml").read_text(encoding="utf-8")\n' - ' assert "python -m build" in release and ' - '"python -m twine check --strict" in release\n' - ' assert "--dist" in release and ' - '"--candidate" in release\n' - ' assert "--delivery" not in release\n' - ' assert ' - '"recursive-integrity-toolkit-phase4-step8-candidate" ' - 'in release\n' - ' assert "rit-phase4-step4" not in release\n' - ' assert "--minimum-tests 3620" in release ' - 'and "--minimum-tests 3623" in release\n' - ' assert release.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2', - 'node': 'test_phase4_step8_current_workflows_preserve_matrix_and_use_active_dispatch', - 'old': 'def ' - 'test_phase4_step8_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root):\n' - ' names = {"ci.yml", "golden.yml", ' - '"security.yml", "release.yml"}\n' - ' root = repo_root / ".github/workflows"\n' - ' expected_timeouts = {"ci.yml": [60, 20], ' - '"golden.yml": [15], "security.yml": [20], ' - '"release.yml": [40]}\n' - ' assert {path.name for path in ' - 'root.glob("*.yml")} == names\n' - ' for name in sorted(names):\n' - ' text = (root / ' - 'name).read_text(encoding="utf-8")\n' - ' assert [int(line.split(":", 1)[1]) ' - 'for line in text.splitlines()\n' - ' if ' - 'line.strip().startswith("timeout-minutes:")] ' - '== expected_timeouts[name]\n' - ' assert "Phase 4 Step 8" in text, ' - 'name\n' - ' assert "--phase 4 --step 8" in text, ' - 'name\n' - ' for older in ("--phase 4 --step 7", ' - '"--phase 4 --step 6", "--phase 4 --step 5", ' - '"--phase 4 --step 4", "--phase 4 --step 3", ' - '"--phase 4 --step 2", "--phase 4 --step 1", ' - '"--phase 3 --step 11"):\n' - ' assert older not in text, (name, ' - 'older)\n' - ' assert "permissions:\\n contents: ' - 'read" in text\n' - ' assert "persist-credentials: false" ' - 'in text\n' - ' assert "timeout-minutes:" in text and ' - '"set -euo pipefail" in text\n' - ' assert "actions/upload-artifact@v4" ' - 'in text and "if-no-files-found: error" in ' - 'text\n' - ' for line in text.splitlines():\n' - ' if "python ' - 'scripts/release_check.py" in line:\n' - ' assert "--phase 4 --step 8" ' - 'in line, (name, line)\n' - ' for forbidden in ' - '("continue-on-error:", "|| true", "contents: ' - 'write", "id-token: write", "twine upload", ' - '"git push"):\n' - ' assert forbidden not in text, ' - '(name, forbidden)\n' - ' ci = (root / ' - '"ci.yml").read_text(encoding="utf-8")\n' - " for required in ('os: [ubuntu-latest, " - 'windows-latest]\', \'python-version: ["3.11", ' - '"3.12"]\',\n' - " 'dependencies: [current, " - 'minimum]\', \'"numpy==2.0.0" ' - '"pandas==2.2.2"\',\n' - " 'RIT_TEST_PARQUET: " - '"0"\', \'RIT_TEST_PARQUET: "1"\', ' - "'--require-parquet',\n" - " '--baseline-evidence', " - "'python -m pip check',\n" - " '--minimum-tests 3620', " - "'--minimum-tests 3623',\n" - ' "find_spec(\'pyarrow\') ' - 'is None", \'import pyarrow\'):\n' - ' assert required in ci\n' - ' assert ci.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2\n' - ' assert " -k " not in ci\n' - ' golden = (root / ' - '"golden.yml").read_text(encoding="utf-8")\n' - ' for required in ' - '("tests/golden/test_phase3_math.py", ' - '"tests/integration/test_hero_structure.py",\n' - ' ' - '"tests/integration/test_phase3_metric_pipeline.py", ' - '"tests/integration/test_hero_end_to_end.py"):\n' - ' assert required in golden\n' - ' security = (root / ' - '"security.yml").read_text(encoding="utf-8")\n' - ' for required in ' - "('tests/integration/test_no_network.py', " - "'tests/integration/test_optional_dependency.py',\n" - ' ' - "'tests/unit/test_PR012_evidence_classes.py', " - "'tests/unit/test_PR013_report_schema.py',\n" - ' ' - "'tests/unit/test_PR014_unavailable.py', " - "'tests/unit/test_PR015_redaction.py',\n" - ' ' - "'tests/unit/test_PR016_determinism.py', " - "'tests/unit/test_PR018_language.py',\n" - ' ' - "'tests/integration/test_partial_provenance_report.py',\n" - ' ' - "'tests/integration/test_partial_lineage_report.py',\n" - ' ' - "'tests/integration/test_phase4_cli.py', " - "'tests/unit/test_phase4_output_safety.py', " - "'tests/unit/test_phase4_contracts.py', " - "'tests/integration/test_phase4_gates.py'):\n" - ' assert required in security\n' - ' release = (root / ' - '"release.yml").read_text(encoding="utf-8")\n' - ' assert "python -m build" in release and ' - '"python -m twine check --strict" in release\n' - ' assert "--dist" in release and ' - '"--candidate" in release\n' - ' assert "--delivery" not in release\n' - ' assert ' - '"recursive-integrity-toolkit-phase4-step8-candidate" ' - 'in release\n' - ' assert "rit-phase4-step4" not in release\n' - ' assert "--minimum-tests 3620" in release ' - 'and "--minimum-tests 3623" in release\n' - ' assert release.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2'}, - {'new': 'def ' - 'test_phase4_step8_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step8_snapshot):\n' - ' import json\n' - '\n' - ' paths = ' - '{path.relative_to(phase4_step8_snapshot).as_posix()\n' - ' for path in ' - 'phase4_step8_snapshot.rglob("*") if ' - 'path.is_file()}\n' - ' assert len(paths) == 236\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' ' - 'destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step8_snapshot ' - '/ relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step8_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] ' - '== 38\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert ' - 'baseline["result_contracts_enabled"] is True\n' - ' assert baseline["adapters_enabled"] is ' - 'True\n' - ' assert baseline["privacy_views_enabled"] ' - 'is True\n' - ' assert baseline["renderers_enabled"] is ' - 'True\n' - ' assert ' - 'baseline["output_publication_enabled"] is ' - 'True\n' - ' assert baseline["cli_analysis_enabled"] ' - 'is True\n' - ' assert baseline["phase_complete"] is ' - 'False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - ' - 'concentration", b"diversity = 0.5 - ' - 'concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ' - '("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", ' - 'b"numpy>=2.0", b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and ' - 'control["active_step"] == 4\\n\',\n' - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, ' - 'step: int = 4) -> None:",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, ' - 'step: int = 4) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n repo_root = ' - 'phase4_step4_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n"),\n' - ' "historical_document": ' - '("docs/cli.md", b"# CLI\\n",\n' - ' b"# ' - 'Unauthorized historical document\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def ' - 'resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": ' - '"schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "html": ' - '"src/recursive_integrity_toolkit/reports/html_report.py",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "config": ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' "hashing": ' - '"src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' "logging": ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' ' - 'path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 8 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized ' - 'later-stage file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / ' - '"PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 9\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py")\n' - ' else:\n' - ' ' - 'control["schema_changes_authorized"] = True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active ' - 'mutation case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)', - 'node': 'test_phase4_step8_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step8_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation):\n' - ' import json\n' - '\n' - ' tracked = subprocess.check_output(["git", ' - '"-C", str(repo_root), "ls-files", "-z"])\n' - ' paths = ' - 'set(tracked.decode().split("\\0")) - {""}\n' - ' assert len(paths) == 236\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' ' - 'destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(repo_root / relative, ' - 'destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step8_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] ' - '== 38\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert ' - 'baseline["result_contracts_enabled"] is True\n' - ' assert baseline["adapters_enabled"] is ' - 'True\n' - ' assert baseline["privacy_views_enabled"] ' - 'is True\n' - ' assert baseline["renderers_enabled"] is ' - 'True\n' - ' assert ' - 'baseline["output_publication_enabled"] is ' - 'True\n' - ' assert baseline["cli_analysis_enabled"] ' - 'is True\n' - ' assert baseline["phase_complete"] is ' - 'False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - ' - 'concentration", b"diversity = 0.5 - ' - 'concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ' - '("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", ' - 'b"numpy>=2.0", b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and ' - 'control["active_step"] == 4\\n\',\n' - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, ' - 'step: int = 4) -> None:",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, ' - 'step: int = 4) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n repo_root = ' - 'phase4_step4_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n"),\n' - ' "historical_document": ' - '("docs/cli.md", b"# CLI\\n",\n' - ' b"# ' - 'Unauthorized historical document\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def ' - 'resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": ' - '"schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "html": ' - '"src/recursive_integrity_toolkit/reports/html_report.py",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "config": ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' "hashing": ' - '"src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' "logging": ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' ' - 'path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 8 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized ' - 'later-stage file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / ' - '"PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 9\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py")\n' - ' else:\n' - ' ' - 'control["schema_changes_authorized"] = True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active ' - 'mutation case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)'}], - 'tests/unit/test_phase4_contracts.py': [{'new': 'def ' - 'test_phase4_step8_approved_control_keeps_independent_step7_anchors(phase4_tools, ' - 'phase4_step8_snapshot):\n' - ' control = json.loads((phase4_step8_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' assert phase4_tools["PHASE4_STEP7_FINAL"] == ' - '"bacd33ae65e392872776c6d976401fdc3d565f63"\n' - ' assert phase4_tools["PHASE4_STEP7_TREE"] == ' - '"4c16914bb0f0c5281efc36624efb8b1a1d4a174e"\n' - ' assert ' - 'phase4_tools["PHASE4_STEP7_TEST_TREE"] == ' - '"96317cb7b25236b79efc0f2d93e1e0f5f132fdb2"\n' - ' assert control["active_phase"] == 4 and ' - 'control["active_step"] == 8\n' - ' assert control["approval_basis"] == "phase 4 ' - 'step 8 开始"\n' - ' assert control["baseline_commit"] == ' - 'BASELINE\n' - ' assert control["previous_step_commit"] == ' - '"bacd33ae65e392872776c6d976401fdc3d565f63"\n' - ' assert control["previous_step_tree"] == ' - '"4c16914bb0f0c5281efc36624efb8b1a1d4a174e"\n' - ' assert control["previous_step_test_tree"] == ' - '"96317cb7b25236b79efc0f2d93e1e0f5f132fdb2"\n' - ' assert control["previous_step_core_tests"] ' - '== 3620\n' - ' assert ' - 'control["previous_step_parquet_tests"] == 3623\n' - ' assert ' - 'len(control["previous_step_files_sha256"]) == ' - '229\n' - ' assert ' - 'set(control["runtime_paths_authorized"]) == {\n' - ' ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' }\n' - ' assert control["schema_changes_authorized"] ' - 'is False\n' - ' assert control["schema_paths_authorized"] == ' - '[]\n' - ' assert control["new_files_permitted"] == ' - "['src/recursive_integrity_toolkit/data/hero/EXPECTED_OUTPUTS.md', " - "'src/recursive_integrity_toolkit/data/hero/config.json', " - "'src/recursive_integrity_toolkit/data/hero/provenance.csv', " - "'src/recursive_integrity_toolkit/data/hero/records_v1.csv', " - "'src/recursive_integrity_toolkit/data/hero/records_v2.csv', " - "'src/recursive_integrity_toolkit/data/hero/version_order.json', " - "'src/recursive_integrity_toolkit/data/report.schema.json']\n" - ' for name in ("phase_complete", ' - '"next_step_authorized", "main_merge_authorized", ' - '"publication_authorized"):\n' - ' assert control[name] is False\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control, ' - 'step=8)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step4_control"](control, ' - 'step=4)', - 'node': 'test_phase4_step8_approved_control_keeps_independent_step7_anchors', - 'old': 'def ' - 'test_phase4_step8_approved_control_keeps_independent_step7_anchors(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' assert phase4_tools["PHASE4_STEP7_FINAL"] == ' - '"bacd33ae65e392872776c6d976401fdc3d565f63"\n' - ' assert phase4_tools["PHASE4_STEP7_TREE"] == ' - '"4c16914bb0f0c5281efc36624efb8b1a1d4a174e"\n' - ' assert ' - 'phase4_tools["PHASE4_STEP7_TEST_TREE"] == ' - '"96317cb7b25236b79efc0f2d93e1e0f5f132fdb2"\n' - ' assert control["active_phase"] == 4 and ' - 'control["active_step"] == 8\n' - ' assert control["approval_basis"] == "phase 4 ' - 'step 8 开始"\n' - ' assert control["baseline_commit"] == ' - 'BASELINE\n' - ' assert control["previous_step_commit"] == ' - '"bacd33ae65e392872776c6d976401fdc3d565f63"\n' - ' assert control["previous_step_tree"] == ' - '"4c16914bb0f0c5281efc36624efb8b1a1d4a174e"\n' - ' assert control["previous_step_test_tree"] == ' - '"96317cb7b25236b79efc0f2d93e1e0f5f132fdb2"\n' - ' assert control["previous_step_core_tests"] ' - '== 3620\n' - ' assert ' - 'control["previous_step_parquet_tests"] == 3623\n' - ' assert ' - 'len(control["previous_step_files_sha256"]) == ' - '229\n' - ' assert ' - 'set(control["runtime_paths_authorized"]) == {\n' - ' ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' }\n' - ' assert control["schema_changes_authorized"] ' - 'is False\n' - ' assert control["schema_paths_authorized"] == ' - '[]\n' - ' assert control["new_files_permitted"] == ' - "['src/recursive_integrity_toolkit/data/hero/EXPECTED_OUTPUTS.md', " - "'src/recursive_integrity_toolkit/data/hero/config.json', " - "'src/recursive_integrity_toolkit/data/hero/provenance.csv', " - "'src/recursive_integrity_toolkit/data/hero/records_v1.csv', " - "'src/recursive_integrity_toolkit/data/hero/records_v2.csv', " - "'src/recursive_integrity_toolkit/data/hero/version_order.json', " - "'src/recursive_integrity_toolkit/data/report.schema.json']\n" - ' for name in ("phase_complete", ' - '"next_step_authorized", "main_merge_authorized", ' - '"publication_authorized"):\n' - ' assert control[name] is False\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control, ' - 'step=8)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step4_control"](control, ' - 'step=4)'}, - {'new': 'def ' - 'test_phase4_step8_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value, phase4_step8_snapshot):\n' - ' control = json.loads((phase4_step8_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' verify = ' - 'phase4_tools["verify_phase4_step8_control"]\n' - ' verify(control, step=8)\n' - ' control[field] = value\n' - ' with pytest.raises(ValueError):\n' - ' verify(control, step=8)', - 'node': 'test_phase4_step8_control_rejects_forged_scope_and_stage', - 'old': 'def ' - 'test_phase4_step8_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' verify = ' - 'phase4_tools["verify_phase4_step8_control"]\n' - ' verify(control, step=8)\n' - ' control[field] = value\n' - ' with pytest.raises(ValueError):\n' - ' verify(control, step=8)'}, - {'new': 'def ' - 'test_phase4_step8_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step, phase4_step8_snapshot):\n' - ' control = json.loads((phase4_step8_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control, ' - 'step=8)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control, ' - 'step=step)', - 'node': 'test_phase4_step8_control_rejects_unapproved_dispatch', - 'old': 'def ' - 'test_phase4_step8_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control, ' - 'step=8)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control, ' - 'step=step)'}, - {'new': 'def ' - 'test_phase4_step8_control_requires_explicit_approval_fields(phase4_tools, ' - 'field, phase4_step8_snapshot):\n' - ' control = json.loads((phase4_step8_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control)\n' - ' del control[field]\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control)', - 'node': 'test_phase4_step8_control_requires_explicit_approval_fields', - 'old': 'def ' - 'test_phase4_step8_control_requires_explicit_approval_fields(phase4_tools, ' - 'field):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control)\n' - ' del control[field]\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control)'}, - {'new': 'def ' - 'test_phase4_step8_control_cannot_mint_extra_permission(phase4_tools, ' - 'phase4_step8_snapshot):\n' - ' control = json.loads((phase4_step8_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control)\n' - ' control["approved_scope_expansion"] = ' - '{"step": 8, "publication": True}\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control)', - 'node': 'test_phase4_step8_control_cannot_mint_extra_permission', - 'old': 'def ' - 'test_phase4_step8_control_cannot_mint_extra_permission(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control)\n' - ' control["approved_scope_expansion"] = ' - '{"step": 8, "publication": True}\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control)'}, - {'new': 'def ' - 'test_phase4_step8_historical_bindings_preserve_assertions(phase4_tools, ' - 'phase4_step7_snapshot, phase4_step8_snapshot):\n' - ' import ast\n' - ' registry = ' - 'phase4_tools["PHASE4_STEP8_MIGRATIONS"]\n' - ' assert sorted(len(rows) for rows in ' - 'registry.values()) == [4, 7]\n' - ' for path, rows in registry.items():\n' - ' ' - 'phase4_tools["verify_phase4_step8_test_migration"](path, ' - '(phase4_step7_snapshot / path).read_bytes(), ' - '(phase4_step8_snapshot / path).read_bytes())\n' - ' for row in rows:\n' - ' trees = [ast.parse(row[key]) for key ' - 'in ("old", "new")]\n' - ' assert trees[0].body[0].name == ' - 'trees[1].body[0].name == row["node"]\n' - ' assert [[ast.dump(n) for n in ' - 'ast.walk(t) if isinstance(n, ast.Assert)] for t ' - 'in trees][0] == [[ast.dump(n) for n in ' - 'ast.walk(t) if isinstance(n, ast.Assert)] for t ' - 'in trees][1]', - 'node': 'test_phase4_step8_historical_bindings_preserve_assertions', - 'old': 'def ' - 'test_phase4_step8_historical_bindings_preserve_assertions(phase4_tools, ' - 'phase4_step7_snapshot):\n' - ' import ast\n' - ' registry = ' - 'phase4_tools["PHASE4_STEP8_MIGRATIONS"]\n' - ' assert sorted(len(rows) for rows in ' - 'registry.values()) == [4, 7]\n' - ' for path, rows in registry.items():\n' - ' ' - 'phase4_tools["verify_phase4_step8_test_migration"](path, ' - '(phase4_step7_snapshot / path).read_bytes(), ' - '(ROOT / path).read_bytes())\n' - ' for row in rows:\n' - ' trees = [ast.parse(row[key]) for key ' - 'in ("old", "new")]\n' - ' assert trees[0].body[0].name == ' - 'trees[1].body[0].name == row["node"]\n' - ' assert [[ast.dump(n) for n in ' - 'ast.walk(t) if isinstance(n, ast.Assert)] for t ' - 'in trees][0] == [[ast.dump(n) for n in ' - 'ast.walk(t) if isinstance(n, ast.Assert)] for t ' - 'in trees][1]'}, - {'new': 'def ' - 'test_phase4_step8_historical_guard_rejects_weakening(phase4_tools, ' - 'phase4_step7_snapshot, mutation, ' - 'phase4_step8_snapshot):\n' - ' path = ' - '"tests/unit/test_phase4_contracts.py"\n' - ' before, after = (phase4_step7_snapshot / ' - 'path).read_bytes(), (phase4_step8_snapshot / ' - 'path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step8_test_migration"]\n' - ' verify(path, before, after)\n' - ' if mutation == "assertion":\n' - " needle = b' assert " - 'control["active_phase"] == 4 and ' - 'control["active_step"] == 7\\n\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b" ' - 'assert True\\n", 1)\n' - ' elif mutation == "binding":\n' - ' needle = ' - "b'json.loads((phase4_step7_snapshot / " - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\'\n' - ' assert needle in after\n' - ' after = after.replace(needle, ' - 'needle.replace(b"phase4_step7_snapshot", ' - 'b"ROOT"), 1)\n' - ' elif mutation == "shadow":\n' - ' after += b"\\ndef ' - 'test_phase4_step7_fixed_boundary_opens_only_approved_existing_paths():\\n ' - 'assert True\\n"\n' - ' else:\n' - ' after += b"\\ndef ' - 'test_phase4_step8_bad(value=globals().clear()):\\n ' - 'pass\\n"\n' - ' with pytest.raises(ValueError): verify(path, ' - 'before, after)', - 'node': 'test_phase4_step8_historical_guard_rejects_weakening', - 'old': 'def ' - 'test_phase4_step8_historical_guard_rejects_weakening(phase4_tools, ' - 'phase4_step7_snapshot, mutation):\n' - ' path = ' - '"tests/unit/test_phase4_contracts.py"\n' - ' before, after = (phase4_step7_snapshot / ' - 'path).read_bytes(), (ROOT / path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step8_test_migration"]\n' - ' verify(path, before, after)\n' - ' if mutation == "assertion":\n' - " needle = b' assert " - 'control["active_phase"] == 4 and ' - 'control["active_step"] == 7\\n\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b" ' - 'assert True\\n", 1)\n' - ' elif mutation == "binding":\n' - ' needle = ' - "b'json.loads((phase4_step7_snapshot / " - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\'\n' - ' assert needle in after\n' - ' after = after.replace(needle, ' - 'needle.replace(b"phase4_step7_snapshot", ' - 'b"ROOT"), 1)\n' - ' elif mutation == "shadow":\n' - ' after += b"\\ndef ' - 'test_phase4_step7_fixed_boundary_opens_only_approved_existing_paths():\\n ' - 'assert True\\n"\n' - ' else:\n' - ' after += b"\\ndef ' - 'test_phase4_step8_bad(value=globals().clear()):\\n ' - 'pass\\n"\n' - ' with pytest.raises(ValueError): verify(path, ' - 'before, after)'}]} - - -PHASE4_STEP9_GOLDEN_SHA256 = {'tests/golden/phase4_hero_redacted.json': 'd2138b4f982d09c967cc3268323511ba29add6420ed6b64d91cf13379652e523', - 'tests/golden/phase4_hero_redacted.md': 'df9a0cfabf6eb95f2f494eddf23adb55cc161e2df197b16fbcc36b654eeba3f9', - 'tests/golden/phase4_hero_report.json': 'ce259014260e296e13707693745308c958b829c26373a0ed27c0826690abe633', - 'tests/golden/phase4_hero_report.md': 'bcab1cf1c3008ec86ee1c2a11cac0b130d668911f8e00868209451d74807da80', - 'tests/golden/phase4_report_cases.md': '3ca7005e2dddc990ea1c18fa3408343f9b51a943e0210fe0f611771111ce9709', - 'tests/golden/phase4_report_expected.json': '94fb170bc4095a8f765f23c9b8b87ffb784020734315d0704f36ec4ba3eba10f'} - - -def _phase4_step8_files(): - """Read the exact accepted Step 8 Git tree and verify every blob identity.""" - for suffix, expected in (("^{commit}", PHASE4_STEP8_FINAL), ("^{tree}", PHASE4_STEP8_TREE), - (":tests", PHASE4_STEP8_TEST_TREE)): - if git("rev-parse", PHASE4_STEP8_FINAL + suffix).decode().strip() != expected: - raise ValueError("Pinned Phase 4 Step 8 identity mismatch") - objects = {} - for entry in git("ls-tree", "-rz", PHASE4_STEP8_FINAL).split(b"\0"): - if not entry: - continue - metadata, raw_path = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - path = raw_path.decode("utf-8") - if (mode not in (b"100644", b"100755") or kind != b"blob" or path in objects - or path.startswith("/") or ".." in path.split("/") or ".git" in path.split("/")): - raise ValueError("Unsafe pinned Step 8 Git object") - objects[path] = oid.decode("ascii") - if len(objects) != 236: - raise ValueError("Pinned Step 8 must contain exactly 236 files") - files = {} - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_STEP8_FINAL))) as archive: - names = [item.filename for item in archive.infolist() if not item.is_dir()] - if len(names) != len(objects) or set(names) != set(objects): - raise ValueError("Pinned Step 8 archive identity mismatch") - for name in names: - raw = archive.read(name) - oid = hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - if oid != objects[name]: - raise ValueError(f"Pinned Step 8 blob mismatch: {name}") - files[name] = raw - return MappingProxyType(files) - - -PHASE4_CURRENT_STEP = 11 -PHASE4_CURRENT_APPROVAL = "继续 Phase 4 Step 11" -PHASE4_CURRENT_BASE = "9142b344bfc2dc2b33dbc696159f07bbc81b223a" -PHASE4_CURRENT_TREE = "a485078d3e7cf739e10b087acb604f317ee19854" -PHASE4_CURRENT_VERSION_PATHS = { - "src/recursive_integrity_toolkit/__init__.py", "pyproject.toml", -} -PHASE4_CURRENT_NEW = { - "PHASE_4_COMPLETION.md", "PHASE_4_VALIDATION_REPORT.md", - "PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md", -} -PHASE4_CURRENT_ALLOWED = PHASE4_G | PHASE4_CURRENT_VERSION_PATHS | PHASE4_CURRENT_NEW | { - "README.md", "CHANGELOG.md", "docs/cli.md", "docs/report_schema.md", - "docs/data_schema.md", "docs/privacy.md", "docs/release_process.md", - "tests/integration/test_cli_validation.py", "tests/integration/test_phase4_cli.py", - "tests/golden/test_phase4_reports.py", -} - - -def _phase4_current_files() -> dict[str, bytes]: - """Use accepted Git content directly, without another source-hash registry.""" - if git("rev-parse", PHASE4_CURRENT_BASE + "^{tree}").decode().strip() != PHASE4_CURRENT_TREE: - raise ValueError("Accepted Phase 4 source tree mismatch") - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_CURRENT_BASE))) as archive: - return {item.filename: archive.read(item) for item in archive.infolist() if not item.is_dir()} - - -def phase4_current_expected_control() -> dict: - """Bind current permission to the user's authorization and accepted Git source.""" - result = json.loads(git("show", PHASE4_CURRENT_BASE + ":PHASE_4_BASELINE.json")) - result.update({ - "control_version": "1.10", "active_step": PHASE4_CURRENT_STEP, - "approval_date": "2026-09-22", "approval_basis": PHASE4_CURRENT_APPROVAL, - "previous_step_commit": PHASE4_CURRENT_BASE, "previous_step_tree": PHASE4_CURRENT_TREE, - "previous_step_test_tree": "2fc4e86c4dee9f069f93af59caab138f0ea63224", - "permitted_paths": sorted(PHASE4_CURRENT_ALLOWED), "new_files_permitted": sorted(PHASE4_CURRENT_NEW), - "report_goldens_authorized": False, - "runtime_changes_authorized": True, - "runtime_paths_authorized": ["src/recursive_integrity_toolkit/__init__.py"], - }) - # Git preserves this accepted snapshot; avoid growing parallel hash registries. - for key in ("previous_step_files_sha256", "previous_step_core_tests", - "previous_step_parquet_tests", "runtime_repair_approval_basis"): - result.pop(key, None) - return result - - -def verify_phase4_current_control(control: dict, step: int = PHASE4_CURRENT_STEP) -> None: - if type(step) is not int or step != PHASE4_CURRENT_STEP or type(control) is not dict: - raise ValueError("Unsupported current Phase 4 stage/control") - try: - actual = json.dumps(control, sort_keys=True, ensure_ascii=True, allow_nan=False) - expected = json.dumps(phase4_current_expected_control(), sort_keys=True, ensure_ascii=True, allow_nan=False) - except (TypeError, ValueError) as error: - raise ValueError("Invalid current Phase 4 control") from error - if actual != expected: - raise ValueError("Phase 4 control differs from the independently authorized current scope") - - -def verify_phase4_current_changes(changes: list[tuple[str, str]]) -> None: - seen = set() - for status, path in changes: - if (path in seen or path not in PHASE4_CURRENT_ALLOWED - or status != ("A" if path in PHASE4_CURRENT_NEW else "M")): - raise ValueError(f"Unapproved current Phase 4 path/operation: {status} {path}") - seen.add(path) - - -def _phase4_step9_header(node, path): - """Allow explicit test parametrization/fixtures, never definition-time effects.""" - for decorator in node.decorator_list: - if not isinstance(decorator, ast.Call) or ast.unparse(decorator.func) not in { - "pytest.fixture", "pytest.mark.parametrize"}: - raise ValueError(f"Unapproved Step 9 test decorator: {path}:{node.name}") - if ast.unparse(decorator.func) == "pytest.fixture" and not node.name.startswith("phase4_step"): - raise ValueError("A Step 9 fixture must have an explicit scoped name") - if any(keyword.arg is None for keyword in decorator.keywords): - raise ValueError("Decorator expansion is outside the Step 9 contract") - for value in [*decorator.args, *(keyword.value for keyword in decorator.keywords)]: - try: - ast.literal_eval(value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError("Step 9 decorator arguments must be literal") from error - clone = ast.parse(ast.unparse(node)).body[0] - clone.decorator_list = [] - _phase4_preserve_function_header(clone, path) - - -def _phase4_step9_append_only(before: bytes, after: bytes, path: str) -> None: - if not after.startswith(before): - raise ValueError(f"Inherited Step 8 prefix changed: {path}") - old, new = ast.parse(before), ast.parse(after) - bindings = set() - for node in old.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - bindings.add(node.name) - elif isinstance(node, (ast.Import, ast.ImportFrom)): - bindings.update(alias.asname or alias.name.split(".")[0] for alias in node.names) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - bindings.update(child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)) - for node in new.body[len(old.body):]: - if not isinstance(node, ast.FunctionDef) or node.name in bindings: - raise ValueError(f"Step 9 addition executes, rebinds or shadows inherited source: {path}") - allowed_snapshot = path == "tests/conftest.py" and node.name == "phase4_step8_snapshot" - if not allowed_snapshot and not node.name.startswith(("test_phase4_step9_", "phase4_step9_")): - raise ValueError(f"Step 9 added test/helper is not explicitly scoped: {path}:{node.name}") - if path == "tests/conftest.py" and node.name != "phase4_step8_snapshot": - raise ValueError("Only the pinned Step 8 shared fixture is authorized") - bindings.add(node.name) - _phase4_step9_header(node, path) - - -def verify_phase4_step9_test_migration(path: str, before: bytes, after: bytes) -> None: - prior = _phase4_step8_files() - if path not in prior or path not in PHASE4_STEP9_ALLOWED or before != prior[path] or not path.startswith("tests/"): - raise ValueError("Step 9 migration requires the exact named Step 8 source") - _phase4_step9_check_test_migration(path, before, after) - - -def _phase4_step9_check_test_migration(path: str, before: bytes, after: bytes) -> None: - """Check a source already read from the verified Step 8 immutable mapping.""" - expected = before - for row in PHASE4_STEP9_MIGRATIONS.get(path, []): - old, new = row["old"].encode(), row["new"].encode() - if expected.count(old) != 1: - raise ValueError("Step 8 historical binding is not unique") - expected = expected.replace(old, new, 1) - _phase4_step9_append_only(expected, after, path) - - -def _phase4_step9_preserve_tooling(before: bytes, after: bytes, path: str) -> None: - """Preserve exact inherited statements using one source-line split per tree.""" - if path == "scripts/release_check.py": - old = ' return phase4_cli_main() if explicit_phase4 else main()' - new = (' explicit_step9 = "--step=9" in argv or any(a == "--step" and b == "9" for a, b in zip(argv, argv[1:]))\n' - ' if explicit_phase4 and explicit_step9:\n' - ' return phase4_step9_cli_main()\n' + old) - # Earlier preserved dispatch functions contain this source too; bind cli_main only. - parsed = ast.parse(before) - entry = next(node for node in parsed.body if isinstance(node, ast.FunctionDef) and node.name == "cli_main") - old_entry = ast.get_source_segment(before.decode(), entry) - if old_entry.count(old) != 1: - raise ValueError("Step 8 release dispatcher identity mismatch") - replacements = [(old_entry, old_entry.replace(old, new, 1))] - else: - old = ' if args.phase == 4 and args.step == 8:\n return phase4_step8_main(step=args.step)\n' - new = old + ' if args.phase == 4 and args.step == 9:\n return phase4_step9_main(step=args.step)\n' - replacements = [(old, new), - ('Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5 or --phase 4 --step 6 or --phase 4 --step 7 or --phase 4 --step 8', - 'Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5 or --phase 4 --step 6 or --phase 4 --step 7 or --phase 4 --step 8 or --phase 4 --step 9')] - expected = before - for old, new in replacements: - if expected.count(old.encode()) != 1: - raise ValueError(f"Historical dispatcher identity mismatch: {path}") - expected = expected.replace(old.encode(), new.encode(), 1) - trees = [ast.parse(expected), ast.parse(after)] - lines = [expected.splitlines(keepends=True), after.splitlines(keepends=True)] - def segment(index, node): - if node.lineno == node.end_lineno: - return lines[index][node.lineno - 1][node.col_offset:node.end_col_offset] - return (lines[index][node.lineno - 1][node.col_offset:] - + b"".join(lines[index][node.lineno:node.end_lineno - 1]) - + lines[index][node.end_lineno - 1][:node.end_col_offset]) - def entry_guard(node): - return isinstance(node, ast.If) and isinstance(node.test, ast.Compare) and "__name__" in ast.unparse(node.test) - guards = [n for n in trees[1].body if entry_guard(n)] - expected_guard = ast.parse('if __name__ == "__main__":\n raise SystemExit(cli_main())').body[0] - if len(guards) != 1 or ast.dump(guards[0]) != ast.dump(expected_guard): - raise ValueError(f"Unapproved Step 9 maintainer entrypoint: {path}") - historical = [n for n in trees[0].body if not entry_guard(n)] - current = [n for n in trees[1].body if not entry_guard(n)] - old_entries = [(segment(0, n), ast.dump(n)) for n in historical] - cursor = 0 - for node in current: - if cursor < len(old_entries) and (segment(1, node), ast.dump(node)) == old_entries[cursor]: - cursor += 1 - continue - if isinstance(node, ast.FunctionDef) and (node.name == "_phase4_step8_files" or node.name.startswith( - ("phase4_step9_", "_phase4_step9_", "verify_phase4_step9_", "audit_phase4_step9"))): - _phase4_preserve_function_header(node, path) - continue - if isinstance(node, ast.Assign) and all(isinstance(target, ast.Name) and target.id.startswith( - ("PHASE4_STEP9_", "PHASE4_STEP8_")) for target in node.targets): - try: - ast.literal_eval(node.value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError(f"Nonliteral added Step 9 maintainer constant: {path}") from error - continue - raise ValueError(f"Unapproved Step 9 maintainer addition: {path}") - if cursor != len(historical): - raise ValueError(f"Inherited Step 8 maintainer statement changed: {path}") - def binding_counts(tree): - counts = {} - for node in tree.body: - names = [] - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - names = [node.name] - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names = [child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)] - elif isinstance(node, (ast.Import, ast.ImportFrom)): - names = [alias.asname or alias.name.split(".")[0] for alias in node.names] - for name in names: - counts[name] = counts.get(name, 0) + 1 - return counts - inherited_counts = binding_counts(trees[0]) - for name, count in binding_counts(trees[1]).items(): - if count > inherited_counts.get(name, 1): - raise ValueError(f"Duplicate Step 9 maintainer binding: {path}:{name}") - - -def phase4_step9_golden_contract(root: Path = ROOT) -> dict: - """Check separately frozen rationale and literal expected reports byte for byte.""" - expected = { - "tests/golden/phase4_report_cases.md", "tests/golden/phase4_report_expected.json", - "tests/golden/phase4_hero_report.json", "tests/golden/phase4_hero_report.md", - "tests/golden/phase4_hero_redacted.json", "tests/golden/phase4_hero_redacted.md", - } - if set(PHASE4_STEP9_GOLDEN_SHA256) != expected: - raise ValueError("Step 9 independently authored golden identities are not frozen") - for name, digest in PHASE4_STEP9_GOLDEN_SHA256.items(): - path = root / name - if not path.is_file() or path.is_symlink() or hashlib.sha256(path.read_bytes()).hexdigest() != digest: - raise ValueError(f"Step 9 golden rationale or expected bytes changed: {name}") - return {"frozen_report_oracle_files": len(expected)} - - -def verify_phase4_current_snapshot(root: Path = ROOT) -> dict: - """Protect unchanged product/authority bytes with the accepted Git snapshot. - - Current maintenance is reviewed and tested by behavior. Obsolete source forms - are retained in Git instead of adding another source-binding migration layer. - """ - prior = _phase4_current_files() - for path, raw in prior.items(): - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError(f"Missing or aliased accepted file: {path}") - if path in PHASE4_CURRENT_VERSION_PATHS: - if raw.count(b"0.1.0.dev2") != 1 or target.read_bytes() != raw.replace(b"0.1.0.dev2", b"0.1.0.dev3", 1): - raise ValueError(f"Only the approved dev3 version literal may change: {path}") - if path not in PHASE4_CURRENT_ALLOWED and target.read_bytes() != raw: - raise ValueError(f"Protected accepted bytes changed: {path}") - for prefix in ("src/recursive_integrity_toolkit/", "schemas/", "examples/hero/"): - paths = list((root / prefix).rglob("*")) - if any(path.is_symlink() for path in paths): - raise ValueError(f"Aliased protected content: {prefix}") - current = {path.relative_to(root).as_posix() for path in paths - if path.is_file() and "__pycache__" not in path.parts} - if current != {path for path in prior if path.startswith(prefix)}: - raise ValueError(f"Protected accepted file set changed: {prefix}") - verify_phase4_current_control(json.loads((root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))) - decisions = (root / "PHASE_4_DECISIONS.md").read_text(encoding="utf-8") - if PHASE4_CURRENT_APPROVAL not in decisions or PHASE4_CURRENT_BASE not in decisions: - raise ValueError("Actual authorization or accepted source anchor missing") - for name in PHASE4_CURRENT_NEW: - path = root / name - if not path.is_file() or path.is_symlink() or not path.read_text(encoding="utf-8").strip(): - raise ValueError(f"Step 11 requires a nonempty regular completion record: {name}") - if any((root / name).exists() for name in PHASE4_FORBIDDEN_OUTPUTS - PHASE4_CURRENT_NEW): - raise ValueError("Audit outputs cannot be committed as completion records") - return {"package_modules": 40, "frozen_runtime_modules": 39, "version_literal_changes": 2, "frozen_schemas": 5, - "hero_files_unchanged": 6, "packaged_resources": 7, "phase_complete": False, - **phase4_step9_golden_contract(root)} - - -def phase4_step9_golden_evidence(junit: Path) -> dict: - """Require all collected report goldens and all twenty unchanged math cases.""" - verify_junit(junit) - selected = ["tests/golden/test_phase4_reports.py", "tests/golden/test_phase3_math.py"] - process = subprocess.run([sys.executable, "-m", "pytest", "-p", "no:cacheprovider", "--collect-only", "-q", *selected], cwd=ROOT, capture_output=True, text=True, check=True) - nodes = [line.strip() for line in process.stdout.splitlines() if line.startswith("tests/") and "::" in line] - oracle_path = "tests/golden/phase3_math_cases.json" - oracle_bytes = (ROOT / oracle_path).read_bytes() - if hashlib.sha256(oracle_bytes).hexdigest() != STEP10_FROZEN_FILES[oracle_path]: - raise ValueError("Step 9 mathematical oracle bytes differ from the frozen authority") - expected_math = {selected[1] + "::test_phase3_frozen_case[" + case["case_id"] + "]" - for case in json.loads(oracle_bytes)["cases"]} - registry = selected[1] + "::test_phase3_all_twenty_oracles_have_unique_traceable_identity" - actual_math = {n for n in nodes if n.startswith(selected[1] + "::")} - if len(nodes) != len(set(nodes)) or len(expected_math) != 20 or actual_math != expected_math | {registry}: - raise ValueError("Step 9 golden collection lost mathematical cases or their registry identity") - if not any(n.startswith(selected[0] + "::") for n in nodes): - raise ValueError("Step 9 report golden suite is empty") - expected = set() + source = extract_sdist(sdist, work / "source") / "src" + hashes = {destination.removeprefix("src/recursive_integrity_toolkit/"): hashlib.sha256((ROOT/original).read_bytes()).hexdigest() for destination,original in RESOURCES.items()} + subprocess.run([sys.executable, "-I", "-c", installed_example_program(), str(source), str(work), str(ROOT), json.dumps(hashes)], cwd=work, check=True) + + +def smoke_all(wheel: Path, sdist: Path) -> None: + for check in (smoke_installed, smoke_installed_duplicates, smoke_installed_contract, + smoke_installed_assembly, smoke_installed_privacy, smoke_installed_renderers, + smoke_installed_publication, smoke_installed_cli, smoke_installed_example): + check(wheel) + smoke_sdist(sdist) + + +def _canonical_identities() -> set[tuple[str, str]]: + env = dict(os.environ, PYTHONPATH=str(ROOT / "src"), PYTHONDONTWRITEBYTECODE="1", RIT_TEST_PARQUET="1") + result = subprocess.run([sys.executable, "-m", "pytest", "-p", "no:cacheprovider", + "--collect-only", "-q", "--ignore=tests/performance"], + cwd=ROOT, env=env, capture_output=True, text=True, check=True) + nodes = [line.strip() for line in result.stdout.splitlines() if line.startswith("tests/") and "::" in line] + if not nodes or len(nodes) != len(set(nodes)): + raise ValueError("Canonical collection is empty or has duplicate identities") + identities = set() for node in nodes: base, bracket, parameter = node.partition("[") owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(junit).getroot().iter("testcase")} - selected_owners = {p.removesuffix(".py").replace("/", ".") for p in selected} - actual = {case for case in cases if case[0] in selected_owners} - if actual != expected: - raise ValueError("Step 9 JUnit does not execute every collected golden and mathematical case") - return {"mathematical_cases": 20, "report_golden_tests": sum(n.startswith(selected[0] + "::") for n in nodes), - "report_golden_nodeids": [n for n in nodes if n.startswith(selected[0] + "::")], - "mathematical_nodeids": [n for n in nodes if n in expected_math], - "mathematical_registry_nodeid": registry, - "missing_nodeids": [], "golden_nodeids_sha256": hashlib.sha256(("\n".join(nodes) + "\n").encode()).hexdigest()} - - -def audit_phase4_current(step: int = PHASE4_CURRENT_STEP) -> dict: - verify_phase4_current_control(json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")), step) - result = {"phase": 4, "active_step": step, **verify_phase4_current_snapshot(), - "phase0_hashes_verified": 16, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_phase4_current_diff(step: int = PHASE4_CURRENT_STEP) -> dict: - if type(step) is not int or step != PHASE4_CURRENT_STEP: - raise ValueError("Unsupported current Phase 4 stage") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE4_CURRENT_BASE, "HEAD"], check=True) - raw = [part.decode("utf-8") for part in git("diff", "--name-status", "--no-renames", "-z", PHASE4_CURRENT_BASE, "--").split(b"\0") if part] - if len(raw) % 2: - raise ValueError("Malformed current Git difference records") - changes = list(zip(raw[::2], raw[1::2])) - changes.extend(("A", p.decode()) for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") if p) - verify_phase4_current_changes(changes) - result = {"previous_step_commit": PHASE4_CURRENT_BASE, "changed_files": len(changes), - "changes": changes, "step": step, "scope": "PASS"} - print(json.dumps(result, indent=2)) - return result + identities.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) + return identities -def phase4_current_suite_evidence(output: Path) -> dict: - """Collect canonical current identities once; historical receipts remain archived.""" - output.mkdir(parents=True, exist_ok=True) - current, log = _collect(ROOT) - result = {"current_nodeids": current, "current_tests": len(current), - "nodeids_sha256": hashlib.sha256(("\n".join(current)+"\n").encode()).hexdigest()} - (output / "phase4_current_test_identity_manifest.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4-current-collection.log").write_text(log, encoding="utf-8") - print(json.dumps({"current_tests": len(current)}, indent=2)) - return result - +def candidate(output: Path) -> None: + """Verify a committed candidate; workflow separately requires its scale job. -def phase4_current_candidate(output: Path, step: int = PHASE4_CURRENT_STEP) -> None: - """Build tested current Phase 4 candidate evidence without declaring Phase 4 complete.""" + Core/Parquet JUnit must belong to this source revision, checked by the existing + workflow before invocation. This entry point does not prove their origin. + """ if os.environ.get("RIT_TEST_PARQUET") != "1" or importlib.util.find_spec("pyarrow") is None: - raise ValueError("current Phase 4 candidate evidence requires RIT_TEST_PARQUET=1 and real PyArrow") - output.mkdir(parents=True, exist_ok=True) - result = audit_phase4_current(step) - result["diff"] = audit_phase4_current_diff(step) + raise ValueError("Candidate verification requires RIT_TEST_PARQUET=1 and real PyArrow") if git("status", "--porcelain").strip(): - raise ValueError("current Phase 4 candidate archive requires committed, clean source") - result["core"] = verify_junit(output / "core.xml", minimum=3768) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=3771) - result["core_math_measurements"] = verify_step10_evidence(output / "core.xml", output / "phase4-current-core-observations.json") - result["parquet_math_measurements"] = verify_step10_evidence(output / "parquet.xml", output / "phase4-current-parquet-observations.json") - result["core_report_goldens"] = phase4_step9_golden_evidence(output / "core.xml") - result["parquet_report_goldens"] = phase4_step9_golden_evidence(output / "parquet.xml") - print(json.dumps({name: result[name] for name in ("core_report_goldens", "parquet_report_goldens")}, indent=2)) - identity = phase4_current_suite_evidence(output) - result["test_identity"] = {k: identity[k] for k in ("current_tests", "nodeids_sha256")} - expected = set() - for node in identity["current_nodeids"]: - base, bracket, parameter = node.partition("[") - owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) + raise ValueError("Candidate archive requires committed, clean source") + result = audit() + expected = _canonical_identities() for name, parquet in (("core.xml", False), ("parquet.xml", True)): - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(output / name).getroot().iter("testcase")} - target = expected if parquet else {c for c in expected if c[1] not in PARQUET_CASES} - if cases != target: - raise ValueError(f"current Phase 4 JUnit does not execute the entire current suite: {name}") - wheel, sdist = phase4_step8_distributions(output / "dist", expected_version="0.1.0.dev3") - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - phase4_step6_installed_publication_smoke(wheel) - phase4_step7_installed_cli_smoke(wheel) - phase4_step8_installed_example_smoke(wheel) - phase4_step8_sdist_smoke(sdist) - archive = output / "recursive-integrity-toolkit-phase4-current-candidate.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {p for p in git("ls-files", "-z").decode().split("\0") if p} + result[name] = verify_junit(output / name, require_parquet=parquet) + actual = {(case.get("classname", ""), case.get("name", "")) + for case in ET.parse(output / name).getroot().iter("testcase")} + required = expected if parquet else {item for item in expected if item[1] not in PARQUET_CASES} + if actual != required: + raise ValueError(f"Candidate JUnit does not execute the complete canonical regression suite: {name}") + wheel, sdist = verify_distributions(output / "dist") + smoke_all(wheel, sdist) + archive = output / "recursive-integrity-toolkit-source.zip" + subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", + "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) + tracked = {name for name in git("ls-files", "-z").decode().split("\0") if name} with zipfile.ZipFile(archive) as zipped: - entries = [item.filename for item in zipped.infolist() if not item.is_dir()] + entries = [member.filename for member in zipped.infolist() if not member.is_dir()] prefix = "recursive-integrity-toolkit/" - if (len(entries) != len(set(entries)) or any(not name.startswith(prefix) for name in entries)): - raise ValueError("current Phase 4 source archive has duplicate or unprefixed entries") - names = {name.removeprefix(prefix) for name in entries} - if names != tracked or len(tracked) != 246 or len(entries) != 246: - raise ValueError("current Phase 4 source archive file set mismatch") - for name in names: - if zipped.read("recursive-integrity-toolkit/"+name) != (ROOT / name).read_bytes(): - raise ValueError(f"current Phase 4 source archive byte mismatch: {name}") - result.update({"commit": git("rev-parse", "HEAD").decode().strip(), - "tree": git("rev-parse", "HEAD^{tree}").decode().strip(), - "source_archive": archive.name, "tracked_files": len(tracked), - "python": sys.version, "platform": sys.platform, - "versions": {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "pytest")}}) - (output / "phase4_current_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4_current_repository_files.sha256").write_text("".join(f"{hashlib.sha256((ROOT/name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - paths = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase4_current_artifacts.sha256") - (output / "phase4_current_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in paths), encoding="utf-8") - print(f"Phase 4 current candidate: {len(tracked)} source files verified; Phase 4 remains incomplete.") - - -def phase4_current_cli_main() -> int: - parser = argparse.ArgumentParser(description="Current Phase 4 maintainer gate") - parser.add_argument("--phase", type=int, choices=(4,), required=True) - parser.add_argument("--step", type=int, choices=(PHASE4_CURRENT_STEP,), required=True) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) - parser.add_argument("--golden-junit", type=Path) - parser.add_argument("--require-parquet", action="store_true") - parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - raise ValueError("Current Phase 4 cannot certify a final phase delivery") - if args.golden_junit is not None: - print(json.dumps(phase4_step9_golden_evidence(args.golden_junit), indent=2)) - elif args.junit is not None: - verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - phase4_current_suite_evidence(args.baseline_evidence) - elif args.dist is not None: - audit_phase4_current(args.step) - wheel, sdist = phase4_step8_distributions(args.dist, expected_version="0.1.0.dev3") - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - phase4_step6_installed_publication_smoke(wheel) - phase4_step7_installed_cli_smoke(wheel) - phase4_step8_installed_example_smoke(wheel) - phase4_step8_sdist_smoke(sdist) - elif args.smoke_wheel is not None: - smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - phase4_current_candidate(args.candidate, args.step) - else: - audit_phase4_current(args.step) - if args.diff: - audit_phase4_current_diff(args.step) - return 0 - - -def cli_main() -> int: - argv = sys.argv[1:] - explicit_phase4 = "--phase=4" in argv or any(a == "--phase" and b == "4" for a, b in zip(argv, argv[1:])) - explicit_step2 = "--step=2" in argv or any(a == "--step" and b == "2" for a, b in zip(argv, argv[1:])) - if explicit_phase4 and explicit_step2: - return phase4_step2_cli_main() - explicit_step3 = "--step=3" in argv or any(a == "--step" and b == "3" for a, b in zip(argv, argv[1:])) - if explicit_phase4 and explicit_step3: - return phase4_step3_cli_main() - explicit_step4 = "--step=4" in argv or any(a == "--step" and b == "4" for a, b in zip(argv, argv[1:])) - if explicit_phase4 and explicit_step4: - return phase4_step4_cli_main() - explicit_step5 = "--step=5" in argv or any(a == "--step" and b == "5" for a, b in zip(argv, argv[1:])) - if explicit_phase4 and explicit_step5: - return phase4_step5_cli_main() - explicit_step6 = "--step=6" in argv or any(a == "--step" and b == "6" for a, b in zip(argv, argv[1:])) - if explicit_phase4 and explicit_step6: - return phase4_step6_cli_main() - explicit_step7 = "--step=7" in argv or any(a == "--step" and b == "7" for a, b in zip(argv, argv[1:])) - if explicit_phase4 and explicit_step7: - return phase4_step7_cli_main() - explicit_step8 = "--step=8" in argv or any(a == "--step" and b == "8" for a, b in zip(argv, argv[1:])) - if explicit_phase4 and explicit_step8: - return phase4_step8_cli_main() - explicit_current = "--step=11" in argv or any(a == "--step" and b == "11" for a, b in zip(argv, argv[1:])) - if explicit_phase4 and explicit_current: - return phase4_current_cli_main() - return phase4_cli_main() if explicit_phase4 else main() - - -def main() -> int: + if len(entries) != len(set(entries)) or {name.removeprefix(prefix) for name in entries} != tracked: + raise ValueError("Candidate source archive file set mismatch") + for name in tracked: + if zipped.read(prefix + name) != (ROOT / name).read_bytes(): + raise ValueError(f"Candidate source archive byte mismatch: {name}") + result.update(commit=git("rev-parse", "HEAD").decode().strip(), + source_archive=archive.name, tracked_files=len(tracked), + canonical_scope="tests excluding performance; designated scale job required separately", + python=sys.version, platform=sys.platform) + (output / "candidate.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print("Current candidate package and installation checks: PASS; no publication performed") + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + if any(arg in {"--phase", "--step"} or arg.startswith(("--phase=", "--step=")) for arg in argv): + raise SystemExit("Historical phase dispatch has been retired. Run the current command without --phase/--step; use Git history at the accepted commit to inspect prior gates.") parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--phase", type=int, choices=(2, 3), default=2) - parser.add_argument("--step", type=int, default=ACTIVE_STEP) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) + modes = parser.add_mutually_exclusive_group() + modes.add_argument("--junit", type=Path) + modes.add_argument("--dist", type=Path) + modes.add_argument("--smoke-wheel", type=Path) + modes.add_argument("--candidate", type=Path) parser.add_argument("--require-parquet", action="store_true") parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - if args.phase != 3 or args.step != 11: - raise ValueError("Phase 3 delivery requires explicit Step 11") - candidate(args.delivery, args.step) - elif args.junit is not None: + args = parser.parse_args(argv) + if args.minimum_tests < 1: + parser.error("--minimum-tests must be positive") + if args.junit: verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - baseline_evidence(args.baseline_evidence) - elif args.dist is not None: - wheel, _ = verify_distributions(args.dist); smoke_installed(wheel); smoke_installed_duplicates(wheel) - elif args.smoke_wheel is not None: + elif args.dist: + audit() + smoke_all(*verify_distributions(args.dist)) + elif args.smoke_wheel: smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - if args.phase != 3: - raise ValueError("Candidate requires explicit --phase 3") - candidate(args.candidate, args.step) + elif args.candidate: + candidate(args.candidate) else: - audit(args.phase, args.step) - if args.diff: - audit_phase3_diff(args.step) if args.phase == 3 else audit_step10_diff() + audit() return 0 if __name__ == "__main__": - raise SystemExit(cli_main()) + raise SystemExit(main()) diff --git a/tests/conftest.py b/tests/conftest.py index 8b0e0fb..1fe3fe5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,25 +1,10 @@ -"""Shared scaffold fixtures and explicit Phase 3 Step 10 test-only composition. - -Owner IDs: - Technical Maintainer, Phase 1 acceptance gate - -Current scope: - Repository structure, safe import, CLI startup, JSON parsing, owner IDs, - prohibited paths, optional dependency isolation, no-network import, and - absence of analytical implementation. - -Limits: - Existing scaffold fixtures stay unchanged. New helpers use only approved Hero - or synthetic data. Nothing here is an importable product orchestrator. -""" +"""Shared fixtures for current product behavior and bounded performance runs.""" from __future__ import annotations -import ast import os import sys from pathlib import Path -from typing import Callable import pytest @@ -61,35 +46,6 @@ def subprocess_env() -> dict[str, str]: return env -@pytest.fixture(scope="session") -def owner_checker(package_root: Path) -> Callable[[str, str], None]: - """Return a checker for approved owner IDs in module docstrings.""" - def check(relative_path: str, owner_id: str) -> None: - path = package_root / relative_path - assert path.is_file(), f"Missing approved module: {relative_path}" - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - doc = ast.get_docstring(tree) or "" - assert "Owner IDs:" in doc, f"Owner section missing in {relative_path}" - assert owner_id in doc, f"{owner_id} missing from {relative_path} owner section" - assert "Current phase status:" in doc, f"Phase status missing in {relative_path}" - assert "No analytical" in doc or "No " in doc, f"No-implementation limit missing in {relative_path}" - return check - - -@pytest.fixture(scope="session") -def placeholder_checker(package_root: Path) -> Callable[[str], None]: - """Return a checker that requires a module to contain only its docstring.""" - def check(relative_path: str) -> None: - path = package_root / relative_path - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - assert len(tree.body) == 1, f"Unexpected executable body in {relative_path}" - node = tree.body[0] - assert isinstance(node, ast.Expr) - assert isinstance(node.value, ast.Constant) - assert isinstance(node.value.value, str) - return check - - @pytest.fixture def phase3_hero_pipeline(repo_root): """Return a test-only callable; ordinary validation still performs no metrics.""" @@ -169,669 +125,6 @@ def measure(name, operation, **details): return measure -@pytest.fixture(scope="session") -def phase3_step10_snapshot(repo_root, tmp_path_factory): - """Materialize immutable Step 10 source for its historical stage assertions.""" - import io - import subprocess - import zipfile - root = tmp_path_factory.mktemp("phase3-step10") - commit = "150a2a105e01883672ef0c2300b41b0de3be352e" - tree = subprocess.check_output(["git", "-C", str(repo_root), "rev-parse", commit+"^{tree}"], text=True).strip() - assert tree == "b609cb75913fc6352729086819a24795caae1dd8" - raw = subprocess.check_output(["git", "-C", str(repo_root), "archive", "--format=zip", commit]) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - archive.extractall(root) - return root - - -@pytest.fixture(scope="session") -def phase3_final_snapshot(repo_root, tmp_path_factory): - """Provide verified final Phase 3 bytes only to listed historical tests. - - The source checkout is never rebound. The private archive has no Git working - tree, accepts only pinned regular-file blobs, and is checked again after use. - Files remain writable so copies in inherited mutation tests work on Windows; - changing this snapshot itself fails the closing byte and file-set check. - """ - import hashlib - import io - import subprocess - import zipfile - from pathlib import PurePosixPath - - commit = "e3ffb8c0a88bfe31f669f9662d9b5213da628b3a" - tree = "e2a25f8cfdc66c3317809c479f80fdae162e6ba9" - tests_tree = "6ab22cb9197a8f094f455b29a07a851062bb26c9" - - def git(*arguments): - return subprocess.check_output(["git", "-C", str(repo_root), *arguments]) - - assert git("rev-parse", commit + "^{commit}").decode().strip() == commit - assert git("rev-parse", commit + "^{tree}").decode().strip() == tree - assert git("rev-parse", commit + ":tests").decode().strip() == tests_tree - blobs = {} - for entry in git("ls-tree", "-rz", commit).split(b"\0"): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - relative = name.decode("utf-8") - parts = PurePosixPath(relative).parts - assert mode in (b"100644", b"100755") and kind == b"blob" - assert parts and not relative.startswith("/") and ".." not in parts and ".git" not in parts - assert relative not in blobs - blobs[relative] = oid.decode("ascii") - - def blob_oid(raw): - return hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - - root = tmp_path_factory.mktemp("phase3-final") - raw = git("archive", "--format=zip", commit) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = [member for member in archive.infolist() if not member.is_dir()] - assert len(members) == len(blobs) - assert {member.filename for member in members} == set(blobs) - for member in members: - content = archive.read(member) - assert blob_oid(content) == blobs[member.filename], member.filename - destination = root.joinpath(*PurePosixPath(member.filename).parts) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(content) - - def verify_unchanged(): - paths = list(root.rglob("*")) - assert not any(path.is_symlink() for path in paths), "Historical snapshot gained a symlink" - files = {path.relative_to(root).as_posix(): path for path in paths if path.is_file()} - assert set(files) == set(blobs), "Historical snapshot file set changed" - for name, path in files.items(): - assert blob_oid(path.read_bytes()) == blobs[name], f"Historical snapshot changed: {name}" - - verify_unchanged() - yield root - verify_unchanged() - - -@pytest.fixture(scope="session") -def phase3_final_package_root(phase3_final_snapshot): - """Return the pinned package only to explicitly migrated historical checks.""" - return phase3_final_snapshot / "src/recursive_integrity_toolkit" - - -@pytest.fixture(scope="session") -def phase3_final_owner_checker(phase3_final_package_root): - """Run the inherited ownership assertions against pinned Phase 3 source.""" - def check(relative_path, owner_id): - path = phase3_final_package_root / relative_path - assert path.is_file(), f"Missing approved module: {relative_path}" - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - doc = ast.get_docstring(tree) or "" - assert "Owner IDs:" in doc, f"Owner section missing in {relative_path}" - assert owner_id in doc, f"{owner_id} missing from {relative_path} owner section" - assert "Current phase status:" in doc, f"Phase status missing in {relative_path}" - assert "No analytical" in doc or "No " in doc, f"No-implementation limit missing in {relative_path}" - return check - - -@pytest.fixture(scope="session") -def phase3_final_placeholder_checker(phase3_final_package_root): - """Run the inherited empty-module assertions against pinned Phase 3 source.""" - def check(relative_path): - path = phase3_final_package_root / relative_path - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - assert len(tree.body) == 1, f"Unexpected executable body in {relative_path}" - node = tree.body[0] - assert isinstance(node, ast.Expr) - assert isinstance(node.value, ast.Constant) - assert isinstance(node.value.value, str) - return check - - -@pytest.fixture(scope="session") -def phase3_final_subprocess_env(phase3_final_snapshot): - """Import only the pinned src tree for historical help, without bytecode writes.""" - env = os.environ.copy() - env["PYTHONPATH"] = str(phase3_final_snapshot / "src") - env["PYTHONDONTWRITEBYTECODE"] = "1" - return env - - -@pytest.fixture(scope="session") -def phase4_step1_snapshot(repo_root, tmp_path_factory): - """Provide verified accepted Phase 4 Step 1 bytes only to listed historical tests. - - The source checkout is never rebound. The private archive has no Git working - tree, accepts only pinned regular-file blobs, and is checked again after use. - Files remain writable so copies in inherited mutation tests work on Windows; - changing this snapshot itself fails the closing byte and file-set check. - """ - import hashlib - import io - import subprocess - import zipfile - from pathlib import PurePosixPath - - commit = "a7f3c46d6ca05de36bfcb60f496ebb2d5ab4a37c" - tree = "7f8ef492568456bf5d46fb1b7e2631cb9b94e3fd" - tests_tree = "2ce77e331a0fc377387735bb55dffd3be630b59c" - - def git(*arguments): - return subprocess.check_output(["git", "-C", str(repo_root), *arguments]) - - assert git("rev-parse", commit + "^{commit}").decode().strip() == commit - assert git("rev-parse", commit + "^{tree}").decode().strip() == tree - assert git("rev-parse", commit + ":tests").decode().strip() == tests_tree - blobs = {} - for entry in git("ls-tree", "-rz", commit).split(b"\0"): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - relative = name.decode("utf-8") - parts = PurePosixPath(relative).parts - assert mode in (b"100644", b"100755") and kind == b"blob" - assert parts and not relative.startswith("/") and ".." not in parts and ".git" not in parts - assert relative not in blobs - blobs[relative] = oid.decode("ascii") - - def blob_oid(raw): - return hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - - root = tmp_path_factory.mktemp("phase4-step1") - raw = git("archive", "--format=zip", commit) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = [member for member in archive.infolist() if not member.is_dir()] - assert len(members) == len(blobs) - assert {member.filename for member in members} == set(blobs) - for member in members: - content = archive.read(member) - assert blob_oid(content) == blobs[member.filename], member.filename - destination = root.joinpath(*PurePosixPath(member.filename).parts) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(content) - - def verify_unchanged(): - paths = list(root.rglob("*")) - assert not any(path.is_symlink() for path in paths), "Historical snapshot gained a symlink" - files = {path.relative_to(root).as_posix(): path for path in paths if path.is_file()} - assert set(files) == set(blobs), "Historical snapshot file set changed" - for name, path in files.items(): - assert blob_oid(path.read_bytes()) == blobs[name], f"Historical snapshot changed: {name}" - - verify_unchanged() - yield root - verify_unchanged() - - -@pytest.fixture(scope="session") -def phase4_step2_snapshot(repo_root, tmp_path_factory): - """Provide verified accepted Phase 4 Step 2 bytes only to listed historical tests. - - The source checkout is never rebound. The private archive has no Git working - tree, accepts only pinned regular-file blobs, and is checked again after use. - Files remain writable so copies in inherited mutation tests work on Windows; - changing this snapshot itself fails the closing byte and file-set check. - """ - import hashlib - import io - import subprocess - import zipfile - from pathlib import PurePosixPath - - commit = "fcea74e2b1858e83cdbfd1b8212d15343d63ff08" - tree = "c3ce887fd393cfc6c18364546005f2b74a63b746" - tests_tree = "9d6ac0cbf3827ab54c7852503d99d442a3796259" - - def git(*arguments): - return subprocess.check_output(["git", "-C", str(repo_root), *arguments]) - - assert git("rev-parse", commit + "^{commit}").decode().strip() == commit - assert git("rev-parse", commit + "^{tree}").decode().strip() == tree - assert git("rev-parse", commit + ":tests").decode().strip() == tests_tree - blobs = {} - for entry in git("ls-tree", "-rz", commit).split(b"\0"): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - relative = name.decode("utf-8") - parts = PurePosixPath(relative).parts - assert mode in (b"100644", b"100755") and kind == b"blob" - assert parts and not relative.startswith("/") and ".." not in parts and ".git" not in parts - assert relative not in blobs - blobs[relative] = oid.decode("ascii") - - def blob_oid(raw): - return hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - - root = tmp_path_factory.mktemp("phase4-step2") - raw = git("archive", "--format=zip", commit) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = [member for member in archive.infolist() if not member.is_dir()] - assert len(members) == len(blobs) - assert {member.filename for member in members} == set(blobs) - for member in members: - content = archive.read(member) - assert blob_oid(content) == blobs[member.filename], member.filename - destination = root.joinpath(*PurePosixPath(member.filename).parts) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(content) - - def verify_unchanged(): - paths = list(root.rglob("*")) - assert not any(path.is_symlink() for path in paths), "Historical snapshot gained a symlink" - files = {path.relative_to(root).as_posix(): path for path in paths if path.is_file()} - assert set(files) == set(blobs), "Historical snapshot file set changed" - for name, path in files.items(): - assert blob_oid(path.read_bytes()) == blobs[name], f"Historical snapshot changed: {name}" - - verify_unchanged() - yield root - verify_unchanged() - - -@pytest.fixture(scope="session") -def phase4_step3_snapshot(repo_root, tmp_path_factory): - """Provide verified accepted Phase 4 Step 3 bytes only to listed historical tests. - - The source checkout is never rebound. The private archive has no Git working - tree, accepts only pinned regular-file blobs, and is checked again after use. - Files remain writable so copies in inherited mutation tests work on Windows; - changing this snapshot itself fails the closing byte and file-set check. - """ - import hashlib - import io - import subprocess - import zipfile - from pathlib import PurePosixPath - - commit = "974545c456e535ba1e1c5b6bf4ae0ce25bc04b57" - tree = "15498f3f00d0ce048c3cba2156eb7a44f096ae88" - tests_tree = "17b87858f77ee4413c98cfd76f16f52170246741" - - def git(*arguments): - return subprocess.check_output(["git", "-C", str(repo_root), *arguments]) - - assert git("rev-parse", commit + "^{commit}").decode().strip() == commit - assert git("rev-parse", commit + "^{tree}").decode().strip() == tree - assert git("rev-parse", commit + ":tests").decode().strip() == tests_tree - blobs = {} - for entry in git("ls-tree", "-rz", commit).split(b"\0"): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - relative = name.decode("utf-8") - parts = PurePosixPath(relative).parts - assert mode in (b"100644", b"100755") and kind == b"blob" - assert parts and not relative.startswith("/") and ".." not in parts and ".git" not in parts - assert relative not in blobs - blobs[relative] = oid.decode("ascii") - - def blob_oid(raw): - return hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - - root = tmp_path_factory.mktemp("phase4-step3") - raw = git("archive", "--format=zip", commit) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = [member for member in archive.infolist() if not member.is_dir()] - assert len(members) == len(blobs) - assert {member.filename for member in members} == set(blobs) - for member in members: - content = archive.read(member) - assert blob_oid(content) == blobs[member.filename], member.filename - destination = root.joinpath(*PurePosixPath(member.filename).parts) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(content) - - def verify_unchanged(): - paths = list(root.rglob("*")) - assert not any(path.is_symlink() for path in paths), "Historical snapshot gained a symlink" - files = {path.relative_to(root).as_posix(): path for path in paths if path.is_file()} - assert set(files) == set(blobs), "Historical snapshot file set changed" - for name, path in files.items(): - assert blob_oid(path.read_bytes()) == blobs[name], f"Historical snapshot changed: {name}" - - verify_unchanged() - yield root - verify_unchanged() - - -@pytest.fixture(scope="session") -def phase4_step4_snapshot(repo_root, tmp_path_factory): - """Provide verified accepted Phase 4 Step 4 bytes only to listed historical tests. - - The source checkout is never rebound. The private archive has no Git working - tree, accepts only pinned regular-file blobs, and is checked again after use. - Files remain writable so copies in inherited mutation tests work on Windows; - changing this snapshot itself fails the closing byte and file-set check. - """ - import hashlib - import io - import subprocess - import zipfile - from pathlib import PurePosixPath - - commit = "49454a9b162cb8d35e38d1cb1ae32cb208d3a01c" - tree = "79264d845e08f73ee68e160dea04f7786f18fd0a" - tests_tree = "4aa4ca0b22f8b8c25afe27feeaba713d6f0830bf" - - def git(*arguments): - return subprocess.check_output(["git", "-C", str(repo_root), *arguments]) - - assert git("rev-parse", commit + "^{commit}").decode().strip() == commit - assert git("rev-parse", commit + "^{tree}").decode().strip() == tree - assert git("rev-parse", commit + ":tests").decode().strip() == tests_tree - blobs = {} - for entry in git("ls-tree", "-rz", commit).split(b"\0"): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - relative = name.decode("utf-8") - parts = PurePosixPath(relative).parts - assert mode in (b"100644", b"100755") and kind == b"blob" - assert parts and not relative.startswith("/") and ".." not in parts and ".git" not in parts - assert relative not in blobs - blobs[relative] = oid.decode("ascii") - - def blob_oid(raw): - return hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - - root = tmp_path_factory.mktemp("phase4-step4") - raw = git("archive", "--format=zip", commit) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = [member for member in archive.infolist() if not member.is_dir()] - assert len(members) == len(blobs) - assert {member.filename for member in members} == set(blobs) - for member in members: - content = archive.read(member) - assert blob_oid(content) == blobs[member.filename], member.filename - destination = root.joinpath(*PurePosixPath(member.filename).parts) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(content) - - def verify_unchanged(): - paths = list(root.rglob("*")) - assert not any(path.is_symlink() for path in paths), "Historical snapshot gained a symlink" - files = {path.relative_to(root).as_posix(): path for path in paths if path.is_file()} - assert set(files) == set(blobs), "Historical snapshot file set changed" - for name, path in files.items(): - assert blob_oid(path.read_bytes()) == blobs[name], f"Historical snapshot changed: {name}" - - verify_unchanged() - yield root - verify_unchanged() - - -@pytest.fixture(scope="session") -def phase4_step5_snapshot(repo_root, tmp_path_factory): - """Provide verified accepted Phase 4 Step 5 bytes only to listed historical tests. - - The source checkout is never rebound. The private archive has no Git working - tree, accepts only pinned regular-file blobs, and is checked again after use. - Files remain writable so copies in inherited mutation tests work on Windows; - changing this snapshot itself fails the closing byte and file-set check. - """ - import hashlib - import io - import subprocess - import zipfile - from pathlib import PurePosixPath - - commit = "c1667179e895a798bba2349960162542efa5ef87" - tree = "d83f9fc8e0b569f2d796bd2a83b87422fe350fba" - tests_tree = "974f8e65904c24ed4e226996eaf571aab6123965" - - def git(*arguments): - return subprocess.check_output(["git", "-C", str(repo_root), *arguments]) - - assert git("rev-parse", commit + "^{commit}").decode().strip() == commit - assert git("rev-parse", commit + "^{tree}").decode().strip() == tree - assert git("rev-parse", commit + ":tests").decode().strip() == tests_tree - blobs = {} - for entry in git("ls-tree", "-rz", commit).split(b"\0"): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - relative = name.decode("utf-8") - parts = PurePosixPath(relative).parts - assert mode in (b"100644", b"100755") and kind == b"blob" - assert parts and not relative.startswith("/") and ".." not in parts and ".git" not in parts - assert relative not in blobs - blobs[relative] = oid.decode("ascii") - - def blob_oid(raw): - return hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - - root = tmp_path_factory.mktemp("phase4-step5") - raw = git("archive", "--format=zip", commit) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = [member for member in archive.infolist() if not member.is_dir()] - assert len(members) == len(blobs) - assert {member.filename for member in members} == set(blobs) - for member in members: - content = archive.read(member) - assert blob_oid(content) == blobs[member.filename], member.filename - destination = root.joinpath(*PurePosixPath(member.filename).parts) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(content) - - def verify_unchanged(): - paths = list(root.rglob("*")) - assert not any(path.is_symlink() for path in paths), "Historical snapshot gained a symlink" - files = {path.relative_to(root).as_posix(): path for path in paths if path.is_file()} - assert set(files) == set(blobs), "Historical snapshot file set changed" - for name, path in files.items(): - assert blob_oid(path.read_bytes()) == blobs[name], f"Historical snapshot changed: {name}" - - verify_unchanged() - yield root - verify_unchanged() - - -@pytest.fixture(scope="session") -def phase4_step6_snapshot(repo_root, tmp_path_factory): - """Provide verified accepted Phase 4 Step 6 bytes only to listed historical tests. - - The source checkout is never rebound. The private archive has no Git working - tree, accepts only pinned regular-file blobs, and is checked again after use. - Files remain writable so copies in inherited mutation tests work on Windows; - changing this snapshot itself fails the closing byte and file-set check. - """ - import hashlib - import io - import subprocess - import zipfile - from pathlib import PurePosixPath - - commit = "aa2355f2359c3af4fc05345c792f9903b0d5858c" - tree = "d9b6910079f49a2c739f86eceec05040918a41a5" - tests_tree = "8d3a03c61da807d1fef8d26fc4af0d6f72e4546e" - - def git(*arguments): - return subprocess.check_output(["git", "-C", str(repo_root), *arguments]) - - assert git("rev-parse", commit + "^{commit}").decode().strip() == commit - assert git("rev-parse", commit + "^{tree}").decode().strip() == tree - assert git("rev-parse", commit + ":tests").decode().strip() == tests_tree - blobs = {} - for entry in git("ls-tree", "-rz", commit).split(b"\0"): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - relative = name.decode("utf-8") - parts = PurePosixPath(relative).parts - assert mode in (b"100644", b"100755") and kind == b"blob" - assert parts and not relative.startswith("/") and ".." not in parts and ".git" not in parts - assert relative not in blobs - blobs[relative] = oid.decode("ascii") - - def blob_oid(raw): - return hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - - root = tmp_path_factory.mktemp("phase4-step6") - raw = git("archive", "--format=zip", commit) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = [member for member in archive.infolist() if not member.is_dir()] - assert len(members) == len(blobs) - assert {member.filename for member in members} == set(blobs) - for member in members: - content = archive.read(member) - assert blob_oid(content) == blobs[member.filename], member.filename - destination = root.joinpath(*PurePosixPath(member.filename).parts) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(content) - - def verify_unchanged(): - paths = list(root.rglob("*")) - assert not any(path.is_symlink() for path in paths), "Historical snapshot gained a symlink" - files = {path.relative_to(root).as_posix(): path for path in paths if path.is_file()} - assert set(files) == set(blobs), "Historical snapshot file set changed" - for name, path in files.items(): - assert blob_oid(path.read_bytes()) == blobs[name], f"Historical snapshot changed: {name}" - - verify_unchanged() - yield root - verify_unchanged() - - -@pytest.fixture(scope="session") -def phase4_step7_snapshot(repo_root, tmp_path_factory): - """Provide verified accepted Phase 4 Step 7 bytes only to listed historical tests. - - The source checkout is never rebound. The private archive has no Git working - tree, accepts only pinned regular-file blobs, and is checked again after use. - Files remain writable so copies in inherited mutation tests work on Windows; - changing this snapshot itself fails the closing byte and file-set check. - """ - import hashlib - import io - import subprocess - import zipfile - from pathlib import PurePosixPath - - commit = "bacd33ae65e392872776c6d976401fdc3d565f63" - tree = "4c16914bb0f0c5281efc36624efb8b1a1d4a174e" - tests_tree = "96317cb7b25236b79efc0f2d93e1e0f5f132fdb2" - - def git(*arguments): - return subprocess.check_output(["git", "-C", str(repo_root), *arguments]) - - assert git("rev-parse", commit + "^{commit}").decode().strip() == commit - assert git("rev-parse", commit + "^{tree}").decode().strip() == tree - assert git("rev-parse", commit + ":tests").decode().strip() == tests_tree - blobs = {} - for entry in git("ls-tree", "-rz", commit).split(b"\0"): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - relative = name.decode("utf-8") - parts = PurePosixPath(relative).parts - assert mode in (b"100644", b"100755") and kind == b"blob" - assert parts and not relative.startswith("/") and ".." not in parts and ".git" not in parts - assert relative not in blobs - blobs[relative] = oid.decode("ascii") - - def blob_oid(raw): - return hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - - root = tmp_path_factory.mktemp("phase4-step7") - raw = git("archive", "--format=zip", commit) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = [member for member in archive.infolist() if not member.is_dir()] - assert len(members) == len(blobs) - assert {member.filename for member in members} == set(blobs) - for member in members: - content = archive.read(member) - assert blob_oid(content) == blobs[member.filename], member.filename - destination = root.joinpath(*PurePosixPath(member.filename).parts) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(content) - - def verify_unchanged(): - paths = list(root.rglob("*")) - assert not any(path.is_symlink() for path in paths), "Historical snapshot gained a symlink" - files = {path.relative_to(root).as_posix(): path for path in paths if path.is_file()} - assert set(files) == set(blobs), "Historical snapshot file set changed" - for name, path in files.items(): - assert blob_oid(path.read_bytes()) == blobs[name], f"Historical snapshot changed: {name}" - - verify_unchanged() - yield root - verify_unchanged() - - -@pytest.fixture(scope="session") -def phase4_step8_snapshot(repo_root, tmp_path_factory): - """Provide verified accepted Phase 4 Step 8 bytes only to listed historical tests. - - The source checkout is never rebound. The private archive has no Git working - tree, accepts only pinned regular-file blobs, and is checked again after use. - Files remain writable so copies in inherited mutation tests work on Windows; - changing this snapshot itself fails the closing byte and file-set check. - """ - import hashlib - import io - import subprocess - import zipfile - from pathlib import PurePosixPath - - commit = "63e347a785838b64a9c32e636ea48c664458cee4" - tree = "3ac450b5f67cad738849de18601999fefee98f63" - tests_tree = "ddaaab24d83080b7c1a27229c15e6e4a3f5b3cd8" - - def git(*arguments): - return subprocess.check_output(["git", "-C", str(repo_root), *arguments]) - - assert git("rev-parse", commit + "^{commit}").decode().strip() == commit - assert git("rev-parse", commit + "^{tree}").decode().strip() == tree - assert git("rev-parse", commit + ":tests").decode().strip() == tests_tree - blobs = {} - for entry in git("ls-tree", "-rz", commit).split(b"\0"): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - relative = name.decode("utf-8") - parts = PurePosixPath(relative).parts - assert mode in (b"100644", b"100755") and kind == b"blob" - assert parts and not relative.startswith("/") and ".." not in parts and ".git" not in parts - assert relative not in blobs - blobs[relative] = oid.decode("ascii") - - def blob_oid(raw): - return hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - - root = tmp_path_factory.mktemp("phase4-step8") - raw = git("archive", "--format=zip", commit) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = [member for member in archive.infolist() if not member.is_dir()] - assert len(members) == len(blobs) - assert {member.filename for member in members} == set(blobs) - for member in members: - content = archive.read(member) - assert blob_oid(content) == blobs[member.filename], member.filename - destination = root.joinpath(*PurePosixPath(member.filename).parts) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(content) - - def verify_unchanged(): - paths = list(root.rglob("*")) - assert not any(path.is_symlink() for path in paths), "Historical snapshot gained a symlink" - files = {path.relative_to(root).as_posix(): path for path in paths if path.is_file()} - assert set(files) == set(blobs), "Historical snapshot file set changed" - for name, path in files.items(): - assert blob_oid(path.read_bytes()) == blobs[name], f"Historical snapshot changed: {name}" - - verify_unchanged() - yield root - verify_unchanged() - - @pytest.fixture def phase4_step10_measure_reports(request, tmp_path, subprocess_env): """Measure complete CLI publication in fresh processes; retain every attempt. diff --git a/tests/fixtures/lineage_complete/README.md b/tests/fixtures/lineage_complete/README.md index 9ea078c..67019ac 100644 --- a/tests/fixtures/lineage_complete/README.md +++ b/tests/fixtures/lineage_complete/README.md @@ -1,7 +1,38 @@ -# lineage_complete fixture placeholder +# Complete and boundary lineage acceptance inputs -Phase 1 status: structure only. +Phase 5 Step 1 specifies synthetic inputs and independent future acceptance +values. `cases.json` contains canonical record/provenance rows, explicit version +order, selected target keys, current input-validation expectations, and future +lineage expectations. These files are test data, not report-schema documents. -This directory is reserved for the approved `lineage_complete` validation fixture. -It contains no real user data and no analytical expected values in Phase 1. -Data may be added only in the phase that owns the corresponding behavior. +The cases separate root support from strict ancestry, preserve known-empty root +sets, distinguish unknown grounding from absent/null parent declarations, and +cover single-parent carryover, duplicate aliases, diamond paths, and conservative +grounded-parent handling. The empty-target case selects no observations from a +loaded context; it concerns the future analysis API and makes no claim about +current empty-file CLI support. + +`hero_expected.json` transcribes the approved Hero arithmetic into exact rational +values for later lineage tests. The six canonical files in `examples/hero` remain +unchanged. The three roots with incidence one are ordered by canonical key. + +Conventions shared by these three lineage fixture directories: + +- Rational strings are exact values. `lineage_bounds` lists lower, upper, width. +- `complete_root_sets` omits unresolved targets. An empty array is a completely + resolved known-empty set. Roots on incomplete paths contribute no exact mass. +- Counts distinguish selected targets from loaded supporting records. +- Conceptual unresolved reasons are explanatory fixture values; later runtime + tests map them to the approved typed reason codes. +- `shared_ancestry_proxy` uses existing report levels: `present`, `not_present`, + or `indeterminate`. The last includes unavailable empty-target results. +- The Step 1 loader test validates only `expected_input`. It uses the existing + comparison input role to load supporting records until the dedicated context + role is implemented. It executes no graph or lineage formula. +- `expected_lineage` becomes a product oracle when Steps 2 through 8 implement + the corresponding behavior. Its presence is not a passing lineage result. + +Arithmetic was derived from the approved P5-D01 through P5-D07 contracts before +graph implementation. No output from the future implementation generated these +expected values. General permutation, renaming, long-chain and resource-boundary +cases belong in the later direct tests instead of expanded static fixture copies. diff --git a/tests/fixtures/lineage_complete/cases.json b/tests/fixtures/lineage_complete/cases.json new file mode 100644 index 0000000..ed59ac2 --- /dev/null +++ b/tests/fixtures/lineage_complete/cases.json @@ -0,0 +1,566 @@ +{ + "purpose": "Independent Phase 5 acceptance inputs and future lineage oracles; no lineage execution is claimed by Step 1.", + "fraction_encoding": "Exact rational strings; null means unavailable and requires the adjacent reason.", + "cases": [ + { + "case_id": "external_root_supports_itself", + "records": [ + { + "dataset_version": "v2", + "record_id": "a", + "content": "Synthetic fixture a" + } + ], + "provenance": [ + { + "dataset_version": "v2", + "record_id": "a", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + } + ], + "version_order": [ + "v2" + ], + "target_record_keys": [ + "v2::a" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [], + "target_declared_references": 0, + "target_resolved_references": 0 + }, + "expected_lineage": { + "complete_root_sets": { + "v2::a": [ + "v2::a" + ] + }, + "strict_ancestors": { + "v2::a": [] + }, + "counts": { + "target": 1, + "grounded": 1, + "closed": 0, + "unresolved": 0 + }, + "depth_by_record": { + "v2::a": 0 + }, + "lineage_depth": 0, + "no_declared_parents": true, + "edge_coverage": "1", + "ancestry_hhi": "1", + "effective_external_root_count": "1", + "shared_ancestry_proxy": "not_present", + "lineage_bounds": [ + "0", + "0", + "0" + ] + } + }, + { + "case_id": "all_closed", + "records": [ + { + "dataset_version": "v2", + "record_id": "a", + "content": "Synthetic fixture a" + }, + { + "dataset_version": "v2", + "record_id": "b", + "content": "Synthetic fixture b" + } + ], + "provenance": [ + { + "dataset_version": "v2", + "record_id": "a", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v2", + "record_id": "b", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::a" + ] + } + ], + "version_order": [ + "v2" + ], + "target_record_keys": [ + "v2::a", + "v2::b" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [], + "target_declared_references": 1, + "target_resolved_references": 1 + }, + "expected_lineage": { + "complete_root_sets": { + "v2::a": [], + "v2::b": [] + }, + "counts": { + "target": 2, + "grounded": 0, + "closed": 2, + "unresolved": 0 + }, + "depth_by_record": { + "v2::a": 0, + "v2::b": 1 + }, + "lineage_depth": 1, + "resolved_lineage_coverage": "1", + "external_ancestry_coverage": "0", + "distinct_external_roots": 0, + "ancestry_hhi": null, + "effective_external_root_count": null, + "concentration_unavailable_reason": "no_completely_resolved_nonempty_root_sets", + "shared_ancestry_proxy": "not_present", + "lineage_bounds": [ + "1", + "1", + "0" + ] + } + }, + { + "case_id": "unknown_absent_and_null_boundaries", + "records": [ + { + "dataset_version": "v2", + "record_id": "unknown", + "content": "Synthetic fixture unknown" + }, + { + "dataset_version": "v2", + "record_id": "absent", + "content": "Synthetic fixture absent" + }, + { + "dataset_version": "v2", + "record_id": "null", + "content": "Synthetic fixture null" + }, + { + "dataset_version": "v2", + "record_id": "grounded_absent", + "content": "Synthetic fixture grounded_absent" + } + ], + "provenance": [ + { + "dataset_version": "v2", + "record_id": "unknown", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "unknown", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v2", + "record_id": "absent", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate" + }, + { + "dataset_version": "v2", + "record_id": "null", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": null + }, + { + "dataset_version": "v2", + "record_id": "grounded_absent", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate" + } + ], + "version_order": [ + "v2" + ], + "target_record_keys": [ + "v2::absent", + "v2::grounded_absent", + "v2::null", + "v2::unknown" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [ + "W_GROUNDING_UNKNOWN" + ], + "target_declared_references": 0, + "target_resolved_references": 0, + "parent_field_states": { + "v2::unknown": "value", + "v2::absent": "absent", + "v2::null": "null", + "v2::grounded_absent": "absent" + } + }, + "expected_lineage": { + "complete_root_sets": {}, + "unresolved_reasons": { + "v2::unknown": [ + "unknown_grounding" + ], + "v2::absent": [ + "undeclared_parent_boundary" + ], + "v2::null": [ + "undeclared_parent_boundary" + ], + "v2::grounded_absent": [ + "undeclared_parent_boundary" + ] + }, + "counts": { + "target": 4, + "grounded": 0, + "closed": 0, + "unresolved": 4 + }, + "depth_by_record": { + "v2::unknown": 0, + "v2::absent": null, + "v2::null": null, + "v2::grounded_absent": null + }, + "lineage_depth": null, + "depth_unavailable_reason": "incomplete_target_parent_paths", + "no_declared_parents": true, + "edge_coverage": "1", + "resolved_lineage_coverage": "0", + "external_ancestry_coverage": "0", + "ancestry_hhi": null, + "effective_external_root_count": null, + "concentration_unavailable_reason": "no_completely_resolved_nonempty_root_sets", + "shared_ancestry_proxy": "indeterminate", + "lineage_bounds": [ + "0", + "1", + "1" + ] + } + }, + { + "case_id": "carryover_diamond_and_grounded_parent_limits", + "records": [ + { + "dataset_version": "v1", + "record_id": "a", + "content": "Synthetic fixture a" + }, + { + "dataset_version": "v1", + "record_id": "b", + "content": "Synthetic fixture b" + }, + { + "dataset_version": "v2", + "record_id": "carry", + "content": "Synthetic fixture carry" + }, + { + "dataset_version": "v2", + "record_id": "left", + "content": "Synthetic fixture left" + }, + { + "dataset_version": "v2", + "record_id": "right", + "content": "Synthetic fixture right" + }, + { + "dataset_version": "v2", + "record_id": "diamond", + "content": "Synthetic fixture diamond" + }, + { + "dataset_version": "v2", + "record_id": "grounded_transform", + "content": "Synthetic fixture grounded_transform" + }, + { + "dataset_version": "v2", + "record_id": "multi_carry", + "content": "Synthetic fixture multi_carry" + }, + { + "dataset_version": "v2", + "record_id": "incomplete_carry", + "content": "Synthetic fixture incomplete_carry" + } + ], + "provenance": [ + { + "dataset_version": "v1", + "record_id": "a", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v1", + "record_id": "b", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v2", + "record_id": "carry", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "carryover", + "parent_ids": [ + "v1::a", + "a" + ] + }, + { + "dataset_version": "v2", + "record_id": "left", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "right", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "diamond", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::left", + "v2::right" + ] + }, + { + "dataset_version": "v2", + "record_id": "grounded_transform", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [ + "v1::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "multi_carry", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "carryover", + "parent_ids": [ + "v1::a", + "v1::b" + ] + }, + { + "dataset_version": "v2", + "record_id": "incomplete_carry", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "carryover", + "parent_ids": [ + "v1::a", + "v1::missing" + ] + } + ], + "version_order": [ + "v1", + "v2" + ], + "target_record_keys": [ + "v2::carry", + "v2::diamond", + "v2::grounded_transform", + "v2::incomplete_carry", + "v2::left", + "v2::multi_carry", + "v2::right" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [ + "W_PARENT_BARE_COMPATIBILITY", + "W_PARENT_DUPLICATE_REFERENCE", + "W_PARENT_UNRESOLVED" + ], + "target_declared_references": 11, + "target_resolved_references": 10 + }, + "expected_lineage": { + "complete_root_sets": { + "v2::carry": [ + "v1::a" + ], + "v2::diamond": [ + "v1::a" + ], + "v2::left": [ + "v1::a" + ], + "v2::right": [ + "v1::a" + ] + }, + "unresolved_reasons": { + "v2::grounded_transform": [ + "unsupported_grounded_parent_rule" + ], + "v2::multi_carry": [ + "unsupported_grounded_parent_rule" + ], + "v2::incomplete_carry": [ + "missing_parent" + ] + }, + "counts": { + "target": 7, + "grounded": 4, + "closed": 0, + "unresolved": 3 + }, + "carryover_generation": 0, + "carryover_depth": 1, + "diamond_depth": 2, + "distinct_external_roots": 1, + "root_order": [ + "v1::a" + ], + "incidence": [ + 4 + ], + "incidence_shares": [ + "4/7" + ], + "fractional_masses": [ + "4" + ], + "normalized_weights": [ + "1" + ], + "ancestry_hhi": "1", + "effective_external_root_count": "1", + "resolved_lineage_coverage": "4/7", + "external_ancestry_coverage": "4/7", + "edge_coverage": "10/11", + "lineage_bounds": [ + "0", + "3/7", + "3/7" + ], + "shared_ancestry_proxy": "present" + }, + "note": "Carryover has two original declarations for one canonical parent. Diamond paths both reach a. Loaded root b has no complete grounded target and contributes no exact root mass." + }, + { + "case_id": "empty_selected_scope", + "records": [ + { + "dataset_version": "v1", + "record_id": "context", + "content": "Synthetic fixture context" + } + ], + "provenance": [ + { + "dataset_version": "v1", + "record_id": "context", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + } + ], + "version_order": [ + "v1" + ], + "target_record_keys": [], + "expected_input": { + "error_codes": [], + "required_warning_codes": [], + "target_declared_references": 0, + "target_resolved_references": 0 + }, + "expected_lineage": { + "target_dataset_version": null, + "counts": { + "target": 0, + "grounded": 0, + "closed": 0, + "unresolved": 0 + }, + "complete_root_sets": {}, + "distinct_external_roots": 0, + "no_declared_parents": true, + "edge_coverage": "1", + "resolved_lineage_coverage": null, + "external_ancestry_coverage": null, + "ancestry_hhi": null, + "effective_external_root_count": null, + "lineage_bounds": null, + "ratios_unavailable_reason": "empty_target", + "shared_ancestry_proxy": "indeterminate" + }, + "note": "Future analysis API empty selection over a loaded context. This case makes no claim that the current CLI accepts an empty primary input file." + } + ] +} diff --git a/tests/fixtures/lineage_complete/hero_expected.json b/tests/fixtures/lineage_complete/hero_expected.json new file mode 100644 index 0000000..dfdfb62 --- /dev/null +++ b/tests/fixtures/lineage_complete/hero_expected.json @@ -0,0 +1,60 @@ +{ + "purpose": "Future lineage oracle copied from the approved Hero arithmetic; canonical Hero files are unchanged.", + "canonical_source": "examples/hero/EXPECTED_OUTPUTS.md", + "target_dataset_version": "v2", + "target_records": 8, + "loaded_records": 16, + "declared_target_references": 8, + "resolved_target_references": 8, + "root_order": [ + "v1::v1_01", + "v1::v1_02", + "v1::v1_03", + "v1::v1_04", + "v1::v1_07" + ], + "incidence": [ + 3, + 2, + 1, + 1, + 1 + ], + "incidence_shares": [ + "3/8", + "1/4", + "1/8", + "1/8", + "1/8" + ], + "fractional_masses": [ + "3", + "2", + "1", + "1", + "1" + ], + "normalized_weights": [ + "3/8", + "1/4", + "1/8", + "1/8", + "1/8" + ], + "ancestry_hhi": "1/4", + "effective_external_root_count": "4", + "lineage_bounds": [ + "0", + "0", + "0" + ], + "direct_bounds": [ + "1/2", + "1/2", + "0" + ], + "distinct_external_roots": 5, + "cycle_detected": false, + "shared_ancestry_proxy": "present", + "lineage_execution": "future_completed_when_requested" +} diff --git a/tests/fixtures/lineage_cycles/README.md b/tests/fixtures/lineage_cycles/README.md index 87291d0..49249ce 100644 --- a/tests/fixtures/lineage_cycles/README.md +++ b/tests/fixtures/lineage_cycles/README.md @@ -1,7 +1,20 @@ -# lineage_cycles fixture placeholder +# Cycle components and affected descendants -Phase 1 status: structure only. +These Phase 5 Step 1 fixtures distinguish cycle members from descendants affected +along parent-to-child edges and from independent target records. A disconnected +cycle in the loaded context remains a whole-graph lineage error even when every +selected target has complete ancestry. General cycle detection starts in Step 3. -This directory is reserved for the approved `lineage_cycles` validation fixture. -It contains no real user data and no analytical expected values in Phase 1. -Data may be added only in the phase that owns the corresponding behavior. +Current input validation accepts valid same-version references in the general +cycle cases, without establishing acyclicity. A direct self-parent already +produces `E_LINEAGE_CYCLE`; its expected input error is recorded separately. +Future graph construction must retain self-reference evidence for diagnosis while +rejecting it as a valid ancestry edge. + +`cycle_count` counts cyclic strongly connected components. It never counts all +simple cycles, individual members or affected descendants. Future cycle failures +must preserve independent results and return an unsuccessful audit exit status. +Step 1 does not execute or claim any of those future graph results. + +Shared fixture format and execution limits are documented in +`../lineage_complete/README.md`. diff --git a/tests/fixtures/lineage_cycles/cases.json b/tests/fixtures/lineage_cycles/cases.json new file mode 100644 index 0000000..bd2499d --- /dev/null +++ b/tests/fixtures/lineage_cycles/cases.json @@ -0,0 +1,530 @@ +{ + "purpose": "Independent Phase 5 acceptance inputs and future lineage oracles; no lineage execution is claimed by Step 1.", + "fraction_encoding": "Exact rational strings; null means unavailable and requires the adjacent reason.", + "cases": [ + { + "case_id": "two_node_cycle_and_descendants", + "records": [ + { + "dataset_version": "v2", + "record_id": "a", + "content": "Synthetic fixture a" + }, + { + "dataset_version": "v2", + "record_id": "b", + "content": "Synthetic fixture b" + }, + { + "dataset_version": "v2", + "record_id": "descendant", + "content": "Synthetic fixture descendant" + }, + { + "dataset_version": "v2", + "record_id": "independent", + "content": "Synthetic fixture independent" + } + ], + "provenance": [ + { + "dataset_version": "v2", + "record_id": "a", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::b" + ] + }, + { + "dataset_version": "v2", + "record_id": "b", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "descendant", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "independent", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + } + ], + "version_order": [ + "v2" + ], + "target_record_keys": [ + "v2::a", + "v2::b", + "v2::descendant", + "v2::independent" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [], + "target_declared_references": 3, + "target_resolved_references": 3 + }, + "expected_lineage": { + "cycle_detected": true, + "cycle_count": 1, + "cycle_count_method": "cyclic_strongly_connected_components", + "cyclic_components": [ + [ + "v2::a", + "v2::b" + ] + ], + "cycle_member_keys": [ + "v2::a", + "v2::b" + ], + "affected_record_keys": [ + "v2::a", + "v2::b", + "v2::descendant" + ], + "complete_root_sets": { + "v2::independent": [ + "v2::independent" + ] + }, + "counts": { + "target": 4, + "grounded": 1, + "closed": 0, + "unresolved": 3 + }, + "lineage_depth": null, + "depth_unavailable_reason": "cycle_affected_target_paths", + "lineage_error": true, + "cli_audit_success": false, + "edge_coverage": "1", + "lineage_bounds": [ + "0", + "3/4", + "3/4" + ] + }, + "note": "Current immediate-reference validation accepts these references. General cycle detection remains Phase 5 Step 3." + }, + { + "case_id": "disconnected_context_cycle", + "records": [ + { + "dataset_version": "v1", + "record_id": "a", + "content": "Synthetic fixture a" + }, + { + "dataset_version": "v1", + "record_id": "b", + "content": "Synthetic fixture b" + }, + { + "dataset_version": "v2", + "record_id": "root", + "content": "Synthetic fixture root" + } + ], + "provenance": [ + { + "dataset_version": "v1", + "record_id": "a", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::b" + ] + }, + { + "dataset_version": "v1", + "record_id": "b", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "root", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + } + ], + "version_order": [ + "v1", + "v2" + ], + "target_record_keys": [ + "v2::root" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [], + "target_declared_references": 0, + "target_resolved_references": 0 + }, + "expected_lineage": { + "cycle_detected": true, + "cycle_count": 1, + "cycle_count_method": "cyclic_strongly_connected_components", + "cyclic_components": [ + [ + "v1::a", + "v1::b" + ] + ], + "cycle_member_keys": [ + "v1::a", + "v1::b" + ], + "affected_record_keys": [ + "v1::a", + "v1::b" + ], + "affected_target_record_keys": [], + "complete_root_sets": { + "v2::root": [ + "v2::root" + ] + }, + "counts": { + "target": 1, + "grounded": 1, + "closed": 0, + "unresolved": 0 + }, + "lineage_depth": 0, + "lineage_error": true, + "cli_audit_success": false, + "no_declared_parents": true, + "edge_coverage": "1", + "lineage_bounds": [ + "0", + "0", + "0" + ] + }, + "note": "A loaded disconnected cycle remains a lineage error even though target ancestry is fully resolved." + }, + { + "case_id": "self_cycle_and_descendant", + "records": [ + { + "dataset_version": "v2", + "record_id": "self", + "content": "Synthetic fixture self" + }, + { + "dataset_version": "v2", + "record_id": "descendant", + "content": "Synthetic fixture descendant" + }, + { + "dataset_version": "v2", + "record_id": "closed", + "content": "Synthetic fixture closed" + } + ], + "provenance": [ + { + "dataset_version": "v2", + "record_id": "self", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::self" + ] + }, + { + "dataset_version": "v2", + "record_id": "descendant", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::self" + ] + }, + { + "dataset_version": "v2", + "record_id": "closed", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [] + } + ], + "version_order": [ + "v2" + ], + "target_record_keys": [ + "v2::closed", + "v2::descendant", + "v2::self" + ], + "expected_input": { + "error_codes": [ + "E_LINEAGE_CYCLE" + ], + "required_warning_codes": [] + }, + "expected_lineage": { + "cycle_detected": true, + "cycle_count": 1, + "cycle_count_method": "cyclic_strongly_connected_components", + "cyclic_components": [ + [ + "v2::self" + ] + ], + "cycle_member_keys": [ + "v2::self" + ], + "affected_record_keys": [ + "v2::descendant", + "v2::self" + ], + "complete_root_sets": { + "v2::closed": [] + }, + "counts": { + "target": 3, + "grounded": 0, + "closed": 1, + "unresolved": 2 + }, + "lineage_error": true, + "cli_audit_success": false, + "lineage_bounds": [ + "1/3", + "1", + "2/3" + ] + }, + "note": "Current validator rejects the direct self-parent. Future graph construction must retain explicit self-reference evidence without accepting it as a valid ancestry edge." + }, + { + "case_id": "two_components_with_multiple_simple_cycles", + "records": [ + { + "dataset_version": "v2", + "record_id": "a", + "content": "Synthetic fixture a" + }, + { + "dataset_version": "v2", + "record_id": "b", + "content": "Synthetic fixture b" + }, + { + "dataset_version": "v2", + "record_id": "c", + "content": "Synthetic fixture c" + }, + { + "dataset_version": "v2", + "record_id": "d", + "content": "Synthetic fixture d" + }, + { + "dataset_version": "v2", + "record_id": "e", + "content": "Synthetic fixture e" + }, + { + "dataset_version": "v2", + "record_id": "f", + "content": "Synthetic fixture f" + }, + { + "dataset_version": "v2", + "record_id": "root", + "content": "Synthetic fixture root" + } + ], + "provenance": [ + { + "dataset_version": "v2", + "record_id": "a", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::b", + "v2::c" + ] + }, + { + "dataset_version": "v2", + "record_id": "b", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::a", + "v2::c" + ] + }, + { + "dataset_version": "v2", + "record_id": "c", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::a", + "v2::b" + ] + }, + { + "dataset_version": "v2", + "record_id": "d", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::e" + ] + }, + { + "dataset_version": "v2", + "record_id": "e", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::d" + ] + }, + { + "dataset_version": "v2", + "record_id": "f", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "root", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + } + ], + "version_order": [ + "v2" + ], + "target_record_keys": [ + "v2::a", + "v2::b", + "v2::c", + "v2::d", + "v2::e", + "v2::f", + "v2::root" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [], + "target_declared_references": 9, + "target_resolved_references": 9 + }, + "expected_lineage": { + "cycle_detected": true, + "cycle_count": 2, + "cycle_count_method": "cyclic_strongly_connected_components", + "cyclic_components": [ + [ + "v2::a", + "v2::b", + "v2::c" + ], + [ + "v2::d", + "v2::e" + ] + ], + "cycle_member_keys": [ + "v2::a", + "v2::b", + "v2::c", + "v2::d", + "v2::e" + ], + "affected_record_keys": [ + "v2::a", + "v2::b", + "v2::c", + "v2::d", + "v2::e", + "v2::f" + ], + "complete_root_sets": { + "v2::root": [ + "v2::root" + ] + }, + "counts": { + "target": 7, + "grounded": 1, + "closed": 0, + "unresolved": 6 + }, + "lineage_error": true, + "cli_audit_success": false, + "edge_coverage": "1", + "lineage_bounds": [ + "0", + "6/7", + "6/7" + ] + }, + "note": "The three-node component contains several simple cycles but counts once; the second component counts once." + } + ] +} diff --git a/tests/fixtures/lineage_multi_root/README.md b/tests/fixtures/lineage_multi_root/README.md index 91bde3d..16da13f 100644 --- a/tests/fixtures/lineage_multi_root/README.md +++ b/tests/fixtures/lineage_multi_root/README.md @@ -1,7 +1,23 @@ -# lineage_multi_root fixture placeholder +# Multi-root and independent uncertainty inputs -Phase 1 status: structure only. +These three Phase 5 Step 1 cases each load seven records and select four targets. +Targets t1 and t2 have complete roots {a} and {a,b}; t3 is known-empty; t4 remains +unresolved. Context records a, b and u never enlarge the target denominator. -This directory is reserved for the approved `lineage_multi_root` validation fixture. -It contains no real user data and no analytical expected values in Phase 1. -Data may be added only in the phase that owns the corresponding behavior. +The mixed case gives t4 both an unknown-grounding ancestor and a missing parent. +The two variants isolate each failure: remove the missing reference while keeping +u unknown, or ground u while retaining the missing reference. Either defect alone +keeps t4 out of exact root allocation. Grounding u in the latter variant does not +make u a contributing root while its only target remains unresolved. + +Hand arithmetic: incidence is (2,1), with target shares (1/2,1/4). Fractional mass +is (3/2,1/2); normalization by two externally rooted targets yields (3/4,1/4). +Thus HHI is 9/16 + 1/16 = 5/8 and effective roots are 8/5. Resolved coverage is +3/4, external ancestry coverage is 1/2, and closure bounds are [1/4,1/2]. Width +is 1/4. Reference coverage is 5/6 with a missing reference and 1 in the +unknown-only variant. Perfect reference coverage alone does not resolve unknown +grounding. + +`cases.json` is future acceptance data. Step 1 executes only its input-validation +expectations. Shared format and execution limits are documented in +`../lineage_complete/README.md`. diff --git a/tests/fixtures/lineage_multi_root/cases.json b/tests/fixtures/lineage_multi_root/cases.json new file mode 100644 index 0000000..7901641 --- /dev/null +++ b/tests/fixtures/lineage_multi_root/cases.json @@ -0,0 +1,583 @@ +{ + "purpose": "Independent Phase 5 acceptance inputs and future lineage oracles; no lineage execution is claimed by Step 1.", + "fraction_encoding": "Exact rational strings; null means unavailable and requires the adjacent reason.", + "cases": [ + { + "case_id": "mixed_missing_and_unknown", + "records": [ + { + "dataset_version": "v1", + "record_id": "a", + "content": "Synthetic fixture a" + }, + { + "dataset_version": "v1", + "record_id": "b", + "content": "Synthetic fixture b" + }, + { + "dataset_version": "v1", + "record_id": "u", + "content": "Synthetic fixture u" + }, + { + "dataset_version": "v2", + "record_id": "t1", + "content": "Synthetic fixture t1" + }, + { + "dataset_version": "v2", + "record_id": "t2", + "content": "Synthetic fixture t2" + }, + { + "dataset_version": "v2", + "record_id": "t3", + "content": "Synthetic fixture t3" + }, + { + "dataset_version": "v2", + "record_id": "t4", + "content": "Synthetic fixture t4" + } + ], + "provenance": [ + { + "dataset_version": "v1", + "record_id": "a", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v1", + "record_id": "b", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v1", + "record_id": "u", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "unknown", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v2", + "record_id": "t1", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "t2", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a", + "v1::b" + ] + }, + { + "dataset_version": "v2", + "record_id": "t3", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v2", + "record_id": "t4", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a", + "v1::u", + "v1::missing" + ] + } + ], + "version_order": [ + "v1", + "v2" + ], + "target_record_keys": [ + "v2::t1", + "v2::t2", + "v2::t3", + "v2::t4" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [ + "W_GROUNDING_UNKNOWN", + "W_PARENT_UNRESOLVED" + ], + "target_declared_references": 6, + "target_resolved_references": 5 + }, + "expected_lineage": { + "complete_root_sets": { + "v2::t1": [ + "v1::a" + ], + "v2::t2": [ + "v1::a", + "v1::b" + ], + "v2::t3": [] + }, + "unresolved_reasons": { + "v2::t4": [ + "unknown_grounding", + "missing_parent" + ] + }, + "counts": { + "target": 4, + "grounded": 2, + "closed": 1, + "unresolved": 1 + }, + "loaded_records": 7, + "records_with_resolved_external_ancestry": 3, + "distinct_external_roots": 2, + "root_order": [ + "v1::a", + "v1::b" + ], + "incidence": [ + 2, + 1 + ], + "incidence_shares": [ + "1/2", + "1/4" + ], + "incidence_denominator": 4, + "fractional_masses": [ + "3/2", + "1/2" + ], + "normalized_weights": [ + "3/4", + "1/4" + ], + "mass_denominator": 2, + "ancestry_hhi": "5/8", + "effective_external_root_count": "8/5", + "resolved_lineage_coverage": "3/4", + "external_ancestry_coverage": "1/2", + "edge_coverage": "5/6", + "lineage_bounds": [ + "1/4", + "1/2", + "1/4" + ], + "shared_ancestry_proxy": "present", + "cycle_detected": false + } + }, + { + "case_id": "unknown_without_missing", + "records": [ + { + "dataset_version": "v1", + "record_id": "a", + "content": "Synthetic fixture a" + }, + { + "dataset_version": "v1", + "record_id": "b", + "content": "Synthetic fixture b" + }, + { + "dataset_version": "v1", + "record_id": "u", + "content": "Synthetic fixture u" + }, + { + "dataset_version": "v2", + "record_id": "t1", + "content": "Synthetic fixture t1" + }, + { + "dataset_version": "v2", + "record_id": "t2", + "content": "Synthetic fixture t2" + }, + { + "dataset_version": "v2", + "record_id": "t3", + "content": "Synthetic fixture t3" + }, + { + "dataset_version": "v2", + "record_id": "t4", + "content": "Synthetic fixture t4" + } + ], + "provenance": [ + { + "dataset_version": "v1", + "record_id": "a", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v1", + "record_id": "b", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v1", + "record_id": "u", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "unknown", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v2", + "record_id": "t1", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "t2", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a", + "v1::b" + ] + }, + { + "dataset_version": "v2", + "record_id": "t3", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v2", + "record_id": "t4", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a", + "v1::u" + ] + } + ], + "version_order": [ + "v1", + "v2" + ], + "target_record_keys": [ + "v2::t1", + "v2::t2", + "v2::t3", + "v2::t4" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [ + "W_GROUNDING_UNKNOWN" + ], + "target_declared_references": 5, + "target_resolved_references": 5 + }, + "expected_lineage": { + "complete_root_sets": { + "v2::t1": [ + "v1::a" + ], + "v2::t2": [ + "v1::a", + "v1::b" + ], + "v2::t3": [] + }, + "unresolved_reasons": { + "v2::t4": [ + "unknown_grounding" + ] + }, + "counts": { + "target": 4, + "grounded": 2, + "closed": 1, + "unresolved": 1 + }, + "loaded_records": 7, + "records_with_resolved_external_ancestry": 3, + "distinct_external_roots": 2, + "root_order": [ + "v1::a", + "v1::b" + ], + "incidence": [ + 2, + 1 + ], + "incidence_shares": [ + "1/2", + "1/4" + ], + "incidence_denominator": 4, + "fractional_masses": [ + "3/2", + "1/2" + ], + "normalized_weights": [ + "3/4", + "1/4" + ], + "mass_denominator": 2, + "ancestry_hhi": "5/8", + "effective_external_root_count": "8/5", + "resolved_lineage_coverage": "3/4", + "external_ancestry_coverage": "1/2", + "edge_coverage": "1", + "lineage_bounds": [ + "1/4", + "1/2", + "1/4" + ], + "shared_ancestry_proxy": "present", + "cycle_detected": false + } + }, + { + "case_id": "missing_without_unknown", + "records": [ + { + "dataset_version": "v1", + "record_id": "a", + "content": "Synthetic fixture a" + }, + { + "dataset_version": "v1", + "record_id": "b", + "content": "Synthetic fixture b" + }, + { + "dataset_version": "v1", + "record_id": "u", + "content": "Synthetic fixture u" + }, + { + "dataset_version": "v2", + "record_id": "t1", + "content": "Synthetic fixture t1" + }, + { + "dataset_version": "v2", + "record_id": "t2", + "content": "Synthetic fixture t2" + }, + { + "dataset_version": "v2", + "record_id": "t3", + "content": "Synthetic fixture t3" + }, + { + "dataset_version": "v2", + "record_id": "t4", + "content": "Synthetic fixture t4" + } + ], + "provenance": [ + { + "dataset_version": "v1", + "record_id": "a", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v1", + "record_id": "b", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v1", + "record_id": "u", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v2", + "record_id": "t1", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "t2", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a", + "v1::b" + ] + }, + { + "dataset_version": "v2", + "record_id": "t3", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v2", + "record_id": "t4", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a", + "v1::u", + "v1::missing" + ] + } + ], + "version_order": [ + "v1", + "v2" + ], + "target_record_keys": [ + "v2::t1", + "v2::t2", + "v2::t3", + "v2::t4" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [ + "W_PARENT_UNRESOLVED" + ], + "target_declared_references": 6, + "target_resolved_references": 5 + }, + "expected_lineage": { + "complete_root_sets": { + "v2::t1": [ + "v1::a" + ], + "v2::t2": [ + "v1::a", + "v1::b" + ], + "v2::t3": [] + }, + "unresolved_reasons": { + "v2::t4": [ + "missing_parent" + ] + }, + "counts": { + "target": 4, + "grounded": 2, + "closed": 1, + "unresolved": 1 + }, + "loaded_records": 7, + "records_with_resolved_external_ancestry": 3, + "distinct_external_roots": 2, + "root_order": [ + "v1::a", + "v1::b" + ], + "incidence": [ + 2, + 1 + ], + "incidence_shares": [ + "1/2", + "1/4" + ], + "incidence_denominator": 4, + "fractional_masses": [ + "3/2", + "1/2" + ], + "normalized_weights": [ + "3/4", + "1/4" + ], + "mass_denominator": 2, + "ancestry_hhi": "5/8", + "effective_external_root_count": "8/5", + "resolved_lineage_coverage": "3/4", + "external_ancestry_coverage": "1/2", + "edge_coverage": "5/6", + "lineage_bounds": [ + "1/4", + "1/2", + "1/4" + ], + "shared_ancestry_proxy": "present", + "cycle_detected": false + } + } + ] +} diff --git a/tests/integration/test_ci_workflows.py b/tests/integration/test_ci_workflows.py index e3ac3a9..1b81ccc 100644 --- a/tests/integration/test_ci_workflows.py +++ b/tests/integration/test_ci_workflows.py @@ -1,112 +1,91 @@ -"""Preserve workflow guarantees while moving active gates to Phase 3 Step 11.""" - -from pathlib import Path - -EXPECTED = {"ci.yml", "golden.yml", "security.yml", "release.yml"} -FORBIDDEN_WORKFLOW_TERMS = {"services:", "docker run", "fastapi", "streamlit", "telemetry", "publish-package"} - - -def test_workflow_file_set_and_basic_contract(repo_root: Path) -> None: - workflow_root = repo_root / ".github" / "workflows" - assert {path.name for path in workflow_root.glob("*.yml")} == EXPECTED - for path in sorted(workflow_root.glob("*.yml")): - text = path.read_text(encoding="utf-8") - assert text.startswith("name:") - assert "permissions:\n contents: read" in text - assert "actions/checkout@v4" in text - assert "actions/setup-python@v5" in text - assert "\t" not in text - - -def test_workflows_preserve_phase_boundary(repo_root: Path,phase3_final_snapshot) -> None: - repo_root = phase3_final_snapshot - combined = "\n".join(p.read_text(encoding="utf-8").lower() for p in (repo_root / ".github/workflows").glob("*.yml")) - for term in FORBIDDEN_WORKFLOW_TERMS: - assert term not in combined - assert "test_no_algorithms.py" in combined - assert "test_no_network.py" in combined - assert "python -m build" in combined - assert "rit version" in combined - assert "--phase 3 --step 11" in combined - assert "release_check.py --diff" not in combined +"""Current workflow scheduling, frozen specifications and release result integrity. +Complete compatibility and delivery gates run for an explicitly selected candidate. +Historical dispatch and source-body migration tests remain in Git history. +""" import ast import hashlib import re import runpy import tomllib -import pytest - -def _release_tools(repo_root): - return runpy.run_path(str(repo_root / "scripts/release_check.py"), run_name="phase2_release_test") +import pytest -def test_phase2_workflows_never_mask_failed_commands(repo_root): - for path in (repo_root / ".github/workflows").glob("*.yml"): +def test_workflow_file_set_and_read_only_permissions(repo_root): + paths = list((repo_root / ".github/workflows").glob("*.yml")) + assert {p.name for p in paths} == {"ci.yml", "golden.yml", "security.yml", "release.yml"} + for path in paths: text = path.read_text(encoding="utf-8") - for unsafe in ("continue-on-error:", "|| true", "allow-failure:", "contents: write", - "id-token: write", "pull_request_target:", "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION"): - assert unsafe not in text, (path.name, unsafe) - assert "shell: bash" in text - assert "timeout-minutes:" in text + assert "permissions:\n contents: read" in text assert "persist-credentials: false" in text + assert "--phase " not in text and "--step " not in text + assert "twine upload" not in text and "git push" not in text -def test_phase2_core_and_real_parquet_have_independent_complete_runs(repo_root): +def test_full_matrix_requires_explicit_candidate_selection(repo_root): text = (repo_root / ".github/workflows/ci.yml").read_text(encoding="utf-8") + assert "pull_request:" in text + assert "docs/**" not in text and "docs/lineage_contract.md" not in text.split(" workflow_dispatch:", 1)[0] + assert "default: focused" in text and "options: [focused, candidate]" in text + for job in ("core", "parquet", "performance", "hero", "security"): + body = re.search(r"^ " + job + r":\n(.*?)(?=^ [a-z_]+:|\Z)", text, re.M | re.S).group(1) + assert "github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate'" in body + focused = text.split(" focused:\n", 1)[1].split("\n core:", 1)[0] + assert "inputs.gate != 'candidate'" in focused + assert "test_current_verification.py" in focused and "test_lineage_fixture_inputs.py" in focused + assert "--ignore=tests/performance" not in focused and "tests/performance " not in focused assert 'os: [ubuntu-latest, windows-latest]' in text assert 'python-version: ["3.11", "3.12"]' in text - assert 'RIT_TEST_PARQUET: "0"' in text and 'RIT_TEST_PARQUET: "1"' in text - assert 'python -m pip install ".[test,parquet]"' in text - assert "--require-parquet" in text - assert "find_spec('pyarrow') is None" in text - assert text.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in text assert 'dependencies: [current, minimum]' in text assert '"numpy==2.0.0" "pandas==2.2.2"' in text - assert 'python -m pip check' in text - assert 'dependencies.txt' in text - - -def test_phase2_delivery_builds_both_formats_and_tests_installed_wheel(repo_root,phase3_final_snapshot): - repo_root = phase3_final_snapshot - # Same identity and safety guarantees; only the active artifact stage changes. - text = (repo_root / ".github/workflows/release.yml").read_text(encoding="utf-8") - for required in ("python -m build", "python -m twine check --strict", "--dist", "--delivery", - "--require-parquet", "actions/upload-artifact@v4", "retention-days: 90", - "if-no-files-found: error", "phase3_test_results.log", "phase3_build_results.log"): - assert required in text - assert "twine upload" not in text and "git push" not in text - script = (repo_root / "scripts/release_check.py").read_text(encoding="utf-8") - for required in ('"--no-index", "--no-deps"', '"-I"', '"--prefix=recursive-integrity-toolkit/"', - '"recursive-integrity-toolkit-phase3.zip"', '"phase3_artifacts.sha256"'): - assert required in script - + assert "find_spec('pyarrow') is None" in text and "--require-parquet" in text + assert text.count("--ignore=tests/performance") == 2 + assert text.count("-q tests/performance --junitxml=") == 1 + assert "needs: [core, parquet, performance, hero, security]" in text + + +def test_candidate_checks_preserve_behavior_and_package_roles(repo_root): + workflows = repo_root / ".github/workflows" + for filename in ("golden.yml", "security.yml"): + text = (workflows / filename).read_text(encoding="utf-8") + assert "workflow_call:" in text and "workflow_dispatch:" in text + assert "pull_request:" not in text + golden = (workflows / "golden.yml").read_text(encoding="utf-8") + assert "test_phase3_math.py" in golden and "test_phase4_reports.py" in golden + assert "build_golden.py" not in golden + security = (workflows / "security.yml").read_text(encoding="utf-8") + for name in ("test_no_network.py", "test_PR015_redaction.py", "test_PR017_content_refs.py", "test_phase4_output_safety.py"): + assert name in security + release = (workflows / "release.yml").read_text(encoding="utf-8") + for required in ("python -m build", "python -m twine check --strict", "--candidate", "SOURCE_DATE_EPOCH", "assert same_bytes", "assert payload_equal", "actions/download-artifact@v4", "source-commit.txt"): + assert required in release + assert "python -m pytest" not in release -def test_phase2_hero_workflow_preserves_scaffold_only_golden_boundary(repo_root,phase3_final_snapshot): - repo_root = phase3_final_snapshot - text = (repo_root / ".github/workflows/golden.yml").read_text(encoding="utf-8") - body = "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#")) - assert "test_hero_structure.py" in body - assert "build_golden.py" not in body and "normalize_golden.py" not in body +def _release_tools(repo_root): + return runpy.run_path(str(repo_root / "scripts/release_check.py"), run_name="phase2_release_test") -def test_phase2_completion_records_exist_and_do_not_authorize_phase3(repo_root): - for name in ("PHASE_2_COMPLETION.md", "PHASE_2_VALIDATION_REPORT.md", "PHASE_2_ARCHITECTURE_COMPLIANCE_REPORT.md"): - text = (repo_root / name).read_text(encoding="utf-8") - assert "Phase 3" in text and "not authorized" in text +def test_current_workflows_never_mask_failed_commands(repo_root): + for path in (repo_root / ".github/workflows").glob("*.yml"): + text = path.read_text(encoding="utf-8") + for unsafe in ("continue-on-error:", "|| true", "allow-failure:", "contents: write", + "id-token: write", "pull_request_target:", "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION"): + assert unsafe not in text, (path.name, unsafe) + assert "shell: bash" in text + assert "timeout-minutes:" in text + assert "persist-credentials: false" in text -def test_phase2_approved_phase0_hashes_match_current_sources(repo_root): +def test_current_approved_phase0_hashes_match_current_sources(repo_root): entries = re.findall(r"\| `([^`]+\.md)` \| `([0-9a-f]{64})` \|", (repo_root / "PHASE_0_APPROVAL.md").read_text(encoding="utf-8")) assert len(entries) == 16 for name, expected in entries: assert hashlib.sha256((repo_root / name).read_bytes()).hexdigest() == expected, name -def test_phase2_package_version_is_consistent_without_protected_module_edit(repo_root): +def test_current_package_version_is_consistent_without_protected_module_edit(repo_root): project = tomllib.loads((repo_root / "pyproject.toml").read_text(encoding="utf-8"))["project"] tree = ast.parse((repo_root / "src/recursive_integrity_toolkit/__init__.py").read_text(encoding="utf-8")) value = [ast.literal_eval(n.value) for n in tree.body if isinstance(n, ast.Assign) @@ -121,84 +100,45 @@ def _junit(tmp_path, body, counts='tests="1" errors="0" failures="0" skipped="0" return path -def test_phase2_junit_gate_counts_passes(repo_root, tmp_path): +def test_current_junit_gate_counts_passes(repo_root, tmp_path): result = _release_tools(repo_root)["verify_junit"](_junit(tmp_path, '')) assert result == {"tests": 1, "passed": 1, "failed": 0, "skipped": 0, "real_parquet_cases": 0} @pytest.mark.parametrize("tag", ["failure", "error", "skipped"]) -def test_phase2_junit_gate_rejects_hidden_failures_and_skips(repo_root, tmp_path, tag): +def test_current_junit_gate_rejects_hidden_failures_and_skips(repo_root, tmp_path, tag): with pytest.raises(ValueError): _release_tools(repo_root)["verify_junit"](_junit(tmp_path, f'<{tag}/>')) @pytest.mark.parametrize("counter", ["failures", "errors", "skipped"]) -def test_phase2_junit_gate_checks_suite_counters(repo_root, tmp_path, counter): +def test_current_junit_gate_checks_suite_counters(repo_root, tmp_path, counter): with pytest.raises(ValueError): _release_tools(repo_root)["verify_junit"](_junit(tmp_path, '', f'{counter}="1"')) -def test_phase2_junit_gate_requires_collected_cases(repo_root, tmp_path): +def test_current_junit_gate_requires_collected_cases(repo_root, tmp_path): with pytest.raises(ValueError): _release_tools(repo_root)["verify_junit"](_junit(tmp_path, "")) -def test_phase2_junit_gate_rejects_duplicate_case_identity(repo_root, tmp_path): +def test_current_junit_gate_rejects_duplicate_case_identity(repo_root, tmp_path): with pytest.raises(ValueError): _release_tools(repo_root)["verify_junit"](_junit(tmp_path, '')) -def test_phase2_junit_gate_requires_full_core_count(repo_root, tmp_path): +def test_current_junit_gate_rejects_insufficient_requested_count(repo_root, tmp_path): with pytest.raises(ValueError): - _release_tools(repo_root)["verify_junit"](_junit(tmp_path, ''), minimum=1131) + _release_tools(repo_root)["verify_junit"](_junit(tmp_path, ''), minimum=2) -def test_phase2_junit_gate_requires_actual_parquet_cases(repo_root, tmp_path): +def test_current_junit_gate_requires_actual_parquet_cases(repo_root, tmp_path): with pytest.raises(ValueError): _release_tools(repo_root)["verify_junit"](_junit(tmp_path, ''), require_parquet=True) -def test_phase2_junit_gate_accepts_three_real_case_names(repo_root, tmp_path): +def test_current_junit_gate_accepts_three_real_case_names(repo_root, tmp_path): names = ("test_PR002_parquet_real_roundtrip", "test_PR002_parquet_real_row_limit", "test_PR002_parquet_real_invalid_file") body = "".join(f'' for name in names) result = _release_tools(repo_root)["verify_junit"](_junit(tmp_path, body, 'tests="3"'), require_parquet=True) assert result["real_parquet_cases"] == 3 - - -def test_phase2_runtime_gates_remain_exact_allowlists(repo_root): - script = (repo_root / "scripts/check_traceability.py").read_text(encoding="utf-8") - assert "ALLOWED_FUNCTIONS" in script and "ALLOWED_CORE_IMPORTS" in script - for term in ("numpy", "pandas", "networkx", "transformers"): - assert term in script - assert "protected docstring-only modules" in script - - -def _restoration_bytes(repo_root): - after = (repo_root / "VALIDATION_PLAN.md").read_bytes() - before = after.replace(b"| Status | APPROVED PHASE 0 BASELINE |", b"| Status | DRAFT VALIDATION BASELINE |", 1) - before = before.replace(b"Definitions marked `APPROVED DECISION`", b"Definitions marked `PENDING DECISION`", 1) - return before, after - - -def test_phase2_restoration_accepts_only_original_approved_transition(repo_root): - tools = _release_tools(repo_root) - before, after = _restoration_bytes(repo_root) - result = tools["verify_approved_restoration"]("VALIDATION_PLAN.md", "M", before, after) - assert result["after_sha256"] == "16f5fe539da2b3cff1c3e0a2854208bd7fbc1e4c5b332684265b06a03a90cf2f" - assert "VALIDATION_PLAN.md" not in tools["STEP10_ALLOWED"] - - -@pytest.mark.parametrize("change", ["path", "added", "deleted", "before", "after", "unchanged", "type"]) -def test_phase2_restoration_rejects_wider_changes(repo_root, change): - verify = _release_tools(repo_root)["verify_approved_restoration"] - before, after = _restoration_bytes(repo_root) - path, status = "VALIDATION_PLAN.md", "M" - if change == "path": path = "DEFINITIONS_AND_UNITS.md" - elif change == "added": status = "A" - elif change == "deleted": status = "D" - elif change == "before": before += b"\n" - elif change == "after": after += b"\n" - elif change == "unchanged": after = before - elif change == "type": after = after.decode("utf-8") - with pytest.raises(ValueError): - verify(path, status, before, after) diff --git a/tests/integration/test_cli_validation.py b/tests/integration/test_cli_validation.py index 527d90d..b4fa136 100644 --- a/tests/integration/test_cli_validation.py +++ b/tests/integration/test_cli_validation.py @@ -1,24 +1,9 @@ -"""Check the Phase 1 help and version startup commands.""" +"""Check help and version startup against the current package.""" import subprocess import sys -def test_cli_help_runs(subprocess_env,phase3_final_subprocess_env) -> None: - subprocess_env = phase3_final_subprocess_env - result = subprocess.run( - [sys.executable, "-m", "recursive_integrity_toolkit", "--help"], - capture_output=True, - text=True, - env=subprocess_env, - check=False, - ) - assert result.returncode == 0, result.stderr - normalized = " ".join(result.stdout.split()) - assert "Phase 1 scaffold" in normalized - assert "Analytical audit functionality is not implemented" in normalized - - def test_cli_version_runs(subprocess_env) -> None: result = subprocess.run( [sys.executable, "-m", "recursive_integrity_toolkit", "version"], @@ -32,7 +17,7 @@ def test_cli_version_runs(subprocess_env) -> None: def test_phase4_cli_help_uses_current_source(subprocess_env) -> None: - """Current startup remains exercised after preserving the historical help text.""" + """The current CLI starts with a useful help response and no diagnostics.""" result = subprocess.run( [sys.executable, "-m", "recursive_integrity_toolkit", "--help"], capture_output=True, diff --git a/tests/integration/test_current_verification.py b/tests/integration/test_current_verification.py new file mode 100644 index 0000000..fd70e4a --- /dev/null +++ b/tests/integration/test_current_verification.py @@ -0,0 +1,155 @@ +"""Current integrity controls protect source scope, specifications and resources. + +These negatives exercise the live controls against disposable copies. Historical +phase tests and their source-binding migrations remain recoverable in Git. +""" + +import json +import runpy +import shutil + +import pytest + + +@pytest.fixture(scope="module") +def current_tools(repo_root): + return runpy.run_path(str(repo_root / "scripts/release_check.py")) + + +@pytest.fixture +def protected_copy(repo_root, tmp_path): + for relative in ("src", "schemas", "examples/hero"): + shutil.copytree(repo_root / relative, tmp_path / relative, + ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + shutil.copy2(repo_root / "pyproject.toml", tmp_path / "pyproject.toml") + return tmp_path + + +@pytest.mark.parametrize("mutation", ["formula", "new_module", "deleted_module"]) +def test_step1_rejects_unauthorized_product_changes(current_tools, protected_copy, mutation): + """Step 1 grants contracts and fixture work without granting runtime changes.""" + verify = current_tools["verify_source_scope"] + verify(protected_copy) + target = protected_copy / "src/recursive_integrity_toolkit/metrics/diversity.py" + if mutation == "formula": + source = target.read_text(encoding="utf-8") + assert "1.0 -" in source + target.write_text(source.replace("1.0 -", "1.0 +", 1), encoding="utf-8") + elif mutation == "new_module": + target.with_name("unauthorized_metric.py").write_text("def score(): return 1\n", encoding="utf-8") + else: + target.unlink() + with pytest.raises(ValueError): + verify(protected_copy) + + +def test_specification_changes_cannot_self_authorize(current_tools, repo_root, tmp_path): + """A changed specification and forged working manifest still fail the gate.""" + control = json.loads((repo_root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) + for relative in control["phase0_sha256"]: + shutil.copy2(repo_root / relative, tmp_path / relative) + current_tools["verify_frozen_specifications"](tmp_path) + specification = tmp_path / "DEFINITIONS_AND_UNITS.md" + specification.write_bytes(specification.read_bytes() + b"\nUnauthorized formula change.\n") + control["phase0_sha256"][specification.name] = "0" * 64 + (tmp_path / "PHASE_4_BASELINE.json").write_text(json.dumps(control), encoding="utf-8") + with pytest.raises(ValueError): + current_tools["verify_frozen_specifications"](tmp_path) + + +@pytest.mark.parametrize("relative", [ + "src/recursive_integrity_toolkit/data/hero/config.json", + "src/recursive_integrity_toolkit/data/report.schema.json", +]) +def test_packaged_resource_corruption_is_rejected(current_tools, protected_copy, relative): + verify = current_tools["verify_resources"] + verify(protected_copy) + (protected_copy / relative).write_text("{}\n", encoding="utf-8") + with pytest.raises(ValueError): + verify(protected_copy) + + +def test_unlisted_packaged_resource_is_rejected(current_tools, protected_copy): + current_tools["verify_resources"](protected_copy) + (protected_copy / "src/recursive_integrity_toolkit/data/unlisted.json").write_text("{}", encoding="utf-8") + with pytest.raises(ValueError): + current_tools["verify_resources"](protected_copy) + + +def _write_test_archives(directory, wheel_entries, sdist_entries): + """Create validator inputs only; these are never release or install evidence.""" + import io + import tarfile + import warnings + import zipfile + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="Duplicate name:.*", category=UserWarning) + with zipfile.ZipFile(directory / "synthetic.whl", "w") as archive: + for name, content in wheel_entries: + archive.writestr(name, content) + with tarfile.open(directory / "synthetic.tar.gz", "w:gz") as archive: + for name, content in sdist_entries: + member = tarfile.TarInfo("synthetic/" + name) + member.size = len(content) + archive.addfile(member, io.BytesIO(content)) + + +@pytest.mark.parametrize("kind", ["wheel", "sdist"]) +@pytest.mark.parametrize("mutation", ["code", "resource", "missing_resource", "duplicate_member"]) +def test_distribution_integrity_rejects_changed_payloads(current_tools, repo_root, tmp_path, kind, mutation): + runtime = {path.relative_to(repo_root / "src").as_posix(): path.read_bytes() + for path in (repo_root / "src/recursive_integrity_toolkit").rglob("*.py")} + resources = {name.removeprefix("src/"): (repo_root / source).read_bytes() + for name, source in current_tools["RESOURCES"].items()} + wheel_entries = list(runtime.items()) + list(resources.items()) + [ + ("synthetic.dist-info/METADATA", b"Version: 0.1.0.test\n"), + ("synthetic.dist-info/LICENSE", b"synthetic notice"), + ("synthetic.dist-info/NOTICE", b"synthetic notice"), + ] + sdist_entries = [("src/" + name, raw) for name, raw in (*runtime.items(), *resources.items())] + sdist_entries += [("LICENSE", b"synthetic notice"), ("NOTICE", b"synthetic notice")] + verify = current_tools["verify_distributions"] + _write_test_archives(tmp_path, wheel_entries, sdist_entries) + verify(tmp_path, expected_version="0.1.0.test") + entries = wheel_entries if kind == "wheel" else sdist_entries + target = "recursive_integrity_toolkit/metrics/diversity.py" if mutation == "code" else "recursive_integrity_toolkit/data/report.schema.json" + if kind == "sdist": + target = "src/" + target + original = next(raw for name, raw in entries if name == target) + if mutation == "duplicate_member": + entries.append((target, original)) + elif mutation == "missing_resource": + entries[:] = [(name, raw) for name, raw in entries if name != target] + else: + entries[:] = [(name, b"corrupted payload" if name == target else raw) for name, raw in entries] + _write_test_archives(tmp_path, wheel_entries, sdist_entries) + with pytest.raises(ValueError): + verify(tmp_path, expected_version="0.1.0.test") + + +@pytest.mark.parametrize("mutation", ["parent", "absolute", "backslash", "drive", "symlink", "hardlink"]) +def test_sdist_extraction_rejects_unsafe_members_before_writes(current_tools, tmp_path, mutation): + import io + import tarfile + + path = tmp_path / "unsafe.tar.gz" + names = {"parent": "root/../outside", "absolute": "/outside", + "backslash": "root\\outside", "drive": "C:/outside"} + with tarfile.open(path, "w:gz") as archive: + first = tarfile.TarInfo("root/valid.txt") + first.size = 5 + archive.addfile(first, io.BytesIO(b"valid")) + member = tarfile.TarInfo(names.get(mutation, "root/link")) + if mutation in {"symlink", "hardlink"}: + member.type = tarfile.SYMTYPE if mutation == "symlink" else tarfile.LNKTYPE + member.linkname = "../outside" + archive.addfile(member) + else: + member.size = 6 + archive.addfile(member, io.BytesIO(b"unsafe")) + destination = tmp_path / "extracted" + destination.mkdir() + with pytest.raises(ValueError): + current_tools["extract_sdist"](path, destination) + assert list(destination.iterdir()) == [] + assert not (tmp_path / "outside").exists() diff --git a/tests/integration/test_no_algorithms.py b/tests/integration/test_no_algorithms.py index 73d816d..34f40a1 100644 --- a/tests/integration/test_no_algorithms.py +++ b/tests/integration/test_no_algorithms.py @@ -1,49 +1,13 @@ -"""Protect Phase 3 Step 9 scoped metrics from premature analytical implementation.""" +"""Current package dependency boundary. + +Direct product suites protect mathematical behavior. Historical staged module +allowlists and exact implementation forms are recoverable from Git history. +""" import ast -from pathlib import Path -STEP1_EXECUTABLE = {"__init__.py", "__main__.py", "cli.py", "config.py", "errors.py", "models.py"} -STEP2_EXECUTABLE = {"io/loaders.py", "utils/hashing.py", "utils/paths.py"} -STEP3_EXECUTABLE = {"io/schema_mapping.py"} -STEP8_EXECUTABLE = {"observability/levels.py"} -PHASE3_FIELD_EXECUTABLE = {"representations/base.py", "representations/field.py"} -PHASE3_DISTRIBUTION_EXECUTABLE = {"metrics/diversity.py"} -PHASE3_PROVENANCE_EXECUTABLE = {"metrics/provenance.py"} -PHASE3_BOUNDS_EXECUTABLE = {"metrics/bounds.py"} -PHASE3_TAIL_EXECUTABLE = {"metrics/tail.py"} -PHASE3_RESAMPLING_EXECUTABLE = {"metrics/resampling.py"} -PHASE3_PAIR_EXECUTABLE = {"representations/compatibility.py"} -PHASE3_EXACT_EXECUTABLE = {"representations/content_hash.py", "metrics/duplicates.py"} -STEP4_EXECUTABLE = {"io/normalization.py", "io/validation.py", "utils/ordering.py"} -PROTECTED_PREFIXES = {"io", "lineage", "metrics", "observability", "reports", "representations", "utils"} FORBIDDEN_ANALYTICAL_IMPORT_ROOTS = {"networkx", "scipy", "sklearn", "torch", "tensorflow", "transformers"} -FORBIDDEN_ANALYTICAL_NAMES = {"support_size", "gini_simpson", "tail_fragility", "source_type_shares", "closure_bounds", "ancestry_hhi", "effective_external_roots", "resample", "reopening"} - - -def _is_docstring_only(path: Path) -> bool: - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - return len(tree.body) == 1 and isinstance(tree.body[0], ast.Expr) and isinstance(tree.body[0].value, ast.Constant) and isinstance(tree.body[0].value.value, str) - - -def test_only_step8_authorized_modules_gain_behavior(package_root,phase3_final_package_root) -> None: - package_root = phase3_final_package_root - for path in sorted(package_root.rglob("*.py")): - relative = path.relative_to(package_root) - if relative.as_posix() in STEP2_EXECUTABLE | STEP3_EXECUTABLE | STEP4_EXECUTABLE | STEP8_EXECUTABLE | PHASE3_FIELD_EXECUTABLE | PHASE3_EXACT_EXECUTABLE | PHASE3_DISTRIBUTION_EXECUTABLE | PHASE3_PROVENANCE_EXECUTABLE | PHASE3_BOUNDS_EXECUTABLE | PHASE3_TAIL_EXECUTABLE | PHASE3_RESAMPLING_EXECUTABLE | PHASE3_PAIR_EXECUTABLE or (len(relative.parts) == 1 and path.name in STEP1_EXECUTABLE): - continue - assert _is_docstring_only(path), f"Premature executable body: {relative}" - - -def test_protected_phase3_plus_modules_remain_placeholders(package_root,phase3_final_package_root) -> None: - package_root = phase3_final_package_root - for prefix in PROTECTED_PREFIXES: - for path in sorted((package_root / prefix).rglob("*.py")): - if path.relative_to(package_root).as_posix() in STEP2_EXECUTABLE | STEP3_EXECUTABLE | STEP4_EXECUTABLE | STEP8_EXECUTABLE | PHASE3_FIELD_EXECUTABLE | PHASE3_EXACT_EXECUTABLE | PHASE3_DISTRIBUTION_EXECUTABLE | PHASE3_PROVENANCE_EXECUTABLE | PHASE3_BOUNDS_EXECUTABLE | PHASE3_TAIL_EXECUTABLE | PHASE3_RESAMPLING_EXECUTABLE | PHASE3_PAIR_EXECUTABLE: - continue - assert _is_docstring_only(path), f"Protected module changed after Step 8: {path}" - assert _is_docstring_only(package_root / "result.py") def test_no_prohibited_analytical_dependency_imports(package_root) -> None: @@ -56,563 +20,3 @@ def test_no_prohibited_analytical_dependency_imports(package_root) -> None: elif isinstance(node, ast.ImportFrom) and node.module: imported_roots.add(node.module.split(".")[0]) assert not (imported_roots & FORBIDDEN_ANALYTICAL_IMPORT_ROOTS) - - -def test_no_analytical_function_names_exist(package_root) -> None: - names = set() - for path in sorted(package_root.rglob("*.py")): - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - names.update(node.name for node in ast.walk(tree) if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))) - assert not (names & FORBIDDEN_ANALYTICAL_NAMES) - - -def test_PR003_traceability_script_enforces_step8_scope(repo_root,phase3_final_snapshot) -> None: - repo_root = phase3_final_snapshot - import subprocess - import sys - result = subprocess.run([sys.executable, "scripts/check_traceability.py"], - cwd=repo_root, capture_output=True, text=True, check=False) - assert result.returncode == 0, result.stdout + result.stderr - assert "protected docstring-only modules: 16" in result.stdout - - -# The checker itself must reject new work outside the current Step 8 boundary. -import shutil -import subprocess -import sys -import pytest - - -@pytest.mark.parametrize("relative,injected", [ - ("io/validation.py", '_read_source(None, None)\n'), - ("io/validation.py", 'load_table(None)\n'), - ("io/validation.py", 'inventory_source(None)\n'), - ("io/validation.py", "load_content_reference('x')\n"), - ("io/validation.py", 'validate_bundle(None)\n'), - ("io/validation.py", '_bundle_control(None, None)\n'), - ("io/validation.py", 'from .loaders import load_table\n'), - ("io/validation.py", 'from .normalization import normalize_row\n'), - ("io/validation.py", 'from .loaders import _parse_csv\n'), - ("io/validation.py", "Path('x').write_text('data')\n"), - - ("observability/levels.py", "import socket\n"), - ("observability/levels.py", "from ..metrics import diversity\n"), - ("observability/levels.py", "from ..lineage import graph\n"), - ("observability/levels.py", "from ..io.loaders import load_table\n"), - ("observability/levels.py", "eval('1')\n"), - ("observability/levels.py", "open('unexpected')\n"), - ("observability/levels.py", "lambda: None\n"), - ("observability/levels.py", "math.sqrt(4)\n"), - ("observability/levels.py", "math.fsum((0.5, 0.5))\n"), - ("observability/levels.py", "1 - 0.5 ** 2\n"), - ("observability/levels.py", "1 / 2\n"), - ("observability/levels.py", "def universal_score(): pass\n"), - - ("utils/paths.py", "import socket\n"), - ("utils/paths.py", "eval('1')\n"), - ("utils/paths.py", "os.system('unapproved')\n"), - ("utils/paths.py", "Path('x').write_text('unapproved')\n"), - ("io/loaders.py", "__import__('socket')\n"), - ("io/loaders.py", "os.remove('unapproved')\n"), - ("io/schema_mapping.py", "def premature(): pass\n"), - ("io/normalization.py", "def premature(): pass\n"), - ("io/validation.py", "def premature(): pass\n"), - ("observability/levels.py", "def premature(): pass\n"), - ("metrics/diversity.py", "def premature(): pass\n"), - ("lineage/graph.py", "def premature(): pass\n"), - ("representations/field.py", "def premature(): pass\n"), - ("reports/assembly.py", "def premature(): pass\n"), - ("utils/ordering.py", "def premature(): pass\n"), - ("result.py", "def premature(): pass\n"), - ("io/loaders.py", "def premature(): pass\n"), - ("io/loaders.py", "import pyarrow\n"), - ("io/loaders.py", "import pandas\n"), - ("models.py", "import math\n"), - ("io/normalization.py", "import socket\n"), - ("io/normalization.py", "import pandas\n"), - ("io/normalization.py", "eval('1')\n"), - ("io/normalization.py", "open('unapproved')\n"), - ("io/validation.py", "def derive_generation(): pass\n"), - ("io/validation.py", "import networkx\n"), - ("io/validation.py", "open('unapproved')\n"), - ("io/validation.py", "def source_type_shares(): pass\n"), - ("io/validation.py", "def closure_bounds(): pass\n"), - ("io/validation.py", "def resolve_parents(): pass\n"), - ("models.py", "class UniversalScore: pass\n"), - ("io/validation.py", "def detect_cycles(): pass\n"), - ("io/validation.py", "def lineage_depth(): pass\n"), - ("io/validation.py", "import socket\n"), - ("models.py", "__import__('socket')\n"), - ("models.py", "eval('1')\n"), - ("models.py", "lambda: None\n"), - ("utils/ordering.py", "def version_rank(): pass\n"), - ("io/schema_mapping.py", "import os\n"), - ("io/schema_mapping.py", "import socket\n"), - ("io/schema_mapping.py", "eval('1')\n"), - ("io/schema_mapping.py", "exec('pass')\n"), - ("io/schema_mapping.py", "__import__('os')\n"), - ("io/schema_mapping.py", "lambda: None\n"), - ("io/schema_mapping.py", "Path('unexpected').write_text('x')\n"), -]) -def test_PR002_checker_rejects_unauthorized_mutations(repo_root, package_root, tmp_path, relative, injected): - target = tmp_path / "repo" - script = target / "scripts/check_traceability.py" - script.parent.mkdir(parents=True) - script.write_bytes((repo_root / "scripts/check_traceability.py").read_bytes()) - copied = target / "src/recursive_integrity_toolkit" - shutil.copytree(package_root, copied, ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) - path = copied / relative - path.write_text(path.read_text(encoding="utf-8") + "\n" + injected, encoding="utf-8") - result = subprocess.run([sys.executable, str(script)], capture_output=True, text=True, check=False) - assert result.returncode != 0, f"Unauthorized implementation accepted: {relative}" - assert any(word in result.stderr for word in ("Protected", "Unexpected", "Dependency", "Import")), result.stderr - - -# Phase 3 Step 2: retain all prior negative identities and add pure-field gates. -@pytest.mark.parametrize("relative", ["representations/base.py", "representations/field.py"]) -@pytest.mark.parametrize("injection", [ - "import socket\n", "from ..metrics import diversity\n", "from ..models import __builtins__\n", - "from ..config import RepresentationConfig as eval\n", "value = 1\n", - "open('private')\n", "eval('1')\n", "lambda: None\n", - "def support_size(): return 1\n", "from ..io.loaders import load_table\n", -]) -def test_phase3_step2_injected_later_feature_is_rejected(repo_root, tmp_path, relative, injection): - target = tmp_path / "gate" - shutil.copytree(repo_root / "src", target / "src", ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) - (target / "scripts").mkdir() - shutil.copy2(repo_root / "scripts/check_traceability.py", target / "scripts/check_traceability.py") - file = target / "src/recursive_integrity_toolkit" / relative - file.write_text(file.read_text(encoding="utf-8") + "\n" + injection, encoding="utf-8") - result = subprocess.run([sys.executable, str(target / "scripts/check_traceability.py")], - cwd=target, text=True, capture_output=True) - assert result.returncode != 0, result.stdout + result.stderr - - -@pytest.mark.parametrize("body", [ - "return sum(values)", "return values ** 2", "return value / 2", "return value * value", - "return open(value)", "return values.pop()", "return callback(values)", - "return __import__('numpy')", "return (lambda: value)()", "return value.lower()", -]) -def test_phase3_step2_hidden_computation_is_rejected(repo_root, body): - import runpy - checker = runpy.run_path(str(repo_root / "scripts/check_traceability.py"), run_name="field_gate") - with pytest.raises(SystemExit): - checker["_phase3_representation_boundary"]( - ast.parse("def _selection():\n " + body + "\n"), "representations/field.py") - - -@pytest.mark.parametrize("path", ["PHASE_3_PLAN.md", "tests/golden/phase3_math_cases.json", - "src/recursive_integrity_toolkit/metrics/diversity.py", "src/recursive_integrity_toolkit/io/validation.py", - "VALIDATION_PLAN.md", "pyproject.toml"]) -def test_phase3_step2_incremental_scope_rejects_unapproved_edits(repo_root, path): - import runpy - checker = runpy.run_path(str(repo_root / "scripts/release_check.py"), run_name="step2_paths") - with pytest.raises(ValueError): - checker["verify_prior_step_changes"]([("M", path)], step=2) - - -@pytest.mark.parametrize("change", ["none", "extra_newline", "skip", "numeric_expectation", "wrong_type"]) -def test_phase3_step2_contract_test_exception_is_exact(repo_root, change): - import runpy - checker = runpy.run_path(str(repo_root / "scripts/release_check.py"), run_name="step2_exception") - after = (repo_root / "tests/unit/test_phase3_contracts.py").read_bytes() - before = after.replace(b"control[key]+=1", b"control[key]=2", 1) - checker["verify_step2_contract_test_migration"](before, after) - if change == "none": bad = before - elif change == "extra_newline": bad = after + b"\n" - elif change == "skip": bad = b"# skipped\n" + after - elif change == "numeric_expectation": bad = after.replace(b'=="5/8"', b'=="1/2"') - else: bad = after.decode() - with pytest.raises(ValueError): - checker["verify_step2_contract_test_migration"](before, bad) - - -# Phase 3 Step 3: inherited identities stay; exact-content modules have bounded calls. -@pytest.mark.parametrize("relative", ["representations/content_hash.py", "metrics/duplicates.py"]) -@pytest.mark.parametrize("injection", [ - "import socket\n", "from ..io.loaders import load_table\n", "from ..metrics import diversity\n", - "from ..utils.hashing import sha256_bytes as eval\n", "from ..models import __builtins__\n", - "open('private')\n", "def semantic_support(): return 1\n", "x = 1\n", - "lambda: None\n", "eval('1')\n", -]) -def test_phase3_step3_later_feature_injection_is_rejected(repo_root, tmp_path, relative, injection): - target = tmp_path / "gate" - shutil.copytree(repo_root / "src", target / "src", ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) - (target / "scripts").mkdir() - shutil.copy2(repo_root / "scripts/check_traceability.py", target / "scripts/check_traceability.py") - file = target / "src/recursive_integrity_toolkit" / relative - file.write_text(file.read_text(encoding="utf-8") + "\n" + injection, encoding="utf-8") - result = subprocess.run([sys.executable, str(target / "scripts/check_traceability.py")], - cwd=target, text=True, capture_output=True) - assert result.returncode != 0, result.stdout + result.stderr - - -@pytest.mark.parametrize("relative", ["representations/content_hash.py", "metrics/duplicates.py"]) -@pytest.mark.parametrize("body", [ - "return value ** 2", "return value / 2", "return sum(values)", "return callback(values)", - "return text.lower()", "return text.casefold()", "return text.replace('a', 'b')", - "return __import__('numpy')", "return open(value)", "return members.pop()", "value += 1", -]) -def test_phase3_step3_hidden_operation_is_rejected(repo_root, relative, body): - import runpy - checker = runpy.run_path(str(repo_root / "scripts/check_traceability.py"), run_name="exact_gate") - with pytest.raises(SystemExit): - checker["_phase3_exact_boundary"](ast.parse("def helper():\n " + body + "\n"), relative) - - -@pytest.mark.parametrize("path", ["tests/unit/test_phase3_contracts.py", - "src/recursive_integrity_toolkit/representations/base.py", - "src/recursive_integrity_toolkit/representations/field.py", - "src/recursive_integrity_toolkit/utils/hashing.py", "tests/golden/phase3_math_cases.json"]) -def test_phase3_step3_incremental_scope_keeps_prior_implementations_frozen(repo_root, path): - import runpy - checker = runpy.run_path(str(repo_root / "scripts/release_check.py"), run_name="step3_paths") - with pytest.raises(ValueError): - checker["verify_prior_step_changes"]([("M", path)], step=3) - - -# Step 4 adds a reviewed-body pin to all inherited phase boundaries. -@pytest.mark.parametrize("injection", [ - "import socket", "from math import log", "from ..io.loaders import load_table", - "from ..metrics.resampling import resample", "from ..lineage import graph", - "from math import fsum as eval", "open('private')", "x=1", - "def shannon_entropy(): return 1", "def support_delta(): return 1", - "def closure_bounds(): return 1", -]) -def test_phase3_step4_later_injection_rejected(repo_root,tmp_path,injection): - target=tmp_path/"step4-gate" - shutil.copytree(repo_root/"src",target/"src",ignore=shutil.ignore_patterns("__pycache__","*.pyc")) - (target/"scripts").mkdir() - shutil.copy2(repo_root/"scripts/check_traceability.py",target/"scripts/check_traceability.py") - p=target/"src/recursive_integrity_toolkit/metrics/diversity.py" - p.write_text(p.read_text(encoding="utf-8")+"\n"+injection+"\n",encoding="utf-8") - result=subprocess.run([sys.executable,str(target/"scripts/check_traceability.py")],cwd=target,capture_output=True,text=True) - assert result.returncode!=0 - - -@pytest.mark.parametrize("old,new", [ - ("value * value", "value ** 3"), ("1.0 - concentration", "1.0 - concentration / 2"), - ("count / total", "count / (total + 1)"), ("if value > 0", "if value >= 0"), - ("counts.get(state, 0) + 1", "counts.get(state, 0) + 2"), - ("return value", "return eval(value)"), -]) -def test_phase3_step4_hidden_formula_changes_rejected(repo_root,old,new): - import runpy - gate=runpy.run_path(str(repo_root/"scripts/check_traceability.py"),run_name="step4-ast") - source=(repo_root/"src/recursive_integrity_toolkit/metrics/diversity.py").read_text(encoding="utf-8") - assert old in source - gate["_phase3_distribution_boundary"](ast.parse(source)) - with pytest.raises(SystemExit): gate["_phase3_distribution_boundary"](ast.parse(source.replace(old,new,1))) - - -@pytest.mark.parametrize("path", [ - "src/recursive_integrity_toolkit/metrics/resampling.py", "src/recursive_integrity_toolkit/metrics/provenance.py", - "src/recursive_integrity_toolkit/metrics/duplicates.py", "src/recursive_integrity_toolkit/io/validation.py", - "src/recursive_integrity_toolkit/representations/base.py", "docs/data_schema.md", "docs/privacy.md", - "pyproject.toml", "PHASE_3_PLAN.md", "tests/golden/phase3_math_cases.json", -]) -def test_phase3_step4_unapproved_paths_rejected(repo_root,path): - import runpy - gate=runpy.run_path(str(repo_root/"scripts/release_check.py"),run_name="step4-path") - with pytest.raises(ValueError): gate["verify_prior_step_changes"]([("M",path)],step=4) - - -def test_phase3_step4_current_and_historical_permissions_separate(repo_root): - import runpy - gate=runpy.run_path(str(repo_root/"scripts/release_check.py"),run_name="step4-authorization") - path="src/recursive_integrity_toolkit/metrics/diversity.py" - gate["verify_prior_step_changes"]([("M",path)],step=4) - for step in (1,2,3): - with pytest.raises(ValueError): gate["verify_prior_step_changes"]([("M",path)],step=step) - - -# Step 5 retains the reviewed positive control and rejects future behavior. -@pytest.mark.parametrize('injection',[ - 'import socket', 'from ..io.loaders import load_table', 'from ..lineage import graph', - 'from .bounds import direct_closure_bounds', 'from math import log', 'from math import fsum as eval', - "open('private')", 'x = 1', 'def confidence_score(): return 1', - 'def effective_source_diversity(): return 1', 'def closure_bounds(): return 1', 'def resample(): return 1', -]) -def test_phase3_step5_rejects_later_features(repo_root,injection): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py'),run_name='step5-boundary') - source=(repo_root/'src/recursive_integrity_toolkit/metrics/provenance.py').read_text(encoding='utf-8') - gate['_phase3_provenance_boundary'](ast.parse(source)) - with pytest.raises(SystemExit):gate['_phase3_provenance_boundary'](ast.parse(source+'\n'+injection+'\n')) - - -@pytest.mark.parametrize('old,new',[ - ('counts[value] += 1','counts[value] += 2'), - ('counts[c] / len(entries)','counts[c] / (len(entries) + 1)'), - ('elif not row.required_fields_valid:', 'elif False:'), - ('mass / total','mass / (total + 1)'), - ('if row is None:\n continue','if row is None:\n counts["unknown"] += 1\n continue'), -]) -def test_phase3_step5_rejects_in_body_changes(repo_root,old,new): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py'),run_name='step5-formula') - source=(repo_root/'src/recursive_integrity_toolkit/metrics/provenance.py').read_text(encoding='utf-8') - assert old in source - gate['_phase3_provenance_boundary'](ast.parse(source)) - with pytest.raises(SystemExit):gate['_phase3_provenance_boundary'](ast.parse(source.replace(old,new,1))) - - -@pytest.mark.parametrize('path',[ - 'src/recursive_integrity_toolkit/metrics/bounds.py','src/recursive_integrity_toolkit/metrics/diversity.py', - 'src/recursive_integrity_toolkit/metrics/resampling.py','src/recursive_integrity_toolkit/metrics/duplicates.py', - 'src/recursive_integrity_toolkit/io/validation.py','src/recursive_integrity_toolkit/lineage/graph.py', - 'docs/data_schema.md','docs/privacy.md','pyproject.toml','PHASE_3_PLAN.md', - 'tests/unit/test_phase3_contracts.py','tests/golden/phase3_math_cases.json', -]) -def test_phase3_step5_unapproved_paths_rejected(repo_root,path): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/release_check.py'),run_name='step5-path') - with pytest.raises(ValueError):gate['verify_prior_step_changes']([('M',path)],step=5) - - -def test_phase3_step5_permissions_keep_prior_boundaries(repo_root): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/release_check.py'),run_name='step5-permissions') - path='src/recursive_integrity_toolkit/metrics/provenance.py' - gate['verify_prior_step_changes']([('M',path)],step=5) - for step in (1,2,3,4): - with pytest.raises(ValueError):gate['verify_prior_step_changes']([('M',path)],step=step) - assert gate['STEP5_NEW']=={ - 'tests/fixtures/provenance_partial/phase3_composition.json', - 'tests/fixtures/provenance_unknown/phase3_grounding_crossed.json', - 'tests/fixtures/weighted/phase3_source_weights.json'} - - -# Step 6 historical gates remain; only direct interval computation is newly opened. -@pytest.mark.parametrize('injection',[ - 'import socket','from ..lineage import graph','from .resampling import resample', - 'open("private")','x = 1','def midpoint(): return 0.5','def risk_score(): return 1', - 'def classify_lineage_grounding(): return 1','def ancestry_hhi(): return 1', -]) -def test_phase3_step6_rejects_later_bounds_features(repo_root,injection): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/bounds.py').read_text() - gate['_phase3_bounds_boundary'](ast.parse(source)) - with pytest.raises(SystemExit):gate['_phase3_bounds_boundary'](ast.parse(source+'\n'+injection+'\n')) - - -@pytest.mark.parametrize('old,new',[ - ('lower = known_closed / total','lower = 0.0'), - ('upper = (known_closed + unresolved) / total','upper = known_closed / total'), - ('width = unresolved / total','width = 0.0'), - ('required_cov.numerator == 0','False'), -]) -def test_phase3_step6_rejects_formula_and_availability_edits(repo_root,old,new): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/bounds.py').read_text() - assert old in source - with pytest.raises(SystemExit):gate['_phase3_bounds_boundary'](ast.parse(source.replace(old,new,1))) - - -@pytest.mark.parametrize('path',[ - 'src/recursive_integrity_toolkit/metrics/provenance.py','src/recursive_integrity_toolkit/metrics/diversity.py', - 'src/recursive_integrity_toolkit/metrics/resampling.py','src/recursive_integrity_toolkit/lineage/graph.py', - 'tests/unit/test_phase3_contracts.py','docs/data_schema.md','docs/privacy.md','PHASE_3_PLAN.md','pyproject.toml', -]) -def test_phase3_step6_unapproved_file_changes_fail(repo_root,path): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/release_check.py')) - with pytest.raises(ValueError):gate['verify_prior_step_changes']([('M',path)],step=6) - - -def test_phase3_step6_requires_reviewed_source_bytes(repo_root,tmp_path): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/bounds.py').read_text() - (tmp_path/'metrics').mkdir();(tmp_path/'metrics/bounds.py').write_text(source+'\nx = 1\n') - gate['_phase3_bounds_boundary'].__globals__['PACKAGE']=tmp_path - with pytest.raises(SystemExit):gate['_phase3_bounds_boundary'](ast.parse(source+'\nx = 1\n')) - - -# Step 7: only observed tail/rank and analytic one-step F-014 are authorized. -@pytest.mark.parametrize('injection',[ - 'import socket','from ..lineage import graph','from .resampling import resample', - 'import numpy.random','open("private")','x = 1','def risk_score(): return 1', - 'def reopen(): return 1','def tail_fragility_signal(): return "high"', - 'def bottom_frequency_quantile(): return 1','def simulate(): return 1', -]) -def test_phase3_step7_rejects_later_tail_features(repo_root,injection): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/tail.py').read_text() - gate['_phase3_tail_boundary'](ast.parse(source)) - with pytest.raises(SystemExit):gate['_phase3_tail_boundary'](ast.parse(source+'\n'+injection+'\n')) - - -@pytest.mark.parametrize('old,new',[ - ('probability = exp(resample_size * log1p(-p))','probability = 0.5'), - ('row.state_count == 1','row.state_count <= 1'), - ('row.state_frequency <= options.frequency_threshold','row.state_frequency < options.frequency_threshold'), - ('row.state_count > 0','row.state_count >= 0'), - ('CalculationEvidenceClass.SIMULATION if scenario','CalculationEvidenceClass.DERIVED_METRIC if scenario'), -]) -def test_phase3_step7_rejects_formula_or_rule_rewrite(repo_root,old,new): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/tail.py').read_text() - assert old in source - with pytest.raises(SystemExit):gate['_phase3_tail_boundary'](ast.parse(source.replace(old,new,1))) - - -@pytest.mark.parametrize('path',[ - 'src/recursive_integrity_toolkit/metrics/provenance.py','src/recursive_integrity_toolkit/metrics/diversity.py', - 'src/recursive_integrity_toolkit/metrics/bounds.py','src/recursive_integrity_toolkit/metrics/resampling.py', - 'src/recursive_integrity_toolkit/lineage/graph.py','src/recursive_integrity_toolkit/io/validation.py', - 'docs/data_schema.md','docs/privacy.md','PHASE_3_PLAN.md','pyproject.toml', - 'tests/golden/phase3_math_cases.json', -]) -def test_phase3_step7_unapproved_file_changes_fail(repo_root,path): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/release_check.py')) - with pytest.raises(ValueError):gate['verify_prior_step_changes']([('M',path)],step=7) - - -def test_phase3_step7_requires_reviewed_source_bytes(repo_root,tmp_path): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/tail.py').read_text() - (tmp_path/'metrics').mkdir();(tmp_path/'metrics/tail.py').write_text(source+'\nx = 1\n') - gate['_phase3_tail_boundary'].__globals__['PACKAGE']=tmp_path - with pytest.raises(SystemExit):gate['_phase3_tail_boundary'](ast.parse(source+'\nx = 1\n')) - - -# Step 8 authorizes only F-015 and the explicit fixed-state closed sampler. -@pytest.mark.parametrize('injection',[ - 'import socket', 'import numpy as np', 'np.random.seed(1)', - 'from ..lineage import graph', 'from ..io.loaders import load_table', - 'open("private")', 'def simulate_reopening(): return 1', - 'def external_reference_loss(): return 1', 'def report(): return "ok"', - 'def auto_simulate(): return 1', 'def universal_score(): return 1', -]) -def test_phase3_step8_rejects_unapproved_resampling_features(repo_root,injection): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/resampling.py').read_text() - gate['_phase3_resampling_boundary'](ast.parse(source)) - with pytest.raises(SystemExit):gate['_phase3_resampling_boundary'](ast.parse(source+'\n'+injection+'\n')) - - -@pytest.mark.parametrize('old,new',[ - ('factor = 1.0 - 1.0 / resample_size','factor = 1.0'), - ('masses = counts','masses = probabilities'), - ('np.random.PCG64(seed)','np.random.PCG64()'), - ('mass == 0 or remaining == 0','remaining == 0'), - ('corrected = sampled and total != 1.0','corrected = True'), - ('abs(total - 1.0) > tolerance','abs(total - 1.0) > .1'), - ('CalculationEvidenceClass.SIMULATION','CalculationEvidenceClass.OBSERVED_FACT'), - ('replicates: int, scope:', 'replicates: int = 1, scope:'), -]) -def test_phase3_step8_rejects_formula_seed_and_evidence_edits(repo_root,old,new): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/resampling.py').read_text() - assert old in source - with pytest.raises(SystemExit):gate['_phase3_resampling_boundary'](ast.parse(source.replace(old,new,1))) - - -@pytest.mark.parametrize('path',[ - 'src/recursive_integrity_toolkit/metrics/tail.py','src/recursive_integrity_toolkit/metrics/diversity.py', - 'src/recursive_integrity_toolkit/metrics/provenance.py','src/recursive_integrity_toolkit/metrics/bounds.py', - 'src/recursive_integrity_toolkit/io/validation.py','src/recursive_integrity_toolkit/lineage/graph.py', - 'src/recursive_integrity_toolkit/representations/compatibility.py', - 'PHASE_3_PLAN.md','pyproject.toml','tests/golden/phase3_math_cases.json','docs/data_schema.md', -]) -def test_phase3_step8_unapproved_file_changes_fail(repo_root,path): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/release_check.py')) - with pytest.raises(ValueError):gate['verify_prior_step_changes']([('M',path)],step=8) - - -def test_phase3_step8_current_permission_does_not_change_prior_permission(repo_root): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/release_check.py')) - path='src/recursive_integrity_toolkit/metrics/resampling.py' - gate['verify_prior_step_changes']([('M',path)],step=8) - for step in range(1,8): - with pytest.raises(ValueError):gate['verify_prior_step_changes']([('M',path)],step=step) - assert gate['STEP8_NEW']=={'tests/fixtures/resampling/phase3_closed_cases.json'} - - -def test_phase3_step8_reviewed_source_bytes_required(repo_root,tmp_path): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/resampling.py').read_text() - (tmp_path/'metrics').mkdir();(tmp_path/'metrics/resampling.py').write_text(source+'\nx = 1\n') - gate['_phase3_resampling_boundary'].__globals__['PACKAGE']=tmp_path - with pytest.raises(SystemExit):gate['_phase3_resampling_boundary'](ast.parse(source+'\nx = 1\n')) - - -# Step 9 opens only compatibility declarations and an explicitly selected pair. -@pytest.mark.parametrize('path,boundary',[ - ('representations/compatibility.py','_phase3_pair_boundary'), - ('metrics/diversity.py','_phase3_distribution_boundary'), -]) -@pytest.mark.parametrize('injection',[ - 'import socket', 'from ..io.loaders import load_table', 'from ..lineage import graph', - 'open("private")', 'def compare_all_versions(): return []', - 'def model_performance_delta(): return 1', 'def render_report(): return "ok"', - 'def infer_state_mapping(): return {}', -]) -def test_phase3_step9_rejects_unapproved_pair_features(repo_root,path,boundary,injection): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit'/path).read_text(encoding='utf-8') - gate[boundary](ast.parse(source)) - with pytest.raises(SystemExit):gate[boundary](ast.parse(source+'\n'+injection+'\n')) - - -@pytest.mark.parametrize('old,new',[ - ('len(right) - len(left)','len(left) - len(right)'), - ('len(left & right) / len(left)','len(left & right) / len(right)'), - ('b.gini_simpson_diversity.value - a.gini_simpson_diversity.value','0.0'), - ('a.scope != context.earlier_scope','False'), - ('if any(row.state_id not in mapping for row in value.states):','if False:'), -]) -def test_phase3_step9_rejects_formula_or_scope_bypass(repo_root,old,new): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/diversity.py').read_text(encoding='utf-8') - assert old in source - with pytest.raises(SystemExit):gate['_phase3_distribution_boundary'](ast.parse(source.replace(old,new,1))) - - -@pytest.mark.parametrize('path',[ - 'src/recursive_integrity_toolkit/metrics/resampling.py','src/recursive_integrity_toolkit/metrics/tail.py', - 'src/recursive_integrity_toolkit/metrics/provenance.py','src/recursive_integrity_toolkit/metrics/bounds.py', - 'src/recursive_integrity_toolkit/io/validation.py','src/recursive_integrity_toolkit/lineage/graph.py', - 'src/recursive_integrity_toolkit/reports/json_report.py','src/recursive_integrity_toolkit/cli.py', - 'tests/unit/test_phase3_contracts.py','PHASE_3_PLAN.md','tests/golden/phase3_math_cases.json', - 'docs/data_schema.md','docs/privacy.md','pyproject.toml', -]) -def test_phase3_step9_prohibits_unlisted_modifications(repo_root,path): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/release_check.py')) - with pytest.raises(ValueError):gate['verify_prior_step_changes']([('M',path)],step=9) - - -def test_phase3_step9_current_scope_does_not_change_step8_scope(repo_root): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/release_check.py')) - path='src/recursive_integrity_toolkit/representations/compatibility.py' - gate['verify_prior_step_changes']([('M',path)],step=9) - with pytest.raises(ValueError):gate['verify_prior_step_changes']([('M',path)],step=8) - assert gate['STEP9_NEW']=={ - 'tests/unit/test_T1_compatibility.py','tests/fixtures/representation_compatible/phase3_pair.json', - 'tests/fixtures/representation_incompatible/phase3_pair.json'} - - -def test_phase3_step9_compatibility_source_bytes_must_be_reviewed(repo_root,tmp_path): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/representations/compatibility.py').read_text(encoding='utf-8') - (tmp_path/'representations').mkdir() - (tmp_path/'representations/compatibility.py').write_text(source+'\nx = 1\n',encoding='utf-8') - gate['_phase3_pair_boundary'].__globals__['PACKAGE']=tmp_path - with pytest.raises(SystemExit):gate['_phase3_pair_boundary'](ast.parse(source+'\nx = 1\n')) diff --git a/tests/integration/test_phase3_metric_pipeline.py b/tests/integration/test_phase3_metric_pipeline.py index 7575361..628d2bd 100644 --- a/tests/integration/test_phase3_metric_pipeline.py +++ b/tests/integration/test_phase3_metric_pipeline.py @@ -6,12 +6,10 @@ from dataclasses import replace from fractions import Fraction from pathlib import Path -import ast import builtins import hashlib import importlib import json -import runpy import socket import pytest @@ -218,186 +216,3 @@ def test_phase3_hero_source_and_oracle_bytes_are_not_rewritten(repo_root,phase3_ assert before == {p:hashlib.sha256(p.read_bytes()).hexdigest() for p in before} assert len(result["bundle"].records) == 16 assert not any((repo_root/name).exists() for name in ("hero_report.json","hero_report.md","lineage_report.json","cycle_report.json")) - - -@pytest.mark.parametrize("module",["lineage/graph.py","lineage/cycles.py","lineage/ancestry.py","reports/assembly.py","reports/json_report.py","reports/markdown_report.py","reports/html_report.py","result.py"]) -def test_phase3_later_implementations_remain_empty(package_root,module,phase3_final_package_root): - package_root = phase3_final_package_root - tree = ast.parse((package_root/module).read_bytes()) - assert len(tree.body) == 1 and isinstance(tree.body[0],ast.Expr) and isinstance(tree.body[0].value,ast.Constant) - assert isinstance(tree.body[0].value.value,str) - - -@pytest.mark.parametrize("path",[ - "src/recursive_integrity_toolkit/metrics/diversity.py", - "src/recursive_integrity_toolkit/metrics/provenance.py", - "src/recursive_integrity_toolkit/io/validation.py", - "src/recursive_integrity_toolkit/reports/assembly.py", - "src/recursive_integrity_toolkit/cli.py", - "tests/unit/test_T1_compatibility.py","tests/unit/test_phase3_contracts.py", - "PHASE_3_PLAN.md","DEFINITIONS_AND_UNITS.md","pyproject.toml", - "examples/hero/EXPECTED_OUTPUTS.md","schemas/report.schema.json", -]) -def test_phase3_step10_rejects_unopened_file_changes(repo_root,path): - gate = runpy.run_path(str(repo_root/"scripts/release_check.py")) - with pytest.raises(ValueError): - gate["verify_prior_step_changes"]([("M",path)],step=10) - - -def test_phase3_step10_scope_is_fixed_not_self_authorized(repo_root,phase3_step10_snapshot): - gate = runpy.run_path(str(repo_root/"scripts/release_check.py")) - control = json.loads((phase3_step10_snapshot/"PHASE_3_BASELINE.json").read_bytes()) - gate["verify_step10_control"](control) - assert gate["PHASE3_STEP10_NEW"] == {"tests/integration/test_phase3_metric_pipeline.py","tests/golden/test_phase3_math.py"} - assert not any(p.startswith("src/") for p in gate["PHASE3_STEP10_ALLOWED"]) - for key,value in (("active_step",11),("previous_step_commit","0"*40),("phase_complete",True),("main_merge_authorized",True)): - altered = dict(control,**{key:value}) - with pytest.raises(ValueError): - gate["verify_step10_control"](altered) - altered = dict(control,permitted_paths=control["permitted_paths"]+["src/new.py"]) - with pytest.raises(ValueError): - gate["verify_step10_control"](altered) - assert gate["verify_step10_snapshot"](phase3_step10_snapshot)["runtime_modules_unchanged"] == 40 - - -@pytest.mark.parametrize("path",["src/recursive_integrity_toolkit/metrics/diversity.py","tests/golden/phase3_math_cases.json","examples/hero/records_v2.csv","pyproject.toml"]) -def test_phase3_step10_snapshot_rejects_byte_changes(repo_root,tmp_path,path,phase3_step10_snapshot): - import shutil - gate = runpy.run_path(str(repo_root/"scripts/release_check.py")) - shutil.copytree(phase3_step10_snapshot/"src/recursive_integrity_toolkit",tmp_path/"src/recursive_integrity_toolkit",ignore=shutil.ignore_patterns("__pycache__")) - for name in gate["STEP10_FROZEN_FILES"]: - target = tmp_path/name - target.parent.mkdir(parents=True,exist_ok=True) - target.write_bytes((phase3_step10_snapshot/name).read_bytes()) - gate["verify_step10_snapshot"](tmp_path) - target = tmp_path/path - target.write_bytes(target.read_bytes()+b"\n") - with pytest.raises(ValueError): - gate["verify_step10_snapshot"](tmp_path) - - -def _synthetic_evidence(repo_root,tmp_path,change=None): - # Synthetic XML solely for testing the evidence validator, never run evidence. - import xml.etree.ElementTree as ET - root = ET.Element("testsuites") - suite = ET.SubElement(root,"testsuite",tests="21",failures="0",errors="0",skipped="0") - cases = json.loads((repo_root/"tests/golden/phase3_math_cases.json").read_bytes())["cases"] - for c in cases: - ET.SubElement(suite,"testcase",classname="tests.golden.test_phase3_math",name="test_phase3_frozen_case["+c["case_id"]+"]") - test = ET.SubElement(suite,"testcase",classname="synthetic",name="synthetic_performance_evidence") - properties = ET.SubElement(test,"properties") - names = ["hero_input_plus_calculations","metadata_100k_setup","metadata_100k_field_representation","metadata_100k_support_diversity", - "metadata_100k_provenance_join","metadata_100k_composition","metadata_100k_exact_duplicates"] - for name in names: - d = dict(name=name,elapsed_seconds=0.0,traced_current_bytes=0,traced_peak_bytes=0, - record_count=16 if name.startswith("hero") else 100000,whole_product_report_target_certified=False, - memory_method="synthetic validator test",timing_method="synthetic validator test",python="synthetic",platform="synthetic") - if change and name == names[-1]: - d.update(change) - ET.SubElement(properties,"property",name="phase3_performance",value=json.dumps(d)) - path = tmp_path/"synthetic-evidence.xml" - ET.ElementTree(root).write(path,encoding="utf-8") - return path - - -@pytest.mark.parametrize("change",[{"elapsed_seconds":-1},{"elapsed_seconds":True},{"traced_peak_bytes":float("nan")}, - {"record_count":1000},{"whole_product_report_target_certified":True},{"memory_method":""},{"name":"metadata_100k_setup"}]) -def test_phase3_step10_rejects_invalid_performance_evidence(repo_root,tmp_path,change): - gate = runpy.run_path(str(repo_root/"scripts/release_check.py")) - with pytest.raises(ValueError): - gate["verify_step10_evidence"](_synthetic_evidence(repo_root,tmp_path,change)) - - -def test_phase3_step10_evidence_requires_executed_cases_and_all_measurements(repo_root,tmp_path): - import xml.etree.ElementTree as ET - gate = runpy.run_path(str(repo_root/"scripts/release_check.py")) - path = _synthetic_evidence(repo_root,tmp_path) - assert gate["verify_step10_evidence"](path)["frozen_cases_executed"] == 20 - tree = ET.parse(path) - suite = tree.getroot().find("testsuite") - suite.remove(suite.find("testcase")) - tree.write(path) - with pytest.raises(ValueError): - gate["verify_step10_evidence"](path) - path = _synthetic_evidence(repo_root,tmp_path) - tree = ET.parse(path) - props = tree.getroot().find(".//properties") - props.remove(props.find("property")) - tree.write(path) - with pytest.raises(ValueError): - gate["verify_step10_evidence"](path) - - -@pytest.mark.parametrize("path",[ - "src/recursive_integrity_toolkit/metrics/diversity.py", - "src/recursive_integrity_toolkit/models.py", - "src/recursive_integrity_toolkit/io/validation.py", - "src/recursive_integrity_toolkit/cli.py", - "tests/unit/test_T1_compatibility.py", - "tests/golden/phase3_math_cases.json", - "tests/golden/phase3_math_cases.md", - "PHASE_3_PLAN.md","PHASE_2_COMPLETION.md", - "schemas/report.schema.json","examples/hero/EXPECTED_OUTPUTS.md", -]) -def test_phase3_step11_rejects_unopened_changes(repo_root,path): - gate = runpy.run_path(str(repo_root/"scripts/release_check.py")) - with pytest.raises(ValueError): - gate["verify_phase3_changes"]([("M",path)],incremental=True) - - -def test_phase3_step11_control_and_new_files_are_fixed(repo_root): - gate = runpy.run_path(str(repo_root/"scripts/release_check.py")) - control = json.loads((repo_root/"PHASE_3_BASELINE.json").read_bytes()) - gate["verify_phase3_control"](control,11) - assert gate["PHASE3_STEP11_NEW"] == {"PHASE_3_COMPLETION.md","PHASE_3_VALIDATION_REPORT.md","PHASE_3_ARCHITECTURE_COMPLIANCE_REPORT.md"} - assert gate["STEP11_TEST_EXCEPTIONS"] == {"tests/unit/test_phase3_contracts.py","tests/integration/test_phase3_metric_pipeline.py"} - for key,value in (("active_step",12),("phase_complete",not control["phase_complete"]), - ("previous_step_commit","0"*40),("next_step_authorized",True), - ("main_merge_authorized",True),("publication_authorized",True)): - with pytest.raises(ValueError): - gate["verify_phase3_control"](dict(control,**{key:value}),11) - for status,path in (("A","src/new.py"),("A","tests/unit/test_phase3_contracts.py"), - ("D","README.md"),("R100","docs/privacy.md")): - with pytest.raises(ValueError): - gate["verify_phase3_changes"]([(status,path)],incremental=True) - - -@pytest.mark.parametrize("path",[ - "src/recursive_integrity_toolkit/__init__.py", - "src/recursive_integrity_toolkit/metrics/resampling.py", - "pyproject.toml","tests/golden/phase3_math_cases.json", -]) -def test_phase3_step11_rejects_changes_beyond_version_literals(repo_root,tmp_path,path,phase3_final_snapshot): - repo_root = phase3_final_snapshot - import shutil - gate = runpy.run_path(str(repo_root/"scripts/release_check.py")) - shutil.copytree(repo_root/"src/recursive_integrity_toolkit",tmp_path/"src/recursive_integrity_toolkit",ignore=shutil.ignore_patterns("__pycache__")) - for name in gate["STEP10_FROZEN_FILES"]: - target = tmp_path/name - target.parent.mkdir(parents=True,exist_ok=True) - target.write_bytes((repo_root/name).read_bytes()) - assert gate["verify_step11_snapshot"](tmp_path)["approved_version_literals"] == 2 - target = tmp_path/path - original = target.read_bytes() - for altered in (original+b"\n", original.replace(b"0.1.0.dev2",b"0.1.0.dev3")): - if altered == original: - continue - target.write_bytes(altered) - with pytest.raises(ValueError): - gate["verify_step11_snapshot"](tmp_path) - - -def test_phase3_step11_test_exceptions_are_exact(repo_root,phase3_step10_snapshot,phase3_final_snapshot): - repo_root = phase3_final_snapshot - gate = runpy.run_path(str(repo_root/"scripts/release_check.py")) - unit = "tests/unit/test_phase3_contracts.py" - pipeline = "tests/integration/test_phase3_metric_pipeline.py" - for name,verify in ((unit,gate["verify_step11_contract_test_migration"]), - (pipeline,gate["verify_step11_pipeline_test_migration"])): - before = (phase3_step10_snapshot/name).read_bytes() - current = (repo_root/name).read_bytes() - verify(before,current) - with pytest.raises(ValueError): - verify(before,b"# unauthorized inherited edit\n"+current) - with pytest.raises(ValueError): - verify(before,current+b"\nFORGED_TEST = True\n") diff --git a/tests/integration/test_phase4_gates.py b/tests/integration/test_phase4_gates.py deleted file mode 100644 index f2bbfac..0000000 --- a/tests/integration/test_phase4_gates.py +++ /dev/null @@ -1,1862 +0,0 @@ -"""Current Phase 4 governance, immutable-source mutations and active workflows.""" -import hashlib -import runpy -import shutil -import subprocess -import sys - -import pytest - - -@pytest.fixture(scope="module") -def phase4_gate_tools(repo_root): - return runpy.run_path(str(repo_root / "scripts/release_check.py"), run_name="phase4_gate_tests") - - -@pytest.fixture(scope="module") -def phase4_mutation_tree(repo_root, tmp_path_factory, phase4_gate_tools, phase4_step1_snapshot): - """One source copy; each individual mutation is restored in a finally block.""" - root = tmp_path_factory.mktemp("phase4-active-mutations") - paths = {path.relative_to(phase4_step1_snapshot).as_posix() - for path in phase4_step1_snapshot.rglob("*") if path.is_file()} - paths.update(phase4_gate_tools["PHASE4_STEP1_NEW"]) - for relative in sorted(paths): - destination = root / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(phase4_step1_snapshot / relative, destination) - return root - - -def test_phase4_current_tree_passes_governance_snapshot(phase4_gate_tools, phase4_mutation_tree): - phase4_gate_tools["verify_phase4_snapshot"](phase4_mutation_tree) - - -@pytest.mark.parametrize("relative", [ - "PROJECT_INSTRUCTIONS.md", - "SPEC_AUDIT.md", - "PHASE_3_BASELINE.json", - "PHASE_3_COMPLETION.md", - "PHASE_4_PLAN.md", - "pyproject.toml", - "schemas/report.schema.json", - "tests/golden/phase3_math_cases.json", - "examples/hero/records_v2.csv", - "examples/hero/EXPECTED_OUTPUTS.md", - "src/recursive_integrity_toolkit/models.py", - "src/recursive_integrity_toolkit/metrics/diversity.py", - "src/recursive_integrity_toolkit/io/validation.py", - "src/recursive_integrity_toolkit/reports/assembly.py", - "src/recursive_integrity_toolkit/cli.py", -]) -def test_phase4_snapshot_rejects_protected_byte_changes(phase4_gate_tools, phase4_mutation_tree, relative): - path = phase4_mutation_tree / relative - original = path.read_bytes() - # Exercise actual formula/oracle/dependency changes as well as any-byte - # preservation. These mutations never enter the source checkout. - replacements = { - "src/recursive_integrity_toolkit/metrics/diversity.py": - (b"diversity = 1.0 - concentration", b"diversity = 0.5 - concentration"), - "tests/golden/phase3_math_cases.json": - (b'"v2_support":5', b'"v2_support":3'), - "pyproject.toml": (b"numpy>=2.0", b"numpy>=3.0"), - "examples/hero/records_v2.csv": (b"v2_08,v2,", b"v2_99,v2,"), - } - if relative in replacements: - before, after = replacements[relative] - assert original.count(before) == 1 - changed = original.replace(before, after, 1) - else: - changed = original + b"\nunauthorized Phase 4 Step 1 mutation\n" - try: - path.write_bytes(changed) - with pytest.raises(ValueError): - phase4_gate_tools["verify_phase4_snapshot"](phase4_mutation_tree) - finally: - path.write_bytes(original) - - -@pytest.mark.parametrize("relative", [ - "PHASE_4_COMPLETION.md", "PHASE_4_VALIDATION_REPORT.md", - "PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md", "report.json", "report.md", - "src/recursive_integrity_toolkit/reports/unauthorized.py", -]) -def test_phase4_snapshot_rejects_premature_outputs_and_new_runtime(phase4_gate_tools, phase4_mutation_tree, relative): - path = phase4_mutation_tree / relative - assert not path.exists() - try: - path.write_text("Unauthorized later-stage artifact\n", encoding="utf-8") - with pytest.raises(ValueError): - phase4_gate_tools["verify_phase4_snapshot"](phase4_mutation_tree) - finally: - path.unlink() - - -def test_phase4_snapshot_rejects_missing_protected_runtime(phase4_gate_tools, phase4_mutation_tree): - path = phase4_mutation_tree / "src/recursive_integrity_toolkit/metrics/bounds.py" - original = path.read_bytes() - try: - path.unlink() - with pytest.raises(ValueError): - phase4_gate_tools["verify_phase4_snapshot"](phase4_mutation_tree) - finally: - path.write_bytes(original) - - -def test_phase4_current_snapshot_enforces_migrated_historical_assertions(phase4_gate_tools, phase4_mutation_tree): - path = phase4_mutation_tree / "tests/integration/test_cli_validation.py" - original = path.read_bytes() - assertion = b"assert result.returncode == 0, result.stderr" - assert assertion in original - try: - path.write_bytes(original.replace(assertion, b"assert True", 1)) - with pytest.raises(ValueError): - phase4_gate_tools["verify_phase4_snapshot"](phase4_mutation_tree) - finally: - path.write_bytes(original) - - -def test_phase4_final_snapshot_preserves_all_current_runtime_bytes(repo_root, phase3_final_snapshot, phase4_step1_snapshot): - repo_root = phase4_step1_snapshot - package = "src/recursive_integrity_toolkit" - current = {path.relative_to(repo_root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (repo_root / package).rglob("*.py")} - frozen = {path.relative_to(phase3_final_snapshot).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (phase3_final_snapshot / package).rglob("*.py")} - assert len(current) == len(frozen) == 40 - assert current == frozen - - -def test_phase4_active_workflows_preserve_full_matrix_and_use_current_dispatch(repo_root, phase4_step1_snapshot): - repo_root = phase4_step1_snapshot - names = {"ci.yml", "golden.yml", "security.yml", "release.yml"} - root = repo_root / ".github/workflows" - assert {path.name for path in root.glob("*.yml")} == names - for name in sorted(names): - text = (root / name).read_text(encoding="utf-8") - assert "--phase 4 --step 1" in text, name - assert "--phase 3 --step 11" not in text, name - assert "permissions:\n contents: read" in text - assert "persist-credentials: false" in text - assert "timeout-minutes:" in text and "set -euo pipefail" in text - assert "actions/upload-artifact@v4" in text - assert "if-no-files-found: error" in text - for forbidden in ("continue-on-error:", "|| true", "contents: write", "id-token: write", "twine upload", "git push"): - assert forbidden not in text, (name, forbidden) - ci = (root / "ci.yml").read_text(encoding="utf-8") - for required in ('os: [ubuntu-latest, windows-latest]', 'python-version: ["3.11", "3.12"]', - 'dependencies: [current, minimum]', '"numpy==2.0.0" "pandas==2.2.2"', - 'RIT_TEST_PARQUET: "0"', 'RIT_TEST_PARQUET: "1"', '--require-parquet', - '--baseline-evidence', 'python -m pip check'): - assert required in ci - assert ci.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in ci - release = (root / "release.yml").read_text(encoding="utf-8") - assert "python -m build" in release and "python -m twine check --strict" in release - assert "--dist" in release and "--candidate" in release - - -def test_phase4_step1_keeps_current_help_and_future_commands_unopened(subprocess_env, tmp_path, phase4_step6_snapshot): - subprocess_env = dict(subprocess_env, PYTHONPATH=str(phase4_step6_snapshot / "src")) - before = set(tmp_path.iterdir()) - result = subprocess.run( - [sys.executable, "-m", "recursive_integrity_toolkit", "--help"], - cwd=tmp_path, env=subprocess_env, text=True, capture_output=True, check=False, - ) - assert result.returncode == 0 and "version" in result.stdout - assert "Analytical audit functionality is not implemented" in " ".join(result.stdout.split()) - rejected = subprocess.run( - [sys.executable, "-m", "recursive_integrity_toolkit", "audit"], - cwd=tmp_path, env=subprocess_env, text=True, capture_output=True, check=False, - ) - assert rejected.returncode == 2 - assert set(tmp_path.iterdir()) == before - - -def test_phase4_step1_has_no_completion_records_or_reports(repo_root, phase4_step1_snapshot): - repo_root = phase4_step1_snapshot - for relative in ("PHASE_4_COMPLETION.md", "PHASE_4_VALIDATION_REPORT.md", - "PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md", "report.json", "report.md"): - assert not (repo_root / relative).exists(), relative - - -def test_phase4_step2_current_runtime_opens_only_the_canonical_result(repo_root, phase3_final_snapshot, phase4_step2_snapshot): - repo_root = phase4_step2_snapshot - package = "src/recursive_integrity_toolkit" - current = {path.relative_to(repo_root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (repo_root / package).rglob("*.py")} - frozen = {path.relative_to(phase3_final_snapshot).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (phase3_final_snapshot / package).rglob("*.py")} - assert len(current) == len(frozen) == 40 - assert current.keys() == frozen.keys() - opened = "src/recursive_integrity_toolkit/result.py" - assert current[opened] != frozen[opened] - assert {path: digest for path, digest in current.items() if path != opened} == { - path: digest for path, digest in frozen.items() if path != opened - } - - -def test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step2_snapshot): - repo_root = phase4_step2_snapshot - names = {"ci.yml", "golden.yml", "security.yml", "release.yml"} - root = repo_root / ".github/workflows" - assert {path.name for path in root.glob("*.yml")} == names - for name in sorted(names): - text = (root / name).read_text(encoding="utf-8") - assert "Phase 4 Step 2" in text, name - assert "--phase 4 --step 2" in text, name - assert "--phase 4 --step 1" not in text, name - assert "--phase 3 --step 11" not in text, name - assert "permissions:\n contents: read" in text - assert "persist-credentials: false" in text - assert "timeout-minutes:" in text and "set -euo pipefail" in text - assert "actions/upload-artifact@v4" in text - assert "if-no-files-found: error" in text - for line in text.splitlines(): - if "python scripts/release_check.py" in line: - assert "--phase 4 --step 2" in line, (name, line) - for forbidden in ("continue-on-error:", "|| true", "contents: write", "id-token: write", "twine upload", "git push"): - assert forbidden not in text, (name, forbidden) - ci = (root / "ci.yml").read_text(encoding="utf-8") - for required in ('os: [ubuntu-latest, windows-latest]', 'python-version: ["3.11", "3.12"]', - 'dependencies: [current, minimum]', '"numpy==2.0.0" "pandas==2.2.2"', - 'RIT_TEST_PARQUET: "0"', 'RIT_TEST_PARQUET: "1"', '--require-parquet', - '--baseline-evidence', 'python -m pip check', - '--minimum-tests 2584', '--minimum-tests 2587', - "find_spec('pyarrow') is None", 'import pyarrow'): - assert required in ci - assert ci.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in ci - golden = (root / "golden.yml").read_text(encoding="utf-8") - assert "tests/golden/test_phase3_math.py" in golden - assert "tests/integration/test_hero_structure.py" in golden - assert "tests/integration/test_phase3_metric_pipeline.py" in golden - security = (root / "security.yml").read_text(encoding="utf-8") - for required in ('tests/integration/test_no_network.py', 'tests/integration/test_optional_dependency.py', - 'tests/unit/test_PR012_evidence_classes.py', 'tests/unit/test_PR013_report_schema.py', - 'tests/unit/test_phase4_contracts.py', 'tests/integration/test_phase4_gates.py'): - assert required in security - release = (root / "release.yml").read_text(encoding="utf-8") - assert "python -m build" in release and "python -m twine check --strict" in release - assert "--dist" in release and "--candidate" in release - assert "--delivery" not in release - assert "recursive-integrity-toolkit-phase4-step2-candidate" in release - assert "rit-phase4-step1" not in release - assert release.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - - -@pytest.mark.parametrize("mutation", [ - "unchanged", "formula", "math_oracle", "frozen_model", "frozen_schema", - "hero", "dependency", "plan", "assembly", "cli", "unknown_runtime", - "missing_runtime", "premature_completion", "premature_report", - "control_step", "control_scope", "control_schema_scope", - "schema_open_root", "schema_wrong_dialect", "model_computation_import", - "historical_assertion", "historical_tooling", -]) -def test_phase4_step2_current_snapshot_checks_valid_tree_before_mutations(repo_root, tmp_path, phase4_gate_tools, mutation, phase4_step2_snapshot): - import json - - paths = {path.relative_to(phase4_step2_snapshot).as_posix() - for path in phase4_step2_snapshot.rglob("*") if path.is_file()} - assert len(paths) == 227 - for relative in sorted(paths): - destination = tmp_path / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(phase4_step2_snapshot / relative, destination) - verify = phase4_gate_tools["verify_phase4_step2_snapshot"] - baseline = verify(tmp_path) - assert baseline["frozen_runtime_modules"] == 39 - assert baseline["frozen_schemas"] == 4 - assert baseline["result_contracts_enabled"] is True - assert baseline["adapters_enabled"] is False - assert baseline["cli_analysis_enabled"] is False - assert baseline["phase_complete"] is False - if mutation == "unchanged": - return - replacements = { - "formula": ("src/recursive_integrity_toolkit/metrics/diversity.py", - b"diversity = 1.0 - concentration", b"diversity = 0.5 - concentration"), - "math_oracle": ("tests/golden/phase3_math_cases.json", b'"v2_support":5', b'"v2_support":3'), - "hero": ("examples/hero/records_v2.csv", b"v2_08,v2,", b"v2_99,v2,"), - "dependency": ("pyproject.toml", b"numpy>=2.0", b"numpy>=3.0"), - "historical_assertion": ("tests/unit/test_phase4_contracts.py", - b' assert phase4_control["active_step"] == 1\n', b" assert True\n"), - "historical_tooling": ("scripts/release_check.py", - b"def verify_phase4_control(control: dict, step: int = 1) -> None:", - b"def verify_phase4_control(control: dict, step: int = 1) -> None:\n return"), - } - appended = { - "frozen_model": "src/recursive_integrity_toolkit/models.py", - "frozen_schema": "schemas/config.schema.json", - "plan": "PHASE_4_PLAN.md", - "assembly": "src/recursive_integrity_toolkit/reports/assembly.py", - "cli": "src/recursive_integrity_toolkit/cli.py", - } - added = { - "unknown_runtime": "src/recursive_integrity_toolkit/reports/unauthorized.py", - "premature_completion": "PHASE_4_COMPLETION.md", - "premature_report": "report.json", - } - if mutation in replacements: - relative, original, replacement = replacements[mutation] - path = tmp_path / relative - source = path.read_bytes() - assert source.count(original) == 1 - path.write_bytes(source.replace(original, replacement, 1)) - elif mutation in appended: - path = tmp_path / appended[mutation] - path.write_bytes(path.read_bytes() + b"\nUnauthorized Step 2 mutation\n") - elif mutation in added: - path = tmp_path / added[mutation] - assert not path.exists() - path.write_text("Unauthorized later-stage file\n", encoding="utf-8") - elif mutation == "missing_runtime": - (tmp_path / "src/recursive_integrity_toolkit/metrics/bounds.py").unlink() - elif mutation.startswith("control_"): - path = tmp_path / "PHASE_4_BASELINE.json" - control = json.loads(path.read_text(encoding="utf-8")) - if mutation == "control_step": - control["active_step"] = 3 - elif mutation == "control_scope": - control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/cli.py") - else: - control["schema_paths_authorized"].append("schemas/config.schema.json") - path.write_text(json.dumps(control), encoding="utf-8") - elif mutation.startswith("schema_"): - path = tmp_path / "schemas/report.schema.json" - schema = json.loads(path.read_text(encoding="utf-8")) - if mutation == "schema_open_root": - schema["additionalProperties"] = True - else: - schema["$schema"] = "http://json-schema.org/draft-07/schema#" - path.write_text(json.dumps(schema), encoding="utf-8") - elif mutation == "model_computation_import": - path = tmp_path / "src/recursive_integrity_toolkit/result.py" - path.write_bytes(path.read_bytes() + b"\nfrom .metrics.diversity import effective_state_diversity\n") - else: - raise AssertionError(f"Unknown active mutation case: {mutation}") - with pytest.raises(ValueError): - verify(tmp_path) - - -def test_phase4_step2_junit_dispatch_accepts_valid_subsets_and_rejects_bad_evidence(repo_root, tmp_path): - import json - - junit = tmp_path / "subset.xml" - command = [sys.executable, str(repo_root / "scripts/release_check.py"), - "--phase", "4", "--step", "2", "--junit", str(junit)] - valid = ('' - '' - '') - junit.write_text(valid, encoding="utf-8") - passed = subprocess.run(command, cwd=tmp_path, text=True, capture_output=True, check=False) - assert passed.returncode == 0, passed.stderr - assert json.loads(passed.stdout)["tests"] == 1 - assert json.loads(passed.stdout)["passed"] == 1 - too_small = subprocess.run(command + ["--minimum-tests", "2584"], cwd=tmp_path, - text=True, capture_output=True, check=False) - assert too_small.returncode != 0 - assert "Insufficient or duplicate test cases" in too_small.stderr - for outcome in ("failure", "skipped"): - invalid = ('' - '' - f'<{outcome} message="independent rejection case" />' - '') - junit.write_text(invalid, encoding="utf-8") - rejected = subprocess.run(command, cwd=tmp_path, text=True, capture_output=True, check=False) - assert rejected.returncode != 0 - assert "Failed, errored or skipped test" in rejected.stderr - - -def test_phase4_step3_current_runtime_opens_only_assembly_and_freezes_schema(repo_root, phase4_step2_snapshot, phase4_step3_snapshot): - repo_root = phase4_step3_snapshot - package = "src/recursive_integrity_toolkit" - current = {path.relative_to(repo_root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (repo_root / package).rglob("*.py")} - frozen = {path.relative_to(phase4_step2_snapshot).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (phase4_step2_snapshot / package).rglob("*.py")} - assert len(current) == len(frozen) == 40 - assert current.keys() == frozen.keys() - opened = "src/recursive_integrity_toolkit/reports/assembly.py" - assert current[opened] != frozen[opened] - assert {path: digest for path, digest in current.items() if path != opened} == { - path: digest for path, digest in frozen.items() if path != opened - } - schemas = {path.name: path.read_bytes() for path in (repo_root / "schemas").glob("*.json")} - assert len(schemas) == 5 - assert schemas == {path.name: path.read_bytes() for path in (phase4_step2_snapshot / "schemas").glob("*.json")} - - -def test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step3_snapshot): - repo_root = phase4_step3_snapshot - names = {"ci.yml", "golden.yml", "security.yml", "release.yml"} - root = repo_root / ".github/workflows" - assert {path.name for path in root.glob("*.yml")} == names - for name in sorted(names): - text = (root / name).read_text(encoding="utf-8") - assert "Phase 4 Step 3" in text, name - assert "--phase 4 --step 3" in text, name - assert "--phase 4 --step 2" not in text, name - assert "--phase 4 --step 1" not in text, name - assert "--phase 3 --step 11" not in text, name - assert "permissions:\n contents: read" in text - assert "persist-credentials: false" in text - assert "timeout-minutes:" in text and "set -euo pipefail" in text - assert "actions/upload-artifact@v4" in text - assert "if-no-files-found: error" in text - for line in text.splitlines(): - if "python scripts/release_check.py" in line: - assert "--phase 4 --step 3" in line, (name, line) - for forbidden in ("continue-on-error:", "|| true", "contents: write", "id-token: write", "twine upload", "git push"): - assert forbidden not in text, (name, forbidden) - ci = (root / "ci.yml").read_text(encoding="utf-8") - for required in ('os: [ubuntu-latest, windows-latest]', 'python-version: ["3.11", "3.12"]', - 'dependencies: [current, minimum]', '"numpy==2.0.0" "pandas==2.2.2"', - 'RIT_TEST_PARQUET: "0"', 'RIT_TEST_PARQUET: "1"', '--require-parquet', - '--baseline-evidence', 'python -m pip check', - '--minimum-tests 2728', '--minimum-tests 2731', - "find_spec('pyarrow') is None", 'import pyarrow'): - assert required in ci - assert ci.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in ci - golden = (root / "golden.yml").read_text(encoding="utf-8") - assert "tests/golden/test_phase3_math.py" in golden - assert "tests/integration/test_hero_structure.py" in golden - assert "tests/integration/test_phase3_metric_pipeline.py" in golden - security = (root / "security.yml").read_text(encoding="utf-8") - for required in ('tests/integration/test_no_network.py', 'tests/integration/test_optional_dependency.py', - 'tests/unit/test_PR012_evidence_classes.py', 'tests/unit/test_PR013_report_schema.py', - 'tests/unit/test_PR014_unavailable.py', 'tests/unit/test_PR018_language.py', - 'tests/integration/test_partial_provenance_report.py', - 'tests/integration/test_partial_lineage_report.py', - 'tests/unit/test_phase4_contracts.py', 'tests/integration/test_phase4_gates.py'): - assert required in security - release = (root / "release.yml").read_text(encoding="utf-8") - assert "python -m build" in release and "python -m twine check --strict" in release - assert "--dist" in release and "--candidate" in release - assert "--delivery" not in release - assert "recursive-integrity-toolkit-phase4-step3-candidate" in release - assert "rit-phase4-step2" not in release - assert "--minimum-tests 2728" in release and "--minimum-tests 2731" in release - assert release.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - - -@pytest.mark.parametrize("mutation", [ - "unchanged", "formula", "math_oracle", "frozen_model", "frozen_result", "frozen_schema", - "hero", "dependency", "plan", "cli", "renderer", "unknown_runtime", "missing_runtime", - "premature_completion", "premature_report", "control_step", "control_scope", "control_schema_scope", - "schema_open_root", "historical_assertion", "historical_tooling", "historical_binding", "historical_document", - "assembly_metric_import", "assembly_calculation_import", "assembly_numeric_import", - "assembly_network_import", "assembly_file_io", "assembly_dynamic_execution", "assembly_new_formula", -]) -def test_phase4_step3_current_snapshot_checks_valid_tree_before_mutations(repo_root, tmp_path, phase4_gate_tools, mutation, phase4_step3_snapshot): - import json - - paths = {path.relative_to(phase4_step3_snapshot).as_posix() - for path in phase4_step3_snapshot.rglob("*") if path.is_file()} - assert len(paths) == 227 - for relative in sorted(paths): - destination = tmp_path / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(phase4_step3_snapshot / relative, destination) - verify = phase4_gate_tools["verify_phase4_step3_snapshot"] - baseline = verify(tmp_path) - assert baseline["package_modules"] == 40 - assert baseline["frozen_runtime_modules"] == 39 - assert baseline["frozen_schemas"] == 5 - assert baseline["result_contracts_enabled"] is True - assert baseline["adapters_enabled"] is True - assert baseline["cli_analysis_enabled"] is False - assert baseline["phase_complete"] is False - if mutation == "unchanged": - return - replacements = { - "formula": ("src/recursive_integrity_toolkit/metrics/diversity.py", - b"diversity = 1.0 - concentration", b"diversity = 0.5 - concentration"), - "math_oracle": ("tests/golden/phase3_math_cases.json", b'"v2_support":5', b'"v2_support":3'), - "hero": ("examples/hero/records_v2.csv", b"v2_08,v2,", b"v2_99,v2,"), - "dependency": ("pyproject.toml", b"numpy>=2.0", b"numpy>=3.0"), - "historical_assertion": ("tests/unit/test_phase4_contracts.py", - b' assert control["active_phase"] == 4 and control["active_step"] == 2\n', - b" assert True\n"), - "historical_tooling": ("scripts/release_check.py", - b"def verify_phase4_step2_control(control: dict, step: int = 2) -> None:", - b"def verify_phase4_step2_control(control: dict, step: int = 2) -> None:\n return"), - "historical_binding": ("tests/integration/test_phase4_gates.py", - b"def test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step2_snapshot):\n repo_root = phase4_step2_snapshot\n", - b"def test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step2_snapshot):\n"), - "historical_document": ("docs/report_schema.md", b"# Report Schema\n", b"# Modified Report Schema\n"), - } - appended = { - "frozen_model": "src/recursive_integrity_toolkit/models.py", - "frozen_result": "src/recursive_integrity_toolkit/result.py", - "frozen_schema": "schemas/config.schema.json", - "plan": "PHASE_4_PLAN.md", - "cli": "src/recursive_integrity_toolkit/cli.py", - "renderer": "src/recursive_integrity_toolkit/reports/json_report.py", - } - added = { - "unknown_runtime": "src/recursive_integrity_toolkit/reports/unauthorized.py", - "premature_completion": "PHASE_4_COMPLETION.md", - "premature_report": "report.json", - } - assembly_injections = { - "assembly_metric_import": b"\nfrom ..metrics.diversity import effective_state_diversity\n", - "assembly_calculation_import": b"\nfrom ..metrics import coverage\n", - "assembly_numeric_import": b"\nimport numpy\n", - "assembly_network_import": b"\nimport socket\n", - "assembly_file_io": b"\nopen('unapproved.txt', 'w').write('data')\n", - "assembly_dynamic_execution": b"\nexec('forged = True')\n", - "assembly_new_formula": b"\ndef invented_metric(values):\n return sum(value * value for value in values)\n", - } - if mutation in replacements: - relative, original, replacement = replacements[mutation] - path = tmp_path / relative - source = path.read_bytes() - assert source.count(original) == 1 - path.write_bytes(source.replace(original, replacement, 1)) - elif mutation in appended: - path = tmp_path / appended[mutation] - path.write_bytes(path.read_bytes() + b"\nUnauthorized Step 3 mutation\n") - elif mutation in added: - path = tmp_path / added[mutation] - assert not path.exists() - path.write_text("Unauthorized later-stage file\n", encoding="utf-8") - elif mutation == "missing_runtime": - (tmp_path / "src/recursive_integrity_toolkit/metrics/bounds.py").unlink() - elif mutation.startswith("control_"): - path = tmp_path / "PHASE_4_BASELINE.json" - control = json.loads(path.read_text(encoding="utf-8")) - if mutation == "control_step": - control["active_step"] = 4 - elif mutation == "control_scope": - control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/result.py") - else: - control["schema_changes_authorized"] = True - control["schema_paths_authorized"].append("schemas/report.schema.json") - path.write_text(json.dumps(control), encoding="utf-8") - elif mutation == "schema_open_root": - path = tmp_path / "schemas/report.schema.json" - schema = json.loads(path.read_text(encoding="utf-8")) - schema["additionalProperties"] = True - path.write_text(json.dumps(schema), encoding="utf-8") - elif mutation in assembly_injections: - path = tmp_path / "src/recursive_integrity_toolkit/reports/assembly.py" - path.write_bytes(path.read_bytes() + assembly_injections[mutation]) - else: - raise AssertionError(f"Unknown active mutation case: {mutation}") - with pytest.raises(ValueError): - verify(tmp_path) - - -def test_phase4_step3_junit_dispatch_accepts_valid_subsets_and_rejects_bad_evidence(repo_root, tmp_path): - import json - - junit = tmp_path / "subset.xml" - command = [sys.executable, str(repo_root / "scripts/release_check.py"), - "--phase", "4", "--step", "3", "--junit", str(junit)] - valid = ('' - '' - '') - junit.write_text(valid, encoding="utf-8") - passed = subprocess.run(command, cwd=tmp_path, text=True, capture_output=True, check=False) - assert passed.returncode == 0, passed.stderr - assert json.loads(passed.stdout)["tests"] == 1 - assert json.loads(passed.stdout)["passed"] == 1 - too_small = subprocess.run(command + ["--minimum-tests", "2728"], cwd=tmp_path, - text=True, capture_output=True, check=False) - assert too_small.returncode != 0 - assert "Insufficient or duplicate test cases" in too_small.stderr - for outcome in ("failure", "skipped"): - invalid = ('' - '' - f'<{outcome} message="independent rejection case" />' - '') - junit.write_text(invalid, encoding="utf-8") - rejected = subprocess.run(command, cwd=tmp_path, text=True, capture_output=True, check=False) - assert rejected.returncode != 0 - assert "Failed, errored or skipped test" in rejected.stderr - - -def test_phase4_step4_current_runtime_opens_only_privacy_metadata_modules_and_freezes_schema(repo_root, phase4_step3_snapshot, phase4_step4_snapshot): - repo_root = phase4_step4_snapshot - package = "src/recursive_integrity_toolkit" - current = {path.relative_to(repo_root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (repo_root / package).rglob("*.py")} - frozen = {path.relative_to(phase4_step3_snapshot).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (phase4_step3_snapshot / package).rglob("*.py")} - assert len(current) == len(frozen) == 40 - assert current.keys() == frozen.keys() - opened = {package + "/" + relative for relative in ( - "result.py", "reports/assembly.py", "config.py", "utils/hashing.py", "utils/logging.py", - )} - assert {path for path in current if current[path] != frozen[path]} == opened - assert {path: digest for path, digest in current.items() if path not in opened} == { - path: digest for path, digest in frozen.items() if path not in opened - } - schemas = {path.name: path.read_bytes() for path in (repo_root / "schemas").glob("*.json")} - assert len(schemas) == 5 - assert schemas == {path.name: path.read_bytes() for path in (phase4_step3_snapshot / "schemas").glob("*.json")} - - -def test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step4_snapshot): - repo_root = phase4_step4_snapshot - names = {"ci.yml", "golden.yml", "security.yml", "release.yml"} - root = repo_root / ".github/workflows" - assert {path.name for path in root.glob("*.yml")} == names - for name in sorted(names): - text = (root / name).read_text(encoding="utf-8") - assert "Phase 4 Step 4" in text, name - assert "--phase 4 --step 4" in text, name - for older in ("--phase 4 --step 3", "--phase 4 --step 2", "--phase 4 --step 1", "--phase 3 --step 11"): - assert older not in text, (name, older) - assert "permissions:\n contents: read" in text - assert "persist-credentials: false" in text - assert "timeout-minutes:" in text and "set -euo pipefail" in text - assert "actions/upload-artifact@v4" in text and "if-no-files-found: error" in text - for line in text.splitlines(): - if "python scripts/release_check.py" in line: - assert "--phase 4 --step 4" in line, (name, line) - for forbidden in ("continue-on-error:", "|| true", "contents: write", "id-token: write", "twine upload", "git push"): - assert forbidden not in text, (name, forbidden) - ci = (root / "ci.yml").read_text(encoding="utf-8") - for required in ('os: [ubuntu-latest, windows-latest]', 'python-version: ["3.11", "3.12"]', - 'dependencies: [current, minimum]', '"numpy==2.0.0" "pandas==2.2.2"', - 'RIT_TEST_PARQUET: "0"', 'RIT_TEST_PARQUET: "1"', '--require-parquet', - '--baseline-evidence', 'python -m pip check', - '--minimum-tests 2876', '--minimum-tests 2879', - "find_spec('pyarrow') is None", 'import pyarrow'): - assert required in ci - assert ci.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in ci - golden = (root / "golden.yml").read_text(encoding="utf-8") - for required in ("tests/golden/test_phase3_math.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_phase3_metric_pipeline.py"): - assert required in golden - security = (root / "security.yml").read_text(encoding="utf-8") - for required in ('tests/integration/test_no_network.py', 'tests/integration/test_optional_dependency.py', - 'tests/unit/test_PR012_evidence_classes.py', 'tests/unit/test_PR013_report_schema.py', - 'tests/unit/test_PR014_unavailable.py', 'tests/unit/test_PR015_redaction.py', - 'tests/unit/test_PR016_determinism.py', 'tests/unit/test_PR018_language.py', - 'tests/integration/test_partial_provenance_report.py', - 'tests/integration/test_partial_lineage_report.py', - 'tests/unit/test_phase4_contracts.py', 'tests/integration/test_phase4_gates.py'): - assert required in security - release = (root / "release.yml").read_text(encoding="utf-8") - assert "python -m build" in release and "python -m twine check --strict" in release - assert "--dist" in release and "--candidate" in release - assert "--delivery" not in release - assert "recursive-integrity-toolkit-phase4-step4-candidate" in release - assert "rit-phase4-step3" not in release - assert "--minimum-tests 2876" in release and "--minimum-tests 2879" in release - assert release.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - - -@pytest.mark.parametrize("mutation", [ - "unchanged", "formula", "math_oracle", "frozen_model", "frozen_schema", "hero", "dependency", "plan", - "cli", "renderer", "output_paths", "unknown_runtime", "missing_runtime", "premature_completion", "premature_report", - "control_step", "control_scope", "control_schema_scope", "historical_assertion", "historical_tooling", - "historical_binding", "historical_document", "input_hash_helper", "config_resolver_helper", -]) -def test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations(repo_root, tmp_path, phase4_gate_tools, mutation, phase4_step4_snapshot): - import json - - paths = {path.relative_to(phase4_step4_snapshot).as_posix() - for path in phase4_step4_snapshot.rglob("*") if path.is_file()} - assert len(paths) == 227 - for relative in sorted(paths): - destination = tmp_path / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(phase4_step4_snapshot / relative, destination) - verify = phase4_gate_tools["verify_phase4_step4_snapshot"] - baseline = verify(tmp_path) - assert baseline["package_modules"] == 40 - assert baseline["frozen_runtime_modules"] == 35 - assert baseline["frozen_schemas"] == 5 - assert baseline["result_contracts_enabled"] is True - assert baseline["adapters_enabled"] is True - assert baseline["privacy_views_enabled"] is True - assert baseline["cli_analysis_enabled"] is False - assert baseline["phase_complete"] is False - if mutation == "unchanged": - return - replacements = { - "formula": ("src/recursive_integrity_toolkit/metrics/diversity.py", - b"diversity = 1.0 - concentration", b"diversity = 0.5 - concentration"), - "math_oracle": ("tests/golden/phase3_math_cases.json", b'"v2_support":5', b'"v2_support":3'), - "hero": ("examples/hero/records_v2.csv", b"v2_08,v2,", b"v2_99,v2,"), - "dependency": ("pyproject.toml", b"numpy>=2.0", b"numpy>=3.0"), - "historical_assertion": ("tests/unit/test_phase4_contracts.py", - b' assert control["active_phase"] == 4 and control["active_step"] == 3\n', - b" assert True\n"), - "historical_tooling": ("scripts/release_check.py", - b"def verify_phase4_step3_control(control: dict, step: int = 3) -> None:", - b"def verify_phase4_step3_control(control: dict, step: int = 3) -> None:\n return"), - "historical_binding": ("tests/integration/test_phase4_gates.py", - b"def test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step3_snapshot):\n repo_root = phase4_step3_snapshot\n", - b"def test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step3_snapshot):\n"), - "historical_document": ("docs/privacy.md", b"# Privacy and Local Input Boundaries\n", - b"# Unauthorized Privacy and Local Input Boundaries\n"), - "input_hash_helper": ("src/recursive_integrity_toolkit/utils/hashing.py", - b"return hashlib.sha256(data).hexdigest()", b"return hashlib.sha256(b'changed').hexdigest()"), - "config_resolver_helper": ("src/recursive_integrity_toolkit/config.py", - b"def resolve_config(", b"def resolve_config_disabled("), - } - appended = { - "frozen_model": "src/recursive_integrity_toolkit/models.py", - "frozen_schema": "schemas/report.schema.json", - "plan": "PHASE_4_PLAN.md", - "cli": "src/recursive_integrity_toolkit/cli.py", - "renderer": "src/recursive_integrity_toolkit/reports/json_report.py", - "output_paths": "src/recursive_integrity_toolkit/utils/paths.py", - } - added = { - "unknown_runtime": "src/recursive_integrity_toolkit/reports/unauthorized.py", - "premature_completion": "PHASE_4_COMPLETION.md", "premature_report": "report.json", - } - if mutation in replacements: - relative, original, replacement = replacements[mutation] - path = tmp_path / relative - source = path.read_bytes() - assert source.count(original) == 1 - path.write_bytes(source.replace(original, replacement, 1)) - elif mutation in appended: - path = tmp_path / appended[mutation] - path.write_bytes(path.read_bytes() + b"\nUnauthorized Step 4 mutation\n") - elif mutation in added: - path = tmp_path / added[mutation] - assert not path.exists() - path.write_text("Unauthorized later-stage file\n", encoding="utf-8") - elif mutation == "missing_runtime": - (tmp_path / "src/recursive_integrity_toolkit/metrics/bounds.py").unlink() - elif mutation.startswith("control_"): - path = tmp_path / "PHASE_4_BASELINE.json" - control = json.loads(path.read_text(encoding="utf-8")) - if mutation == "control_step": - control["active_step"] = 5 - elif mutation == "control_scope": - control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/json_report.py") - else: - control["schema_changes_authorized"] = True - control["schema_paths_authorized"].append("schemas/report.schema.json") - path.write_text(json.dumps(control), encoding="utf-8") - else: - raise AssertionError(f"Unknown active mutation case: {mutation}") - with pytest.raises(ValueError): - verify(tmp_path) - - -@pytest.mark.parametrize("relative,injection", [ - ("result.py", b"\nimport socket\n"), - ("result.py", b"\nfrom .metrics.diversity import effective_state_diversity\n"), - ("reports/assembly.py", b"\nimport numpy\n"), - ("reports/assembly.py", b"\ndef invented_metric(values):\n return sum(value * value for value in values)\n"), - ("reports/assembly.py", b"\nfrom .json_report import render_json\n"), - ("config.py", b"\nfrom .metrics import coverage\n"), - ("config.py", b"\nresolve_config = lambda *args, **kwargs: None\n"), - ("utils/hashing.py", b"\nimport urllib.request\n"), - ("utils/hashing.py", b"\ndef expose_secret(secret):\n print(secret)\n"), - ("utils/hashing.py", b"\nsha256_bytes = lambda data: '0' * 64\n"), - ("utils/logging.py", b"\nopen('private.txt', 'w').write('raw content')\n"), - ("utils/logging.py", b"\nexec('forged = True')\n"), - ("utils/logging.py", b"\ndef dynamic_call(value):\n return eval(value)\n"), - ("utils/logging.py", b"\ndef effect_at_definition(value=globals().clear()):\n pass\n"), - ("utils/logging.py", b"\n@print('raw content')\ndef decorated_effect():\n pass\n"), -]) -def test_phase4_step4_privacy_ast_checks_current_source_before_rejecting_scope_injection(repo_root, tmp_path, relative, injection, phase4_step5_snapshot, phase4_step6_snapshot): - checker = runpy.run_path(str(repo_root / "scripts/check_traceability.py"), run_name="phase4_step4_ast_tests") - verify = checker["phase4_step4_runtime_boundary"] - repository_path = "src/recursive_integrity_toolkit/" + relative - current = (phase4_step5_snapshot if relative in ("utils/logging.py", "reports/assembly.py") else phase4_step6_snapshot if relative == "config.py" else repo_root) / repository_path - verify(current, repository_path) - copy = tmp_path / "source.py" - copy.write_bytes(current.read_bytes() + injection) - with pytest.raises(ValueError): - verify(copy, repository_path) - - -def test_phase4_step4_junit_dispatch_accepts_valid_subsets_and_rejects_bad_evidence(repo_root, tmp_path): - import json - - junit = tmp_path / "subset.xml" - command = [sys.executable, str(repo_root / "scripts/release_check.py"), - "--phase", "4", "--step", "4", "--junit", str(junit)] - valid = ('' - '' - '') - junit.write_text(valid, encoding="utf-8") - passed = subprocess.run(command, cwd=tmp_path, text=True, capture_output=True, check=False) - assert passed.returncode == 0, passed.stderr - assert json.loads(passed.stdout)["tests"] == 1 - assert json.loads(passed.stdout)["passed"] == 1 - too_small = subprocess.run(command + ["--minimum-tests", "2876"], cwd=tmp_path, - text=True, capture_output=True, check=False) - assert too_small.returncode != 0 - assert "Insufficient or duplicate test cases" in too_small.stderr - for outcome in ("failure", "error", "skipped"): - invalid = ('' - '' - f'<{outcome} message="independent rejection case" />' - '') - junit.write_text(invalid, encoding="utf-8") - rejected = subprocess.run(command, cwd=tmp_path, text=True, capture_output=True, check=False) - assert rejected.returncode != 0 - assert "Failed, errored or skipped test" in rejected.stderr - - -def test_phase4_step5_current_runtime_opens_only_renderers_and_freezes_schema(repo_root, phase4_step4_snapshot, phase4_step5_snapshot): - repo_root = phase4_step5_snapshot - package = "src/recursive_integrity_toolkit" - current = {path.relative_to(repo_root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (repo_root / package).rglob("*.py")} - frozen = {path.relative_to(phase4_step4_snapshot).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (phase4_step4_snapshot / package).rglob("*.py")} - assert len(current) == len(frozen) == 40 - assert current.keys() == frozen.keys() - opened = {package + "/" + relative for relative in ( - "reports/json_report.py", "reports/markdown_report.py", - )} - assert {path for path in current if current[path] != frozen[path]} == opened - assert {path: digest for path, digest in current.items() if path not in opened} == { - path: digest for path, digest in frozen.items() if path not in opened - } - schemas = {path.name: path.read_bytes() for path in (repo_root / "schemas").glob("*.json")} - assert len(schemas) == 5 - assert schemas == {path.name: path.read_bytes() for path in (phase4_step4_snapshot / "schemas").glob("*.json")} - - -def test_phase4_step5_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step5_snapshot): - repo_root = phase4_step5_snapshot - names = {"ci.yml", "golden.yml", "security.yml", "release.yml"} - root = repo_root / ".github/workflows" - expected_timeouts = {"ci.yml": [60, 20], "golden.yml": [15], "security.yml": [20], "release.yml": [25]} - assert {path.name for path in root.glob("*.yml")} == names - for name in sorted(names): - text = (root / name).read_text(encoding="utf-8") - assert [int(line.split(":", 1)[1]) for line in text.splitlines() - if line.strip().startswith("timeout-minutes:")] == expected_timeouts[name] - assert "Phase 4 Step 5" in text, name - assert "--phase 4 --step 5" in text, name - for older in ("--phase 4 --step 4", "--phase 4 --step 3", "--phase 4 --step 2", "--phase 4 --step 1", "--phase 3 --step 11"): - assert older not in text, (name, older) - assert "permissions:\n contents: read" in text - assert "persist-credentials: false" in text - assert "timeout-minutes:" in text and "set -euo pipefail" in text - assert "actions/upload-artifact@v4" in text and "if-no-files-found: error" in text - for line in text.splitlines(): - if "python scripts/release_check.py" in line: - assert "--phase 4 --step 5" in line, (name, line) - for forbidden in ("continue-on-error:", "|| true", "contents: write", "id-token: write", "twine upload", "git push"): - assert forbidden not in text, (name, forbidden) - ci = (root / "ci.yml").read_text(encoding="utf-8") - for required in ('os: [ubuntu-latest, windows-latest]', 'python-version: ["3.11", "3.12"]', - 'dependencies: [current, minimum]', '"numpy==2.0.0" "pandas==2.2.2"', - 'RIT_TEST_PARQUET: "0"', 'RIT_TEST_PARQUET: "1"', '--require-parquet', - '--baseline-evidence', 'python -m pip check', - '--minimum-tests 3071', '--minimum-tests 3074', - "find_spec('pyarrow') is None", 'import pyarrow'): - assert required in ci - assert ci.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in ci - golden = (root / "golden.yml").read_text(encoding="utf-8") - for required in ("tests/golden/test_phase3_math.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_phase3_metric_pipeline.py"): - assert required in golden - security = (root / "security.yml").read_text(encoding="utf-8") - for required in ('tests/integration/test_no_network.py', 'tests/integration/test_optional_dependency.py', - 'tests/unit/test_PR012_evidence_classes.py', 'tests/unit/test_PR013_report_schema.py', - 'tests/unit/test_PR014_unavailable.py', 'tests/unit/test_PR015_redaction.py', - 'tests/unit/test_PR016_determinism.py', 'tests/unit/test_PR018_language.py', - 'tests/integration/test_partial_provenance_report.py', - 'tests/integration/test_partial_lineage_report.py', - 'tests/unit/test_phase4_contracts.py', 'tests/integration/test_phase4_gates.py'): - assert required in security - release = (root / "release.yml").read_text(encoding="utf-8") - assert "python -m build" in release and "python -m twine check --strict" in release - assert "--dist" in release and "--candidate" in release - assert "--delivery" not in release - assert "recursive-integrity-toolkit-phase4-step5-candidate" in release - assert "rit-phase4-step4" not in release - assert "--minimum-tests 3071" in release and "--minimum-tests 3074" in release - assert release.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - - -@pytest.mark.parametrize("mutation", [ - "unchanged", "formula", "math_oracle", "frozen_model", "frozen_schema", "hero", "dependency", "plan", - "cli", "html", "assembly", "result", "config", "hashing", "logging", "output_paths", "unknown_runtime", "missing_runtime", "premature_completion", "premature_report", - "control_step", "control_scope", "control_schema_scope", "historical_assertion", "historical_tooling", - "historical_binding", "historical_document", "input_hash_helper", "config_resolver_helper", -]) -def test_phase4_step5_current_snapshot_checks_valid_tree_before_mutations(repo_root, tmp_path, phase4_gate_tools, mutation, phase4_step5_snapshot): - import json - - paths = {path.relative_to(phase4_step5_snapshot).as_posix() - for path in phase4_step5_snapshot.rglob("*") if path.is_file()} - assert len(paths) == 227 - for relative in sorted(paths): - destination = tmp_path / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(phase4_step5_snapshot / relative, destination) - verify = phase4_gate_tools["verify_phase4_step5_snapshot"] - baseline = verify(tmp_path) - assert baseline["package_modules"] == 40 - assert baseline["frozen_runtime_modules"] == 38 - assert baseline["frozen_schemas"] == 5 - assert baseline["result_contracts_enabled"] is True - assert baseline["adapters_enabled"] is True - assert baseline["privacy_views_enabled"] is True - assert baseline["renderers_enabled"] is True - assert baseline["cli_analysis_enabled"] is False - assert baseline["phase_complete"] is False - if mutation == "unchanged": - return - replacements = { - "formula": ("src/recursive_integrity_toolkit/metrics/diversity.py", - b"diversity = 1.0 - concentration", b"diversity = 0.5 - concentration"), - "math_oracle": ("tests/golden/phase3_math_cases.json", b'"v2_support":5', b'"v2_support":3'), - "hero": ("examples/hero/records_v2.csv", b"v2_08,v2,", b"v2_99,v2,"), - "dependency": ("pyproject.toml", b"numpy>=2.0", b"numpy>=3.0"), - "historical_assertion": ("tests/unit/test_phase4_contracts.py", - b' assert control["active_phase"] == 4 and control["active_step"] == 4\n', - b" assert True\n"), - "historical_tooling": ("scripts/release_check.py", - b"def verify_phase4_step4_control(control: dict, step: int = 4) -> None:", - b"def verify_phase4_step4_control(control: dict, step: int = 4) -> None:\n return"), - "historical_binding": ("tests/integration/test_phase4_gates.py", - b"def test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step4_snapshot):\n repo_root = phase4_step4_snapshot\n", - b"def test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step4_snapshot):\n"), - "historical_document": ("docs/report_schema.md", b"# Report Schema\n", - b"# Unauthorized historical document\n"), - "input_hash_helper": ("src/recursive_integrity_toolkit/utils/hashing.py", - b"return hashlib.sha256(data).hexdigest()", b"return hashlib.sha256(b'changed').hexdigest()"), - "config_resolver_helper": ("src/recursive_integrity_toolkit/config.py", - b"def resolve_config(", b"def resolve_config_disabled("), - } - appended = { - "frozen_model": "src/recursive_integrity_toolkit/models.py", - "frozen_schema": "schemas/report.schema.json", - "plan": "PHASE_4_PLAN.md", - "cli": "src/recursive_integrity_toolkit/cli.py", - "html": "src/recursive_integrity_toolkit/reports/html_report.py", - "assembly": "src/recursive_integrity_toolkit/reports/assembly.py", - "result": "src/recursive_integrity_toolkit/result.py", - "config": "src/recursive_integrity_toolkit/config.py", - "hashing": "src/recursive_integrity_toolkit/utils/hashing.py", - "logging": "src/recursive_integrity_toolkit/utils/logging.py", - "output_paths": "src/recursive_integrity_toolkit/utils/paths.py", - } - added = { - "unknown_runtime": "src/recursive_integrity_toolkit/reports/unauthorized.py", - "premature_completion": "PHASE_4_COMPLETION.md", "premature_report": "report.json", - } - if mutation in replacements: - relative, original, replacement = replacements[mutation] - path = tmp_path / relative - source = path.read_bytes() - assert source.count(original) == 1 - path.write_bytes(source.replace(original, replacement, 1)) - elif mutation in appended: - path = tmp_path / appended[mutation] - path.write_bytes(path.read_bytes() + b"\nUnauthorized Step 5 mutation\n") - elif mutation in added: - path = tmp_path / added[mutation] - assert not path.exists() - path.write_text("Unauthorized later-stage file\n", encoding="utf-8") - elif mutation == "missing_runtime": - (tmp_path / "src/recursive_integrity_toolkit/metrics/bounds.py").unlink() - elif mutation.startswith("control_"): - path = tmp_path / "PHASE_4_BASELINE.json" - control = json.loads(path.read_text(encoding="utf-8")) - if mutation == "control_step": - control["active_step"] = 6 - elif mutation == "control_scope": - control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py") - else: - control["schema_changes_authorized"] = True - control["schema_paths_authorized"].append("schemas/report.schema.json") - path.write_text(json.dumps(control), encoding="utf-8") - else: - raise AssertionError(f"Unknown active mutation case: {mutation}") - with pytest.raises(ValueError): - verify(tmp_path) - - -@pytest.mark.parametrize("relative,injection", [ - ("reports/json_report.py", b"\nimport socket\n"), - ("reports/json_report.py", b"\nfrom ..metrics.diversity import effective_state_diversity\n"), - ("reports/json_report.py", b"\nfrom ..io.validation import validate_bundle\n"), - ("reports/json_report.py", b"\nimport numpy\n"), - ("reports/json_report.py", b"\ndef invented_metric(values):\n return sum(value * value for value in values)\n"), - ("reports/json_report.py", b"\nrender_json = lambda value: '{}'\n"), - ("reports/json_report.py", b"\ndef dynamic_call(value):\n return eval(value)\n"), - ("reports/markdown_report.py", b"\nimport urllib.request\n"), - ("reports/markdown_report.py", b"\nfrom .assembly import assemble_report\n"), - ("reports/markdown_report.py", b"\nfrom ..metrics import coverage\n"), - ("reports/markdown_report.py", b"\nfrom ..io.readers import read_csv\n"), - ("reports/markdown_report.py", b"\nimport pandas\n"), - ("reports/markdown_report.py", b"\nopen('private.txt', 'w').write('raw content')\n"), - ("reports/markdown_report.py", b"\nexec('forged = True')\n"), - ("reports/markdown_report.py", b"\ndef effect_at_definition(value=globals().clear()):\n pass\n"), - ("reports/markdown_report.py", b"\n@print('raw content')\ndef decorated_effect():\n pass\n"), -]) -def test_phase4_step5_renderer_ast_checks_current_source_before_rejecting_scope_injection(repo_root, tmp_path, relative, injection): - checker = runpy.run_path(str(repo_root / "scripts/check_traceability.py"), run_name="phase4_step5_ast_tests") - verify = checker["phase4_step5_runtime_boundary"] - repository_path = "src/recursive_integrity_toolkit/" + relative - current = repo_root / repository_path - verify(current, repository_path) - copy = tmp_path / "source.py" - copy.write_bytes(current.read_bytes() + injection) - with pytest.raises(ValueError): - verify(copy, repository_path) - - -def test_phase4_step5_junit_dispatch_accepts_valid_subsets_and_rejects_bad_evidence(repo_root, tmp_path): - import json - - junit = tmp_path / "subset.xml" - command = [sys.executable, str(repo_root / "scripts/release_check.py"), - "--phase", "4", "--step", "5", "--junit", str(junit)] - valid = ('' - '' - '') - junit.write_text(valid, encoding="utf-8") - passed = subprocess.run(command, cwd=tmp_path, text=True, capture_output=True, check=False) - assert passed.returncode == 0, passed.stderr - assert json.loads(passed.stdout)["tests"] == 1 - assert json.loads(passed.stdout)["passed"] == 1 - too_small = subprocess.run(command + ["--minimum-tests", "3071"], cwd=tmp_path, - text=True, capture_output=True, check=False) - assert too_small.returncode != 0 - assert "Insufficient or duplicate test cases" in too_small.stderr - for outcome in ("failure", "error", "skipped"): - invalid = ('' - '' - f'<{outcome} message="independent rejection case" />' - '') - junit.write_text(invalid, encoding="utf-8") - rejected = subprocess.run(command, cwd=tmp_path, text=True, capture_output=True, check=False) - assert rejected.returncode != 0 - assert "Failed, errored or skipped test" in rejected.stderr - - -def test_phase4_step6_current_runtime_opens_only_output_helpers_and_freezes_schema(repo_root, phase4_step5_snapshot, phase4_step6_snapshot): - repo_root = phase4_step6_snapshot - package = "src/recursive_integrity_toolkit" - current = {path.relative_to(repo_root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (repo_root / package).rglob("*.py")} - frozen = {path.relative_to(phase4_step5_snapshot).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (phase4_step5_snapshot / package).rglob("*.py")} - assert len(current) == len(frozen) == 40 - assert current.keys() == frozen.keys() - opened = {package + "/" + relative for relative in ( - "utils/paths.py", "utils/logging.py", - )} - assert {path for path in current if current[path] != frozen[path]} == opened - assert {path: digest for path, digest in current.items() if path not in opened} == { - path: digest for path, digest in frozen.items() if path not in opened - } - schemas = {path.name: path.read_bytes() for path in (repo_root / "schemas").glob("*.json")} - assert len(schemas) == 5 - assert schemas == {path.name: path.read_bytes() for path in (phase4_step5_snapshot / "schemas").glob("*.json")} - - -def test_phase4_step6_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step6_snapshot): - repo_root = phase4_step6_snapshot - names = {"ci.yml", "golden.yml", "security.yml", "release.yml"} - root = repo_root / ".github/workflows" - expected_timeouts = {"ci.yml": [60, 20], "golden.yml": [15], "security.yml": [20], "release.yml": [25]} - assert {path.name for path in root.glob("*.yml")} == names - for name in sorted(names): - text = (root / name).read_text(encoding="utf-8") - assert [int(line.split(":", 1)[1]) for line in text.splitlines() - if line.strip().startswith("timeout-minutes:")] == expected_timeouts[name] - assert "Phase 4 Step 6" in text, name - assert "--phase 4 --step 6" in text, name - for older in ("--phase 4 --step 5", "--phase 4 --step 4", "--phase 4 --step 3", "--phase 4 --step 2", "--phase 4 --step 1", "--phase 3 --step 11"): - assert older not in text, (name, older) - assert "permissions:\n contents: read" in text - assert "persist-credentials: false" in text - assert "timeout-minutes:" in text and "set -euo pipefail" in text - assert "actions/upload-artifact@v4" in text and "if-no-files-found: error" in text - for line in text.splitlines(): - if "python scripts/release_check.py" in line: - assert "--phase 4 --step 6" in line, (name, line) - for forbidden in ("continue-on-error:", "|| true", "contents: write", "id-token: write", "twine upload", "git push"): - assert forbidden not in text, (name, forbidden) - ci = (root / "ci.yml").read_text(encoding="utf-8") - for required in ('os: [ubuntu-latest, windows-latest]', 'python-version: ["3.11", "3.12"]', - 'dependencies: [current, minimum]', '"numpy==2.0.0" "pandas==2.2.2"', - 'RIT_TEST_PARQUET: "0"', 'RIT_TEST_PARQUET: "1"', '--require-parquet', - '--baseline-evidence', 'python -m pip check', - '--minimum-tests 3215', '--minimum-tests 3218', - "find_spec('pyarrow') is None", 'import pyarrow'): - assert required in ci - assert ci.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in ci - golden = (root / "golden.yml").read_text(encoding="utf-8") - for required in ("tests/golden/test_phase3_math.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_phase3_metric_pipeline.py"): - assert required in golden - security = (root / "security.yml").read_text(encoding="utf-8") - for required in ('tests/integration/test_no_network.py', 'tests/integration/test_optional_dependency.py', - 'tests/unit/test_PR012_evidence_classes.py', 'tests/unit/test_PR013_report_schema.py', - 'tests/unit/test_PR014_unavailable.py', 'tests/unit/test_PR015_redaction.py', - 'tests/unit/test_PR016_determinism.py', 'tests/unit/test_PR018_language.py', - 'tests/integration/test_partial_provenance_report.py', - 'tests/integration/test_partial_lineage_report.py', - 'tests/unit/test_phase4_output_safety.py', 'tests/unit/test_phase4_contracts.py', 'tests/integration/test_phase4_gates.py'): - assert required in security - release = (root / "release.yml").read_text(encoding="utf-8") - assert "python -m build" in release and "python -m twine check --strict" in release - assert "--dist" in release and "--candidate" in release - assert "--delivery" not in release - assert "recursive-integrity-toolkit-phase4-step6-candidate" in release - assert "rit-phase4-step4" not in release - assert "--minimum-tests 3215" in release and "--minimum-tests 3218" in release - assert release.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - - -@pytest.mark.parametrize("mutation", [ - "unchanged", "formula", "math_oracle", "frozen_model", "frozen_schema", "hero", "dependency", "plan", - "cli", "html", "assembly", "result", "config", "hashing", "logging", "output_paths", "unknown_runtime", "missing_runtime", "premature_completion", "premature_report", - "control_step", "control_scope", "control_schema_scope", "historical_assertion", "historical_tooling", - "historical_binding", "historical_document", "input_hash_helper", "config_resolver_helper", -]) -def test_phase4_step6_current_snapshot_checks_valid_tree_before_mutations(repo_root, tmp_path, phase4_gate_tools, mutation, phase4_step6_snapshot): - import json - - paths = {path.relative_to(phase4_step6_snapshot).as_posix() - for path in phase4_step6_snapshot.rglob("*") if path.is_file()} - assert len(paths) == 228 - for relative in sorted(paths): - destination = tmp_path / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(phase4_step6_snapshot / relative, destination) - verify = phase4_gate_tools["verify_phase4_step6_snapshot"] - baseline = verify(tmp_path) - assert baseline["package_modules"] == 40 - assert baseline["frozen_runtime_modules"] == 38 - assert baseline["frozen_schemas"] == 5 - assert baseline["result_contracts_enabled"] is True - assert baseline["adapters_enabled"] is True - assert baseline["privacy_views_enabled"] is True - assert baseline["renderers_enabled"] is True - assert baseline["output_publication_enabled"] is True - assert baseline["cli_analysis_enabled"] is False - assert baseline["phase_complete"] is False - if mutation == "unchanged": - return - replacements = { - "formula": ("src/recursive_integrity_toolkit/metrics/diversity.py", - b"diversity = 1.0 - concentration", b"diversity = 0.5 - concentration"), - "math_oracle": ("tests/golden/phase3_math_cases.json", b'"v2_support":5', b'"v2_support":3'), - "hero": ("examples/hero/records_v2.csv", b"v2_08,v2,", b"v2_99,v2,"), - "dependency": ("pyproject.toml", b"numpy>=2.0", b"numpy>=3.0"), - "historical_assertion": ("tests/unit/test_phase4_contracts.py", - b' assert control["active_phase"] == 4 and control["active_step"] == 4\n', - b" assert True\n"), - "historical_tooling": ("scripts/release_check.py", - b"def verify_phase4_step4_control(control: dict, step: int = 4) -> None:", - b"def verify_phase4_step4_control(control: dict, step: int = 4) -> None:\n return"), - "historical_binding": ("tests/integration/test_phase4_gates.py", - b"def test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step4_snapshot):\n repo_root = phase4_step4_snapshot\n", - b"def test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step4_snapshot):\n"), - "historical_document": ("docs/cli.md", b"# CLI\n", - b"# Unauthorized historical document\n"), - "input_hash_helper": ("src/recursive_integrity_toolkit/utils/hashing.py", - b"return hashlib.sha256(data).hexdigest()", b"return hashlib.sha256(b'changed').hexdigest()"), - "config_resolver_helper": ("src/recursive_integrity_toolkit/config.py", - b"def resolve_config(", b"def resolve_config_disabled("), - } - appended = { - "frozen_model": "src/recursive_integrity_toolkit/models.py", - "frozen_schema": "schemas/report.schema.json", - "plan": "PHASE_4_PLAN.md", - "cli": "src/recursive_integrity_toolkit/cli.py", - "html": "src/recursive_integrity_toolkit/reports/html_report.py", - "assembly": "src/recursive_integrity_toolkit/reports/assembly.py", - "result": "src/recursive_integrity_toolkit/result.py", - "config": "src/recursive_integrity_toolkit/config.py", - "hashing": "src/recursive_integrity_toolkit/utils/hashing.py", - "logging": "src/recursive_integrity_toolkit/utils/logging.py", - "output_paths": "src/recursive_integrity_toolkit/utils/paths.py", - } - added = { - "unknown_runtime": "src/recursive_integrity_toolkit/reports/unauthorized.py", - "premature_completion": "PHASE_4_COMPLETION.md", "premature_report": "report.json", - } - if mutation in replacements: - relative, original, replacement = replacements[mutation] - path = tmp_path / relative - source = path.read_bytes() - assert source.count(original) == 1 - path.write_bytes(source.replace(original, replacement, 1)) - elif mutation in appended: - path = tmp_path / appended[mutation] - path.write_bytes(path.read_bytes() + b"\nUnauthorized Step 6 mutation\n") - elif mutation in added: - path = tmp_path / added[mutation] - assert not path.exists() - path.write_text("Unauthorized later-stage file\n", encoding="utf-8") - elif mutation == "missing_runtime": - (tmp_path / "src/recursive_integrity_toolkit/metrics/bounds.py").unlink() - elif mutation.startswith("control_"): - path = tmp_path / "PHASE_4_BASELINE.json" - control = json.loads(path.read_text(encoding="utf-8")) - if mutation == "control_step": - control["active_step"] = 7 - elif mutation == "control_scope": - control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py") - else: - control["schema_changes_authorized"] = True - control["schema_paths_authorized"].append("schemas/report.schema.json") - path.write_text(json.dumps(control), encoding="utf-8") - else: - raise AssertionError(f"Unknown active mutation case: {mutation}") - with pytest.raises(ValueError): - verify(tmp_path) - - -@pytest.mark.parametrize("relative,injection", [ - ("utils/paths.py", b"\nimport socket\n"), - ("utils/paths.py", b"\nfrom recursive_integrity_toolkit.metrics import diversity\n"), - ("utils/paths.py", b"\ndef new_reader(path):\n return path.read_bytes()\n"), - ("utils/paths.py", b"\nexec('forged=True')\n"), - ("utils/paths.py", b"\ndef effect(value=globals().clear()):\n pass\n"), - ("utils/logging.py", b"\nimport urllib.request\n"), - ("utils/logging.py", b"\ndef disclose(value):\n print(value)\n"), - ("utils/logging.py", b"\n@print('private')\ndef effect():\n pass\n"), -]) -def test_phase4_step6_current_output_ast_rejects_injections(repo_root, tmp_path, relative, injection): - checker = runpy.run_path(str(repo_root / "scripts/check_traceability.py"), run_name="phase4_step6_ast_tests") - verify = checker["phase4_step6_runtime_boundary"] - repository_path = "src/recursive_integrity_toolkit/" + relative - current = repo_root / repository_path - verify(current, repository_path) - target = tmp_path / "changed.py" - target.write_bytes(current.read_bytes() + injection) - with pytest.raises(ValueError): - verify(target, repository_path) - - -@pytest.mark.parametrize("relative,needle,replacement", [ - ("utils/paths.py", b"def resolve_content_reference", b"def disabled_content_reference"), - ("utils/logging.py", b"def format_diagnostic", b"def disabled_diagnostic"), -]) -def test_phase4_step6_preserves_existing_input_and_diagnostic_helpers(repo_root, phase4_step5_snapshot, phase4_gate_tools, relative, needle, replacement): - path = "src/recursive_integrity_toolkit/" + relative - before = (phase4_step5_snapshot / path).read_bytes() - after = (repo_root / path).read_bytes() - verify = phase4_gate_tools["_phase4_step6_preserve_runtime"] - verify(before, after, path) - assert needle in after - with pytest.raises(ValueError): - verify(before, after.replace(needle, replacement, 1), path) - - -@pytest.mark.parametrize("injection", [ - b"\nimport socket\n", b"\npytest.skip('inherited tests')\n", - b"\ndef test_unscoped():\n pass\n", - b"\ndef test_phase4_step6_unsafe(value=globals().clear()):\n pass\n", - b"\n@pytest.mark.parametrize('value', list(range(2)))\ndef test_phase4_step6_unsafe(value):\n pass\n", -]) -def test_phase4_step6_new_test_file_rejects_definition_effects(repo_root, phase4_gate_tools, injection): - path = "tests/unit/test_phase4_output_safety.py" - raw = (repo_root / path).read_bytes() - verify = phase4_gate_tools["_phase4_step6_new_test"] - verify(raw, path) - with pytest.raises(ValueError): - verify(raw + injection, path) - - -def test_phase4_step7_current_runtime_opens_only_cli_configuration_and_freezes_schema(repo_root, phase4_step6_snapshot, phase4_step7_snapshot): - repo_root = phase4_step7_snapshot - package = "src/recursive_integrity_toolkit" - current = {path.relative_to(repo_root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (repo_root / package).rglob("*.py")} - frozen = {path.relative_to(phase4_step6_snapshot).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (phase4_step6_snapshot / package).rglob("*.py")} - assert len(current) == len(frozen) == 40 - assert current.keys() == frozen.keys() - opened = {package + "/" + relative for relative in ( - "cli.py", "config.py", - )} - assert {path for path in current if current[path] != frozen[path]} == opened - assert {path: digest for path, digest in current.items() if path not in opened} == { - path: digest for path, digest in frozen.items() if path not in opened - } - schemas = {path.name: path.read_bytes() for path in (repo_root / "schemas").glob("*.json")} - assert len(schemas) == 5 - assert schemas == {path.name: path.read_bytes() for path in (phase4_step6_snapshot / "schemas").glob("*.json")} - - -def test_phase4_step7_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step7_snapshot): - repo_root = phase4_step7_snapshot - names = {"ci.yml", "golden.yml", "security.yml", "release.yml"} - root = repo_root / ".github/workflows" - expected_timeouts = {"ci.yml": [60, 20], "golden.yml": [15], "security.yml": [20], "release.yml": [40]} - assert {path.name for path in root.glob("*.yml")} == names - for name in sorted(names): - text = (root / name).read_text(encoding="utf-8") - assert [int(line.split(":", 1)[1]) for line in text.splitlines() - if line.strip().startswith("timeout-minutes:")] == expected_timeouts[name] - assert "Phase 4 Step 7" in text, name - assert "--phase 4 --step 7" in text, name - for older in ("--phase 4 --step 6", "--phase 4 --step 5", "--phase 4 --step 4", "--phase 4 --step 3", "--phase 4 --step 2", "--phase 4 --step 1", "--phase 3 --step 11"): - assert older not in text, (name, older) - assert "permissions:\n contents: read" in text - assert "persist-credentials: false" in text - assert "timeout-minutes:" in text and "set -euo pipefail" in text - assert "actions/upload-artifact@v4" in text and "if-no-files-found: error" in text - for line in text.splitlines(): - if "python scripts/release_check.py" in line: - assert "--phase 4 --step 7" in line, (name, line) - for forbidden in ("continue-on-error:", "|| true", "contents: write", "id-token: write", "twine upload", "git push"): - assert forbidden not in text, (name, forbidden) - ci = (root / "ci.yml").read_text(encoding="utf-8") - for required in ('os: [ubuntu-latest, windows-latest]', 'python-version: ["3.11", "3.12"]', - 'dependencies: [current, minimum]', '"numpy==2.0.0" "pandas==2.2.2"', - 'RIT_TEST_PARQUET: "0"', 'RIT_TEST_PARQUET: "1"', '--require-parquet', - '--baseline-evidence', 'python -m pip check', - '--minimum-tests 3420', '--minimum-tests 3423', - "find_spec('pyarrow') is None", 'import pyarrow'): - assert required in ci - assert ci.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in ci - golden = (root / "golden.yml").read_text(encoding="utf-8") - for required in ("tests/golden/test_phase3_math.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_phase3_metric_pipeline.py"): - assert required in golden - security = (root / "security.yml").read_text(encoding="utf-8") - for required in ('tests/integration/test_no_network.py', 'tests/integration/test_optional_dependency.py', - 'tests/unit/test_PR012_evidence_classes.py', 'tests/unit/test_PR013_report_schema.py', - 'tests/unit/test_PR014_unavailable.py', 'tests/unit/test_PR015_redaction.py', - 'tests/unit/test_PR016_determinism.py', 'tests/unit/test_PR018_language.py', - 'tests/integration/test_partial_provenance_report.py', - 'tests/integration/test_partial_lineage_report.py', - 'tests/integration/test_phase4_cli.py', 'tests/unit/test_phase4_output_safety.py', 'tests/unit/test_phase4_contracts.py', 'tests/integration/test_phase4_gates.py'): - assert required in security - release = (root / "release.yml").read_text(encoding="utf-8") - assert "python -m build" in release and "python -m twine check --strict" in release - assert "--dist" in release and "--candidate" in release - assert "--delivery" not in release - assert "recursive-integrity-toolkit-phase4-step7-candidate" in release - assert "rit-phase4-step4" not in release - assert "--minimum-tests 3420" in release and "--minimum-tests 3423" in release - assert release.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - - -@pytest.mark.parametrize("mutation", [ - "unchanged", "formula", "math_oracle", "frozen_model", "frozen_schema", "hero", "dependency", "plan", - "cli", "html", "assembly", "result", "config", "hashing", "logging", "output_paths", "unknown_runtime", "missing_runtime", "premature_completion", "premature_report", - "control_step", "control_scope", "control_schema_scope", "historical_assertion", "historical_tooling", - "historical_binding", "historical_document", "input_hash_helper", "config_resolver_helper", -]) -def test_phase4_step7_current_snapshot_checks_valid_tree_before_mutations(repo_root, tmp_path, phase4_gate_tools, mutation, phase4_step7_snapshot): - import json - - paths = {path.relative_to(phase4_step7_snapshot).as_posix() - for path in phase4_step7_snapshot.rglob("*") if path.is_file()} - assert len(paths) == 229 - for relative in sorted(paths): - destination = tmp_path / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(phase4_step7_snapshot / relative, destination) - verify = phase4_gate_tools["verify_phase4_step7_snapshot"] - baseline = verify(tmp_path) - assert baseline["package_modules"] == 40 - assert baseline["frozen_runtime_modules"] == 38 - assert baseline["frozen_schemas"] == 5 - assert baseline["result_contracts_enabled"] is True - assert baseline["adapters_enabled"] is True - assert baseline["privacy_views_enabled"] is True - assert baseline["renderers_enabled"] is True - assert baseline["output_publication_enabled"] is True - assert baseline["cli_analysis_enabled"] is True - assert baseline["phase_complete"] is False - if mutation == "unchanged": - return - replacements = { - "formula": ("src/recursive_integrity_toolkit/metrics/diversity.py", - b"diversity = 1.0 - concentration", b"diversity = 0.5 - concentration"), - "math_oracle": ("tests/golden/phase3_math_cases.json", b'"v2_support":5', b'"v2_support":3'), - "hero": ("examples/hero/records_v2.csv", b"v2_08,v2,", b"v2_99,v2,"), - "dependency": ("pyproject.toml", b"numpy>=2.0", b"numpy>=3.0"), - "historical_assertion": ("tests/unit/test_phase4_contracts.py", - b' assert control["active_phase"] == 4 and control["active_step"] == 4\n', - b" assert True\n"), - "historical_tooling": ("scripts/release_check.py", - b"def verify_phase4_step4_control(control: dict, step: int = 4) -> None:", - b"def verify_phase4_step4_control(control: dict, step: int = 4) -> None:\n return"), - "historical_binding": ("tests/integration/test_phase4_gates.py", - b"def test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step4_snapshot):\n repo_root = phase4_step4_snapshot\n", - b"def test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step4_snapshot):\n"), - "historical_document": ("docs/cli.md", b"# CLI\n", - b"# Unauthorized historical document\n"), - "input_hash_helper": ("src/recursive_integrity_toolkit/utils/hashing.py", - b"return hashlib.sha256(data).hexdigest()", b"return hashlib.sha256(b'changed').hexdigest()"), - "config_resolver_helper": ("src/recursive_integrity_toolkit/config.py", - b"def resolve_config(", b"def resolve_config_disabled("), - } - appended = { - "frozen_model": "src/recursive_integrity_toolkit/models.py", - "frozen_schema": "schemas/report.schema.json", - "plan": "PHASE_4_PLAN.md", - "cli": "src/recursive_integrity_toolkit/cli.py", - "html": "src/recursive_integrity_toolkit/reports/html_report.py", - "assembly": "src/recursive_integrity_toolkit/reports/assembly.py", - "result": "src/recursive_integrity_toolkit/result.py", - "config": "src/recursive_integrity_toolkit/config.py", - "hashing": "src/recursive_integrity_toolkit/utils/hashing.py", - "logging": "src/recursive_integrity_toolkit/utils/logging.py", - "output_paths": "src/recursive_integrity_toolkit/utils/paths.py", - } - added = { - "unknown_runtime": "src/recursive_integrity_toolkit/reports/unauthorized.py", - "premature_completion": "PHASE_4_COMPLETION.md", "premature_report": "report.json", - } - if mutation in replacements: - relative, original, replacement = replacements[mutation] - path = tmp_path / relative - source = path.read_bytes() - assert source.count(original) == 1 - path.write_bytes(source.replace(original, replacement, 1)) - elif mutation in appended: - path = tmp_path / appended[mutation] - path.write_bytes(path.read_bytes() + b"\nUnauthorized Step 7 mutation\n") - elif mutation in added: - path = tmp_path / added[mutation] - assert not path.exists() - path.write_text("Unauthorized later-stage file\n", encoding="utf-8") - elif mutation == "missing_runtime": - (tmp_path / "src/recursive_integrity_toolkit/metrics/bounds.py").unlink() - elif mutation.startswith("control_"): - path = tmp_path / "PHASE_4_BASELINE.json" - control = json.loads(path.read_text(encoding="utf-8")) - if mutation == "control_step": - control["active_step"] = 8 - elif mutation == "control_scope": - control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py") - else: - control["schema_changes_authorized"] = True - control["schema_paths_authorized"].append("schemas/report.schema.json") - path.write_text(json.dumps(control), encoding="utf-8") - else: - raise AssertionError(f"Unknown active mutation case: {mutation}") - with pytest.raises(ValueError): - verify(tmp_path) - - -@pytest.mark.parametrize("relative,injection", [ - ("cli.py", b"\nimport socket\n"), ("cli.py", b"\nmain()\n"), - ("cli.py", b"\ndef invented_metric():\n return 0.5\n"), - ("cli.py", b"\ndef unsafe(value=globals().clear()):\n pass\n"), - ("config.py", b"\nresolve_config = lambda *args: None\n"), - ("config.py", b"\nimport urllib.request\n"), -]) -def test_phase4_step7_current_cli_ast_rejects_injection(repo_root, tmp_path, relative, injection, phase4_step7_snapshot): - repo_root = phase4_step7_snapshot - checker = runpy.run_path(str(repo_root / "scripts/check_traceability.py"), run_name="phase4_step7_ast_tests") - verify = checker["phase4_step7_runtime_boundary"] - path = "src/recursive_integrity_toolkit/" + relative - verify(repo_root / path, path) - changed = tmp_path / "changed.py" - changed.write_bytes((repo_root / path).read_bytes() + injection) - with pytest.raises(ValueError): - verify(changed, path) - - -def test_phase4_step7_preserves_all_inherited_config_bytes(repo_root, phase4_step6_snapshot, phase4_gate_tools): - path = "src/recursive_integrity_toolkit/config.py" - before, after = (phase4_step6_snapshot / path).read_bytes(), (repo_root / path).read_bytes() - verify = phase4_gate_tools["_phase4_step7_preserve_runtime"] - verify(before, after, path) - for needle in (b"def resolve_config(", b"def resolve_phase4_options(", b"def phase4_config_hash("): - assert after.count(needle) == 1 - with pytest.raises(ValueError): - verify(before, after.replace(needle, b"def changed(", 1), path) - - -@pytest.mark.parametrize("injection", [ - b"\nimport socket\n", b"\npytest.skip('inherited tests')\n", - b"\ndef test_unscoped():\n pass\n", - b"\ndef test_phase4_step7_unsafe(value=globals().clear()):\n pass\n", -]) -def test_phase4_step7_new_cli_tests_reject_definition_effects(repo_root, phase4_gate_tools, injection, phase4_step7_snapshot): - repo_root = phase4_step7_snapshot - path = "tests/integration/test_phase4_cli.py" - raw = (repo_root / path).read_bytes() - verify = phase4_gate_tools["_phase4_step7_new_test"] - verify(raw, path) - with pytest.raises(ValueError): - verify(raw + injection, path) - - -def test_phase4_step8_current_runtime_opens_only_cli_configuration_and_freezes_schema(repo_root, phase4_step7_snapshot, phase4_step8_snapshot): - repo_root = phase4_step8_snapshot - package = "src/recursive_integrity_toolkit" - current = {path.relative_to(repo_root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (repo_root / package).rglob("*.py")} - frozen = {path.relative_to(phase4_step7_snapshot).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (phase4_step7_snapshot / package).rglob("*.py")} - assert len(current) == len(frozen) == 40 - assert current.keys() == frozen.keys() - opened = {package + "/" + relative for relative in ( - "cli.py", "config.py", - )} - assert {path for path in current if current[path] != frozen[path]} == opened - assert {path: digest for path, digest in current.items() if path not in opened} == { - path: digest for path, digest in frozen.items() if path not in opened - } - schemas = {path.name: path.read_bytes() for path in (repo_root / "schemas").glob("*.json")} - assert len(schemas) == 5 - assert schemas == {path.name: path.read_bytes() for path in (phase4_step7_snapshot / "schemas").glob("*.json")} - - -def test_phase4_step8_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step8_snapshot): - repo_root = phase4_step8_snapshot - names = {"ci.yml", "golden.yml", "security.yml", "release.yml"} - root = repo_root / ".github/workflows" - expected_timeouts = {"ci.yml": [60, 20], "golden.yml": [15], "security.yml": [20], "release.yml": [40]} - assert {path.name for path in root.glob("*.yml")} == names - for name in sorted(names): - text = (root / name).read_text(encoding="utf-8") - assert [int(line.split(":", 1)[1]) for line in text.splitlines() - if line.strip().startswith("timeout-minutes:")] == expected_timeouts[name] - assert "Phase 4 Step 8" in text, name - assert "--phase 4 --step 8" in text, name - for older in ("--phase 4 --step 7", "--phase 4 --step 6", "--phase 4 --step 5", "--phase 4 --step 4", "--phase 4 --step 3", "--phase 4 --step 2", "--phase 4 --step 1", "--phase 3 --step 11"): - assert older not in text, (name, older) - assert "permissions:\n contents: read" in text - assert "persist-credentials: false" in text - assert "timeout-minutes:" in text and "set -euo pipefail" in text - assert "actions/upload-artifact@v4" in text and "if-no-files-found: error" in text - for line in text.splitlines(): - if "python scripts/release_check.py" in line: - assert "--phase 4 --step 8" in line, (name, line) - for forbidden in ("continue-on-error:", "|| true", "contents: write", "id-token: write", "twine upload", "git push"): - assert forbidden not in text, (name, forbidden) - ci = (root / "ci.yml").read_text(encoding="utf-8") - for required in ('os: [ubuntu-latest, windows-latest]', 'python-version: ["3.11", "3.12"]', - 'dependencies: [current, minimum]', '"numpy==2.0.0" "pandas==2.2.2"', - 'RIT_TEST_PARQUET: "0"', 'RIT_TEST_PARQUET: "1"', '--require-parquet', - '--baseline-evidence', 'python -m pip check', - '--minimum-tests 3620', '--minimum-tests 3623', - "find_spec('pyarrow') is None", 'import pyarrow'): - assert required in ci - assert ci.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in ci - golden = (root / "golden.yml").read_text(encoding="utf-8") - for required in ("tests/golden/test_phase3_math.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_phase3_metric_pipeline.py", "tests/integration/test_hero_end_to_end.py"): - assert required in golden - security = (root / "security.yml").read_text(encoding="utf-8") - for required in ('tests/integration/test_no_network.py', 'tests/integration/test_optional_dependency.py', - 'tests/unit/test_PR012_evidence_classes.py', 'tests/unit/test_PR013_report_schema.py', - 'tests/unit/test_PR014_unavailable.py', 'tests/unit/test_PR015_redaction.py', - 'tests/unit/test_PR016_determinism.py', 'tests/unit/test_PR018_language.py', - 'tests/integration/test_partial_provenance_report.py', - 'tests/integration/test_partial_lineage_report.py', - 'tests/integration/test_phase4_cli.py', 'tests/unit/test_phase4_output_safety.py', 'tests/unit/test_phase4_contracts.py', 'tests/integration/test_phase4_gates.py'): - assert required in security - release = (root / "release.yml").read_text(encoding="utf-8") - assert "python -m build" in release and "python -m twine check --strict" in release - assert "--dist" in release and "--candidate" in release - assert "--delivery" not in release - assert "recursive-integrity-toolkit-phase4-step8-candidate" in release - assert "rit-phase4-step4" not in release - assert "--minimum-tests 3620" in release and "--minimum-tests 3623" in release - assert release.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - - -@pytest.mark.parametrize("mutation", [ - "unchanged", "formula", "math_oracle", "frozen_model", "frozen_schema", "hero", "dependency", "plan", - "cli", "html", "assembly", "result", "config", "hashing", "logging", "output_paths", "unknown_runtime", "missing_runtime", "premature_completion", "premature_report", - "control_step", "control_scope", "control_schema_scope", "historical_assertion", "historical_tooling", - "historical_binding", "historical_document", "input_hash_helper", "config_resolver_helper", -]) -def test_phase4_step8_current_snapshot_checks_valid_tree_before_mutations(repo_root, tmp_path, phase4_gate_tools, mutation, phase4_step8_snapshot): - import json - - paths = {path.relative_to(phase4_step8_snapshot).as_posix() - for path in phase4_step8_snapshot.rglob("*") if path.is_file()} - assert len(paths) == 236 - for relative in sorted(paths): - destination = tmp_path / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(phase4_step8_snapshot / relative, destination) - verify = phase4_gate_tools["verify_phase4_step8_snapshot"] - baseline = verify(tmp_path) - assert baseline["package_modules"] == 40 - assert baseline["frozen_runtime_modules"] == 38 - assert baseline["frozen_schemas"] == 5 - assert baseline["result_contracts_enabled"] is True - assert baseline["adapters_enabled"] is True - assert baseline["privacy_views_enabled"] is True - assert baseline["renderers_enabled"] is True - assert baseline["output_publication_enabled"] is True - assert baseline["cli_analysis_enabled"] is True - assert baseline["phase_complete"] is False - if mutation == "unchanged": - return - replacements = { - "formula": ("src/recursive_integrity_toolkit/metrics/diversity.py", - b"diversity = 1.0 - concentration", b"diversity = 0.5 - concentration"), - "math_oracle": ("tests/golden/phase3_math_cases.json", b'"v2_support":5', b'"v2_support":3'), - "hero": ("examples/hero/records_v2.csv", b"v2_08,v2,", b"v2_99,v2,"), - "dependency": ("pyproject.toml", b"numpy>=2.0", b"numpy>=3.0"), - "historical_assertion": ("tests/unit/test_phase4_contracts.py", - b' assert control["active_phase"] == 4 and control["active_step"] == 4\n', - b" assert True\n"), - "historical_tooling": ("scripts/release_check.py", - b"def verify_phase4_step4_control(control: dict, step: int = 4) -> None:", - b"def verify_phase4_step4_control(control: dict, step: int = 4) -> None:\n return"), - "historical_binding": ("tests/integration/test_phase4_gates.py", - b"def test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step4_snapshot):\n repo_root = phase4_step4_snapshot\n", - b"def test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step4_snapshot):\n"), - "historical_document": ("docs/cli.md", b"# CLI\n", - b"# Unauthorized historical document\n"), - "input_hash_helper": ("src/recursive_integrity_toolkit/utils/hashing.py", - b"return hashlib.sha256(data).hexdigest()", b"return hashlib.sha256(b'changed').hexdigest()"), - "config_resolver_helper": ("src/recursive_integrity_toolkit/config.py", - b"def resolve_config(", b"def resolve_config_disabled("), - } - appended = { - "frozen_model": "src/recursive_integrity_toolkit/models.py", - "frozen_schema": "schemas/report.schema.json", - "plan": "PHASE_4_PLAN.md", - "cli": "src/recursive_integrity_toolkit/cli.py", - "html": "src/recursive_integrity_toolkit/reports/html_report.py", - "assembly": "src/recursive_integrity_toolkit/reports/assembly.py", - "result": "src/recursive_integrity_toolkit/result.py", - "config": "src/recursive_integrity_toolkit/config.py", - "hashing": "src/recursive_integrity_toolkit/utils/hashing.py", - "logging": "src/recursive_integrity_toolkit/utils/logging.py", - "output_paths": "src/recursive_integrity_toolkit/utils/paths.py", - } - added = { - "unknown_runtime": "src/recursive_integrity_toolkit/reports/unauthorized.py", - "premature_completion": "PHASE_4_COMPLETION.md", "premature_report": "report.json", - } - if mutation in replacements: - relative, original, replacement = replacements[mutation] - path = tmp_path / relative - source = path.read_bytes() - assert source.count(original) == 1 - path.write_bytes(source.replace(original, replacement, 1)) - elif mutation in appended: - path = tmp_path / appended[mutation] - path.write_bytes(path.read_bytes() + b"\nUnauthorized Step 8 mutation\n") - elif mutation in added: - path = tmp_path / added[mutation] - assert not path.exists() - path.write_text("Unauthorized later-stage file\n", encoding="utf-8") - elif mutation == "missing_runtime": - (tmp_path / "src/recursive_integrity_toolkit/metrics/bounds.py").unlink() - elif mutation.startswith("control_"): - path = tmp_path / "PHASE_4_BASELINE.json" - control = json.loads(path.read_text(encoding="utf-8")) - if mutation == "control_step": - control["active_step"] = 9 - elif mutation == "control_scope": - control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py") - else: - control["schema_changes_authorized"] = True - control["schema_paths_authorized"].append("schemas/report.schema.json") - path.write_text(json.dumps(control), encoding="utf-8") - else: - raise AssertionError(f"Unknown active mutation case: {mutation}") - with pytest.raises(ValueError): - verify(tmp_path) - - -@pytest.mark.parametrize("relative,injection", [ - ("cli.py", b"\nimport socket\n"), ("cli.py", b"\nmain()\n"), - ("cli.py", b"\ndef invented_metric():\n return 0.5\n"), - ("cli.py", b"\ndef unsafe(value=globals().clear()):\n pass\n"), - ("config.py", b"\nresolve_config = lambda *args: None\n"), - ("config.py", b"\nimport urllib.request\n"), -]) -def test_phase4_step8_current_cli_ast_rejects_injection(repo_root, tmp_path, relative, injection): - checker = runpy.run_path(str(repo_root / "scripts/check_traceability.py"), run_name="phase4_step8_ast_tests") - verify = checker["phase4_step8_runtime_boundary"] - path = "src/recursive_integrity_toolkit/" + relative - verify(repo_root / path, path) - changed = tmp_path / "changed.py" - changed.write_bytes((repo_root / path).read_bytes() + injection) - with pytest.raises(ValueError): - verify(changed, path) - - -def test_phase4_step8_preserves_all_inherited_config_bytes(repo_root, phase4_step7_snapshot, phase4_gate_tools): - path = "src/recursive_integrity_toolkit/config.py" - before, after = (phase4_step7_snapshot / path).read_bytes(), (repo_root / path).read_bytes() - verify = phase4_gate_tools["_phase4_step8_preserve_runtime"] - verify(before, after, path) - for needle in (b"def resolve_config(", b"def resolve_phase4_options(", b"def phase4_config_hash("): - assert after.count(needle) == 1 - with pytest.raises(ValueError): - verify(before, after.replace(needle, b"def changed(", 1), path) - - -@pytest.mark.parametrize("mutation", ["copy", "schema", "extra", "glob", "version", "dependency", "backend"]) -def test_phase4_step8_resources_and_packaging_have_exact_boundaries(repo_root, tmp_path, phase4_gate_tools, phase4_step7_snapshot, mutation, phase4_step8_snapshot): - resources = phase4_gate_tools["PHASE4_STEP8_RESOURCES"] - for relative in {*resources, *resources.values(), "pyproject.toml"}: - target = tmp_path / relative - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes((phase4_step8_snapshot / relative).read_bytes()) - prior = {"pyproject.toml": (phase4_step7_snapshot / "pyproject.toml").read_bytes()} - verify = phase4_gate_tools["phase4_step8_resources"] - verify(tmp_path, prior) - if mutation in ("copy", "schema"): - path = "src/recursive_integrity_toolkit/data/hero/config.json" if mutation == "copy" else "src/recursive_integrity_toolkit/data/report.schema.json" - (tmp_path / path).write_bytes(b"{}") - elif mutation == "extra": - (tmp_path / "src/recursive_integrity_toolkit/data/extra.json").write_bytes(b"{}") - else: - path = tmp_path / "pyproject.toml" - raw = path.read_bytes() - a, b = {"glob": (b"data/hero/*", b"data/*"), "version": (b"0.1.0.dev2", b"0.1.0.dev3"), "dependency": (b"numpy>=2.0", b"numpy>=3.0"), "backend": (b"setuptools.build_meta", b"other.backend")}[mutation] - assert raw.count(a) == 1 - path.write_bytes(raw.replace(a, b)) - with pytest.raises(ValueError): verify(tmp_path, prior) - - -def test_phase4_step9_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root): - import re - names = {"ci.yml", "golden.yml", "security.yml", "release.yml"} - root = repo_root / ".github/workflows" - expected_timeouts = {"ci.yml": [105, 75], "golden.yml": [15], "security.yml": [20], "release.yml": [40]} - assert {path.name for path in root.glob("*.yml")} == names - for name in sorted(names): - text = (root / name).read_text(encoding="utf-8") - assert [int(line.split(":", 1)[1]) for line in text.splitlines() - if line.strip().startswith("timeout-minutes:")] == expected_timeouts[name] - assert "Phase 4 Step 11" in text, name - assert "--phase 4 --step 11" in text, name - for older in ("--phase 4 --step 10", "--phase 4 --step 9", "--phase 4 --step 8", "--phase 4 --step 7", "--phase 4 --step 6", "--phase 4 --step 5", "--phase 4 --step 4", "--phase 4 --step 3", "--phase 4 --step 2", "--phase 4 --step 1", "--phase 3 --step 11"): - assert re.search(re.escape(older) + r"(?![0-9])", text) is None, (name, older) - assert "permissions:\n contents: read" in text - assert "persist-credentials: false" in text - assert "timeout-minutes:" in text and "set -euo pipefail" in text - assert "actions/upload-artifact@v4" in text and "if-no-files-found: error" in text - for line in text.splitlines(): - if "python scripts/release_check.py" in line: - assert "--phase 4 --step 11" in line, (name, line) - for forbidden in ("continue-on-error:", "|| true", "contents: write", "id-token: write", "twine upload", "git push"): - assert forbidden not in text, (name, forbidden) - ci = (root / "ci.yml").read_text(encoding="utf-8") - for required in ('os: [ubuntu-latest, windows-latest]', 'python-version: ["3.11", "3.12"]', - 'dependencies: [current, minimum]', '"numpy==2.0.0" "pandas==2.2.2"', - 'RIT_TEST_PARQUET: "0"', 'RIT_TEST_PARQUET: "1"', '--require-parquet', - '--baseline-evidence', 'python -m pip check', - '--minimum-tests 3768', '--minimum-tests 3771', - "find_spec('pyarrow') is None", 'import pyarrow'): - assert required in ci - assert ci.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in ci and "--ignore" not in ci - assert "needs: [core, parquet]" in ci - assert "uses: ./.github/workflows/release.yml" in ci - for name in ("ci.yml", "golden.yml", "security.yml"): - text = (root / name).read_text(encoding="utf-8") - assert " push:" not in text - assert " pull_request:" in text and " workflow_dispatch:" in text - golden = (root / "golden.yml").read_text(encoding="utf-8") - for required in ("tests/golden/test_phase3_math.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_phase3_metric_pipeline.py", "tests/integration/test_hero_end_to_end.py", "tests/golden/test_phase4_reports.py", "--golden-junit"): - assert required in golden - security = (root / "security.yml").read_text(encoding="utf-8") - for required in ('tests/golden/test_phase4_reports.py', 'tests/integration/test_no_network.py', 'tests/integration/test_optional_dependency.py', - 'tests/unit/test_PR012_evidence_classes.py', 'tests/unit/test_PR013_report_schema.py', - 'tests/unit/test_PR014_unavailable.py', 'tests/unit/test_PR015_redaction.py', - 'tests/unit/test_PR016_determinism.py', 'tests/unit/test_PR018_language.py', - 'tests/integration/test_partial_provenance_report.py', - 'tests/integration/test_partial_lineage_report.py', - 'tests/integration/test_phase4_cli.py', 'tests/unit/test_phase4_output_safety.py', 'tests/unit/test_phase4_contracts.py', 'tests/integration/test_phase4_gates.py'): - assert required in security - release = (root / "release.yml").read_text(encoding="utf-8") - assert "python -m build" in release and "python -m twine check --strict" in release - assert "--candidate" in release - assert " workflow_call:" in release and " push:" not in release - assert release.count("uses: actions/download-artifact@v4") == 2 - assert "phase4-current-core-ubuntu-latest-3.12-current" in release - assert "phase4-current-real-parquet" in release - assert "core-evidence/source-commit.txt" in release and "parquet-evidence/source-commit.txt" in release - assert "git rev-parse HEAD" in release - assert "--delivery" not in release - assert "recursive-integrity-toolkit-phase4-current-candidate" in release - assert "rit-phase4-step4" not in release - assert "python -m pytest" not in release - assert "SOURCE_DATE_EPOCH" in release and "reproducibility.json" in release - - -def test_phase4_step9_runtime_schemas_resources_and_dependencies_equal_step8(repo_root, phase4_step8_snapshot, phase4_gate_tools): - prefixes = ("src/recursive_integrity_toolkit/", "schemas/", "examples/hero/") - current = {p.relative_to(repo_root).as_posix(): p.read_bytes() for directory in ("src/recursive_integrity_toolkit", "schemas", "examples/hero") for p in (repo_root / directory).rglob("*") if p.is_file() and "__pycache__" not in p.parts} - current["pyproject.toml"] = (repo_root / "pyproject.toml").read_bytes() - accepted = phase4_gate_tools["_phase4_current_files"]() - protected = {k: v for k, v in accepted.items() if k.startswith(prefixes) or k == "pyproject.toml"} - for path in phase4_gate_tools["PHASE4_CURRENT_VERSION_PATHS"]: - protected[path] = protected[path].replace(b"0.1.0.dev2", b"0.1.0.dev3", 1) - assert current == protected - assert sum(p.startswith("src/") and p.endswith(".py") for p in protected) == 40 - - -@pytest.mark.parametrize("relative", ["scripts/release_check.py", "scripts/check_traceability.py", "scripts/check_spec_consistency.py"]) -def test_phase4_step9_maintainer_definitions_preserved_except_exact_dispatch(repo_root, relative): - """Current entry points reject unapproved stages without preserving obsolete bodies.""" - result = subprocess.run([sys.executable, str(repo_root / relative), "--phase", "4", "--step", "12"], - cwd=repo_root, capture_output=True, text=True) - assert result.returncode != 0 - assert "Traceback" not in result.stdout - - -@pytest.mark.parametrize("mutation", [ - "unchanged", "formula", "cli", "assembly", "schema", "resource", "dependency", "version", "plan", "math_oracle", - "golden_json", "golden_markdown", "golden_rationale", "unknown_runtime", "unknown_resource", "missing_runtime", - "completion", "report", "control", -]) -def test_phase4_step9_snapshot_accepts_current_then_rejects_substantive_mutation(repo_root, phase4_step8_snapshot, phase4_gate_tools, tmp_path, mutation): - import json - prior = {p.relative_to(phase4_step8_snapshot).as_posix() for p in phase4_step8_snapshot.rglob("*") if p.is_file()} - paths = set(phase4_gate_tools["_phase4_current_files"]()) | phase4_gate_tools["PHASE4_CURRENT_NEW"] - assert len(prior) == 236 and len(paths) == 246 - for relative in sorted(paths): - target = tmp_path / relative - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes((repo_root / relative).read_bytes()) - verify = phase4_gate_tools["verify_phase4_current_snapshot"] - result = verify(tmp_path) - assert result["frozen_runtime_modules"] == 39 - assert result["package_modules"] == 40 and result["version_literal_changes"] == 2 - assert result["frozen_schemas"] == 5 and result["packaged_resources"] == 7 - assert result["frozen_report_oracle_files"] == 6 and result["phase_complete"] is False - if mutation == "unchanged": - return - paths_by_mutation = { - "formula": "src/recursive_integrity_toolkit/metrics/diversity.py", "cli": "src/recursive_integrity_toolkit/cli.py", - "assembly": "src/recursive_integrity_toolkit/reports/assembly.py", - "schema": "schemas/report.schema.json", "resource": "src/recursive_integrity_toolkit/data/hero/config.json", - "dependency": "pyproject.toml", "version": "src/recursive_integrity_toolkit/__init__.py", "plan": "PHASE_4_PLAN.md", - "math_oracle": "tests/golden/phase3_math_cases.json", "golden_json": "tests/golden/phase4_hero_report.json", - "golden_markdown": "tests/golden/phase4_hero_redacted.md", "golden_rationale": "tests/golden/phase4_report_cases.md", - } - if mutation in paths_by_mutation: - path = tmp_path / paths_by_mutation[mutation] - raw = path.read_bytes() - path.write_bytes(raw + b"\nchanged substantive evidence\n") - elif mutation == "completion": - (tmp_path / "PHASE_4_COMPLETION.md").unlink() - elif mutation in ("unknown_runtime", "unknown_resource", "report"): - relative = {"unknown_runtime": "src/recursive_integrity_toolkit/extra.py", "unknown_resource": "src/recursive_integrity_toolkit/data/extra.json", "report": "report.json"}[mutation] - (tmp_path / relative).write_text("{}", encoding="utf-8") - elif mutation == "missing_runtime": - (tmp_path / "src/recursive_integrity_toolkit/models.py").unlink() - elif mutation == "control": - path = tmp_path / "PHASE_4_BASELINE.json" - value = json.loads(path.read_text(encoding="utf-8")) - value["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/cli.py") - path.write_text(json.dumps(value), encoding="utf-8") - with pytest.raises(ValueError): - verify(tmp_path) diff --git a/tests/performance/README.md b/tests/performance/README.md index e9e092c..2258a13 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -1,5 +1,11 @@ # Performance measurements +Current scheduling: Phase 5 candidate verification runs this directory once on +the designated Ubuntu/Python 3.12 reference profile. Compatibility cells run the +canonical suite without this performance directory. The existing complete 100k +report benchmark remains required. The sparse-lineage placeholder gains actual +execution in Phase 5 Step 9 and currently makes no graph performance claim. + Phase 4 Step 10 extends the retained Phase 3 calculation observations with public CLI audit measurements ending only after both `report.json` and `report.md` have been published. The original Phase 3 test functions and measurement fixture remain diff --git a/tests/unit/test_PR001_identity.py b/tests/unit/test_PR001_identity.py index ea60738..c5fc307 100644 --- a/tests/unit/test_PR001_identity.py +++ b/tests/unit/test_PR001_identity.py @@ -54,11 +54,6 @@ def _csv_row(tmp_path, text): return load_table(InputSource(FileRole.RECORDS_PRIMARY, path)).rows[0] -def test_PR001_step4_owner_metadata(owner_checker): - for path, owner in [("io/normalization.py", "PR-001"), ("io/validation.py", "PR-001")]: - owner_checker(path, owner) - - @pytest.mark.parametrize("field", ["dataset_version", "record_id", "content"]) @pytest.mark.parametrize("missing", [True, False]) def test_PR001_required_field_missing_or_null(field, missing): diff --git a/tests/unit/test_PR002_loaders.py b/tests/unit/test_PR002_loaders.py index 4d57cca..e8c2910 100644 --- a/tests/unit/test_PR002_loaders.py +++ b/tests/unit/test_PR002_loaders.py @@ -28,11 +28,6 @@ def _source(tmp_path, content, suffix=".csv", role=FileRole.RECORDS_PRIMARY, dec return InputSource(role, path, declared) -def test_PR002_loader_owner(owner_checker): - owner_checker("io/loaders.py", "PR-002") - owner_checker("utils/hashing.py", "PR-016") - - @pytest.mark.parametrize("name,fmt", [("step2_records.csv", FileFormat.CSV), ("step2_records.jsonl", FileFormat.JSONL)]) def test_PR002_minimal_local_fixture_loads(name, fmt): source = InputSource(FileRole.RECORDS_PRIMARY, FIXTURES / "minimal_valid" / name) diff --git a/tests/unit/test_PR003_schema_mapping.py b/tests/unit/test_PR003_schema_mapping.py index cdea238..118257d 100644 --- a/tests/unit/test_PR003_schema_mapping.py +++ b/tests/unit/test_PR003_schema_mapping.py @@ -42,8 +42,7 @@ def _map(value, *operations): return map_row({"x": value}, plan) -def test_PR003_owner_and_operation_inventory(owner_checker, schema_root): - owner_checker("io/schema_mapping.py", "PR-003") +def test_PR003_schema_operation_inventory(schema_root): schema = json.loads((schema_root / "schema_mapping.schema.json").read_text()) Draft202012Validator.check_schema(schema) assert {name.removeprefix("op_") for name in schema["$defs"] if name.startswith("op_")} == APPROVED_OPERATIONS diff --git a/tests/unit/test_PR004_coverage.py b/tests/unit/test_PR004_coverage.py index 56c42fb..66600ac 100644 --- a/tests/unit/test_PR004_coverage.py +++ b/tests/unit/test_PR004_coverage.py @@ -5,14 +5,6 @@ """ -def test_PR004_coverage_owner_and_placeholder(owner_checker, repo_root): - owner_checker("metrics/provenance.py", "PR-004") - text = (repo_root / "src/recursive_integrity_toolkit/metrics/provenance.py").read_text(encoding="utf-8") - assert "Phase 3 Step 5" in text - for name in ("closure_bounds", "ancestry_hhi", "confidence_score", "effective_source_diversity"): - assert "def " + name + "(" not in text - - # Step 4 row-field validation only. Joins and coverage remain unimplemented. from datetime import datetime, timezone import json diff --git a/tests/unit/test_PR005_source_shares.py b/tests/unit/test_PR005_source_shares.py index 93313d6..bb4b0ce 100644 --- a/tests/unit/test_PR005_source_shares.py +++ b/tests/unit/test_PR005_source_shares.py @@ -5,14 +5,6 @@ """ -def test_PR005_source_shares_owner_and_placeholder(owner_checker, repo_root): - owner_checker("metrics/provenance.py", "PR-005") - text = (repo_root / "src/recursive_integrity_toolkit/metrics/provenance.py").read_text(encoding="utf-8") - assert "Phase 3 Step 5" in text - for name in ("closure_bounds", "ancestry_hhi", "confidence_score", "effective_source_diversity"): - assert "def " + name + "(" not in text - - # Step 5 tests only declared-field separation. No source counts or shares. def test_PR005_join_keeps_crossed_declarations_without_inference(repo_root): from recursive_integrity_toolkit.io.loaders import load_table diff --git a/tests/unit/test_PR006_duplicates.py b/tests/unit/test_PR006_duplicates.py index e7f5bac..86f8d1e 100644 --- a/tests/unit/test_PR006_duplicates.py +++ b/tests/unit/test_PR006_duplicates.py @@ -7,7 +7,6 @@ from itertools import permutations from pathlib import Path from types import MappingProxyType -import ast import hashlib import json import pytest @@ -37,16 +36,6 @@ def detect(rows, **changes): return detect_exact_duplicates(tuple(rows), **options(**changes)) -def test_PR006_duplicates_owner_and_placeholder(owner_checker, package_root): - # Historical identity retained; only its obsolete placeholder requirement migrates. - owner_checker("metrics/duplicates.py", "PR-006") - owner_checker("representations/content_hash.py", "PR-006") - tree = ast.parse((package_root / "metrics/duplicates.py").read_text(encoding="utf-8")) - names = {n.name for n in tree.body if isinstance(n, ast.FunctionDef)} - assert names == {"detect_exact_duplicates"} - assert callable(detect_exact_duplicates) - - def test_PR006_identical_normalized_content_groups(): result = detect((row(key="c"), row("different", "b"), row(key="a"))) assert result.duplicate_record_count.value == 1 diff --git a/tests/unit/test_PR008_parent_resolution.py b/tests/unit/test_PR008_parent_resolution.py index 5c33cd6..b3cc62b 100644 --- a/tests/unit/test_PR008_parent_resolution.py +++ b/tests/unit/test_PR008_parent_resolution.py @@ -12,11 +12,6 @@ """ -def test_PR008_parent_resolution_owner_and_placeholder(owner_checker, placeholder_checker): - owner_checker("lineage/graph.py", "PR-008") - placeholder_checker("lineage/graph.py") - - # Phase 2 Step 6: PR-008 immediate references, without graph algorithms. import pytest from recursive_integrity_toolkit.errors import CanonicalValidationError, ErrorCode, WarningCode diff --git a/tests/unit/test_PR009_generation.py b/tests/unit/test_PR009_generation.py index 6b4ce1f..c13f5ed 100644 --- a/tests/unit/test_PR009_generation.py +++ b/tests/unit/test_PR009_generation.py @@ -1,6 +1,5 @@ """PR-009 Step 4 declared generation types only; parent-based derivation is deferred.""" -import ast import pytest from recursive_integrity_toolkit.errors import CanonicalValidationError, ErrorCode @@ -14,15 +13,6 @@ def _row(generation): "provenance_confidence": "unknown", "external_grounding": "yes", "generation": generation} -def test_PR009_generation_owner_and_no_derivation(owner_checker, package_root, placeholder_checker): - owner_checker("io/validation.py", "PR-009") - tree = ast.parse((package_root / "io/validation.py").read_text()) - names = {node.name for node in ast.walk(tree) if isinstance(node, ast.FunctionDef)} - assert not names & {"expected_generation", "derive_generation", "lineage_depth", "detect_cycles"} - for path in ["lineage/graph.py", "lineage/cycles.py", "lineage/ancestry.py"]: - placeholder_checker(path) - - @pytest.mark.parametrize("value", [0, 1, 2, 1000, None]) def test_PR009_declared_generation_is_preserved_without_inference(value): row = normalize_row(_row(value), kind="provenance") diff --git a/tests/unit/test_PR012_evidence_classes.py b/tests/unit/test_PR012_evidence_classes.py index 1223096..adebc27 100644 --- a/tests/unit/test_PR012_evidence_classes.py +++ b/tests/unit/test_PR012_evidence_classes.py @@ -12,13 +12,6 @@ """ -def test_PR012_evidence_classes_owner_and_placeholder(owner_checker, placeholder_checker,phase3_final_owner_checker,phase3_final_placeholder_checker): - owner_checker = phase3_final_owner_checker - placeholder_checker = phase3_final_placeholder_checker - owner_checker("reports/assembly.py", "PR-012") - placeholder_checker("reports/assembly.py") - - def phase4_step2_evidence_report_fixture(): """Independent literal control data, never generated from production code.""" return { diff --git a/tests/unit/test_PR013_report_schema.py b/tests/unit/test_PR013_report_schema.py index b87e7dd..86d82e9 100644 --- a/tests/unit/test_PR013_report_schema.py +++ b/tests/unit/test_PR013_report_schema.py @@ -12,13 +12,6 @@ """ -def test_PR013_report_schema_owner_and_placeholder(owner_checker, placeholder_checker,phase3_final_owner_checker,phase3_final_placeholder_checker): - owner_checker = phase3_final_owner_checker - placeholder_checker = phase3_final_placeholder_checker - owner_checker("reports/json_report.py", "PR-013") - placeholder_checker("reports/json_report.py") - - def phase4_step2_report_fixture(): """Hand-authored empty report from reporting sections 8, 10 and 31. diff --git a/tests/unit/test_PR014_unavailable.py b/tests/unit/test_PR014_unavailable.py index d2bb192..8a52535 100644 --- a/tests/unit/test_PR014_unavailable.py +++ b/tests/unit/test_PR014_unavailable.py @@ -12,13 +12,6 @@ """ -def test_PR014_unavailable_owner_and_placeholder(owner_checker, placeholder_checker,phase3_final_owner_checker,phase3_final_placeholder_checker): - owner_checker = phase3_final_owner_checker - placeholder_checker = phase3_final_placeholder_checker - owner_checker("reports/assembly.py", "PR-014") - placeholder_checker("reports/assembly.py") - - # Phase 4 Step 3: independently authored report assembly expectations. def phase4_step3_run(): return { diff --git a/tests/unit/test_PR015_redaction.py b/tests/unit/test_PR015_redaction.py index eac0737..423dcc2 100644 --- a/tests/unit/test_PR015_redaction.py +++ b/tests/unit/test_PR015_redaction.py @@ -12,13 +12,6 @@ """ -def test_PR015_redaction_owner_and_placeholder(owner_checker, placeholder_checker,phase3_final_owner_checker,phase3_final_placeholder_checker): - owner_checker = phase3_final_owner_checker - placeholder_checker = phase3_final_placeholder_checker - owner_checker("utils/logging.py", "PR-015") - placeholder_checker("utils/logging.py") - - # Phase 4 Step 4: independently authored privacy cases. def phase4_step4_private_report(tmp_path, *, scenario=False, duplicates=False, pair=False): import json diff --git a/tests/unit/test_PR016_determinism.py b/tests/unit/test_PR016_determinism.py index 48d89f7..397dca6 100644 --- a/tests/unit/test_PR016_determinism.py +++ b/tests/unit/test_PR016_determinism.py @@ -1,6 +1,5 @@ """PR-016 lexical order retained; Step 8 seeded determinism has explicit boundaries.""" -import ast from itertools import permutations import pytest @@ -13,21 +12,6 @@ def _row(version, identity): kind="records") -def test_PR016_ordering_owner_and_no_later_behavior(owner_checker, package_root, placeholder_checker,phase3_final_placeholder_checker): - placeholder_checker = phase3_final_placeholder_checker - owner_checker("utils/ordering.py", "PR-016") - tree = ast.parse((package_root / "utils/ordering.py").read_text()) - names = {node.name for node in ast.walk(tree) if isinstance(node, ast.FunctionDef)} - assert names == {"_record_order_key", "stable_record_order"} - for path in ["reports/json_report.py", "reports/markdown_report.py"]: - placeholder_checker(path) - # The authorized shared module is checked against its exact reviewed body. - import runpy - checker = runpy.run_path(str(package_root.parents[1] / "scripts/check_traceability.py")) - source = (package_root / "metrics/resampling.py").read_text(encoding="utf-8") - checker["_phase3_resampling_boundary"](ast.parse(source)) - - def test_PR016_stable_exact_lexical_order_is_not_version_chronology(): rows = (_row("v2", "a"), _row("v10", "b"), _row("v1", "0001")) expected = (rows[2], rows[1], rows[0]) diff --git a/tests/unit/test_PR017_content_refs.py b/tests/unit/test_PR017_content_refs.py index 61d118b..99b9206 100644 --- a/tests/unit/test_PR017_content_refs.py +++ b/tests/unit/test_PR017_content_refs.py @@ -9,10 +9,6 @@ from recursive_integrity_toolkit.utils.paths import local_input_path -def test_PR017_paths_owner(owner_checker): - owner_checker("utils/paths.py", "PR-017") - - @pytest.mark.parametrize("value", ["http://invalid.example/data.csv", "https:/invalid/data.csv", "s3://bucket/file.csv", "gs://bucket/file.csv", "ftp://host/f.csv", "file:///tmp/f.csv", "ssh://host/file", "//server/share/file.csv", r"\\server\share\file.csv", r"\\?\C:\file.csv", "", "bad\x00path"]) def test_PR017_nonlocal_paths_rejected_before_io(value, monkeypatch): def forbidden(*args, **kwargs): diff --git a/tests/unit/test_PR018_language.py b/tests/unit/test_PR018_language.py index 81db652..76ba359 100644 --- a/tests/unit/test_PR018_language.py +++ b/tests/unit/test_PR018_language.py @@ -12,13 +12,6 @@ """ -def test_PR018_language_owner_and_placeholder(owner_checker, placeholder_checker,phase3_final_owner_checker,phase3_final_placeholder_checker): - owner_checker = phase3_final_owner_checker - placeholder_checker = phase3_final_placeholder_checker - owner_checker("reports/markdown_report.py", "PR-018") - placeholder_checker("reports/markdown_report.py") - - # Phase 4 Step 3: independently authored report assembly expectations. def phase4_step3_run(): return { diff --git a/tests/unit/test_T1_compatibility.py b/tests/unit/test_T1_compatibility.py index 5182bac..0d2882e 100644 --- a/tests/unit/test_T1_compatibility.py +++ b/tests/unit/test_T1_compatibility.py @@ -74,11 +74,6 @@ def _mapping_pair(direction='earlier_to_later'): return a, b, kw -def test_step9_owner_and_new_module_scope(owner_checker): - for owner in ('T1', 'PR-007', 'PR-011'): - owner_checker('representations/compatibility.py', owner) - - def test_equal_size_support_can_lose_and_add_states(): x = _compare() assert x.support_delta.value == 0 and x.support_loss_count.value == x.support_added_count.value == 1 diff --git a/tests/unit/test_T1_diversity.py b/tests/unit/test_T1_diversity.py index 8486851..ff0cf67 100644 --- a/tests/unit/test_T1_diversity.py +++ b/tests/unit/test_T1_diversity.py @@ -33,13 +33,6 @@ def weighted(r,w): return calculate_state_distribution(r,weighting=WeightingOpti def approx(v): return pytest.approx(float(v),abs=1e-12,rel=1e-12) -def test_T1_diversity_owner_and_placeholder(owner_checker,repo_root): - owner_checker('metrics/diversity.py','T1') - text=(repo_root/'src/recursive_integrity_toolkit/metrics/diversity.py').read_text(encoding='utf-8') - assert 'F-003' in text and 'F-004' in text - assert all('def '+name not in text for name in ('support_delta','shannon_entropy','simulate','ancestry_hhi')) - - @pytest.mark.parametrize('p,d',[(('1',),'0'),(('1/2','1/2'),'1/2'),(('1/4',)*4,'3/4'),(('1/2','1/4','1/4'),'5/8')]) def test_F003_A_to_D(p,d): x=distribution_from_probabilities(tuple((str(i),float(Fraction(v))) for i,v in enumerate(p)),scope=scope(),representation=descriptor()) diff --git a/tests/unit/test_T1_resampling.py b/tests/unit/test_T1_resampling.py index 240a18e..0f9f638 100644 --- a/tests/unit/test_T1_resampling.py +++ b/tests/unit/test_T1_resampling.py @@ -37,15 +37,6 @@ def sample(p=None, **kw): return simulate_closed_resampling({'a':.5,'b':.25,'c':.25} if p is None else p, **args) -def test_T1_resampling_owner_and_placeholder(owner_checker, repo_root): - owner_checker('metrics/resampling.py', 'T1') - text = (repo_root/'src/recursive_integrity_toolkit/metrics/resampling.py').read_text() - assert 'Phase 3 Step 8' in text - assert 'def simulate_closed_resampling(' in text - for name in ('simulate_reopening', 'external_reference_loss', 'ancestry_hhi'): - assert 'def '+name+'(' not in text - - def test_T1_expected_one_step_contraction_matches_formula(): result=analytic({'a':.5,'b':.5},resample_size=2,steps=1) assert result.expected_diversity == (.5,.25) diff --git a/tests/unit/test_T1_support.py b/tests/unit/test_T1_support.py index a799303..3ca88de 100644 --- a/tests/unit/test_T1_support.py +++ b/tests/unit/test_T1_support.py @@ -27,13 +27,6 @@ def represented(fields, policy='exclude', missing=None): return assign_field_states(rows,dataset_versions=('v1',),scope_id='support-test',config=RepresentationConfig('topic','topic_field','topic','v1',policy),missing_state_id=missing) -def test_T1_support_owner_and_placeholder(owner_checker,repo_root): - owner_checker('metrics/diversity.py','T1') - text=(repo_root/'src/recursive_integrity_toolkit/metrics/diversity.py').read_text(encoding='utf-8') - assert 'Phase 3 Step 4' in text - assert all('def '+name not in text for name in ('support_delta','support_retention','resample')) - - def test_F001_counts_and_frequencies(): x=distribution_from_counts({'c':1,'a':2,'b':1},scope=scope(),representation=descriptor()) assert tuple(s.state_count for s in x.states)==(2,1,1) diff --git a/tests/unit/test_T2_tail.py b/tests/unit/test_T2_tail.py index 5eca9d4..c9880ae 100644 --- a/tests/unit/test_T2_tail.py +++ b/tests/unit/test_T2_tail.py @@ -51,13 +51,6 @@ def value(result): return result.one_step_extinction_probability.value -def test_T2_tail_owner_and_placeholder(owner_checker,repo_root): - owner_checker('metrics/tail.py','T2') - text=(repo_root/'src/recursive_integrity_toolkit/metrics/tail.py').read_text() - assert 'Phase 3 Step 7' in text - assert all('def '+name+'(' not in text for name in ('simulate','reopen','risk_score','tail_fragility_signal')) - - @pytest.mark.parametrize('rule,args,expected',[ ('singleton_count',{},('bird','fish','refund')), ('count_at_or_below',{'count_threshold':0},()), diff --git a/tests/unit/test_T3_bounds.py b/tests/unit/test_T3_bounds.py index f9f4a59..22ef1b7 100644 --- a/tests/unit/test_T3_bounds.py +++ b/tests/unit/test_T3_bounds.py @@ -39,14 +39,6 @@ def composition(fields): def values(x): return x.lower_bound.value,x.upper_bound.value,x.interval_width.value -def test_T3_bounds_owner_and_placeholder(owner_checker,repo_root): - owner_checker('metrics/bounds.py','T3') - text=(repo_root/'src/recursive_integrity_toolkit/metrics/bounds.py').read_text() - assert 'Phase 3 Step 6' in text - for name in ('lineage_closure_bounds','midpoint','risk_score','ancestry_hhi'): - assert 'def '+name+'(' not in text - - @pytest.mark.parametrize('c,u,n,expected',[(3,2,8,(3/8,5/8,1/4)),(0,0,8,(0,0,0)), (8,0,8,(1,1,0)),(0,8,8,(0,1,1)),(1,0,1,(1,1,0)),(0,1,1,(0,1,1)),(2,0,8,(1/4,1/4,0))]) def test_exact_intervals(c,u,n,expected): diff --git a/tests/unit/test_T3_provenance.py b/tests/unit/test_T3_provenance.py index 2d97f5b..673b5e3 100644 --- a/tests/unit/test_T3_provenance.py +++ b/tests/unit/test_T3_provenance.py @@ -5,14 +5,6 @@ """ -def test_T3_provenance_owner_and_placeholder(owner_checker, repo_root): - owner_checker("metrics/provenance.py", "T3") - text = (repo_root / "src/recursive_integrity_toolkit/metrics/provenance.py").read_text(encoding="utf-8") - assert "Phase 3 Step 5" in text - for name in ("closure_bounds", "ancestry_hhi", "confidence_score", "effective_source_diversity"): - assert "def " + name + "(" not in text - - # Direct declaration-bound input basis only; no intervals or graph traversal. import json import pytest diff --git a/tests/unit/test_T4_ancestry.py b/tests/unit/test_T4_ancestry.py index b1ff67e..1d75e02 100644 --- a/tests/unit/test_T4_ancestry.py +++ b/tests/unit/test_T4_ancestry.py @@ -1,17 +1,4 @@ -"""Phase 1 placeholder for T4. +"""Reserved for direct Phase 5 ancestry behavior tests. -Planned scope: - Future ancestry tests are deferred to Phase 5. - -Current Phase 1 scope: - Verify the approved owner ID and confirm that the target module remains a - docstring-only, import-safe placeholder. - -Limits: - No mathematical metric, data loading, graph operation, or report result is tested. +Step 1 records independently authored fixtures before implementation. """ - - -def test_T4_ancestry_owner_and_placeholder(owner_checker, placeholder_checker): - owner_checker("lineage/ancestry.py", "T4") - placeholder_checker("lineage/ancestry.py") diff --git a/tests/unit/test_T5_reopening.py b/tests/unit/test_T5_reopening.py index 1bb69c7..bddaa48 100644 --- a/tests/unit/test_T5_reopening.py +++ b/tests/unit/test_T5_reopening.py @@ -3,8 +3,7 @@ import inspect -def test_T5_reopening_owner_and_placeholder(owner_checker, repo_root): - owner_checker("metrics/resampling.py", "T5") +def test_T5_reopening_public_boundary(repo_root): from recursive_integrity_toolkit.metrics import resampling source=(repo_root/"src/recursive_integrity_toolkit/metrics/resampling.py").read_text() public={node.name for node in ast.parse(source).body if isinstance(node,ast.FunctionDef) and not node.name.startswith("_")} diff --git a/tests/unit/test_T6_cycles.py b/tests/unit/test_T6_cycles.py index af647d3..98fc95d 100644 --- a/tests/unit/test_T6_cycles.py +++ b/tests/unit/test_T6_cycles.py @@ -1,17 +1,4 @@ -"""Phase 1 placeholder for T6. +"""Reserved for direct Phase 5 cycle behavior tests. -Planned scope: - Future cycle tests are deferred to Phase 5. - -Current Phase 1 scope: - Verify the approved owner ID and confirm that the target module remains a - docstring-only, import-safe placeholder. - -Limits: - No mathematical metric, data loading, graph operation, or report result is tested. +Step 1 records independently authored fixtures before implementation. """ - - -def test_T6_cycles_owner_and_placeholder(owner_checker, placeholder_checker): - owner_checker("lineage/cycles.py", "T6") - placeholder_checker("lineage/cycles.py") diff --git a/tests/unit/test_lineage_fixture_inputs.py b/tests/unit/test_lineage_fixture_inputs.py new file mode 100644 index 0000000..8b40ed4 --- /dev/null +++ b/tests/unit/test_lineage_fixture_inputs.py @@ -0,0 +1,94 @@ +"""Load Phase 5 fixtures through current input validation only. + +Future lineage oracle values in the fixture documents are not executed here. +These checks detect malformed acceptance inputs, loss of parent declarations, +and incorrect reference counts before the graph implementation consumes them. +""" + +import json +from pathlib import Path + +import pytest + +from recursive_integrity_toolkit.io.validation import validate_bundle +from recursive_integrity_toolkit.models import ( + AuditBundle, + FileRole, + InputSource, + ParentResolutionStatus, + ValidationSeverity, +) + + +FIXTURE_ROOT = Path(__file__).resolve().parents[1] / "fixtures" +CASES = [ + case + for directory in ("lineage_complete", "lineage_multi_root", "lineage_cycles") + for case in json.loads((FIXTURE_ROOT / directory / "cases.json").read_text())["cases"] +] + + +def _write_rows(path, rows): + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + + +@pytest.mark.parametrize("case", CASES, ids=lambda case: case["case_id"]) +def test_lineage_acceptance_inputs_preserve_declared_evidence(case, tmp_path): + target_keys = set(case["target_record_keys"]) + primary = [ + row for row in case["records"] + if f"{row['dataset_version']}::{row['record_id']}" in target_keys + ] + context = [row for row in case["records"] if row not in primary] + if not primary: + # Empty target is a future in-memory selection over loaded context. + # This does not claim support for an empty current CLI primary file. + primary, context = context, [] + primary_path = tmp_path / "records.jsonl" + _write_rows(primary_path, primary) + sources = [InputSource(FileRole.RECORDS_PRIMARY, primary_path)] + if context: + context_path = tmp_path / "context.jsonl" + _write_rows(context_path, context) + # The future context role is not implemented in Step 1. The current + # comparison role loads the same canonical rows for input checks only. + sources.append(InputSource(FileRole.RECORDS_COMPARE, context_path)) + provenance_path = tmp_path / "provenance.jsonl" + _write_rows(provenance_path, case["provenance"]) + sources.append(InputSource(FileRole.PROVENANCE_MANIFEST, provenance_path)) + order_path = tmp_path / "version_order.json" + order_path.write_text(json.dumps({"version_order": case["version_order"]}), encoding="utf-8") + sources.append(InputSource(FileRole.VERSION_ORDER, order_path)) + result = validate_bundle(AuditBundle(tuple(sources))) + + assert {str(row.record_key) for row in result.records} == { + f"{row['dataset_version']}::{row['record_id']}" for row in case["records"] + } + expected = case["expected_input"] + error_codes = { + message.code for message in result.validation_messages + if message.severity in (ValidationSeverity.ERROR, ValidationSeverity.FATAL) + } + assert error_codes == set(expected["error_codes"]) + warning_codes = { + message.code for message in result.validation_messages + if message.severity is ValidationSeverity.WARNING + } + assert set(expected["required_warning_codes"]) <= warning_codes + + if "target_declared_references" in expected: + assert result.generation is not None + parents = {str(item.child_key): item for item in result.generation.parents} + references = [ref for key in target_keys for ref in parents[key].references] + assert sum(len(ref.source_references) for ref in references) == expected[ + "target_declared_references" + ] + assert sum( + len(ref.source_references) for ref in references + if ref.resolution_status is ParentResolutionStatus.RESOLVED + ) == expected["target_resolved_references"] + provenance = {str(row.record_key): row for row in result.provenance} + for key, state in expected.get("parent_field_states", {}).items(): + # Normalization materializes absent parent_ids as null in values, while + # field_states retains the original distinction for the graph adapter. + assert provenance[key].field_states["parent_ids"] == state diff --git a/tests/unit/test_phase3_contracts.py b/tests/unit/test_phase3_contracts.py index 36ac2fe..b86054e 100644 --- a/tests/unit/test_phase3_contracts.py +++ b/tests/unit/test_phase3_contracts.py @@ -1,11 +1,8 @@ -"""Phase 3 Step 1 contract/baseline tests. No metric or state assignment runs.""" -import ast -from copy import deepcopy +"""Direct calculation contract validation and independent mathematical oracles.""" from dataclasses import FrozenInstanceError, replace import hashlib import json from pathlib import Path -import runpy import pytest from recursive_integrity_toolkit.models import ( @@ -31,10 +28,6 @@ def metadata(): "dimensionless", "supplied-not-computed", scope(), descriptor()) -def tools(): - return runpy.run_path(str(ROOT / "scripts/release_check.py"), run_name="contract_test_tools") - - def scenario(**changes): values = dict(method="analytic_expectation", resample_size=8, simulation_horizon=1, representation=descriptor(), state_order=("a", "b"), assumptions=("finite closed categorical model",)) @@ -218,38 +211,6 @@ def blocked(*args,**kwargs): assert not calls and capsys.readouterr() == ("", "") -def test_phase3_approved_plan_and_baseline_control(): - control=json.loads((ROOT/"PHASE_3_BASELINE.json").read_text(encoding="utf-8")) - assert hashlib.sha256((ROOT/"PHASE_3_PLAN.md").read_bytes()).hexdigest()==control["approved_plan_sha256"] - tools()["verify_phase3_control"](control) - - -@pytest.mark.parametrize("key", ["baseline_commit","baseline_tree","baseline_test_tree","approved_plan_sha256", - "active_phase","active_step","permitted_paths","approved_decisions","phase_complete"]) -def test_phase3_forged_manifest_does_not_authorize_a_change(key): - control=deepcopy(json.loads((ROOT/"PHASE_3_BASELINE.json").read_text(encoding="utf-8"))) - if key in ("active_phase","active_step"): control[key]+=1 - elif key=="phase_complete": control[key]=not control[key] - elif key=="permitted_paths": control[key].append("src/recursive_integrity_toolkit/metrics/diversity.py") - elif key=="approved_decisions": control[key].pop() - else: control[key]="0"*40 - with pytest.raises(ValueError): tools()["verify_phase3_control"](control) - - -@pytest.mark.parametrize("status,path", [ - ("M","VALIDATION_PLAN.md"), ("M","src/recursive_integrity_toolkit/io/validation.py"), - ("M","src/recursive_integrity_toolkit/metrics/diversity.py"), ("M","src/recursive_integrity_toolkit/cli.py"), - ("A","tests/unit/test_arbitrary.py"), ("D","tests/conftest.py"), ("M","pyproject.toml"), -]) -def test_phase3_unauthorized_path_change_fails(status,path): - with pytest.raises(ValueError): tools()["verify_prior_step_changes"]([(status,path)], step=1) - - -def test_phase3_exact_new_file_set(): - t=tools(); t["verify_phase3_changes"]([("A",p) for p in t["STEP1_NEW"]]) - assert len(t["STEP1_NEW"])==6 and "VALIDATION_PLAN.md" not in t["STEP1_ALLOWED"] - - def test_phase3_oracle_is_independent_source_transcription(): path=ROOT/"tests/golden/phase3_math_cases.json" assert hashlib.sha256(path.read_bytes()).hexdigest()=="b494d50a1a9bd1009c060c998e4cb3c7433d0e8949f73a8d627e538fa38884b2" @@ -265,21 +226,3 @@ def test_phase3_oracle_is_independent_source_transcription(): assert by_id["HERO-PAIR"]["expected"]["lost_states"]==["battery","lizard","turtle"] assert by_id["HERO-EXTINCTION-CAT"]["expected"]["one_step_extinction_probability"]=="390625/16777216" # Authored literal comparisons only; no implementation calculates the oracle. - - -@pytest.mark.parametrize("body", ["return value ** 2", "return value / 2", "return sum(value)", - "return open('private')", "return eval(value)", "return __import__('numpy')", "return math.sqrt(value)", - "return (lambda: value)()"]) -def test_phase3_contract_gate_rejects_hidden_calculation_or_execution(body): - checker=runpy.run_path(str(ROOT/"scripts/check_traceability.py"),run_name="contract_boundary_test") - with pytest.raises(SystemExit): checker["_phase3_contract_boundary"](ast.parse("def _calculation_number(value):\n "+body+"\n")) - - -def test_phase3_current_model_passes_restricted_contract_gate(): - checker=runpy.run_path(str(ROOT/"scripts/check_traceability.py"),run_name="contract_boundary_test") - checker["_phase3_contract_boundary"](ast.parse((ROOT/"src/recursive_integrity_toolkit/models.py").read_text(encoding="utf-8"))) - - -def test_phase3_final_completion_records_not_created(phase3_step10_snapshot): - for name in ("PHASE_3_COMPLETION.md","PHASE_3_VALIDATION_REPORT.md","PHASE_3_ARCHITECTURE_COMPLIANCE_REPORT.md"): - assert not (phase3_step10_snapshot/name).exists() diff --git a/tests/unit/test_phase4_contracts.py b/tests/unit/test_phase4_contracts.py deleted file mode 100644 index 72d21ce..0000000 --- a/tests/unit/test_phase4_contracts.py +++ /dev/null @@ -1,1627 +0,0 @@ -"""Approved Phase 4 Step 1 controls cannot authorize their own expansion. - -These gates protect the current tree. Reporting contracts and calculations remain -at the accepted Phase 3 bytes until their separately authorized steps. -""" -from copy import deepcopy -import hashlib -import json -from pathlib import Path -import runpy - -import pytest - - -ROOT = Path(__file__).resolve().parents[2] -BASELINE = "e3ffb8c0a88bfe31f669f9662d9b5213da628b3a" -BASELINE_TREE = "e2a25f8cfdc66c3317809c479f80fdae162e6ba9" -BASELINE_TEST_TREE = "6ab22cb9197a8f094f455b29a07a851062bb26c9" -PLAN_SHA256 = "5a6d65696720a426630e876d001937b2837d24774fcb6c2bd43b8e09f5ae93f0" -NEW_FILES = { - "PHASE_4_PLAN.md", "PHASE_4_BASELINE.json", "PHASE_4_DECISIONS.md", - "tests/unit/test_phase4_contracts.py", "tests/integration/test_phase4_gates.py", -} - - -@pytest.fixture(scope="module") -def phase4_tools(): - return runpy.run_path(str(ROOT / "scripts/release_check.py"), run_name="phase4_contract_tests") - - -@pytest.fixture -def phase4_control(phase4_step1_snapshot): - return json.loads((phase4_step1_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - - -def test_phase4_accepted_anchors_and_plan_bytes_are_independent(phase4_tools, phase4_control): - assert phase4_tools["PHASE4_FINAL"] == BASELINE - assert phase4_tools["PHASE4_TREE"] == BASELINE_TREE - assert phase4_tools["PHASE4_TEST_TREE"] == BASELINE_TEST_TREE - assert phase4_tools["PHASE4_PLAN_SHA256"] == PLAN_SHA256 - assert hashlib.sha256((ROOT / "PHASE_4_PLAN.md").read_bytes()).hexdigest() == PLAN_SHA256 - assert phase4_control["baseline_commit"] == BASELINE - assert phase4_control["baseline_core_tests"] == 2489 - assert phase4_control["baseline_parquet_tests"] == 2492 - assert phase4_control["active_phase"] == 4 - assert phase4_control["active_step"] == 1 - assert phase4_control["approved_decisions"] == [f"P4-D{i:02d}" for i in range(1, 12)] - phase4_tools["verify_phase4_control"](phase4_control, step=1) - - -@pytest.mark.parametrize("field,value", [ - ("active_phase", 3), ("active_phase", True), - ("active_step", 2), ("active_step", True), - ("baseline_commit", "0" * 40), ("baseline_tree", "0" * 40), - ("baseline_test_tree", "0" * 40), ("approved_plan_sha256", "0" * 64), - ("baseline_core_tests", 1), ("baseline_parquet_tests", 1), - ("approved_decisions", []), ("approved_decisions", ["P4-D01"]), - ("phase_complete", True), ("next_step_authorized", True), - ("main_merge_authorized", True), ("publication_authorized", True), - ("permitted_paths", ["src/recursive_integrity_toolkit/result.py"]), - ("new_files_permitted", ["PHASE_4_COMPLETION.md"]), -]) -def test_phase4_control_rejects_forged_authority(phase4_tools, phase4_control, field, value): - phase4_control[field] = value - with pytest.raises(ValueError): - phase4_tools["verify_phase4_control"](phase4_control, step=1) - - -@pytest.mark.parametrize("step", [0, 2, 11, True, "1", None]) -def test_phase4_control_rejects_unapproved_dispatch_steps(phase4_tools, phase4_control, step): - with pytest.raises(ValueError): - phase4_tools["verify_phase4_control"](phase4_control, step=step) - - -@pytest.mark.parametrize("field", ["active_step", "baseline_commit", "approved_decisions", "permitted_paths"]) -def test_phase4_control_requires_its_approval_fields(phase4_tools, phase4_control, field): - del phase4_control[field] - with pytest.raises(ValueError): - phase4_tools["verify_phase4_control"](phase4_control, step=1) - - -def test_phase4_manifest_cannot_mint_an_extra_permission(phase4_tools, phase4_control): - forged = deepcopy(phase4_control) - forged["approved_scope_expansion"] = {"step": 2, "result_implementation": True} - with pytest.raises(ValueError): - phase4_tools["verify_phase4_control"](forged, step=1) - - -def test_phase4_fixed_path_boundary_opens_only_governance(phase4_tools): - assert phase4_tools["PHASE4_STEP1_NEW"] == NEW_FILES - allowed = phase4_tools["PHASE4_STEP1_ALLOWED"] - assert not any(path.startswith(("src/", "schemas/", "examples/", "tests/golden/")) for path in allowed) - phase4_tools["verify_phase4_changes"]([ - ("A", "PHASE_4_PLAN.md"), ("A", "PHASE_4_BASELINE.json"), - ("M", "scripts/release_check.py"), ("M", "tests/conftest.py"), - ]) - - -@pytest.mark.parametrize("status,path", [ - ("M", "src/recursive_integrity_toolkit/metrics/diversity.py"), - ("M", "src/recursive_integrity_toolkit/result.py"), - ("M", "schemas/report.schema.json"), - ("M", "tests/golden/phase3_math_cases.json"), - ("M", "PROJECT_INSTRUCTIONS.md"), - ("M", "PHASE_3_BASELINE.json"), - ("M", "PHASE_4_PLAN.md"), - ("A", "scripts/release_check.py"), - ("A", "PHASE_4_COMPLETION.md"), - ("A", "tests/unit/test_unapproved.py"), - ("A", "src/recursive_integrity_toolkit/reports/new.py"), - ("D", "scripts/release_check.py"), - ("R100", "scripts/release_check.py"), - ("T", "scripts/release_check.py"), - ("M", "./scripts/release_check.py"), - ("M", "scripts/../scripts/release_check.py"), -]) -def test_phase4_changes_reject_protected_or_unlisted_operations(phase4_tools, status, path): - with pytest.raises(ValueError): - phase4_tools["verify_phase4_changes"]([(status, path)]) - - -def test_phase4_changes_reject_duplicate_path_operations(phase4_tools): - with pytest.raises(ValueError): - phase4_tools["verify_phase4_changes"]([ - ("M", "scripts/release_check.py"), ("M", "scripts/release_check.py"), - ]) - - -def test_phase4_historical_phase3_control_still_uses_its_original_contract(phase4_tools): - original = json.loads((ROOT / "PHASE_3_BASELINE.json").read_text(encoding="utf-8")) - assert phase4_tools["ACTIVE_PHASE"] == 3 and phase4_tools["ACTIVE_STEP"] == 11 - phase4_tools["verify_phase3_control"](original, step=11) - original["active_phase"] = 4 - with pytest.raises(ValueError): - phase4_tools["verify_phase3_control"](original, step=11) - - -def test_phase4_registered_migrations_accept_only_the_approved_inventory(phase4_tools, phase3_final_snapshot, phase4_step1_snapshot): - approved_files = { - "tests/unit/test_PR012_evidence_classes.py", "tests/unit/test_PR013_report_schema.py", - "tests/unit/test_PR014_unavailable.py", "tests/unit/test_PR015_redaction.py", - "tests/unit/test_PR016_determinism.py", "tests/unit/test_PR018_language.py", - "tests/integration/test_no_algorithms.py", "tests/integration/test_phase3_metric_pipeline.py", - "tests/integration/test_cli_validation.py", "tests/integration/test_ci_workflows.py", - } - registry = phase4_tools["PHASE4_MIGRATIONS"] - assert set(registry) == approved_files - assert sum(len(rows) for rows in registry.values()) == 16 - for path in sorted(approved_files): - phase4_tools["verify_phase4_test_migration"]( - path, (phase3_final_snapshot / path).read_bytes(), (phase4_step1_snapshot / path).read_bytes(), - ) - - -@pytest.mark.parametrize("mutation", ["assertion", "wrong_baseline", "duplicate_node", "unlisted_path"]) -def test_phase4_migration_guard_rejects_weakening_or_wrong_identity(phase4_tools, phase3_final_snapshot, mutation, phase4_step1_snapshot): - path = "tests/integration/test_cli_validation.py" - before = (phase3_final_snapshot / path).read_bytes() - after = (phase4_step1_snapshot / path).read_bytes() - verify = phase4_tools["verify_phase4_test_migration"] - verify(path, before, after) - if mutation == "assertion": - assertion = b"assert result.returncode == 0, result.stderr" - assert assertion in after - after = after.replace(assertion, b"assert True", 1) - elif mutation == "wrong_baseline": - before += b"\n# Claimed replacement baseline.\n" - elif mutation == "duplicate_node": - after += b"\ndef test_cli_help_runs():\n assert True\n" - else: - path = "tests/integration/test_package_install.py" - with pytest.raises(ValueError): - verify(path, before, after) - - -@pytest.mark.parametrize("statement", [ - b"PHASE3_ACTIVE_STEP += 1", - b"ALLOWED_FUNCTIONS.clear()", - b"PHASE4_MUTATION = ALLOWED_FUNCTIONS.clear()", - b"def phase4_mutator(value=ALLOWED_FUNCTIONS.clear()):\n pass", - b"def phase4_mutator(*, value=ALLOWED_FUNCTIONS.clear()):\n pass", - b"@ALLOWED_FUNCTIONS.clear()\ndef phase4_mutator():\n pass", - b"def phase4_mutator() -> ALLOWED_FUNCTIONS.clear():\n pass", - b"def phase4_mutator(value: ALLOWED_FUNCTIONS.clear()):\n pass", -]) -def test_phase4_tooling_guard_rejects_appended_mutation_of_historical_state(phase4_tools, statement): - path = "scripts/check_traceability.py" - current = (ROOT / path).read_bytes() - verify = phase4_tools["_phase4_preserve_tooling"] - verify(current, current, path) - with pytest.raises(ValueError): - verify(current, current + b"\n" + statement + b"\n", path) - - -def test_phase4_append_guard_rejects_import_rebinding_an_inherited_test(phase4_tools): - path = "tests/integration/test_cli_validation.py" - current = (ROOT / path).read_bytes() - verify = phase4_tools["_phase4_append_only"] - verify(current, current, path) - with pytest.raises(ValueError): - verify(current, current + b"\nfrom pathlib import Path as test_cli_help_runs\n", path) - - -@pytest.mark.parametrize("statement", [ - b"def test_phase4_mutator(value=globals().clear()):\n pass", - b"def test_phase4_mutator(*, value=globals().clear()):\n pass", - b"@globals().clear()\ndef test_phase4_mutator():\n pass", -]) -def test_phase4_append_guard_rejects_definition_time_mutations(phase4_tools, statement): - path = "tests/integration/test_cli_validation.py" - current = (ROOT / path).read_bytes() - verify = phase4_tools["_phase4_append_only"] - verify(current, current, path) - with pytest.raises(ValueError): - verify(current, current + b"\n" + statement + b"\n", path) - - -def test_phase4_step2_approved_control_keeps_independent_step1_anchors(phase4_tools, phase4_step2_snapshot): - control = json.loads((phase4_step2_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - assert phase4_tools["PHASE4_STEP1_FINAL"] == "a7f3c46d6ca05de36bfcb60f496ebb2d5ab4a37c" - assert phase4_tools["PHASE4_STEP1_TREE"] == "7f8ef492568456bf5d46fb1b7e2631cb9b94e3fd" - assert phase4_tools["PHASE4_STEP1_TEST_TREE"] == "2ce77e331a0fc377387735bb55dffd3be630b59c" - assert control["active_phase"] == 4 and control["active_step"] == 2 - assert control["approval_basis"] == "批准，开始 **Phase 4 Step 2**" - assert control["baseline_commit"] == BASELINE - assert control["previous_step_commit"] == "a7f3c46d6ca05de36bfcb60f496ebb2d5ab4a37c" - assert control["previous_step_tree"] == "7f8ef492568456bf5d46fb1b7e2631cb9b94e3fd" - assert control["previous_step_test_tree"] == "2ce77e331a0fc377387735bb55dffd3be630b59c" - assert control["previous_step_core_tests"] == 2584 - assert control["previous_step_parquet_tests"] == 2587 - assert len(control["previous_step_files_sha256"]) == 227 - assert control["runtime_paths_authorized"] == ["src/recursive_integrity_toolkit/result.py"] - assert control["schema_paths_authorized"] == ["schemas/report.schema.json"] - assert control["new_files_permitted"] == [] - for name in ("phase_complete", "next_step_authorized", "main_merge_authorized", "publication_authorized"): - assert control[name] is False - phase4_tools["verify_phase4_step2_control"](control, step=2) - with pytest.raises(ValueError): - phase4_tools["verify_phase4_control"](control, step=1) - - -@pytest.mark.parametrize("field,value", [ - ("active_phase", 3), ("active_phase", True), - ("active_step", 1), ("active_step", 3), ("active_step", True), - ("approval_basis", "approved by this manifest"), - ("previous_step_commit", "0000000000000000000000000000000000000000"), ("previous_step_tree", "0000000000000000000000000000000000000000"), - ("previous_step_test_tree", "0000000000000000000000000000000000000000"), ("previous_step_files_sha256", {}), - ("baseline_commit", "0000000000000000000000000000000000000000"), ("baseline_files_sha256", {}), - ("previous_step_core_tests", 1), ("previous_step_parquet_tests", 1), - ("runtime_paths_authorized", ["src/recursive_integrity_toolkit/metrics/diversity.py"]), - ("schema_paths_authorized", ["schemas/config.schema.json"]), - ("new_files_permitted", ["src/recursive_integrity_toolkit/reports/new.py"]), - ("permitted_paths", ["src/recursive_integrity_toolkit/cli.py"]), - ("phase_complete", True), ("next_step_authorized", True), - ("main_merge_authorized", True), ("publication_authorized", True), -]) -def test_phase4_step2_control_rejects_forged_scope_and_stage(phase4_tools, field, value, phase4_step2_snapshot): - control = json.loads((phase4_step2_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - verify = phase4_tools["verify_phase4_step2_control"] - verify(control, step=2) - control[field] = value - with pytest.raises(ValueError): - verify(control, step=2) - - -@pytest.mark.parametrize("step", [0, 1, 3, 11, True, "2", None]) -def test_phase4_step2_control_rejects_unapproved_dispatch(phase4_tools, step, phase4_step2_snapshot): - control = json.loads((phase4_step2_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step2_control"](control, step=step) - - -@pytest.mark.parametrize("field", ["active_step", "previous_step_commit", "runtime_paths_authorized", "schema_paths_authorized"]) -def test_phase4_step2_control_requires_explicit_approval_fields(phase4_tools, field, phase4_step2_snapshot): - control = json.loads((phase4_step2_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step2_control"](control) - del control[field] - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step2_control"](control) - - -def test_phase4_step2_control_cannot_mint_extra_permission(phase4_tools, phase4_step2_snapshot): - control = json.loads((phase4_step2_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step2_control"](control) - control["approved_scope_expansion"] = {"step": 3, "assembly": True} - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step2_control"](control) - - -def test_phase4_step2_fixed_boundary_opens_only_approved_existing_paths(phase4_tools): - approved = { - "PHASE_4_BASELINE.json", "PHASE_4_DECISIONS.md", "scripts/check_traceability.py", - "scripts/check_spec_consistency.py", "scripts/release_check.py", "tests/conftest.py", - "tests/unit/test_phase4_contracts.py", "tests/integration/test_phase4_gates.py", - "tests/integration/test_no_algorithms.py", "tests/integration/test_ci_workflows.py", - "tests/integration/test_repository_structure.py", "tests/integration/test_owner_ids.py", - "tests/integration/test_package_import.py", "tests/integration/test_package_install.py", - "tests/integration/test_optional_dependency.py", "tests/integration/test_no_network.py", - "tests/integration/test_schema_json.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_prohibited_structure.py", "tests/integration/test_license_notices.py", - ".github/workflows/ci.yml", ".github/workflows/security.yml", - ".github/workflows/golden.yml", ".github/workflows/release.yml", - "docs/architecture.md", "docs/theory_traceability.md", - "src/recursive_integrity_toolkit/result.py", "schemas/report.schema.json", - "docs/report_schema.md", "tests/unit/test_PR012_evidence_classes.py", - "tests/unit/test_PR013_report_schema.py", - } - assert set(phase4_tools["PHASE4_STEP2_ALLOWED"]) == approved - assert set(phase4_tools["PHASE4_STEP2_NEW"]) == set() - phase4_tools["verify_phase4_step2_changes"]([("M", path) for path in sorted(approved)]) - - -@pytest.mark.parametrize("status,path", [ - ("M", "src/recursive_integrity_toolkit/metrics/diversity.py"), - ("M", "src/recursive_integrity_toolkit/models.py"), - ("M", "src/recursive_integrity_toolkit/reports/assembly.py"), - ("M", "src/recursive_integrity_toolkit/cli.py"), - ("M", "schemas/config.schema.json"), - ("M", "tests/golden/phase3_math_cases.json"), - ("M", "PHASE_4_PLAN.md"), ("M", "pyproject.toml"), - ("A", "src/recursive_integrity_toolkit/result.py"), - ("A", "tests/unit/test_phase4_step2_new.py"), - ("A", "PHASE_4_COMPLETION.md"), - ("D", "schemas/report.schema.json"), ("R100", "docs/report_schema.md"), - ("T", "scripts/release_check.py"), ("M", "./scripts/release_check.py"), - ("M", "scripts/../scripts/release_check.py"), -]) -def test_phase4_step2_diff_rejects_unopened_paths_and_nonmodification_operations(phase4_tools, status, path): - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step2_changes"]([(status, path)]) - - -def test_phase4_step2_diff_rejects_duplicate_operations(phase4_tools): - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step2_changes"]([ - ("M", "schemas/report.schema.json"), ("M", "schemas/report.schema.json"), - ]) - - -def test_phase4_step2_historical_migrations_are_five_explicit_step1_bindings(phase4_tools, phase4_step1_snapshot, phase4_step2_snapshot): - registry = phase4_tools["PHASE4_STEP2_MIGRATIONS"] - assert {path: {row["node"] for row in rows} for path, rows in registry.items()} == { - "tests/unit/test_phase4_contracts.py": { - "phase4_control", "test_phase4_registered_migrations_accept_only_the_approved_inventory", - }, - "tests/integration/test_phase4_gates.py": { - "phase4_mutation_tree", "test_phase4_final_snapshot_preserves_all_current_runtime_bytes", - "test_phase4_active_workflows_preserve_full_matrix_and_use_current_dispatch", - }, - } - for path in sorted(registry): - phase4_tools["verify_phase4_step2_test_migration"]( - path, (phase4_step1_snapshot / path).read_bytes(), (phase4_step2_snapshot / path).read_bytes(), - ) - - -@pytest.mark.parametrize("mutation", ["old_assertion", "wrong_snapshot", "rebind_node", "definition_effect"]) -def test_phase4_step2_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, phase4_step1_snapshot, mutation, phase4_step2_snapshot): - path = "tests/unit/test_phase4_contracts.py" - before = (phase4_step1_snapshot / path).read_bytes() - after = (phase4_step2_snapshot / path).read_bytes() - verify = phase4_tools["verify_phase4_step2_test_migration"] - verify(path, before, after) - if mutation == "old_assertion": - original = b' assert phase4_control["active_step"] == 1\n' - assert after.count(original) == 1 - after = after.replace(original, b" assert True\n", 1) - elif mutation == "wrong_snapshot": - original = b' return json.loads((phase4_step1_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - assert after.count(original) == 1 - after = after.replace(original, b' return json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', 1) - elif mutation == "rebind_node": - after += b"\ndef test_phase4_fixed_path_boundary_opens_only_governance():\n assert True\n" - else: - after += b"\ndef test_phase4_step2_mutator(value=globals().clear()):\n pass\n" - with pytest.raises(ValueError): - verify(path, before, after) - - -def test_phase4_step3_approved_control_keeps_independent_step2_anchors(phase4_tools, phase4_step3_snapshot): - control = json.loads((phase4_step3_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - assert phase4_tools["PHASE4_STEP2_FINAL"] == "fcea74e2b1858e83cdbfd1b8212d15343d63ff08" - assert phase4_tools["PHASE4_STEP2_TREE"] == "c3ce887fd393cfc6c18364546005f2b74a63b746" - assert phase4_tools["PHASE4_STEP2_TEST_TREE"] == "9d6ac0cbf3827ab54c7852503d99d442a3796259" - assert control["active_phase"] == 4 and control["active_step"] == 3 - assert control["approval_basis"] == "开始 **Phase 4 Step 3**" - assert control["baseline_commit"] == BASELINE - assert control["previous_step_commit"] == "fcea74e2b1858e83cdbfd1b8212d15343d63ff08" - assert control["previous_step_tree"] == "c3ce887fd393cfc6c18364546005f2b74a63b746" - assert control["previous_step_test_tree"] == "9d6ac0cbf3827ab54c7852503d99d442a3796259" - assert control["previous_step_core_tests"] == 2728 - assert control["previous_step_parquet_tests"] == 2731 - assert len(control["previous_step_files_sha256"]) == 227 - assert control["runtime_paths_authorized"] == ["src/recursive_integrity_toolkit/reports/assembly.py"] - assert control["schema_changes_authorized"] is False - assert control["schema_paths_authorized"] == [] - assert control["new_files_permitted"] == [] - for name in ("phase_complete", "next_step_authorized", "main_merge_authorized", "publication_authorized"): - assert control[name] is False - phase4_tools["verify_phase4_step3_control"](control, step=3) - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step2_control"](control, step=2) - - -@pytest.mark.parametrize("field,value", [ - ("active_phase", 3), ("active_phase", True), - ("active_step", 2), ("active_step", 4), ("active_step", True), - ("approval_basis", "approved by this manifest"), - ("previous_step_commit", "0000000000000000000000000000000000000000"), - ("previous_step_tree", "0000000000000000000000000000000000000000"), - ("previous_step_test_tree", "0000000000000000000000000000000000000000"), - ("previous_step_files_sha256", {}), ("baseline_files_sha256", {}), - ("previous_step_core_tests", 1), ("previous_step_parquet_tests", 1), - ("runtime_paths_authorized", ["src/recursive_integrity_toolkit/result.py"]), - ("schema_changes_authorized", True), ("schema_paths_authorized", ["schemas/report.schema.json"]), - ("new_files_permitted", ["src/recursive_integrity_toolkit/reports/new.py"]), - ("permitted_paths", ["src/recursive_integrity_toolkit/cli.py"]), - ("phase_complete", True), ("next_step_authorized", True), - ("main_merge_authorized", True), ("publication_authorized", True), -]) -def test_phase4_step3_control_rejects_forged_scope_and_stage(phase4_tools, field, value, phase4_step3_snapshot): - control = json.loads((phase4_step3_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - verify = phase4_tools["verify_phase4_step3_control"] - verify(control, step=3) - control[field] = value - with pytest.raises(ValueError): - verify(control, step=3) - - -@pytest.mark.parametrize("step", [0, 1, 2, 4, 11, True, "3", None]) -def test_phase4_step3_control_rejects_unapproved_dispatch(phase4_tools, step, phase4_step3_snapshot): - control = json.loads((phase4_step3_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step3_control"](control, step=step) - - -@pytest.mark.parametrize("field", ["active_step", "previous_step_commit", "runtime_paths_authorized", "schema_paths_authorized"]) -def test_phase4_step3_control_requires_explicit_approval_fields(phase4_tools, field, phase4_step3_snapshot): - control = json.loads((phase4_step3_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step3_control"](control) - del control[field] - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step3_control"](control) - - -def test_phase4_step3_control_cannot_mint_extra_permission(phase4_tools, phase4_step3_snapshot): - control = json.loads((phase4_step3_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step3_control"](control) - control["approved_scope_expansion"] = {"step": 4, "privacy": True} - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step3_control"](control) - - -def test_phase4_step3_fixed_boundary_opens_only_approved_existing_paths(phase4_tools): - approved = { - "PHASE_4_BASELINE.json", "PHASE_4_DECISIONS.md", "scripts/check_traceability.py", - "scripts/check_spec_consistency.py", "scripts/release_check.py", "tests/conftest.py", - "tests/unit/test_phase4_contracts.py", "tests/integration/test_phase4_gates.py", - "tests/integration/test_no_algorithms.py", "tests/integration/test_ci_workflows.py", - "tests/integration/test_repository_structure.py", "tests/integration/test_owner_ids.py", - "tests/integration/test_package_import.py", "tests/integration/test_package_install.py", - "tests/integration/test_optional_dependency.py", "tests/integration/test_no_network.py", - "tests/integration/test_schema_json.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_prohibited_structure.py", "tests/integration/test_license_notices.py", - ".github/workflows/ci.yml", ".github/workflows/security.yml", - ".github/workflows/golden.yml", ".github/workflows/release.yml", - "docs/architecture.md", "docs/theory_traceability.md", - "src/recursive_integrity_toolkit/reports/assembly.py", "docs/report_schema.md", - "tests/unit/test_PR012_evidence_classes.py", "tests/unit/test_PR014_unavailable.py", - "tests/unit/test_PR018_language.py", "tests/integration/test_partial_provenance_report.py", - "tests/integration/test_partial_lineage_report.py", - } - assert set(phase4_tools["PHASE4_STEP3_ALLOWED"]) == approved - assert set(phase4_tools["PHASE4_STEP3_NEW"]) == set() - phase4_tools["verify_phase4_step3_changes"]([("M", path) for path in sorted(approved)]) - - -@pytest.mark.parametrize("status,path", [ - ("M", "src/recursive_integrity_toolkit/metrics/diversity.py"), - ("M", "src/recursive_integrity_toolkit/models.py"), - ("M", "src/recursive_integrity_toolkit/result.py"), - ("M", "src/recursive_integrity_toolkit/cli.py"), - ("M", "schemas/report.schema.json"), ("M", "schemas/config.schema.json"), - ("M", "tests/golden/phase3_math_cases.json"), - ("M", "tests/unit/test_PR013_report_schema.py"), - ("M", "PHASE_4_PLAN.md"), ("M", "pyproject.toml"), - ("A", "src/recursive_integrity_toolkit/reports/assembly.py"), - ("A", "tests/unit/test_phase4_step3_new.py"), ("A", "PHASE_4_COMPLETION.md"), - ("D", "docs/report_schema.md"), ("R100", "docs/report_schema.md"), - ("T", "scripts/release_check.py"), ("M", "./scripts/release_check.py"), - ("M", "scripts/../scripts/release_check.py"), -]) -def test_phase4_step3_diff_rejects_unopened_paths_and_nonmodification_operations(phase4_tools, status, path): - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step3_changes"]([(status, path)]) - - -def test_phase4_step3_diff_rejects_duplicate_operations(phase4_tools): - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step3_changes"]([ - ("M", "docs/report_schema.md"), ("M", "docs/report_schema.md"), - ]) - - -def test_phase4_step3_historical_migrations_preserve_ten_step2_gate_nodes(phase4_tools, phase4_step2_snapshot, phase4_step3_snapshot): - registry = phase4_tools["PHASE4_STEP3_MIGRATIONS"] - expected = { - "tests/unit/test_phase4_contracts.py": { - "test_phase4_step2_approved_control_keeps_independent_step1_anchors", - "test_phase4_step2_control_rejects_forged_scope_and_stage", - "test_phase4_step2_control_rejects_unapproved_dispatch", - "test_phase4_step2_control_requires_explicit_approval_fields", - "test_phase4_step2_control_cannot_mint_extra_permission", - "test_phase4_step2_historical_migrations_are_five_explicit_step1_bindings", - "test_phase4_step2_historical_guard_rejects_assertion_and_binding_weakening", - }, - "tests/integration/test_phase4_gates.py": { - "test_phase4_step2_current_runtime_opens_only_the_canonical_result", - "test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch", - "test_phase4_step2_current_snapshot_checks_valid_tree_before_mutations", - }, - } - for path, nodes in expected.items(): - assert {row["node"] for row in registry[path]} == nodes - phase4_tools["verify_phase4_step3_test_migration"]( - path, (phase4_step2_snapshot / path).read_bytes(), (phase4_step3_snapshot / path).read_bytes(), - ) - - -@pytest.mark.parametrize("mutation", ["old_assertion", "wrong_snapshot", "rebind_node", "definition_effect"]) -def test_phase4_step3_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, phase4_step2_snapshot, mutation, phase4_step3_snapshot): - path = "tests/unit/test_phase4_contracts.py" - before = (phase4_step2_snapshot / path).read_bytes() - after = (phase4_step3_snapshot / path).read_bytes() - verify = phase4_tools["verify_phase4_step3_test_migration"] - verify(path, before, after) - if mutation == "old_assertion": - original = b' assert control["active_phase"] == 4 and control["active_step"] == 2\n' - assert after.count(original) == 1 - after = after.replace(original, b" assert True\n", 1) - elif mutation == "wrong_snapshot": - original = (b'def test_phase4_step2_control_cannot_mint_extra_permission(phase4_tools, phase4_step2_snapshot):\n' - b' control = json.loads((phase4_step2_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n') - assert after.count(original) == 1 - after = after.replace(original, original.replace(b'(phase4_step2_snapshot /', b'(ROOT /'), 1) - elif mutation == "rebind_node": - after += b"\ndef test_phase4_step2_fixed_boundary_opens_only_approved_existing_paths():\n assert True\n" - else: - after += b"\ndef test_phase4_step3_mutator(value=globals().clear()):\n pass\n" - with pytest.raises(ValueError): - verify(path, before, after) - - -def test_phase4_step4_approved_control_keeps_independent_step3_anchors(phase4_tools, phase4_step4_snapshot): - control = json.loads((phase4_step4_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - assert phase4_tools["PHASE4_STEP3_FINAL"] == "974545c456e535ba1e1c5b6bf4ae0ce25bc04b57" - assert phase4_tools["PHASE4_STEP3_TREE"] == "15498f3f00d0ce048c3cba2156eb7a44f096ae88" - assert phase4_tools["PHASE4_STEP3_TEST_TREE"] == "17b87858f77ee4413c98cfd76f16f52170246741" - assert control["active_phase"] == 4 and control["active_step"] == 4 - assert control["approval_basis"] == "继续" - assert control["baseline_commit"] == BASELINE - assert control["previous_step_commit"] == "974545c456e535ba1e1c5b6bf4ae0ce25bc04b57" - assert control["previous_step_tree"] == "15498f3f00d0ce048c3cba2156eb7a44f096ae88" - assert control["previous_step_test_tree"] == "17b87858f77ee4413c98cfd76f16f52170246741" - assert control["previous_step_core_tests"] == 2876 - assert control["previous_step_parquet_tests"] == 2879 - assert len(control["previous_step_files_sha256"]) == 227 - assert set(control["runtime_paths_authorized"]) == { - "src/recursive_integrity_toolkit/result.py", - "src/recursive_integrity_toolkit/reports/assembly.py", - "src/recursive_integrity_toolkit/config.py", - "src/recursive_integrity_toolkit/utils/hashing.py", - "src/recursive_integrity_toolkit/utils/logging.py", - } - assert control["schema_changes_authorized"] is False - assert control["schema_paths_authorized"] == [] - assert control["new_files_permitted"] == [] - for name in ("phase_complete", "next_step_authorized", "main_merge_authorized", "publication_authorized"): - assert control[name] is False - phase4_tools["verify_phase4_step4_control"](control, step=4) - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step3_control"](control, step=3) - - -@pytest.mark.parametrize("field,value", [ - ("active_phase", 3), ("active_phase", True), - ("active_step", 3), ("active_step", 5), ("active_step", True), - ("approval_basis", "approved by this manifest"), - ("previous_step_commit", "0000000000000000000000000000000000000000"), - ("previous_step_tree", "0000000000000000000000000000000000000000"), - ("previous_step_test_tree", "0000000000000000000000000000000000000000"), - ("previous_step_files_sha256", {}), ("baseline_files_sha256", {}), - ("previous_step_core_tests", 1), ("previous_step_parquet_tests", 1), - ("runtime_paths_authorized", ["src/recursive_integrity_toolkit/cli.py"]), - ("schema_changes_authorized", True), ("schema_paths_authorized", ["schemas/report.schema.json"]), - ("new_files_permitted", ["src/recursive_integrity_toolkit/reports/privacy.py"]), - ("permitted_paths", ["src/recursive_integrity_toolkit/reports/json_report.py"]), - ("phase_complete", True), ("next_step_authorized", True), - ("main_merge_authorized", True), ("publication_authorized", True), -]) -def test_phase4_step4_control_rejects_forged_scope_and_stage(phase4_tools, field, value, phase4_step4_snapshot): - control = json.loads((phase4_step4_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - verify = phase4_tools["verify_phase4_step4_control"] - verify(control, step=4) - control[field] = value - with pytest.raises(ValueError): - verify(control, step=4) - - -@pytest.mark.parametrize("step", [0, 1, 2, 3, 5, 11, True, "4", None]) -def test_phase4_step4_control_rejects_unapproved_dispatch(phase4_tools, step, phase4_step4_snapshot): - control = json.loads((phase4_step4_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step4_control"](control, step=4) - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step4_control"](control, step=step) - - -@pytest.mark.parametrize("field", ["active_step", "previous_step_commit", "runtime_paths_authorized", "schema_paths_authorized"]) -def test_phase4_step4_control_requires_explicit_approval_fields(phase4_tools, field, phase4_step4_snapshot): - control = json.loads((phase4_step4_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step4_control"](control) - del control[field] - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step4_control"](control) - - -def test_phase4_step4_control_cannot_mint_extra_permission(phase4_tools, phase4_step4_snapshot): - control = json.loads((phase4_step4_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step4_control"](control) - control["approved_scope_expansion"] = {"step": 5, "renderers": True} - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step4_control"](control) - - -def test_phase4_step4_fixed_boundary_opens_only_approved_existing_paths(phase4_tools): - approved = { - "PHASE_4_BASELINE.json", "PHASE_4_DECISIONS.md", "scripts/check_traceability.py", - "scripts/check_spec_consistency.py", "scripts/release_check.py", "tests/conftest.py", - "tests/unit/test_phase4_contracts.py", "tests/integration/test_phase4_gates.py", - "tests/integration/test_no_algorithms.py", "tests/integration/test_ci_workflows.py", - "tests/integration/test_repository_structure.py", "tests/integration/test_owner_ids.py", - "tests/integration/test_package_import.py", "tests/integration/test_package_install.py", - "tests/integration/test_optional_dependency.py", "tests/integration/test_no_network.py", - "tests/integration/test_schema_json.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_prohibited_structure.py", "tests/integration/test_license_notices.py", - ".github/workflows/ci.yml", ".github/workflows/security.yml", - ".github/workflows/golden.yml", ".github/workflows/release.yml", - "docs/architecture.md", "docs/theory_traceability.md", - "src/recursive_integrity_toolkit/result.py", "src/recursive_integrity_toolkit/reports/assembly.py", - "src/recursive_integrity_toolkit/config.py", "src/recursive_integrity_toolkit/utils/hashing.py", - "src/recursive_integrity_toolkit/utils/logging.py", "docs/privacy.md", "docs/report_schema.md", - "tests/unit/test_PR015_redaction.py", "tests/unit/test_PR016_determinism.py", - "tests/unit/test_PR018_language.py", - } - assert set(phase4_tools["PHASE4_STEP4_ALLOWED"]) == approved - assert set(phase4_tools["PHASE4_STEP4_NEW"]) == set() - phase4_tools["verify_phase4_step4_changes"]([("M", path) for path in sorted(approved)]) - - -@pytest.mark.parametrize("status,path", [ - ("M", "src/recursive_integrity_toolkit/metrics/diversity.py"), - ("M", "src/recursive_integrity_toolkit/models.py"), - ("M", "src/recursive_integrity_toolkit/utils/paths.py"), - ("M", "src/recursive_integrity_toolkit/cli.py"), - ("M", "src/recursive_integrity_toolkit/reports/json_report.py"), - ("M", "src/recursive_integrity_toolkit/reports/markdown_report.py"), - ("M", "schemas/report.schema.json"), ("M", "schemas/config.schema.json"), - ("M", "tests/golden/phase3_math_cases.json"), - ("M", "tests/unit/test_PR012_evidence_classes.py"), - ("M", "tests/unit/test_PR013_report_schema.py"), - ("M", "tests/unit/test_PR014_unavailable.py"), - ("M", "PHASE_4_PLAN.md"), ("M", "pyproject.toml"), - ("A", "src/recursive_integrity_toolkit/utils/hashing.py"), - ("A", "tests/unit/test_phase4_step4_new.py"), ("A", "PHASE_4_COMPLETION.md"), - ("D", "docs/privacy.md"), ("R100", "docs/privacy.md"), - ("T", "scripts/release_check.py"), ("M", "./scripts/release_check.py"), - ("M", "scripts/../scripts/release_check.py"), -]) -def test_phase4_step4_diff_rejects_unopened_paths_and_nonmodification_operations(phase4_tools, status, path): - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step4_changes"]([(status, path)]) - - -def test_phase4_step4_diff_rejects_duplicate_operations(phase4_tools): - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step4_changes"]([ - ("M", "docs/privacy.md"), ("M", "docs/privacy.md"), - ]) - - -def test_phase4_step4_historical_migrations_preserve_ten_step3_gate_nodes(phase4_tools, phase4_step3_snapshot, phase4_step4_snapshot): - registry = phase4_tools["PHASE4_STEP4_MIGRATIONS"] - expected = { - "tests/unit/test_phase4_contracts.py": { - "test_phase4_step3_approved_control_keeps_independent_step2_anchors", - "test_phase4_step3_control_rejects_forged_scope_and_stage", - "test_phase4_step3_control_rejects_unapproved_dispatch", - "test_phase4_step3_control_requires_explicit_approval_fields", - "test_phase4_step3_control_cannot_mint_extra_permission", - "test_phase4_step3_historical_migrations_preserve_ten_step2_gate_nodes", - "test_phase4_step3_historical_guard_rejects_assertion_and_binding_weakening", - }, - "tests/integration/test_phase4_gates.py": { - "test_phase4_step3_current_runtime_opens_only_assembly_and_freezes_schema", - "test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch", - "test_phase4_step3_current_snapshot_checks_valid_tree_before_mutations", - }, - } - assert set(registry) == set(expected) - assert sum(len(rows) for rows in registry.values()) == 13 - for path, nodes in expected.items(): - assert {row["node"] for row in registry[path]} == nodes - phase4_tools["verify_phase4_step4_test_migration"]( - path, (phase4_step3_snapshot / path).read_bytes(), (phase4_step4_snapshot / path).read_bytes(), - ) - - -@pytest.mark.parametrize("mutation", ["old_assertion", "wrong_snapshot", "rebind_node", "definition_effect"]) -def test_phase4_step4_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, phase4_step3_snapshot, mutation, phase4_step4_snapshot): - path = "tests/unit/test_phase4_contracts.py" - before = (phase4_step3_snapshot / path).read_bytes() - after = (phase4_step4_snapshot / path).read_bytes() - verify = phase4_tools["verify_phase4_step4_test_migration"] - verify(path, before, after) - if mutation == "old_assertion": - original = b' assert control["active_phase"] == 4 and control["active_step"] == 3\n' - assert after.count(original) == 1 - after = after.replace(original, b" assert True\n", 1) - elif mutation == "wrong_snapshot": - original = (b'def test_phase4_step3_control_cannot_mint_extra_permission(phase4_tools, phase4_step3_snapshot):\n' - b' control = json.loads((phase4_step3_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n') - assert after.count(original) == 1 - after = after.replace(original, original.replace(b'(phase4_step3_snapshot /', b'(ROOT /'), 1) - elif mutation == "rebind_node": - after += b"\ndef test_phase4_step3_fixed_boundary_opens_only_approved_existing_paths():\n assert True\n" - else: - after += b"\ndef test_phase4_step4_mutator(value=globals().clear()):\n pass\n" - with pytest.raises(ValueError): - verify(path, before, after) - - -def test_phase4_step4_inherited_config_and_file_hash_helpers_preserve_exact_bodies(phase4_step3_snapshot): - import ast - - for relative in ("config.py", "utils/hashing.py"): - path = "src/recursive_integrity_toolkit/" + relative - old = (phase4_step3_snapshot / path).read_text(encoding="utf-8") - current = (ROOT / path).read_text(encoding="utf-8") - original_nodes = [node for node in ast.parse(old).body - if isinstance(node, (ast.FunctionDef, ast.ClassDef))] - current_nodes = [node for node in ast.parse(current).body - if isinstance(node, (ast.FunctionDef, ast.ClassDef))] - assert original_nodes - for node in original_nodes: - matches = [candidate for candidate in current_nodes if candidate.name == node.name] - assert len(matches) == 1, (relative, node.name) - assert ast.get_source_segment(old, node) == ast.get_source_segment(current, matches[0]), (relative, node.name) - - -def test_phase4_step4_reviewed_ast_identity_is_portable_without_ignoring_semantics(): - import ast - - checker = runpy.run_path(str(ROOT / "scripts/check_traceability.py"), run_name="phase4_step4_portable_ast") - digest = checker["_phase4_step4_ast_digest"] - source = 'def protect(value: str) -> str:\n return "safe:" + value\n\nclass View:\n enabled = True\n' - with_empty_parameters = ast.parse(source).body - without_parameters = ast.parse(source).body - for node in with_empty_parameters: - if "type_params" not in node._fields: - node._fields = (*node._fields, "type_params") - node.type_params = [] - for node in without_parameters: - if hasattr(node, "type_params"): - del node.type_params - assert digest(with_empty_parameters) == digest(without_parameters) - for changed in (source.replace('"safe:"', '"raw:"'), source.replace("enabled = True", "enabled = False"), - source.replace("value: str", "value: bytes")): - assert digest(ast.parse(changed).body) != digest(without_parameters) - with_empty_parameters[0].type_params = [ast.Name(id="UnapprovedGeneric", ctx=ast.Load())] - assert digest(with_empty_parameters) != digest(without_parameters) - - -def test_phase4_step5_approved_control_keeps_independent_step4_anchors(phase4_tools, phase4_step5_snapshot): - control = json.loads((phase4_step5_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - assert phase4_tools["PHASE4_STEP4_FINAL"] == "49454a9b162cb8d35e38d1cb1ae32cb208d3a01c" - assert phase4_tools["PHASE4_STEP4_TREE"] == "79264d845e08f73ee68e160dea04f7786f18fd0a" - assert phase4_tools["PHASE4_STEP4_TEST_TREE"] == "4aa4ca0b22f8b8c25afe27feeaba713d6f0830bf" - assert control["active_phase"] == 4 and control["active_step"] == 5 - assert control["approval_basis"] == "很好，Phase 4 Step 5 继续" - assert control["baseline_commit"] == BASELINE - assert control["previous_step_commit"] == "49454a9b162cb8d35e38d1cb1ae32cb208d3a01c" - assert control["previous_step_tree"] == "79264d845e08f73ee68e160dea04f7786f18fd0a" - assert control["previous_step_test_tree"] == "4aa4ca0b22f8b8c25afe27feeaba713d6f0830bf" - assert control["previous_step_core_tests"] == 3071 - assert control["previous_step_parquet_tests"] == 3074 - assert len(control["previous_step_files_sha256"]) == 227 - assert set(control["runtime_paths_authorized"]) == { - "src/recursive_integrity_toolkit/reports/json_report.py", - "src/recursive_integrity_toolkit/reports/markdown_report.py", - } - assert control["schema_changes_authorized"] is False - assert control["schema_paths_authorized"] == [] - assert control["new_files_permitted"] == [] - for name in ("phase_complete", "next_step_authorized", "main_merge_authorized", "publication_authorized"): - assert control[name] is False - phase4_tools["verify_phase4_step5_control"](control, step=5) - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step4_control"](control, step=4) - - -@pytest.mark.parametrize("field,value", [ - ("active_phase", 3), ("active_phase", True), - ("active_step", 4), ("active_step", 6), ("active_step", True), - ("approval_basis", "approved by this manifest"), - ("previous_step_commit", "0000000000000000000000000000000000000000"), - ("previous_step_tree", "0000000000000000000000000000000000000000"), - ("previous_step_test_tree", "0000000000000000000000000000000000000000"), - ("previous_step_files_sha256", {}), ("baseline_files_sha256", {}), - ("previous_step_core_tests", 1), ("previous_step_parquet_tests", 1), - ("runtime_paths_authorized", ["src/recursive_integrity_toolkit/cli.py"]), - ("schema_changes_authorized", True), ("schema_paths_authorized", ["schemas/report.schema.json"]), - ("new_files_permitted", ["src/recursive_integrity_toolkit/reports/privacy.py"]), - ("permitted_paths", ["src/recursive_integrity_toolkit/cli.py"]), - ("phase_complete", True), ("next_step_authorized", True), - ("main_merge_authorized", True), ("publication_authorized", True), -]) -def test_phase4_step5_control_rejects_forged_scope_and_stage(phase4_tools, field, value, phase4_step5_snapshot): - control = json.loads((phase4_step5_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - verify = phase4_tools["verify_phase4_step5_control"] - verify(control, step=5) - control[field] = value - with pytest.raises(ValueError): - verify(control, step=5) - - -@pytest.mark.parametrize("step", [0, 1, 2, 3, 4, 6, 11, True, "5", None]) -def test_phase4_step5_control_rejects_unapproved_dispatch(phase4_tools, step, phase4_step5_snapshot): - control = json.loads((phase4_step5_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step5_control"](control, step=5) - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step5_control"](control, step=step) - - -@pytest.mark.parametrize("field", ["active_step", "previous_step_commit", "runtime_paths_authorized", "schema_paths_authorized"]) -def test_phase4_step5_control_requires_explicit_approval_fields(phase4_tools, field, phase4_step5_snapshot): - control = json.loads((phase4_step5_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step5_control"](control) - del control[field] - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step5_control"](control) - - -def test_phase4_step5_control_cannot_mint_extra_permission(phase4_tools, phase4_step5_snapshot): - control = json.loads((phase4_step5_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step5_control"](control) - control["approved_scope_expansion"] = {"step": 6, "publication": True} - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step5_control"](control) - - -def test_phase4_step5_fixed_boundary_opens_only_approved_existing_paths(phase4_tools): - approved = { - "PHASE_4_BASELINE.json", "PHASE_4_DECISIONS.md", "scripts/check_traceability.py", - "scripts/check_spec_consistency.py", "scripts/release_check.py", "tests/conftest.py", - "tests/unit/test_phase4_contracts.py", "tests/integration/test_phase4_gates.py", - "tests/integration/test_no_algorithms.py", "tests/integration/test_ci_workflows.py", - "tests/integration/test_repository_structure.py", "tests/integration/test_owner_ids.py", - "tests/integration/test_package_import.py", "tests/integration/test_package_install.py", - "tests/integration/test_optional_dependency.py", "tests/integration/test_no_network.py", - "tests/integration/test_schema_json.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_prohibited_structure.py", "tests/integration/test_license_notices.py", - ".github/workflows/ci.yml", ".github/workflows/security.yml", - ".github/workflows/golden.yml", ".github/workflows/release.yml", - "docs/architecture.md", "docs/theory_traceability.md", - "src/recursive_integrity_toolkit/reports/json_report.py", - "src/recursive_integrity_toolkit/reports/markdown_report.py", "docs/report_schema.md", - "tests/unit/test_PR013_report_schema.py", "tests/unit/test_PR016_determinism.py", - "tests/unit/test_PR018_language.py", - } - assert set(phase4_tools["PHASE4_STEP5_ALLOWED"]) == approved - assert set(phase4_tools["PHASE4_STEP5_NEW"]) == set() - phase4_tools["verify_phase4_step5_changes"]([("M", path) for path in sorted(approved)]) - - -@pytest.mark.parametrize("status,path", [ - ("M", "src/recursive_integrity_toolkit/metrics/diversity.py"), - ("M", "src/recursive_integrity_toolkit/models.py"), - ("M", "src/recursive_integrity_toolkit/utils/paths.py"), - ("M", "src/recursive_integrity_toolkit/cli.py"), - ("M", "src/recursive_integrity_toolkit/reports/assembly.py"), - ("M", "src/recursive_integrity_toolkit/reports/html_report.py"), - ("M", "src/recursive_integrity_toolkit/result.py"), - ("M", "src/recursive_integrity_toolkit/config.py"), - ("M", "src/recursive_integrity_toolkit/utils/hashing.py"), - ("M", "src/recursive_integrity_toolkit/utils/logging.py"), - ("M", "schemas/report.schema.json"), ("M", "schemas/config.schema.json"), - ("M", "tests/golden/phase3_math_cases.json"), - ("M", "tests/unit/test_PR012_evidence_classes.py"), - ("M", "tests/unit/test_PR015_redaction.py"), - ("M", "tests/unit/test_PR014_unavailable.py"), - ("M", "PHASE_4_PLAN.md"), ("M", "pyproject.toml"), - ("A", "src/recursive_integrity_toolkit/reports/json_report.py"), - ("A", "tests/unit/test_phase4_step5_new.py"), ("A", "PHASE_4_COMPLETION.md"), - ("D", "docs/report_schema.md"), ("R100", "docs/report_schema.md"), - ("T", "scripts/release_check.py"), ("M", "./scripts/release_check.py"), - ("M", "scripts/../scripts/release_check.py"), -]) -def test_phase4_step5_diff_rejects_unopened_paths_and_nonmodification_operations(phase4_tools, status, path): - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step5_changes"]([(status, path)]) - - -def test_phase4_step5_diff_rejects_duplicate_operations(phase4_tools): - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step5_changes"]([ - ("M", "docs/report_schema.md"), ("M", "docs/report_schema.md"), - ]) - - -def test_phase4_step5_historical_migrations_preserve_ten_step4_gate_nodes(phase4_tools, phase4_step4_snapshot, phase4_step5_snapshot): - registry = phase4_tools["PHASE4_STEP5_MIGRATIONS"] - expected = { - "tests/unit/test_phase4_contracts.py": { - "test_phase4_step4_approved_control_keeps_independent_step3_anchors", - "test_phase4_step4_control_rejects_forged_scope_and_stage", - "test_phase4_step4_control_rejects_unapproved_dispatch", - "test_phase4_step4_control_requires_explicit_approval_fields", - "test_phase4_step4_control_cannot_mint_extra_permission", - "test_phase4_step4_historical_migrations_preserve_ten_step3_gate_nodes", - "test_phase4_step4_historical_guard_rejects_assertion_and_binding_weakening", - }, - "tests/integration/test_phase4_gates.py": { - "test_phase4_step4_current_runtime_opens_only_privacy_metadata_modules_and_freezes_schema", - "test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch", - "test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations", - }, - } - assert set(registry) == set(expected) - assert sum(len(rows) for rows in registry.values()) == 13 - for path, nodes in expected.items(): - assert {row["node"] for row in registry[path]} == nodes - phase4_tools["verify_phase4_step5_test_migration"]( - path, (phase4_step4_snapshot / path).read_bytes(), (phase4_step5_snapshot / path).read_bytes(), - ) - - -@pytest.mark.parametrize("mutation", ["old_assertion", "wrong_snapshot", "rebind_node", "definition_effect"]) -def test_phase4_step5_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, phase4_step4_snapshot, mutation, phase4_step5_snapshot): - path = "tests/unit/test_phase4_contracts.py" - before = (phase4_step4_snapshot / path).read_bytes() - after = (phase4_step5_snapshot / path).read_bytes() - verify = phase4_tools["verify_phase4_step5_test_migration"] - verify(path, before, after) - if mutation == "old_assertion": - original = b' assert control["active_phase"] == 4 and control["active_step"] == 4\n' - assert after.count(original) == 1 - after = after.replace(original, b" assert True\n", 1) - elif mutation == "wrong_snapshot": - original = (b'def test_phase4_step4_control_cannot_mint_extra_permission(phase4_tools, phase4_step4_snapshot):\n' - b' control = json.loads((phase4_step4_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n') - assert after.count(original) == 1 - after = after.replace(original, original.replace(b'(phase4_step4_snapshot /', b'(ROOT /'), 1) - elif mutation == "rebind_node": - after += b"\ndef test_phase4_step4_fixed_boundary_opens_only_approved_existing_paths():\n assert True\n" - else: - after += b"\ndef test_phase4_step5_mutator(value=globals().clear()):\n pass\n" - with pytest.raises(ValueError): - verify(path, before, after) - - -def test_phase4_step6_approved_control_keeps_independent_step5_anchors(phase4_tools, phase4_step6_snapshot): - control = json.loads((phase4_step6_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - assert phase4_tools["PHASE4_STEP5_FINAL"] == "c1667179e895a798bba2349960162542efa5ef87" - assert phase4_tools["PHASE4_STEP5_TREE"] == "d83f9fc8e0b569f2d796bd2a83b87422fe350fba" - assert phase4_tools["PHASE4_STEP5_TEST_TREE"] == "974f8e65904c24ed4e226996eaf571aab6123965" - assert control["active_phase"] == 4 and control["active_step"] == 6 - assert control["approval_basis"] == "批准开始Phase 4 **Step 6**" - assert control["baseline_commit"] == BASELINE - assert control["previous_step_commit"] == "c1667179e895a798bba2349960162542efa5ef87" - assert control["previous_step_tree"] == "d83f9fc8e0b569f2d796bd2a83b87422fe350fba" - assert control["previous_step_test_tree"] == "974f8e65904c24ed4e226996eaf571aab6123965" - assert control["previous_step_core_tests"] == 3215 - assert control["previous_step_parquet_tests"] == 3218 - assert len(control["previous_step_files_sha256"]) == 227 - assert set(control["runtime_paths_authorized"]) == { - "src/recursive_integrity_toolkit/utils/paths.py", - "src/recursive_integrity_toolkit/utils/logging.py", - } - assert control["schema_changes_authorized"] is False - assert control["schema_paths_authorized"] == [] - assert control["new_files_permitted"] == ["tests/unit/test_phase4_output_safety.py"] - for name in ("phase_complete", "next_step_authorized", "main_merge_authorized", "publication_authorized"): - assert control[name] is False - phase4_tools["verify_phase4_step6_control"](control, step=6) - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step4_control"](control, step=4) - - -@pytest.mark.parametrize("field,value", [ - ("active_phase", 3), ("active_phase", True), - ("active_step", 5), ("active_step", 7), ("active_step", True), - ("approval_basis", "approved by this manifest"), - ("previous_step_commit", "0000000000000000000000000000000000000000"), - ("previous_step_tree", "0000000000000000000000000000000000000000"), - ("previous_step_test_tree", "0000000000000000000000000000000000000000"), - ("previous_step_files_sha256", {}), ("baseline_files_sha256", {}), - ("previous_step_core_tests", 1), ("previous_step_parquet_tests", 1), - ("runtime_paths_authorized", ["src/recursive_integrity_toolkit/cli.py"]), - ("schema_changes_authorized", True), ("schema_paths_authorized", ["schemas/report.schema.json"]), - ("new_files_permitted", ["src/recursive_integrity_toolkit/reports/privacy.py"]), - ("permitted_paths", ["src/recursive_integrity_toolkit/cli.py"]), - ("phase_complete", True), ("next_step_authorized", True), - ("main_merge_authorized", True), ("publication_authorized", True), -]) -def test_phase4_step6_control_rejects_forged_scope_and_stage(phase4_tools, field, value, phase4_step6_snapshot): - control = json.loads((phase4_step6_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - verify = phase4_tools["verify_phase4_step6_control"] - verify(control, step=6) - control[field] = value - with pytest.raises(ValueError): - verify(control, step=6) - - -@pytest.mark.parametrize("step", [0, 1, 2, 3, 4, 5, 7, 11, True, "6", None]) -def test_phase4_step6_control_rejects_unapproved_dispatch(phase4_tools, step, phase4_step6_snapshot): - control = json.loads((phase4_step6_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step6_control"](control, step=6) - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step6_control"](control, step=step) - - -@pytest.mark.parametrize("field", ["active_step", "previous_step_commit", "runtime_paths_authorized", "schema_paths_authorized"]) -def test_phase4_step6_control_requires_explicit_approval_fields(phase4_tools, field, phase4_step6_snapshot): - control = json.loads((phase4_step6_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step6_control"](control) - del control[field] - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step6_control"](control) - - -def test_phase4_step6_control_cannot_mint_extra_permission(phase4_tools, phase4_step6_snapshot): - control = json.loads((phase4_step6_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step6_control"](control) - control["approved_scope_expansion"] = {"step": 6, "publication": True} - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step6_control"](control) - - -def test_phase4_step6_fixed_boundary_opens_only_approved_existing_paths(phase4_tools): - approved = { - "PHASE_4_BASELINE.json", "PHASE_4_DECISIONS.md", "scripts/check_traceability.py", - "scripts/check_spec_consistency.py", "scripts/release_check.py", "tests/conftest.py", - "tests/unit/test_phase4_contracts.py", "tests/integration/test_phase4_gates.py", - "tests/integration/test_no_algorithms.py", "tests/integration/test_ci_workflows.py", - "tests/integration/test_repository_structure.py", "tests/integration/test_owner_ids.py", - "tests/integration/test_package_import.py", "tests/integration/test_package_install.py", - "tests/integration/test_optional_dependency.py", "tests/integration/test_no_network.py", - "tests/integration/test_schema_json.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_prohibited_structure.py", "tests/integration/test_license_notices.py", - ".github/workflows/ci.yml", ".github/workflows/security.yml", - ".github/workflows/golden.yml", ".github/workflows/release.yml", - "docs/architecture.md", "docs/theory_traceability.md", - "src/recursive_integrity_toolkit/utils/paths.py", - "src/recursive_integrity_toolkit/utils/logging.py", "docs/privacy.md", "docs/cli.md", - "tests/unit/test_phase4_output_safety.py", - } - assert set(phase4_tools["PHASE4_STEP6_ALLOWED"]) == approved - assert set(phase4_tools["PHASE4_STEP6_NEW"]) == {"tests/unit/test_phase4_output_safety.py"} - phase4_tools["verify_phase4_step6_changes"]([("A" if path == "tests/unit/test_phase4_output_safety.py" else "M", path) for path in sorted(approved)]) - - -@pytest.mark.parametrize("status,path", [ - ("M", "src/recursive_integrity_toolkit/cli.py"), - ("M", "src/recursive_integrity_toolkit/reports/json_report.py"), - ("M", "src/recursive_integrity_toolkit/reports/markdown_report.py"), - ("M", "src/recursive_integrity_toolkit/result.py"), - ("M", "src/recursive_integrity_toolkit/metrics/diversity.py"), - ("M", "schemas/report.schema.json"), ("M", "pyproject.toml"), - ("M", "PHASE_4_PLAN.md"), ("M", "tests/unit/test_phase4_output_safety.py"), - ("A", "tests/unit/test_phase4_unknown.py"), ("A", "PHASE_4_COMPLETION.md"), - ("A", "src/recursive_integrity_toolkit/utils/paths.py"), - ("D", "docs/cli.md"), ("R100", "docs/privacy.md"), - ("T", "scripts/release_check.py"), ("M", "./scripts/release_check.py"), -]) -def test_phase4_step6_diff_rejects_paths_and_operations(phase4_tools, status, path): - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step6_changes"]([(status, path)]) - - -def test_phase4_step6_diff_rejects_duplicate_operations(phase4_tools): - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step6_changes"]([("M", "docs/cli.md"), ("M", "docs/cli.md")]) - - -def test_phase4_step6_historical_bindings_preserve_assertions_and_headers(phase4_tools, phase4_step5_snapshot, phase4_step6_snapshot): - import ast - - registry = phase4_tools["PHASE4_STEP6_MIGRATIONS"] - assert set(registry) == {"tests/unit/test_phase4_contracts.py", "tests/integration/test_phase4_gates.py"} - assert sorted(len(rows) for rows in registry.values()) == [7, 8] - expected_inventory = {"phase4_mutation_tree"} | { - "test_phase4_step" + str(step) + "_current_snapshot_checks_valid_tree_before_mutations" - for step in (2, 3, 4, 5) - } - inventory_seen = set() - for path, rows in registry.items(): - phase4_tools["verify_phase4_step6_test_migration"]( - path, (phase4_step5_snapshot / path).read_bytes(), (phase4_step6_snapshot / path).read_bytes()) - for row in rows: - trees = [ast.parse(row[key]) for key in ("old", "new")] - assertions = [[ast.dump(n) for n in ast.walk(tree) if isinstance(n, ast.Assert)] for tree in trees] - assert assertions[0] == assertions[1] - assert trees[0].body[0].name == trees[1].body[0].name == row["node"] - if row["node"] in expected_inventory: - inventory_seen.add(row["node"]) - assert '"ls-files"' in row["old"] and 'rglob("*")' in row["new"] - assert inventory_seen == expected_inventory - - -@pytest.mark.parametrize("mutation", ["assertion", "binding", "inventory", "logging", "shadow", "default"]) -def test_phase4_step6_historical_guard_rejects_weakening(phase4_tools, phase4_step5_snapshot, mutation, phase4_step6_snapshot): - path = "tests/integration/test_phase4_gates.py" if mutation in ("inventory", "logging") else "tests/unit/test_phase4_contracts.py" - before = (phase4_step5_snapshot / path).read_bytes() - after = (phase4_step6_snapshot / path).read_bytes() - verify = phase4_tools["verify_phase4_step6_test_migration"] - verify(path, before, after) - if mutation == "assertion": - needle = b' assert control["active_phase"] == 4 and control["active_step"] == 5\n' - assert after.count(needle) == 1 - after = after.replace(needle, b" assert True\n", 1) - elif mutation == "binding": - needle = b'json.loads((phase4_step5_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))' - assert needle in after - after = after.replace(needle, needle.replace(b"phase4_step5_snapshot", b"ROOT"), 1) - elif mutation == "inventory": - needle = b'for path in phase4_step1_snapshot.rglob("*") if path.is_file()' - assert after.count(needle) == 1 - after = after.replace(needle, needle.replace(b"phase4_step1_snapshot", b"repo_root"), 1) - elif mutation == "logging": - needle = b'(phase4_step5_snapshot if relative == "utils/logging.py" else repo_root)' - assert after.count(needle) == 1 - after = after.replace(needle, b"repo_root", 1) - elif mutation == "shadow": - after += b"\ndef test_phase4_step5_fixed_boundary_opens_only_approved_existing_paths():\n assert True\n" - else: - after += b"\ndef test_phase4_step6_bad(value=globals().clear()):\n pass\n" - with pytest.raises(ValueError): - verify(path, before, after) - - -def test_phase4_step7_approved_control_keeps_independent_step6_anchors(phase4_tools, phase4_step7_snapshot): - control = json.loads((phase4_step7_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - assert phase4_tools["PHASE4_STEP6_FINAL"] == "aa2355f2359c3af4fc05345c792f9903b0d5858c" - assert phase4_tools["PHASE4_STEP6_TREE"] == "d9b6910079f49a2c739f86eceec05040918a41a5" - assert phase4_tools["PHASE4_STEP6_TEST_TREE"] == "8d3a03c61da807d1fef8d26fc4af0d6f72e4546e" - assert control["active_phase"] == 4 and control["active_step"] == 7 - assert control["approval_basis"] == "Phase 4 Step 7 继续" - assert control["baseline_commit"] == BASELINE - assert control["previous_step_commit"] == "aa2355f2359c3af4fc05345c792f9903b0d5858c" - assert control["previous_step_tree"] == "d9b6910079f49a2c739f86eceec05040918a41a5" - assert control["previous_step_test_tree"] == "8d3a03c61da807d1fef8d26fc4af0d6f72e4546e" - assert control["previous_step_core_tests"] == 3420 - assert control["previous_step_parquet_tests"] == 3423 - assert len(control["previous_step_files_sha256"]) == 228 - assert set(control["runtime_paths_authorized"]) == { - "src/recursive_integrity_toolkit/cli.py", - "src/recursive_integrity_toolkit/config.py", - } - assert control["schema_changes_authorized"] is False - assert control["schema_paths_authorized"] == [] - assert control["new_files_permitted"] == ["tests/integration/test_phase4_cli.py"] - for name in ("phase_complete", "next_step_authorized", "main_merge_authorized", "publication_authorized"): - assert control[name] is False - phase4_tools["verify_phase4_step7_control"](control, step=7) - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step4_control"](control, step=4) - - -@pytest.mark.parametrize("field,value", [ - ("active_phase", 3), ("active_phase", True), - ("active_step", 6), ("active_step", 8), ("active_step", True), - ("approval_basis", "approved by this manifest"), - ("previous_step_commit", "0000000000000000000000000000000000000000"), - ("previous_step_tree", "0000000000000000000000000000000000000000"), - ("previous_step_test_tree", "0000000000000000000000000000000000000000"), - ("previous_step_files_sha256", {}), ("baseline_files_sha256", {}), - ("previous_step_core_tests", 1), ("previous_step_parquet_tests", 1), - ("runtime_paths_authorized", ["src/recursive_integrity_toolkit/cli.py"]), - ("schema_changes_authorized", True), ("schema_paths_authorized", ["schemas/report.schema.json"]), - ("new_files_permitted", ["src/recursive_integrity_toolkit/reports/privacy.py"]), - ("permitted_paths", ["src/recursive_integrity_toolkit/cli.py"]), - ("phase_complete", True), ("next_step_authorized", True), - ("main_merge_authorized", True), ("publication_authorized", True), -]) -def test_phase4_step7_control_rejects_forged_scope_and_stage(phase4_tools, field, value, phase4_step7_snapshot): - control = json.loads((phase4_step7_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - verify = phase4_tools["verify_phase4_step7_control"] - verify(control, step=7) - control[field] = value - with pytest.raises(ValueError): - verify(control, step=7) - - -@pytest.mark.parametrize("step", [0, 1, 2, 3, 4, 5, 6, 8, 11, True, "7", None]) -def test_phase4_step7_control_rejects_unapproved_dispatch(phase4_tools, step, phase4_step7_snapshot): - control = json.loads((phase4_step7_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step7_control"](control, step=7) - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step7_control"](control, step=step) - - -@pytest.mark.parametrize("field", ["active_step", "previous_step_commit", "runtime_paths_authorized", "schema_paths_authorized"]) -def test_phase4_step7_control_requires_explicit_approval_fields(phase4_tools, field, phase4_step7_snapshot): - control = json.loads((phase4_step7_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step7_control"](control) - del control[field] - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step7_control"](control) - - -def test_phase4_step7_control_cannot_mint_extra_permission(phase4_tools, phase4_step7_snapshot): - control = json.loads((phase4_step7_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step7_control"](control) - control["approved_scope_expansion"] = {"step": 7, "publication": True} - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step7_control"](control) - - -def test_phase4_step7_fixed_boundary_opens_only_approved_existing_paths(phase4_tools): - approved = { - "PHASE_4_BASELINE.json", "PHASE_4_DECISIONS.md", "scripts/check_traceability.py", - "scripts/check_spec_consistency.py", "scripts/release_check.py", "tests/conftest.py", - "tests/unit/test_phase4_contracts.py", "tests/integration/test_phase4_gates.py", - "tests/integration/test_no_algorithms.py", "tests/integration/test_ci_workflows.py", - "tests/integration/test_repository_structure.py", "tests/integration/test_owner_ids.py", - "tests/integration/test_package_import.py", "tests/integration/test_package_install.py", - "tests/integration/test_optional_dependency.py", "tests/integration/test_no_network.py", - "tests/integration/test_schema_json.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_prohibited_structure.py", "tests/integration/test_license_notices.py", - ".github/workflows/ci.yml", ".github/workflows/security.yml", - ".github/workflows/golden.yml", ".github/workflows/release.yml", - "docs/architecture.md", "docs/theory_traceability.md", - "src/recursive_integrity_toolkit/cli.py", - "src/recursive_integrity_toolkit/config.py", "tests/integration/test_cli_validation.py", "docs/cli.md", - "tests/integration/test_phase4_cli.py", - } - assert set(phase4_tools["PHASE4_STEP7_ALLOWED"]) == approved - assert set(phase4_tools["PHASE4_STEP7_NEW"]) == {"tests/integration/test_phase4_cli.py"} - phase4_tools["verify_phase4_step7_changes"]([("A" if path == "tests/integration/test_phase4_cli.py" else "M", path) for path in sorted(approved)]) - - -@pytest.mark.parametrize("status,path", [ - ("M", "src/recursive_integrity_toolkit/utils/paths.py"), - ("M", "src/recursive_integrity_toolkit/utils/logging.py"), - ("M", "src/recursive_integrity_toolkit/reports/assembly.py"), - ("M", "src/recursive_integrity_toolkit/metrics/diversity.py"), - ("M", "schemas/report.schema.json"), ("M", "pyproject.toml"), - ("M", "PHASE_4_PLAN.md"), ("M", "tests/integration/test_phase4_cli.py"), - ("A", "tests/unit/test_phase4_unknown.py"), ("A", "PHASE_4_COMPLETION.md"), - ("A", "src/recursive_integrity_toolkit/cli.py"), ("M", "docs/privacy.md"), - ("D", "docs/cli.md"), ("R100", "docs/cli.md"), - ("T", "scripts/release_check.py"), ("M", "./scripts/release_check.py"), -]) -def test_phase4_step7_diff_rejects_paths_and_operations(phase4_tools, status, path): - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step7_changes"]([(status, path)]) - - -def test_phase4_step7_diff_rejects_duplicate_operations(phase4_tools): - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step7_changes"]([("M", "docs/cli.md"), ("M", "docs/cli.md")]) - - -def test_phase4_step7_historical_bindings_preserve_assertions(phase4_tools, phase4_step6_snapshot, phase4_step7_snapshot): - import ast - registry = phase4_tools["PHASE4_STEP7_MIGRATIONS"] - assert set(registry) == {"tests/unit/test_phase4_contracts.py", "tests/integration/test_phase4_gates.py"} - assert sorted(len(rows) for rows in registry.values()) == [5, 7] - for path, rows in registry.items(): - phase4_tools["verify_phase4_step7_test_migration"](path, (phase4_step6_snapshot / path).read_bytes(), (phase4_step7_snapshot / path).read_bytes()) - for row in rows: - trees = [ast.parse(row[key]) for key in ("old", "new")] - assert trees[0].body[0].name == trees[1].body[0].name == row["node"] - assertions = [[ast.dump(n) for n in ast.walk(tree) if isinstance(n, ast.Assert)] for tree in trees] - assert assertions[0] == assertions[1] - - -@pytest.mark.parametrize("mutation", ["assertion", "binding", "config", "shadow", "default"]) -def test_phase4_step7_historical_guard_rejects_weakening(phase4_tools, phase4_step6_snapshot, mutation, phase4_step7_snapshot): - path = "tests/integration/test_phase4_gates.py" if mutation == "config" else "tests/unit/test_phase4_contracts.py" - before, after = (phase4_step6_snapshot / path).read_bytes(), (phase4_step7_snapshot / path).read_bytes() - verify = phase4_tools["verify_phase4_step7_test_migration"] - verify(path, before, after) - if mutation == "assertion": - needle = b' assert control["active_phase"] == 4 and control["active_step"] == 6\n' - assert after.count(needle) == 1 - after = after.replace(needle, b" assert True\n", 1) - elif mutation == "binding": - needle = b'json.loads((phase4_step6_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))' - assert needle in after - after = after.replace(needle, needle.replace(b"phase4_step6_snapshot", b"ROOT"), 1) - elif mutation == "config": - needle = b'else phase4_step6_snapshot if relative == "config.py" else repo_root' - assert after.count(needle) == 1 - after = after.replace(needle, b"else repo_root", 1) - elif mutation == "shadow": - after += b"\ndef test_phase4_step6_fixed_boundary_opens_only_approved_existing_paths():\n assert True\n" - else: - after += b"\ndef test_phase4_step7_bad(value=globals().clear()):\n pass\n" - with pytest.raises(ValueError): - verify(path, before, after) - - -def test_phase4_step8_approved_control_keeps_independent_step7_anchors(phase4_tools, phase4_step8_snapshot): - control = json.loads((phase4_step8_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - assert phase4_tools["PHASE4_STEP7_FINAL"] == "bacd33ae65e392872776c6d976401fdc3d565f63" - assert phase4_tools["PHASE4_STEP7_TREE"] == "4c16914bb0f0c5281efc36624efb8b1a1d4a174e" - assert phase4_tools["PHASE4_STEP7_TEST_TREE"] == "96317cb7b25236b79efc0f2d93e1e0f5f132fdb2" - assert control["active_phase"] == 4 and control["active_step"] == 8 - assert control["approval_basis"] == "phase 4 step 8 开始" - assert control["baseline_commit"] == BASELINE - assert control["previous_step_commit"] == "bacd33ae65e392872776c6d976401fdc3d565f63" - assert control["previous_step_tree"] == "4c16914bb0f0c5281efc36624efb8b1a1d4a174e" - assert control["previous_step_test_tree"] == "96317cb7b25236b79efc0f2d93e1e0f5f132fdb2" - assert control["previous_step_core_tests"] == 3620 - assert control["previous_step_parquet_tests"] == 3623 - assert len(control["previous_step_files_sha256"]) == 229 - assert set(control["runtime_paths_authorized"]) == { - "src/recursive_integrity_toolkit/cli.py", - "src/recursive_integrity_toolkit/config.py", - } - assert control["schema_changes_authorized"] is False - assert control["schema_paths_authorized"] == [] - assert control["new_files_permitted"] == ['src/recursive_integrity_toolkit/data/hero/EXPECTED_OUTPUTS.md', 'src/recursive_integrity_toolkit/data/hero/config.json', 'src/recursive_integrity_toolkit/data/hero/provenance.csv', 'src/recursive_integrity_toolkit/data/hero/records_v1.csv', 'src/recursive_integrity_toolkit/data/hero/records_v2.csv', 'src/recursive_integrity_toolkit/data/hero/version_order.json', 'src/recursive_integrity_toolkit/data/report.schema.json'] - for name in ("phase_complete", "next_step_authorized", "main_merge_authorized", "publication_authorized"): - assert control[name] is False - phase4_tools["verify_phase4_step8_control"](control, step=8) - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step4_control"](control, step=4) - - -@pytest.mark.parametrize("field,value", [ - ("active_phase", 3), ("active_phase", True), - ("active_step", 7), ("active_step", 9), ("active_step", True), - ("approval_basis", "approved by this manifest"), - ("previous_step_commit", "0000000000000000000000000000000000000000"), - ("previous_step_tree", "0000000000000000000000000000000000000000"), - ("previous_step_test_tree", "0000000000000000000000000000000000000000"), - ("previous_step_files_sha256", {}), ("baseline_files_sha256", {}), - ("previous_step_core_tests", 1), ("previous_step_parquet_tests", 1), - ("runtime_paths_authorized", ["src/recursive_integrity_toolkit/cli.py"]), - ("schema_changes_authorized", True), ("schema_paths_authorized", ["schemas/report.schema.json"]), - ("new_files_permitted", ["src/recursive_integrity_toolkit/reports/privacy.py"]), - ("permitted_paths", ["src/recursive_integrity_toolkit/cli.py"]), - ("phase_complete", True), ("next_step_authorized", True), - ("main_merge_authorized", True), ("publication_authorized", True), -]) -def test_phase4_step8_control_rejects_forged_scope_and_stage(phase4_tools, field, value, phase4_step8_snapshot): - control = json.loads((phase4_step8_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - verify = phase4_tools["verify_phase4_step8_control"] - verify(control, step=8) - control[field] = value - with pytest.raises(ValueError): - verify(control, step=8) - - -@pytest.mark.parametrize("step", [0, 1, 2, 3, 4, 5, 6, 7, 9, 11, True, "8", None]) -def test_phase4_step8_control_rejects_unapproved_dispatch(phase4_tools, step, phase4_step8_snapshot): - control = json.loads((phase4_step8_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step8_control"](control, step=8) - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step8_control"](control, step=step) - - -@pytest.mark.parametrize("field", ["active_step", "previous_step_commit", "runtime_paths_authorized", "schema_paths_authorized"]) -def test_phase4_step8_control_requires_explicit_approval_fields(phase4_tools, field, phase4_step8_snapshot): - control = json.loads((phase4_step8_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step8_control"](control) - del control[field] - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step8_control"](control) - - -def test_phase4_step8_control_cannot_mint_extra_permission(phase4_tools, phase4_step8_snapshot): - control = json.loads((phase4_step8_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - phase4_tools["verify_phase4_step8_control"](control) - control["approved_scope_expansion"] = {"step": 8, "publication": True} - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step8_control"](control) - - -def test_phase4_step8_fixed_boundary_opens_only_approved_existing_paths(phase4_tools): - approved = { - "PHASE_4_BASELINE.json", "PHASE_4_DECISIONS.md", "scripts/check_traceability.py", - "scripts/check_spec_consistency.py", "scripts/release_check.py", "tests/conftest.py", - "tests/unit/test_phase4_contracts.py", "tests/integration/test_phase4_gates.py", - "tests/integration/test_no_algorithms.py", "tests/integration/test_ci_workflows.py", - "tests/integration/test_repository_structure.py", "tests/integration/test_owner_ids.py", - "tests/integration/test_package_import.py", "tests/integration/test_package_install.py", - "tests/integration/test_optional_dependency.py", "tests/integration/test_no_network.py", - "tests/integration/test_schema_json.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_prohibited_structure.py", "tests/integration/test_license_notices.py", - ".github/workflows/ci.yml", ".github/workflows/security.yml", - ".github/workflows/golden.yml", ".github/workflows/release.yml", - "docs/architecture.md", "docs/theory_traceability.md", - "src/recursive_integrity_toolkit/cli.py", - "src/recursive_integrity_toolkit/config.py", "pyproject.toml", "tests/integration/test_hero_end_to_end.py", "docs/cli.md", - "tests/integration/test_phase4_cli.py", - } - resources = {'src/recursive_integrity_toolkit/data/report.schema.json', 'src/recursive_integrity_toolkit/data/hero/version_order.json', 'src/recursive_integrity_toolkit/data/hero/config.json', 'src/recursive_integrity_toolkit/data/hero/EXPECTED_OUTPUTS.md', 'src/recursive_integrity_toolkit/data/hero/records_v2.csv', 'src/recursive_integrity_toolkit/data/hero/provenance.csv', 'src/recursive_integrity_toolkit/data/hero/records_v1.csv'} - approved.update(resources) - assert len(approved) == 39 - assert set(phase4_tools["PHASE4_STEP8_ALLOWED"]) == approved - assert set(phase4_tools["PHASE4_STEP8_NEW"]) == resources - phase4_tools["verify_phase4_step8_changes"]([("A" if path in resources else "M", path) for path in sorted(approved)]) - - -@pytest.mark.parametrize("status,path", [ - ("M", "src/recursive_integrity_toolkit/metrics/diversity.py"), - ("M", "src/recursive_integrity_toolkit/reports/assembly.py"), - ("M", "schemas/report.schema.json"), ("M", "examples/hero/config.json"), - ("M", "PHASE_4_PLAN.md"), ("A", "tests/integration/test_phase4_cli.py"), - ("M", "src/recursive_integrity_toolkit/data/hero/config.json"), - ("A", "src/recursive_integrity_toolkit/data/other.json"), - ("A", "PHASE_4_COMPLETION.md"), ("D", "docs/cli.md"), - ("R100", "pyproject.toml"), ("T", "scripts/release_check.py"), -]) -def test_phase4_step8_diff_rejects_paths_and_operations(phase4_tools, status, path): - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step8_changes"]([(status, path)]) - - -def test_phase4_step8_historical_bindings_preserve_assertions(phase4_tools, phase4_step7_snapshot, phase4_step8_snapshot): - import ast - registry = phase4_tools["PHASE4_STEP8_MIGRATIONS"] - assert sorted(len(rows) for rows in registry.values()) == [4, 7] - for path, rows in registry.items(): - phase4_tools["verify_phase4_step8_test_migration"](path, (phase4_step7_snapshot / path).read_bytes(), (phase4_step8_snapshot / path).read_bytes()) - for row in rows: - trees = [ast.parse(row[key]) for key in ("old", "new")] - assert trees[0].body[0].name == trees[1].body[0].name == row["node"] - assert [[ast.dump(n) for n in ast.walk(t) if isinstance(n, ast.Assert)] for t in trees][0] == [[ast.dump(n) for n in ast.walk(t) if isinstance(n, ast.Assert)] for t in trees][1] - - -@pytest.mark.parametrize("mutation", ["assertion", "binding", "shadow", "default"]) -def test_phase4_step8_historical_guard_rejects_weakening(phase4_tools, phase4_step7_snapshot, mutation, phase4_step8_snapshot): - path = "tests/unit/test_phase4_contracts.py" - before, after = (phase4_step7_snapshot / path).read_bytes(), (phase4_step8_snapshot / path).read_bytes() - verify = phase4_tools["verify_phase4_step8_test_migration"] - verify(path, before, after) - if mutation == "assertion": - needle = b' assert control["active_phase"] == 4 and control["active_step"] == 7\n' - assert after.count(needle) == 1 - after = after.replace(needle, b" assert True\n", 1) - elif mutation == "binding": - needle = b'json.loads((phase4_step7_snapshot / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))' - assert needle in after - after = after.replace(needle, needle.replace(b"phase4_step7_snapshot", b"ROOT"), 1) - elif mutation == "shadow": - after += b"\ndef test_phase4_step7_fixed_boundary_opens_only_approved_existing_paths():\n assert True\n" - else: - after += b"\ndef test_phase4_step8_bad(value=globals().clear()):\n pass\n" - with pytest.raises(ValueError): verify(path, before, after) - - -def test_phase4_step9_control_keeps_step8_acceptance_and_freezes_runtime(phase4_tools): - control = json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - assert control["active_phase"] == 4 and control["active_step"] == 11 - assert control["approval_basis"] == "继续 Phase 4 Step 11" - assert control["previous_step_commit"] == "9142b344bfc2dc2b33dbc696159f07bbc81b223a" - assert control["previous_step_tree"] == "a485078d3e7cf739e10b087acb604f317ee19854" - assert control["previous_step_test_tree"] == "2fc4e86c4dee9f069f93af59caab138f0ea63224" - assert "previous_step_core_tests" not in control - assert "previous_step_parquet_tests" not in control - assert "previous_step_files_sha256" not in control - assert control["runtime_paths_authorized"] == ["src/recursive_integrity_toolkit/__init__.py"] - assert control["schema_paths_authorized"] == [] - assert control["runtime_changes_authorized"] is True - assert "runtime_repair_approval_basis" not in control - assert control["package_resource_copies"] == {} - assert control["report_goldens_authorized"] is False - for key in ("schema_changes_authorized", "package_data_declaration_only", "phase_complete", "next_step_authorized", "main_merge_authorized", "publication_authorized"): - assert control[key] is False - phase4_tools["verify_phase4_current_control"](control, step=11) - with pytest.raises(ValueError): - phase4_tools["verify_phase4_step8_control"](control, step=8) - - -@pytest.mark.parametrize("field,value", [ - ("active_step", 8), ("active_step", 9), ("active_phase", True), - ("approval_basis", "approved by manifest"), ("previous_step_commit", "0000000000000000000000000000000000000000"), - ("previous_step_tree", "0000000000000000000000000000000000000000"), ("previous_step_test_tree", "0000000000000000000000000000000000000000"), - ("previous_step_files_sha256", {}), ("previous_step_core_tests", 1), - ("runtime_changes_authorized", False), ("runtime_paths_authorized", ["src/recursive_integrity_toolkit/cli.py"]), - ("schema_changes_authorized", True), ("package_data_declaration_only", True), - ("new_files_permitted", ["src/recursive_integrity_toolkit/data/extra.json"]), - ("report_goldens_authorized", True), ("permitted_paths", []), - ("next_step_authorized", True), ("phase_complete", True), ("publication_authorized", True), -]) -def test_phase4_step9_control_rejects_forged_permission(phase4_tools, field, value): - control = json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - verify = phase4_tools["verify_phase4_current_control"] - verify(control) - control[field] = value - with pytest.raises(ValueError): - verify(control) - - -@pytest.mark.parametrize("step", [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, True, "11", None]) -def test_phase4_step9_control_requires_exact_stage(phase4_tools, step): - control = json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) - with pytest.raises(ValueError): - phase4_tools["verify_phase4_current_control"](control, step=step) - - -def test_phase4_step9_allowlist_is_exact_common_g_plus_named_report_paths(phase4_tools): - """Current scope permits final documentation, exact version literals and common maintenance.""" - additional = {"README.md", "CHANGELOG.md", "docs/cli.md", "docs/report_schema.md", - "docs/data_schema.md", "docs/privacy.md", "docs/release_process.md", - "pyproject.toml", "src/recursive_integrity_toolkit/__init__.py", - "tests/integration/test_cli_validation.py", "tests/integration/test_phase4_cli.py", - "tests/golden/test_phase4_reports.py", "PHASE_4_COMPLETION.md", - "PHASE_4_VALIDATION_REPORT.md", "PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md"} - allowed = phase4_tools["PHASE4_G"] | additional - assert len(allowed) == 41 - assert phase4_tools["PHASE4_CURRENT_ALLOWED"] == allowed - phase4_tools["verify_phase4_current_changes"]([ - ("A" if p in phase4_tools["PHASE4_CURRENT_NEW"] else "M", p) for p in sorted(allowed)]) - - -@pytest.mark.parametrize("status,path", [ - ("M", "src/recursive_integrity_toolkit/cli.py"), ("M", "schemas/report.schema.json"), - ("M", "src/recursive_integrity_toolkit/data/hero/config.json"), ("M", "src/recursive_integrity_toolkit/reports/assembly.py"), - ("M", "PHASE_4_PLAN.md"), ("M", "tests/golden/phase3_math_cases.json"), - ("M", "tests/performance/test_hero_runtime.py"), ("M", "PHASE_4_COMPLETION.md"), - ("A", "scripts/build_golden.py"), ("M", "tests/golden/phase4_hero_report.json"), - ("D", "tests/golden/README.md"), ("R100", "scripts/release_check.py"), - ("T", "scripts/normalize_golden.py"), ("M", "./scripts/build_golden.py"), -]) -def test_phase4_step9_diff_rejects_wrong_paths_or_operations(phase4_tools, status, path): - with pytest.raises(ValueError): - phase4_tools["verify_phase4_current_changes"]([(status, path)]) - - -def test_phase4_step9_rejects_duplicate_diff_entries(phase4_tools): - with pytest.raises(ValueError): - phase4_tools["verify_phase4_current_changes"]([("M", "README.md"), ("M", "README.md")]) - - -def test_phase4_step9_exact_historical_migrations_preserve_assertions_and_parameters(phase4_tools, phase4_step8_snapshot): - import ast - registry = phase4_tools["PHASE4_STEP9_MIGRATIONS"] - assert sorted(len(rows) for rows in registry.values()) == [2, 7] - for path, rows in registry.items(): - phase4_tools["verify_phase4_step9_test_migration"](path, (phase4_step8_snapshot / path).read_bytes(), phase4_tools["_phase4_current_files"]()[path]) - for row in rows: - old, new = [ast.parse(row[k]) for k in ("old", "new")] - assert old.body[0].name == new.body[0].name == row["node"] - assert [ast.dump(n) for n in ast.walk(old) if isinstance(n, ast.Assert)] == [ast.dump(n) for n in ast.walk(new) if isinstance(n, ast.Assert)] - assert [n.arg for n in new.body[0].args.args] == [n.arg for n in old.body[0].args.args] + ["phase4_step8_snapshot"] - - -@pytest.mark.parametrize("mutation", ["assertion", "binding", "shadow", "default", "decorator", "import"]) -def test_phase4_step9_historical_migration_rejects_weakening(phase4_tools, phase4_step8_snapshot, mutation): - path = "tests/unit/test_phase4_contracts.py" - before, after = (phase4_step8_snapshot / path).read_bytes(), phase4_tools["_phase4_current_files"]()[path] - verify = phase4_tools["verify_phase4_step9_test_migration"] - verify(path, before, after) - if mutation == "assertion": - after = after.replace(b' assert control["active_phase"] == 4 and control["active_step"] == 8\n', b" assert True\n", 1) - elif mutation == "binding": - after = after.replace(b'json.loads((phase4_step8_snapshot / "PHASE_4_BASELINE.json")', b'json.loads((ROOT / "PHASE_4_BASELINE.json")', 1) - elif mutation == "shadow": - after += b"\ndef test_phase4_step8_control_cannot_mint_extra_permission():\n pass\n" - elif mutation == "default": - after += b"\ndef test_phase4_step9_bad(value=globals().clear()):\n pass\n" - elif mutation == "decorator": - after += b"\n@pytest.mark.parametrize('x', list(range(1)))\ndef test_phase4_step9_bad(x):\n pass\n" - else: - after += b"\nimport socket\n" - with pytest.raises(ValueError): - verify(path, before, after) - - -@pytest.mark.parametrize("mutation", ["complete", "missing_golden", "missing_math", "missing_registry", "extra_golden", "duplicate", "skipped", "empty_golden_collection"]) -def test_phase4_step9_golden_evidence_requires_complete_collected_identity_set(phase4_tools, monkeypatch, tmp_path, mutation): - import subprocess - import xml.etree.ElementTree as ET - from types import SimpleNamespace - cases = json.loads((ROOT / "tests/golden/phase3_math_cases.json").read_bytes())["cases"] - math = ["tests/golden/test_phase3_math.py::test_phase3_frozen_case[" + case["case_id"] + "]" for case in cases] - registry = ["tests/golden/test_phase3_math.py::test_phase3_all_twenty_oracles_have_unique_traceable_identity"] - reports = ["tests/golden/test_phase4_reports.py::test_literal[standard]", "tests/golden/test_phase4_reports.py::test_literal[redacted]"] - collected = math + registry + ([] if mutation == "empty_golden_collection" else reports) - def collect(command, **kwargs): - assert command[-2:] == ["tests/golden/test_phase4_reports.py", "tests/golden/test_phase3_math.py"] - assert "--collect-only" in command - return SimpleNamespace(stdout="\n".join(collected) + "\n") - monkeypatch.setattr(subprocess, "run", collect) - executed = math + registry + reports - if mutation == "missing_golden": - executed = math + registry + reports[:1] - elif mutation == "missing_math": - executed = math[:-1] + registry + reports - elif mutation == "missing_registry": - executed = math + reports - elif mutation == "extra_golden": - executed += ["tests/golden/test_phase4_reports.py::test_uncollected"] - elif mutation == "duplicate": - executed += reports[:1] - root = ET.Element("testsuite", tests=str(len(executed)), failures="0", errors="0", skipped="1" if mutation == "skipped" else "0") - for node in executed: - owner, name = node.split("::", 1) - case = ET.SubElement(root, "testcase", classname=owner.removesuffix(".py").replace("/", "."), name=name) - if mutation == "skipped" and node == reports[0]: - ET.SubElement(case, "skipped") - path = tmp_path / "golden.xml" - ET.ElementTree(root).write(path, encoding="utf-8") - if mutation == "complete": - result = phase4_tools["phase4_step9_golden_evidence"](path) - assert result["mathematical_cases"] == 20 and result["report_golden_tests"] == 2 - assert result["report_golden_nodeids"] == reports and result["mathematical_nodeids"] == math - assert result["mathematical_registry_nodeid"] == registry[0] - assert result["missing_nodeids"] == [] - else: - with pytest.raises(ValueError): - phase4_tools["phase4_step9_golden_evidence"](path) From e40543ff5ed3d2793a41bf58d7e4831bd2a9bfe9 Mon Sep 17 00:00:00 2001 From: DavidWallstructurallaw Date: Tue, 22 Sep 2026 20:05:26 -0700 Subject: [PATCH 02/12] Implement Phase 5 Step 2 retained parent graph --- .github/workflows/ci.yml | 4 +- PHASE_5_DECISIONS.md | 25 +- PHASE_5_STEP_2.md | 90 ++++ docs/lineage_contract.md | 35 +- docs/release_process.md | 15 +- scripts/release_check.py | 21 +- src/recursive_integrity_toolkit/errors.py | 20 + .../io/validation.py | 164 ++++++-- .../lineage/graph.py | 390 +++++++++++++++++- src/recursive_integrity_toolkit/models.py | 81 ++++ .../integration/test_current_verification.py | 4 +- tests/unit/test_lineage_graph.py | 368 +++++++++++++++++ tests/unit/test_parent_batch.py | 188 +++++++++ 13 files changed, 1343 insertions(+), 62 deletions(-) create mode 100644 PHASE_5_STEP_2.md create mode 100644 tests/unit/test_lineage_graph.py create mode 100644 tests/unit/test_parent_batch.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83ffe4a..1da35f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,13 +50,13 @@ jobs: with: python-version: "3.12" - run: python -m pip install ".[test]" - - name: Check Step 1 contracts and affected boundaries + - name: Check Step 2 parent graph and affected boundaries run: | set -euo pipefail python scripts/check_spec_consistency.py python scripts/check_traceability.py python scripts/release_check.py - python -m pytest -p no:cacheprovider -q tests/integration/test_current_verification.py tests/integration/test_ci_workflows.py tests/unit/test_lineage_fixture_inputs.py tests/unit/test_phase3_contracts.py tests/integration/test_phase3_metric_pipeline.py tests/unit/test_PR016_determinism.py tests/integration/test_no_algorithms.py tests/integration/test_repository_structure.py + python -m pytest -p no:cacheprovider -q tests/unit/test_parent_batch.py tests/unit/test_lineage_graph.py tests/unit/test_PR007_version_order.py tests/unit/test_PR008_parent_resolution.py tests/unit/test_PR009_generation.py tests/unit/test_lineage_fixture_inputs.py tests/unit/test_phase3_contracts.py tests/integration/test_phase3_metric_pipeline.py tests/unit/test_PR016_determinism.py tests/integration/test_current_verification.py tests/integration/test_ci_workflows.py tests/integration/test_no_algorithms.py tests/integration/test_repository_structure.py core: if: github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate' diff --git a/PHASE_5_DECISIONS.md b/PHASE_5_DECISIONS.md index 074f9e1..8fe4fa6 100644 --- a/PHASE_5_DECISIONS.md +++ b/PHASE_5_DECISIONS.md @@ -1,6 +1,6 @@ -# Phase 5 Decisions and Step 1 Authorization +# Phase 5 Decisions and Current Authorization -Status: **PLAN APPROVED; STEP 1 AUTHORIZED; PHASE INCOMPLETE**. +Status: **PLAN APPROVED; STEPS 1-2 COMPLETE; PHASE INCOMPLETE**. On 2026-09-23 UTC the Theory Owner instructed: `批准，Phase 5 step 1开始`. @@ -116,4 +116,23 @@ release matrix or demonstrate graph correctness. Actual changes, check results, failures and unresolved limitations belong in the Step 1 completion report. This approval record makes no claim that those checks have already passed. -Stop after Step 1. Phase 5 Step 2 requires its own execution instruction. +Step 1 completed before the following separate execution instruction. + +## Step 2 execution instruction + +On 2026-09-23 UTC the Theory Owner instructed `Phase 5 Step 2开始`. +This authorizes the bounded reference-retention refactor and immutable validated +parent graph in the approved plan. Work starts from Step 1 commit +`5e946d1a42f87f89d4f97e9da4fd76cb05517caa` on `phase5-lineage`. + +The current source boundary opens four existing modules: `models.py`, `errors.py`, +`io/validation.py` and `lineage/graph.py`. Direct reference/graph tests, affected +generation/input neighbors, documentation and the current focused gate are in +scope. This updates the single current control without adding a registry or +historical migration. Package version and executable report schema remain +`0.1.0.dev3` and `1.0`. No cycle/depth/root algorithm, lineage metric, new CLI role, +full environment matrix or performance measurement is authorized by this step. + +Stop after Step 2; subsequent execution needs its own instruction. Existing +authorization to synchronize this branch is retained. Merge, tag and publication +remain separate actions. diff --git a/PHASE_5_STEP_2.md b/PHASE_5_STEP_2.md new file mode 100644 index 0000000..182efad --- /dev/null +++ b/PHASE_5_STEP_2.md @@ -0,0 +1,90 @@ +# Phase 5 Step 2 Completion + +Date: 2026-09-23 UTC. Status: **STEP 2 COMPLETE; PHASE 5 INCOMPLETE**. + +The Theory Owner instructed `Phase 5 Step 2开始`. Work starts from +`5e946d1a42f87f89d4f97e9da4fd76cb05517caa` on `phase5-lineage`. +This step implements retained immediate-parent validation and immutable graph +construction. It does not authorize subsequent steps, a merge, tag or release. + +## Delivered behavior + +`resolve_parent_batch` shares single-reference interpretation with the existing +fail-fast resolver and builds one identity lookup per batch. Its immutable +per-record results preserve accepted and unresolved siblings when another +reference is ambiguous, malformed, a future parent or a direct self-reference. +The existing standalone resolver and generation API retain their behavior. +`validate_bundle` retains the batch without invoking analytical algorithms. + +Original declaration counts include duplicate aliases. Identity-resolved counts +remain known when chronology is unavailable; cross-version graph edges require +chronology. Adjacency deduplicates canonical parent identities. Missing +provenance, absent declarations, null declarations and explicit empty lists stay +distinct. Malformed containers with unknown cardinality never become zero +declared parents. Existing ingestion still rejects malformed serialized input at +its established boundary; the retained batch also handles typed per-reference +failures without losing successful siblings. + +`build_lineage_graph` supplies explicit target/context scope, canonical sorted +nodes, immutable parent and child adjacency, sanitized per-record evidence and +separate self-reference evidence. It revalidates retained input evidence and +rejects stale or forged handoffs. Same-version edges remain available for later +cycle analysis. Graph construction performs no file or network access. + +`LineageLimits` enforces node and unique-edge admission. Exceeding a limit raises +a typed resource error and returns no partial exact graph. Root membership and +union-work limits are validated parameters reserved for Step 4. These limits do +not claim to bound already-loaded input memory or process RSS. + +## Verification + +| Check | Actual result | +|---|---| +| Final exact focused CI selection | 438 passed in 3.98 seconds | +| Provenance, observability, capability, evidence and CLI neighbors | 501 passed in 12.18 seconds | +| Direct parent-batch and graph cases within the focused selection | 15 batch cases and 40 graph cases passed | +| Current specification, traceability and source-scope checks | PASS | +| Diff formatting and independent implementation review | PASS; no remaining concrete blocker | + +The earlier 398-test affected run is superseded by the final focused selection +and is not added to it. Independent review found two direct graph-construction +gaps: target nodes could be omitted with adjusted counts, and resolved reference +multiplicity could be falsified. Both are fixed and protected by direct +regressions. An initial graph static check found a missing implementation-status +annotation; it was restored and the complete static checks passed. The initial +graph test run had two empty-scope fixture-construction errors, corrected to use +the approved typed analytical input before the final passing run. + +The final focused command matches `.github/workflows/ci.yml`, with local JUnit +output. The neighbor selection was: + +```bash +python -m pytest -p no:cacheprovider -q \ + tests/unit/test_PR004_coverage.py \ + tests/unit/test_PR010_observability.py \ + tests/unit/test_PR011_capabilities.py \ + tests/unit/test_PR012_evidence_classes.py \ + tests/integration/test_phase4_cli.py +``` + +The three current static checks protect 55 unchanged product files, permit the +four approved implementation modules, retain the exact product inventory, and +verify 16 frozen specifications, 7 packaged resource copies and 40 owned modules. +The existing source-scope control and focused CI selection are updated in place; +no historical migration, approval registry or evidence framework is added. + +Local environment: Python 3.12.14, pytest 9.1.1, NumPy 2.5.3 and pandas 3.0.6. +This step does not run the full regression, OS/Python/dependency matrix, optional +Parquet profile, release builds, clean installs or 100k performance workloads. +Hosted CI success is not claimed. + +## Handoff + +Package version remains `0.1.0.dev3`; report schema remains `1.0`. Graph execution +is a direct opt-in API. The audit/validate CLI, report assembly and ordinary +metric pipeline do not dispatch lineage analysis. General cycles, depth, roots, +ancestry metrics, schema 1.1 and lineage CLI/context inputs remain later steps. + +Step 3 may consume immediate adjacency and retained self-reference evidence to +implement iterative cycle analysis and structural depth after its separate +execution instruction. diff --git a/docs/lineage_contract.md b/docs/lineage_contract.md index 5682a3d..3760ff0 100644 --- a/docs/lineage_contract.md +++ b/docs/lineage_contract.md @@ -7,10 +7,11 @@ Authority: [Phase 5 decisions](../PHASE_5_DECISIONS.md), approved Phase 5 plan, frozen owners are T4, T6, PR-008, PR-009 and T3 lineage bounds. The original definitions/specifications remain unchanged. -This document defines interfaces to implement in Steps 2-8. Step 1 leaves the -package at `0.1.0.dev3`, the executable report schema at `1.0`, and general -lineage execution deferred. Code-like declarations below are contracts, not -claims about currently importable classes or functions. +This document defines interfaces for Steps 2-8. Step 2 implements retained batch +parent evidence and `build_lineage_graph` with immutable scope, adjacency and +node/edge limits. Cycle, depth, root, metric, report and CLI lineage interfaces +remain later implementation targets. The package remains at `0.1.0.dev3` and the +executable report schema at `1.0`. ## 1. Scope and reference semantics @@ -46,8 +47,12 @@ index for the batch. It never treats an ambiguous/future/invalid reference as an accepted ancestry edge. For each syntactically valid reference list, declared count is the original -list length, including repeated aliases. Resolved count sums the lengths of -`source_references` for accepted resolved references. Unresolved count is +list length, including repeated aliases. Absent/null declarations and missing +provenance have zero original reference entries; this does not establish an +explicitly parentless record. Resolved count sums the lengths of +`source_references` whose existing resolution status is `resolved`, including +known identities with unavailable chronology. Graph edge admission separately +requires known chronology. Unresolved reference count is declared minus resolved and includes invalid references whose entries can be counted. A malformed declaration with unknown reference cardinality makes the whole-target reference totals/coverage unavailable; retain known-subset counts @@ -82,6 +87,24 @@ records. Selection is explicit, revalidated and independent of ordinary metric eligibility. `validate_bundle` remains input-only and never calls this function. CLI orchestration calls it only after an explicit lineage request. +Step 2 supplies the lower-level input-only handoff in `models.py`: +`ParentRecordValidation` retains provenance presence, an optional +`ParentValidationResult`, declared/resolved/self-reference counts and diagnostics; +`ParentBatchValidationResult` covers every loaded key in canonical order. +`io.validation.resolve_parent_batch` shares reference interpretation with the +existing fail-fast resolver and creates one identity lookup per batch. +`BundleValidationResult.parent_validation` is an appended optional field for +source compatibility. Graph construction requires this retained evidence and +revalidates it against the canonical inputs, rejecting stale or forged handoffs. + +`lineage.graph.build_lineage_graph(validation, *, target_dataset_version, limits)` +constructs the immediate graph only. When explicit primary record roles exist, +the selected version must be that primary version. Context-only typed inputs can +represent an empty target with no declared version; an explicit empty version +must be present in the declared version order. Input loading and its limits stay +independent of graph limits. Root-membership and union-work limits are reserved +for the later ancestry implementation. + Ownership stays in existing modules: | Module | Typed handoff and responsibilities | diff --git a/docs/release_process.md b/docs/release_process.md index a7f9c14..e51ce18 100644 --- a/docs/release_process.md +++ b/docs/release_process.md @@ -2,9 +2,10 @@ ## Current Phase 5 boundary -Phase 5 Step 1 is approved. Runtime remains at `0.1.0.dev3`, with executable -report schema 1.0 and general lineage deferred. The Phase 5 contracts describe -future implementation. Stable v0.1 publication, a tag and main merge require +Phase 5 Step 2 is approved. Runtime remains at `0.1.0.dev3`, with executable +report schema 1.0. Parent-graph construction is the current implementation scope; +cycle, depth, ancestry and report integration remain subsequent steps. +Stable v0.1 publication, a tag and main merge require separate authorization. Use one current verification path: @@ -18,8 +19,10 @@ python scripts/release_check.py Historical `--phase` and `--step` dispatch is retired. Earlier source forms, migrations and receipts remain recoverable from Git and archived phase records. The accepted Phase 4 commit supplies the existing historical authority; no new -Phase 5 hash registry is needed. During Step 1 the source check rejects changes -to runtime, schemas, package metadata and canonical Hero files. Sixteen frozen +Phase 5 hash registry is needed. During Step 2 the existing source check permits +the approved models, errors, input-validation and lineage-graph modules, retaining +the exact product file inventory and rejecting unrelated runtime, schema, +package-metadata and canonical Hero changes. Sixteen frozen specifications and seven packaged resource copies remain protected. ## Risk-sensitive gates @@ -43,7 +46,7 @@ Dispatch `ci.yml` with `gate=candidate` for a stable candidate. The workflow run Compatibility cells run complete canonical regression with `--ignore=tests/performance`. The designated performance job runs `tests/performance` once, avoiding nine repetitions of the costly 100k report -workload. The Phase 5 sparse-lineage test is still a placeholder in Step 1; +workload. The Phase 5 sparse-lineage test is still a placeholder in Step 2; it becomes an executable graph benchmark in Step 9. No placeholder is counted as a successful lineage measurement. diff --git a/scripts/release_check.py b/scripts/release_check.py index dbd3025..bf01fe6 100644 --- a/scripts/release_check.py +++ b/scripts/release_check.py @@ -1,6 +1,6 @@ """Current source and candidate checks for Recursive Integrity Toolkit. -Step 1 preserves accepted product bytes. Historical dispatch, source-body +Step 2 opens the approved parent-graph implementation scope. Historical dispatch, source-body migrations and phase registries are recoverable from the accepted Git commit. Installed checks below retain their existing product, privacy and package cases. """ @@ -26,6 +26,12 @@ ROOT = Path(__file__).resolve().parents[1] ACCEPTED_COMMIT = "1db3b1a460c233242ff37fbe45b8fac74d6101ef" PROTECTED_PREFIXES = ("src/", "schemas/", "examples/hero/") +CURRENT_IMPLEMENTATION_PATHS = frozenset({ + "src/recursive_integrity_toolkit/models.py", + "src/recursive_integrity_toolkit/errors.py", + "src/recursive_integrity_toolkit/io/validation.py", + "src/recursive_integrity_toolkit/lineage/graph.py", +}) PARQUET_CASES = {"test_PR002_parquet_real_roundtrip", "test_PR002_parquet_real_row_limit", "test_PR002_parquet_real_invalid_file"} HERO_NAMES = {"records_v1.csv", "records_v2.csv", "provenance.csv", "config.json", "version_order.json", "EXPECTED_OUTPUTS.md"} RESOURCES = {f"src/recursive_integrity_toolkit/data/hero/{name}": f"examples/hero/{name}" for name in HERO_NAMES} @@ -54,7 +60,7 @@ def _regular_file(root: Path, relative: str) -> Path: def verify_source_scope(root: Path = ROOT) -> dict: - """Step 1 is administrative: reject any product mutation, even outside Git's index. + """Reject product changes outside the currently authorized parent-graph scope. Later authorized implementation updates this single current boundary. It does not append another historical dispatcher or grant authority through metadata. @@ -62,8 +68,9 @@ def verify_source_scope(root: Path = ROOT) -> dict: expected = {name: raw for name, raw in _accepted_files().items() if name.startswith(PROTECTED_PREFIXES) or name == "pyproject.toml"} for name, raw in expected.items(): - if _regular_file(root, name).read_bytes() != raw: - raise ValueError(f"Unauthorized product mutation during Step 1: {name}") + current = _regular_file(root, name).read_bytes() + if name not in CURRENT_IMPLEMENTATION_PATHS and current != raw: + raise ValueError(f"Unauthorized product mutation outside Step 2 scope: {name}") for prefix in PROTECTED_PREFIXES: paths = list((root / prefix).rglob("*")) if (root / prefix).is_symlink() or any(path.is_symlink() for path in paths): @@ -71,8 +78,10 @@ def verify_source_scope(root: Path = ROOT) -> dict: actual = {path.relative_to(root).as_posix() for path in paths if path.is_file() and "__pycache__" not in path.parts} if actual != {name for name in expected if name.startswith(prefix)}: - raise ValueError(f"Unauthorized product file-set mutation during Step 1: {prefix}") - return {"protected_product_files": len(expected), "product_bytes": "unchanged"} + raise ValueError(f"Unauthorized product file-set mutation during Step 2: {prefix}") + return {"protected_product_files": len(expected) - len(CURRENT_IMPLEMENTATION_PATHS), + "authorized_implementation_files": len(CURRENT_IMPLEMENTATION_PATHS), + "product_file_inventory": "unchanged"} def verify_frozen_specifications(root: Path = ROOT) -> dict: diff --git a/src/recursive_integrity_toolkit/errors.py b/src/recursive_integrity_toolkit/errors.py index 125b4b2..6578b30 100644 --- a/src/recursive_integrity_toolkit/errors.py +++ b/src/recursive_integrity_toolkit/errors.py @@ -24,6 +24,10 @@ from __future__ import annotations from enum import StrEnum +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .lineage.graph import LineageResourceUsage class ErrorCode(StrEnum): @@ -42,6 +46,7 @@ class ErrorCode(StrEnum): PARENT_AMBIGUOUS = "E_PARENT_AMBIGUOUS" PARENT_FUTURE_VERSION = "E_PARENT_FUTURE_VERSION" LINEAGE_CYCLE = "E_LINEAGE_CYCLE" + LINEAGE_RESOURCE_LIMIT_EXCEEDED = "E_LINEAGE_RESOURCE_LIMIT_EXCEEDED" VERSION_ORDER_CONFLICT = "E_VERSION_ORDER_CONFLICT" REPRESENTATION_INCOMPATIBLE = "E_REPRESENTATION_INCOMPATIBLE" MAPPING_SOURCE_FIELD_MISSING = "E_MAPPING_SOURCE_FIELD_MISSING" @@ -96,6 +101,21 @@ class SecurityError(ToolkitError): """Local security-boundary violation.""" +class LineageResourceLimitError(ToolkitError): + """A bounded lineage stage stopped before admitting its next work unit. + + ``resource_usage`` is the immutable ``LineageResourceUsage`` supplied by the + computation owner. It contains counts and limits only, with no input data. + """ + + reason_code = "LINEAGE_RESOURCE_LIMIT_EXCEEDED" + + def __init__(self, resource_usage: LineageResourceUsage) -> None: + self.resource_usage = resource_usage + super().__init__(ErrorCode.LINEAGE_RESOURCE_LIMIT_EXCEEDED, + "lineage computation exceeded an explicit resource limit") + + class IngestionError(InputError): """Physical input failure with a safe location and no source-value echo.""" diff --git a/src/recursive_integrity_toolkit/io/validation.py b/src/recursive_integrity_toolkit/io/validation.py index 9c39f58..a149213 100644 --- a/src/recursive_integrity_toolkit/io/validation.py +++ b/src/recursive_integrity_toolkit/io/validation.py @@ -43,6 +43,7 @@ ProvenanceAssessment, ProvenanceMatch, ProvenanceJoinResult, ValidationCoverage, ValidationMessage, ValidationSeverity, ParentResolutionStatus, VersionOrderResult, ParentReference, ParentValidationResult, + ParentRecordValidation, ParentBatchValidationResult, GenerationAssessment, GenerationValidationResult, AuditBundle, BundleValidationResult, NormalizationOptions, ScenarioParameters, ) @@ -601,33 +602,53 @@ def _parent_lookup(keys: tuple[RecordKey, ...], order: VersionOrderResult return set(keys), by_id, {version: i for i, version in enumerate(order.order)} +def _interpret_parent_reference(text: str, child: RecordKey, + lookup: tuple[set[RecordKey], dict[str, list[RecordKey]], dict[str, int]], + location: RowLocation) -> tuple[RecordKey | None, bool, bool, str]: + """Single interpretation shared by fail-fast and retained-batch adapters.""" + loaded, by_id, positions = lookup + parsed = _reference_key(text, location, child) + bare = parsed is None + if bare: + candidates = by_id.get(text, []) + if len(candidates) > 1: + raise _fail(ErrorCode.PARENT_AMBIGUOUS, "bare parent matches multiple loaded records", "parent_ids", location, child) + parsed = candidates[0] if candidates else None + resolved = parsed in loaded if parsed is not None else False + if parsed == child: + raise _fail(ErrorCode.LINEAGE_CYCLE, "direct self-parent reference is invalid input", "parent_ids", location, child) + temporal = "unavailable" + if parsed is not None: + if parsed.dataset_version == child.dataset_version: + temporal = "same_version" + elif parsed.dataset_version in positions and child.dataset_version in positions: + if positions[parsed.dataset_version] > positions[child.dataset_version]: + raise _fail(ErrorCode.PARENT_FUTURE_VERSION, "parent refers to a later declared version", "parent_ids", location, child) + temporal = "earlier_version" + return parsed, bare, resolved, temporal + + def _resolve_parent_list(child: RecordKey, values: tuple[str, ...] | None, lookup: tuple[set[RecordKey], dict[str, list[RecordKey]], dict[str, int]], - location: RowLocation, promoted: tuple[str, ...]) -> ParentValidationResult: - loaded, by_id, positions = lookup + location: RowLocation, promoted: tuple[str, ...], *, + retain_errors: bool = False) -> ParentValidationResult: messages: list[ValidationMessage] = [] grouped: dict[tuple[str, str], tuple[RecordKey | None, list[str], bool, str]] = {} graph_deferred = False - for text in sorted(values or ()): - parsed = _reference_key(text, location, child) - bare = parsed is None - if bare: - candidates = by_id.get(text, []) - if len(candidates) > 1: - raise _fail(ErrorCode.PARENT_AMBIGUOUS, "bare parent matches multiple loaded records", "parent_ids", location, child) - parsed = candidates[0] if candidates else None - resolved = parsed in loaded if parsed is not None else False - if parsed == child: - raise _fail(ErrorCode.LINEAGE_CYCLE, "direct self-parent reference is invalid input", "parent_ids", location, child) - temporal = "unavailable" - if parsed is not None: - if parsed.dataset_version == child.dataset_version: - temporal = "same_version" - graph_deferred = True - elif parsed.dataset_version in positions and child.dataset_version in positions: - if positions[parsed.dataset_version] > positions[child.dataset_version]: - raise _fail(ErrorCode.PARENT_FUTURE_VERSION, "parent refers to a later declared version", "parent_ids", location, child) - temporal = "earlier_version" + # Invalid native entries are retained by count with static diagnostics; + # legacy callers still pass a parsed all-string tuple and fail immediately. + for text in sorted(values or (), key=lambda item: item if type(item) is str else ""): + try: + if type(text) is not str: + raise _fail(ErrorCode.PARENT_FORMAT, "parent reference must be a string", "parent_ids", location, child) + parsed, bare, resolved, temporal = _interpret_parent_reference(text, child, lookup, location) + except CanonicalValidationError as error: + if not retain_errors: + raise + messages.append(_join_message(error.code.value, ValidationSeverity.ERROR, + error.safe_message, child, location, "parent_ids")) + continue + graph_deferred = graph_deferred or temporal == "same_version" token = ("key", str(parsed)) if parsed is not None else ("bare", text) if token in grouped: grouped[token][1].append(text) @@ -672,6 +693,99 @@ def resolve_parent_references( return _resolve_parent_list(child_key, parents, _parent_lookup(keys, order), loc, promoted) +def resolve_parent_batch( + loaded_keys: tuple[RecordKey, ...], + provenance: tuple[CanonicalRow | ProvenanceAssessment, ...] | None, *, + version_order: VersionOrderResult | None = None, strict_mode: bool = False, + strict_warning_codes: tuple[str, ...] = (), +) -> ParentBatchValidationResult: + """Retain every record's immediate parent evidence with one lookup index. + + Invalid references are diagnosed individually, preserving valid siblings. + This function performs no graph traversal, cycle detection, or ancestry + classification. Missing chronology does not erase a resolved identity. + """ + from dataclasses import replace + + keys = () if type(loaded_keys) is tuple and not loaded_keys else _parent_keys(loaded_keys) + promoted = _join_options(None, strict_mode, strict_warning_codes) + if keys: + order = _checked_order(version_order, tuple(sorted({key.dataset_version for key in keys}))) + else: + order = VersionOrderResult((), (), "empty_scope", MappingProxyType({}), MappingProxyType({})) + if type(version_order) is VersionOrderResult and (version_order.declarations or + version_order.invocation_order is not None): + declared = _version_labels(version_order.order, field="version_order") + checked = resolve_version_order(declared, document=version_order.declarations or None, + invocation_order=version_order.invocation_order) + order = replace(checked, loaded_versions=()) + if version_order is not None and (type(version_order) is not VersionOrderResult or version_order != order): + raise _version_failure("parent batch ordering evidence disagrees with its declarations") + if provenance is not None and type(provenance) is not tuple: + raise CanonicalValidationError(ErrorCode.SCHEMA_TYPE, "parent batch requires explicit provenance rows") + lookup = _parent_lookup(keys, order) + rows: dict[RecordKey, tuple[ProvenanceAssessment, object, str]] = {} + for row in provenance or (): + if type(row) not in (CanonicalRow, ProvenanceAssessment): + raise CanonicalValidationError(ErrorCode.SCHEMA_TYPE, "parent batch requires canonical or assessed provenance") + loc = _join_location(row.location) + values = _join_fields(row.values, loc) + # Parent failures belong to the retained per-record result. Identity and + # other provenance fields keep their existing fail-closed validation. + raw_parents = values.pop("parent_ids", None) + assessed = assess_provenance_row(replace(row, values=MappingProxyType(values))) + key = assessed.record_key + if key in rows: + raise _fail(ErrorCode.PROVENANCE_DUPLICATE_ROW, "duplicate parent batch provenance identity", "record_id", loc, key) + if key not in lookup[0]: + raise _fail(ErrorCode.PROVENANCE_UNMATCHED_ROW, "parent batch provenance has no loaded record", "record_id", loc, key) + absent = "parent_ids" not in row.values + if type(row) is CanonicalRow: + states = _join_fields(row.field_states, loc) + state = states.get("parent_ids") + if state == "absent": + if raw_parents is not None: + raise _fail(ErrorCode.SCHEMA_TYPE, "absent parent state conflicts with its value", "parent_ids", loc, key) + absent = True + elif state == "null" and raw_parents is not None: + raise _fail(ErrorCode.SCHEMA_TYPE, "null parent state conflicts with its value", "parent_ids", loc, key) + declaration = "absent" if absent else ("null" if raw_parents is None else + "empty" if type(raw_parents) in (list, tuple) and not raw_parents else "declared") + rows[key] = (assessed, raw_parents, declaration) + assessments: list[ParentRecordValidation] = [] + messages = [replace(message, severity=ValidationSeverity.ERROR if message.code in promoted + else message.severity) for message in order.messages] + for key in keys: + item = rows.get(key) + if item is None: + missing = _join_message(WarningCode.PROVENANCE_MISSING_ROW.value, + ValidationSeverity.ERROR if WarningCode.PROVENANCE_MISSING_ROW.value in promoted + else ValidationSeverity.WARNING, "record has no provenance for parent validation", key, RowLocation()) + result = ParentValidationResult(key, "absent", (), False, (missing,)) + assessment = ParentRecordValidation(key, False, result, 0, 0, 0, (missing,)) + else: + row, raw_parents, declaration = item + notices = tuple(_join_message(ErrorCode.SCHEMA_REQUIRED_FIELD.value, ValidationSeverity.ERROR, + "parent input lacks a required provenance field", key, row.location, name) + for name in row.missing_required_fields) + if raw_parents is not None and type(raw_parents) not in (list, tuple): + malformed = _join_message(ErrorCode.PARENT_FORMAT.value, ValidationSeverity.ERROR, + "parent declaration must be an array of strings", key, row.location, "parent_ids") + assessment = ParentRecordValidation(key, True, None, None, 0, 0, notices + (malformed,)) + else: + parents = None if raw_parents is None else tuple(raw_parents) + result = _resolve_parent_list(key, parents, lookup, row.location, promoted, retain_errors=True) + result = replace(result, declaration_state=declaration) + resolved = sum(len(ref.source_references) for ref in result.references + if ref.resolution_status is ParentResolutionStatus.RESOLVED) + self_count = sum(message.code == ErrorCode.LINEAGE_CYCLE.value for message in result.messages) + assessment = ParentRecordValidation(key, True, result, len(parents or ()), + resolved, self_count, notices + result.messages) + assessments.append(assessment) + messages.extend(assessment.messages) + return ParentBatchValidationResult(keys, tuple(assessments), order, tuple(messages), promoted) + + def validate_generation_declarations( loaded_keys: tuple[RecordKey, ...], provenance: tuple[CanonicalRow | ProvenanceAssessment, ...], *, version_order: VersionOrderResult | None = None, strict_mode: bool = False, @@ -1088,6 +1202,9 @@ def validate_bundle(bundle: AuditBundle, *, configuration: dict[str, object] | N order = _bundle_order(versions, order_document, config.version_order, invocation_order) messages.extend(joined.messages) messages.extend(order.messages) + parent_validation = resolve_parent_batch(tuple(row.record_key for row in canonical_records), + canonical_provenance, version_order=order, + strict_mode=config.strict_mode, strict_warning_codes=config.strict_warning_codes) generation = None try: generation = validate_generation_declarations(tuple(row.record_key for row in canonical_records), @@ -1100,6 +1217,7 @@ def validate_bundle(bundle: AuditBundle, *, configuration: dict[str, object] | N raise key = RecordKey.parse(error.record_key) if error.record_key is not None else None messages.append(_bundle_error_message(error, RowLocation(FileRole.PROVENANCE_MANIFEST), key)) + messages.extend(parent_validation.messages) content = {} if options.content_mode is ContentMode.LOCAL_REF else None if resolve_local_content: for row in canonical_records: @@ -1125,7 +1243,7 @@ def validate_bundle(bundle: AuditBundle, *, configuration: dict[str, object] | N classification = replace(classification, validation_messages=final_messages) return BundleValidationResult(tuple(sorted(inventory, key=_bundle_inventory_key)), canonical_records, canonical_provenance, joined, order, generation, classification, tuple(traces), - tuple(sorted(content or {})), final_messages) + tuple(sorted(content or {})), final_messages, parent_validation) def _bundle_row_key(row): diff --git a/src/recursive_integrity_toolkit/lineage/graph.py b/src/recursive_integrity_toolkit/lineage/graph.py index 4063639..d595875 100644 --- a/src/recursive_integrity_toolkit/lineage/graph.py +++ b/src/recursive_integrity_toolkit/lineage/graph.py @@ -1,20 +1,382 @@ -"""Own future record-ancestry graph construction and parent-edge resolution. +"""Build a bounded immutable parent graph from retained input validation. -Owner IDs: - PR-008 +Owner IDs: PR-008; Phase 5 Step 2. The shared input resolver owns reference +identity and chronology. This module selects an explicit target population and +admits accepted edges. No cycle/ancestry traversal, metrics, file or network I/O. +Raw record content, provenance URI fields and locations never enter graph results. +Reference spellings remain identity evidence, including duplicate aliases. -Future inputs: - Future canonical record keys and parent-reference rows. +Current phase status: + Phase 5 Step 2 parent graph construction. General cycle and ancestry + analysis remain deferred to the following implementation steps. +""" +from __future__ import annotations -Future outputs: - Future node, edge, resolution, and graph-scope records. +from dataclasses import dataclass +from types import MappingProxyType +from typing import Mapping -Assumptions: - Parent identity uses explicit composite references. +from ..errors import CanonicalValidationError, ErrorCode, LineageResourceLimitError, WarningCode +from ..models import ( + BundleValidationResult, CanonicalRow, FileRole, ParentBatchValidationResult, + ParentRecordValidation, ParentReference, ParentResolutionStatus, + ParentValidationResult, RecordKey, RowLocation, ValidationMessage, ValidationSeverity, +) -Limits: - No adjacency structure, edge resolution, graph construction, or traversal is implemented here. -Current phase status: - Phase 1 package scaffold only. Import-safe. No analytical or data-processing logic. -""" +def _invalid(message: str, code: ErrorCode = ErrorCode.SCHEMA_TYPE) -> CanonicalValidationError: + return CanonicalValidationError(code, message) + + +def _integer(value: object, *, positive: bool = False) -> None: + if type(value) is not int or value < (1 if positive else 0): + raise _invalid("lineage counts and limits require exact integers in range") + + +def _key(value: object) -> RecordKey: + if type(value) is not RecordKey: + raise _invalid("lineage identities require canonical record keys") + try: + return RecordKey(value.dataset_version, value.record_id) + except (TypeError, ValueError): + raise _invalid("lineage identity violates the canonical contract") from None + + +def _keys(value: object) -> tuple[RecordKey, ...]: + if type(value) is not tuple: + raise _invalid("lineage identity collections require immutable tuples") + checked = tuple(_key(item) for item in value) + if checked != tuple(sorted(set(checked))): + raise _invalid("lineage identities must be unique and canonically ordered") + return checked + + +def _version(value: object) -> str: + try: + return RecordKey(value, "lineage-scope").dataset_version + except (TypeError, ValueError): + raise _invalid("lineage target requires a valid explicit version", ErrorCode.CONFIG_INVALID) from None + + +@dataclass(frozen=True, slots=True) +class LineageLimits: + """Positive work limits, independent of loader limits and performance goals.""" + max_nodes: int = 200000 + max_edges: int = 1000000 + max_root_memberships: int = 1000000 + max_root_union_visits: int = 10000000 + + def __post_init__(self) -> None: + for value in (self.max_nodes, self.max_edges, self.max_root_memberships, self.max_root_union_visits): + _integer(value, positive=True) + + +def _limits(value: object) -> LineageLimits: + if type(value) is not LineageLimits: + raise _invalid("lineage limits require LineageLimits", ErrorCode.CONFIG_INVALID) + return LineageLimits(value.max_nodes, value.max_edges, value.max_root_memberships, value.max_root_union_visits) + + +@dataclass(frozen=True, slots=True) +class LineageResourceUsage: + """Exact admitted work and the next rejected work unit, when exhausted.""" + admitted_node_count: int + admitted_edge_count: int + stored_root_membership_count: int + root_union_visit_count: int + limits: LineageLimits + exhausted_limit: str | None = None + attempted_value: int | None = None + + def __post_init__(self) -> None: + bounded = _limits(self.limits) + pairs = ( + ("max_nodes", self.admitted_node_count, bounded.max_nodes), + ("max_edges", self.admitted_edge_count, bounded.max_edges), + ("max_root_memberships", self.stored_root_membership_count, bounded.max_root_memberships), + ("max_root_union_visits", self.root_union_visit_count, bounded.max_root_union_visits), + ) + for _, count, maximum in pairs: + _integer(count) + if count > maximum: + raise _invalid("lineage usage exceeds its admitted limit") + if self.exhausted_limit is None: + if self.attempted_value is not None: + raise _invalid("completed lineage work cannot have a rejected attempt") + else: + limits_by_name = {name: (count, maximum) for name, count, maximum in pairs} + if type(self.exhausted_limit) is not str or self.exhausted_limit not in limits_by_name: + raise _invalid("lineage exhaustion names an unsupported limit") + _integer(self.attempted_value, positive=True) + count, maximum = limits_by_name[self.exhausted_limit] + if count != maximum or self.attempted_value != maximum + 1: + raise _invalid("lineage exhaustion must identify its next rejected admission") + object.__setattr__(self, "limits", bounded) + + +@dataclass(frozen=True, slots=True) +class LineageScope: + """Target records determine population size; all other nodes are context.""" + target_dataset_version: str | None + target_record_keys: tuple[RecordKey, ...] + target_record_count: int + loaded_record_count: int + context_record_count: int + loaded_dataset_versions: tuple[str, ...] + + def __post_init__(self) -> None: + keys = _keys(self.target_record_keys) + for value in (self.target_record_count, self.loaded_record_count, self.context_record_count): + _integer(value) + if self.target_record_count != len(keys) or self.loaded_record_count != len(keys) + self.context_record_count: + raise _invalid("lineage scope counts do not match their explicit population") + if self.target_dataset_version is None: + if keys: + raise _invalid("a populated lineage target requires an explicit version", ErrorCode.CONFIG_INVALID) + else: + _version(self.target_dataset_version) + if any(key.dataset_version != self.target_dataset_version for key in keys): + raise _invalid("lineage target keys do not match their selected version") + if type(self.loaded_dataset_versions) is not tuple: + raise _invalid("lineage version inventory requires an immutable tuple") + versions = tuple(_version(value) for value in self.loaded_dataset_versions) + if versions != tuple(sorted(set(versions))): + raise _invalid("lineage version inventory must be unique and lexical") + if bool(versions) != bool(self.loaded_record_count) or set(key.dataset_version for key in keys) - set(versions): + raise _invalid("lineage version inventory disagrees with its scope") + + +_DIAGNOSTIC_CODES = frozenset(code.value for code in ErrorCode) | frozenset(code.value for code in WarningCode) + + +def _safe_message(value: object) -> ValidationMessage: + if (type(value) is not ValidationMessage or type(value.code) is not str + or value.code not in _DIAGNOSTIC_CODES or type(value.severity) is not ValidationSeverity): + raise _invalid("lineage diagnostics require known typed validation codes") + key = None if value.record_key is None else _key(value.record_key) + return ValidationMessage(value.code, value.severity, + "Validation evidence reported this diagnostic.", record_key=key) + + +def _messages(values: object) -> tuple[ValidationMessage, ...]: + if type(values) is not tuple: + raise _invalid("lineage diagnostics require an immutable tuple") + messages = {_safe_message(value) for value in values} + return tuple(sorted(messages, key=lambda item: ( + "" if item.record_key is None else item.record_key.dataset_version, + "" if item.record_key is None else item.record_key.record_id, item.code, item.severity.value))) + + +def _safe_evidence(value: object, child: RecordKey) -> ParentRecordValidation: + if type(value) is not ParentRecordValidation or _key(value.child_key) != child: + raise _invalid("lineage parent evidence must match its record") + if type(value.provenance_available) is not bool: + raise _invalid("lineage provenance availability requires an explicit boolean") + if value.declared_reference_count is not None: + _integer(value.declared_reference_count) + _integer(value.resolved_reference_count) + _integer(value.invalid_self_reference_count) + if (value.declared_reference_count is not None + and value.resolved_reference_count + value.invalid_self_reference_count > value.declared_reference_count): + raise _invalid("lineage reference counts exceed their declarations") + result = value.result + if result is not None: + if (type(result) is not ParentValidationResult or _key(result.child_key) != child + or result.declaration_state not in ("absent", "null", "empty", "declared") + or type(result.references) is not tuple or type(result.graph_validation_deferred) is not bool): + raise _invalid("lineage requires typed parent declaration evidence") + refs = [] + for ref in result.references: + if (type(ref) is not ParentReference or type(ref.source_references) is not tuple + or not ref.source_references or any(type(text) is not str for text in ref.source_references) + or type(ref.resolution_status) is not ParentResolutionStatus + or ref.temporal_status not in ("same_version", "earlier_version", "unavailable")): + raise _invalid("lineage reference evidence is invalid") + parent = None if ref.parent_key is None else _key(ref.parent_key) + if ref.canonical_reference != (None if parent is None else str(parent)): + raise _invalid("lineage reference identity disagrees with its canonical spelling") + for text in ref.source_references: + try: + RecordKey.parse(text) if "::" in text else RecordKey(child.dataset_version, text) + except (TypeError, ValueError): + raise _invalid("lineage evidence contains an invalid reference spelling") from None + refs.append(ParentReference(tuple(ref.source_references), ref.canonical_reference, + parent, ref.resolution_status, ref.temporal_status)) + resolved = sum(len(ref.source_references) for ref in refs + if ref.resolution_status is ParentResolutionStatus.RESOLVED) + represented = sum(len(ref.source_references) for ref in refs) + value.invalid_self_reference_count + if (resolved != value.resolved_reference_count or value.declared_reference_count is None + or represented > value.declared_reference_count + or (result.declaration_state != "declared" and value.declared_reference_count != 0)): + raise _invalid("lineage reference evidence disagrees with its declared counts") + result = ParentValidationResult(child, result.declaration_state, tuple(refs), + result.graph_validation_deferred, _messages(result.messages)) + elif value.declared_reference_count is not None or value.resolved_reference_count or value.invalid_self_reference_count: + raise _invalid("missing parent result requires unknown declaration cardinality") + return ParentRecordValidation(child, value.provenance_available, result, + value.declared_reference_count, value.resolved_reference_count, + value.invalid_self_reference_count, _messages(value.messages)) + + +def _accepted_parents(evidence: ParentRecordValidation) -> tuple[RecordKey, ...]: + if evidence.result is None: + return () + return tuple(sorted({ref.parent_key for ref in evidence.result.references + if ref.resolution_status is ParentResolutionStatus.RESOLVED + and ref.temporal_status in ("same_version", "earlier_version") + and ref.parent_key is not None and ref.parent_key != evidence.child_key})) + + +@dataclass(frozen=True, slots=True) +class LineageGraph: + """Accepted edges and rejected-reference evidence, without an acyclic claim. + + Both adjacency mappings include isolated nodes. All nested containers are + detached immutable values; raw source locations are removed from diagnostics. + """ + scope: LineageScope + node_keys: tuple[RecordKey, ...] + parents_by_child: Mapping[RecordKey, tuple[RecordKey, ...]] + children_by_parent: Mapping[RecordKey, tuple[RecordKey, ...]] + parent_evidence_by_record: Mapping[RecordKey, ParentRecordValidation] + self_parent_record_keys: tuple[RecordKey, ...] + messages: tuple[ValidationMessage, ...] + resource_usage: LineageResourceUsage + + def __post_init__(self) -> None: + nodes = _keys(self.node_keys) + if type(self.scope) is not LineageScope: + raise _invalid("lineage graph requires an explicit typed scope") + scope = LineageScope(self.scope.target_dataset_version, self.scope.target_record_keys, + self.scope.target_record_count, self.scope.loaded_record_count, + self.scope.context_record_count, self.scope.loaded_dataset_versions) + node_set = set(nodes) + if (scope.loaded_record_count != len(nodes) + or scope.target_record_keys != tuple(key for key in nodes if key.dataset_version == scope.target_dataset_version) + or scope.loaded_dataset_versions != tuple(sorted({key.dataset_version for key in nodes}))): + raise _invalid("lineage graph nodes disagree with its scope") + mappings = (self.parents_by_child, self.children_by_parent, self.parent_evidence_by_record) + if any(type(value) not in (dict, MappingProxyType) or set(value) != node_set for value in mappings): + raise _invalid("lineage mappings must cover exactly the loaded nodes") + evidence = {key: _safe_evidence(self.parent_evidence_by_record[key], key) for key in nodes} + parents, children = {}, {key: [] for key in nodes} + for child in nodes: + adjacent = _keys(self.parents_by_child[child]) + if set(adjacent) - node_set or adjacent != _accepted_parents(evidence[child]): + raise _invalid("lineage adjacency disagrees with accepted reference evidence") + parents[child] = adjacent + for parent in adjacent: + children[parent].append(child) + for parent in nodes: + if _keys(self.children_by_parent[parent]) != tuple(children[parent]): + raise _invalid("lineage forward and reverse adjacency disagree") + self_keys = _keys(self.self_parent_record_keys) + if self_keys != tuple(key for key in nodes if evidence[key].invalid_self_reference_count): + raise _invalid("lineage self-reference diagnostics disagree with retained evidence") + usage = self.resource_usage + if type(usage) is not LineageResourceUsage: + raise _invalid("lineage graph requires typed resource usage") + usage = LineageResourceUsage(usage.admitted_node_count, usage.admitted_edge_count, + usage.stored_root_membership_count, usage.root_union_visit_count, + usage.limits, usage.exhausted_limit, usage.attempted_value) + if (usage.admitted_node_count != len(nodes) or usage.admitted_edge_count != sum(map(len, parents.values())) + or usage.stored_root_membership_count or usage.root_union_visit_count or usage.exhausted_limit is not None): + raise _invalid("completed parent graph has inconsistent resource usage") + object.__setattr__(self, "scope", scope) + object.__setattr__(self, "node_keys", nodes) + object.__setattr__(self, "parents_by_child", MappingProxyType(parents)) + object.__setattr__(self, "children_by_parent", MappingProxyType({key: tuple(values) for key, values in children.items()})) + object.__setattr__(self, "parent_evidence_by_record", MappingProxyType(evidence)) + object.__setattr__(self, "messages", _messages(self.messages)) + object.__setattr__(self, "resource_usage", usage) + + +def build_lineage_graph( + validation: BundleValidationResult, *, target_dataset_version: str | None, + limits: LineageLimits = LineageLimits(), +) -> LineageGraph: + """Revalidate the shared batch once, then admit bounded deduplicated edges. + + Older handoffs without retained ``parent_validation`` must be rebuilt through + ``validate_bundle``. Rechecking uses one identity lookup for the entire batch, + avoiding a per-record rebuild. Reference count evidence includes original + aliases; accepted graph edges additionally require known chronology. + """ + from ..io.validation import resolve_parent_batch + + bounded = _limits(limits) + if type(validation) is not BundleValidationResult or type(validation.records) is not tuple: + raise _invalid("lineage graph requires a typed bundle validation result") + if len(validation.records) > bounded.max_nodes: + raise LineageResourceLimitError(LineageResourceUsage(bounded.max_nodes, 0, 0, 0, + bounded, "max_nodes", bounded.max_nodes + 1)) + keys, primary_versions, roles_by_version = [], set(), {} + for row in validation.records: + if type(row) is not CanonicalRow or row.kind != "records": + raise _invalid("lineage graph requires canonical record rows") + key = _key(row.record_key) + if (type(row.values) not in (dict, MappingProxyType) + or row.values.get("dataset_version") != key.dataset_version + or row.values.get("record_id") != key.record_id): + raise _invalid("lineage record identity disagrees with its retained values") + if type(row.location) is not RowLocation or (row.location.file_role is not None and + (type(row.location.file_role) is not FileRole or row.location.file_role not in + (FileRole.RECORDS_PRIMARY, FileRole.RECORDS_COMPARE))): + raise _invalid("lineage records have an unsupported input role") + role = row.location.file_role + roles_by_version.setdefault(key.dataset_version, set()).add(role) + if role is FileRole.RECORDS_PRIMARY: + primary_versions.add(key.dataset_version) + keys.append(key) + nodes = tuple(sorted(keys)) + if len(set(nodes)) != len(nodes): + raise _invalid("duplicate canonical record identity in lineage input", ErrorCode.RECORD_DUPLICATE_ID) + if len(primary_versions) > 1 or any(FileRole.RECORDS_PRIMARY in roles and FileRole.RECORDS_COMPARE in roles + for roles in roles_by_version.values()): + raise _invalid("lineage primary and context version roles must be disjoint", ErrorCode.CONFIG_INVALID) + if target_dataset_version is None: + if primary_versions or (nodes and all(roles == {None} for roles in roles_by_version.values())): + raise _invalid("loaded primary records require an explicit target version", ErrorCode.CONFIG_INVALID) + else: + _version(target_dataset_version) + if primary_versions and target_dataset_version not in primary_versions: + raise _invalid("lineage target must select the primary version", ErrorCode.CONFIG_INVALID) + if not primary_versions and target_dataset_version in roles_by_version and FileRole.RECORDS_COMPARE in roles_by_version[target_dataset_version]: + raise _invalid("lineage context records cannot define the primary target", ErrorCode.CONFIG_INVALID) + retained = validation.parent_validation + if type(retained) is not ParentBatchValidationResult: + raise _invalid("lineage needs retained parent validation; call validate_bundle again") + promoted = retained.promoted_warning_codes + if type(promoted) is not tuple: + raise _invalid("lineage warning promotion evidence is invalid") + rebuilt = resolve_parent_batch(nodes, validation.provenance, version_order=validation.version_order, + strict_mode=bool(promoted), strict_warning_codes=promoted) + if retained != rebuilt: + raise _invalid("retained parent validation disagrees with its original declarations") + versions = tuple(sorted({key.dataset_version for key in nodes})) + if target_dataset_version is not None and target_dataset_version not in versions and target_dataset_version not in rebuilt.version_order.order: + raise _invalid("lineage target version is outside the declared input scope", ErrorCode.CONFIG_INVALID) + targets = tuple(key for key in nodes if key.dataset_version == target_dataset_version) + scope = LineageScope(target_dataset_version, targets, len(targets), len(nodes), len(nodes) - len(targets), versions) + evidence = {item.child_key: item for item in rebuilt.assessments} + if tuple(evidence) != nodes: + raise _invalid("retained parent validation does not cover the loaded scope") + parents, children = {}, {key: [] for key in nodes} + edges = 0 + for child in nodes: + adjacent = _accepted_parents(evidence[child]) + accepted = [] + for parent in adjacent: + if edges == bounded.max_edges: + raise LineageResourceLimitError(LineageResourceUsage(len(nodes), edges, 0, 0, + bounded, "max_edges", edges + 1)) + accepted.append(parent) + children[parent].append(child) + edges += 1 + parents[child] = tuple(accepted) + usage = LineageResourceUsage(len(nodes), edges, 0, 0, bounded) + if type(validation.validation_messages) is not tuple: + raise _invalid("lineage validation diagnostics require an immutable tuple") + return LineageGraph(scope, nodes, parents, {key: tuple(value) for key, value in children.items()}, + evidence, tuple(key for key in nodes if evidence[key].invalid_self_reference_count), + validation.validation_messages + rebuilt.messages, usage) diff --git a/src/recursive_integrity_toolkit/models.py b/src/recursive_integrity_toolkit/models.py index 53d58d0..9798c18 100644 --- a/src/recursive_integrity_toolkit/models.py +++ b/src/recursive_integrity_toolkit/models.py @@ -473,6 +473,86 @@ class ParentValidationResult: messages: tuple[ValidationMessage, ...] = () +@dataclass(frozen=True, slots=True) +class ParentRecordValidation: + """Retained immediate evidence for one loaded record, including failures. + + Missing and null declarations have zero declared references, while an + unparseable container has unknown cardinality. Identity resolution counts + remain distinct from chronological eligibility for ancestry edges. + """ + + child_key: RecordKey + provenance_available: bool + result: ParentValidationResult | None = field(repr=False) + declared_reference_count: int | None + resolved_reference_count: int + invalid_self_reference_count: int + messages: tuple[ValidationMessage, ...] = () + + def __post_init__(self) -> None: + if type(self.child_key) is not RecordKey or type(self.provenance_available) is not bool: + raise TypeError("parent evidence requires a canonical key and explicit provenance flag") + if self.result is not None and (type(self.result) is not ParentValidationResult + or self.result.child_key != self.child_key): + raise ValueError("parent result must belong to its loaded record") + if self.result is not None: + if (self.result.declaration_state not in ("absent", "null", "empty", "declared") + or type(self.result.graph_validation_deferred) is not bool): + raise ValueError("parent evidence requires an explicit declaration state") + if type(self.result.references) is not tuple: + raise TypeError("parent references require an immutable tuple") + for reference in self.result.references: + if (type(reference) is not ParentReference + or type(reference.source_references) is not tuple + or any(type(text) is not str for text in reference.source_references)): + raise TypeError("parent source references require immutable literal spellings") + if type(self.result.messages) is not tuple or any(type(item) is not ValidationMessage + for item in self.result.messages): + raise TypeError("parent result messages require an immutable tuple") + counts = (self.resolved_reference_count, self.invalid_self_reference_count) + if self.declared_reference_count is not None: + counts += (self.declared_reference_count,) + for value in counts: + if type(value) is not int or value < 0: + raise ValueError("parent reference counts require nonnegative integers") + if self.declared_reference_count is not None and ( + self.resolved_reference_count + self.invalid_self_reference_count + > self.declared_reference_count): + raise ValueError("parent evidence counts exceed the declaration count") + if type(self.messages) is not tuple or any(type(item) is not ValidationMessage for item in self.messages): + raise TypeError("parent messages require an immutable tuple") + + +@dataclass(frozen=True, slots=True) +class ParentBatchValidationResult: + """Input-only immediate evidence retained for every loaded canonical key.""" + + record_keys: tuple[RecordKey, ...] + assessments: tuple[ParentRecordValidation, ...] = field(repr=False) + version_order: VersionOrderResult + messages: tuple[ValidationMessage, ...] + promoted_warning_codes: tuple[str, ...] = () + + def __post_init__(self) -> None: + if type(self.record_keys) is not tuple or any(type(key) is not RecordKey for key in self.record_keys): + raise TypeError("parent batch keys require an immutable canonical tuple") + if tuple(sorted(set(self.record_keys))) != self.record_keys: + raise ValueError("parent batch keys must be unique and sorted") + if type(self.assessments) is not tuple or any(type(item) is not ParentRecordValidation + for item in self.assessments): + raise TypeError("parent batch assessments require an immutable tuple") + if tuple(item.child_key for item in self.assessments) != self.record_keys: + raise ValueError("parent batch assessments must cover every loaded key exactly once") + if type(self.version_order) is not VersionOrderResult: + raise TypeError("parent batch chronology requires typed evidence") + if type(self.messages) is not tuple or any(type(item) is not ValidationMessage for item in self.messages): + raise TypeError("parent batch messages require an immutable tuple") + if type(self.promoted_warning_codes) is not tuple or any(type(code) is not str + for code in self.promoted_warning_codes): + raise TypeError("parent warning policy requires an immutable tuple") + + @dataclass(frozen=True, slots=True) class GenerationAssessment: """PR-009 declaration versus safely established non-grounding step count.""" @@ -555,6 +635,7 @@ class BundleValidationResult: mapping_traces: tuple[RowMappingEvidence, ...] = field(repr=False) content_read_keys: tuple[RecordKey, ...] validation_messages: tuple[ValidationMessage, ...] + parent_validation: ParentBatchValidationResult | None = field(default=None, repr=False) @property def has_errors(self) -> bool: diff --git a/tests/integration/test_current_verification.py b/tests/integration/test_current_verification.py index fd70e4a..0bcc4bb 100644 --- a/tests/integration/test_current_verification.py +++ b/tests/integration/test_current_verification.py @@ -26,8 +26,8 @@ def protected_copy(repo_root, tmp_path): @pytest.mark.parametrize("mutation", ["formula", "new_module", "deleted_module"]) -def test_step1_rejects_unauthorized_product_changes(current_tools, protected_copy, mutation): - """Step 1 grants contracts and fixture work without granting runtime changes.""" +def test_current_scope_rejects_unauthorized_product_changes(current_tools, protected_copy, mutation): + """Parent-graph implementation does not authorize unrelated product changes.""" verify = current_tools["verify_source_scope"] verify(protected_copy) target = protected_copy / "src/recursive_integrity_toolkit/metrics/diversity.py" diff --git a/tests/unit/test_lineage_graph.py b/tests/unit/test_lineage_graph.py new file mode 100644 index 0000000..178982a --- /dev/null +++ b/tests/unit/test_lineage_graph.py @@ -0,0 +1,368 @@ +"""Direct Phase 5 graph behavior, reference boundaries and resource admission. + +These tests execute graph construction only. They make no ancestry, cycle, +concentration or report-completion claim. +""" + +from dataclasses import FrozenInstanceError, replace +import json +from pathlib import Path +import socket + +import pytest + +from recursive_integrity_toolkit.errors import CanonicalValidationError +from recursive_integrity_toolkit.io.validation import resolve_version_order, validate_bundle +from recursive_integrity_toolkit.lineage.graph import ( + LineageLimits, + LineageResourceLimitError, + build_lineage_graph, +) +from recursive_integrity_toolkit.models import ( + AuditBundle, + FileRole, + InputSource, + RecordKey, + ValidationSeverity, +) + + +ABSENT = object() +MISSING_PROVENANCE = object() + + +def _key(value): + return RecordKey.parse(value) + + +def _write_rows(path, rows): + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + + +def _validation(tmp_path, declarations, *, target="v2", chronology=True, + strict=False, reverse=False): + """Use the real loader and validator so fixture evidence is never forged.""" + tmp_path.mkdir(parents=True, exist_ok=True) + records = [] + provenance = [] + for identity, parents in declarations.items(): + key = _key(identity) + identity_fields = {"dataset_version": key.dataset_version, "record_id": key.record_id} + records.append({**identity_fields, "content": "Private fixture content"}) + if parents is MISSING_PROVENANCE: + continue + row = { + **identity_fields, + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + } + if parents is not ABSENT: + row["parent_ids"] = list(reversed(parents)) if reverse and isinstance(parents, list) else parents + provenance.append(row) + if reverse: + records.reverse() + provenance.reverse() + primary = [row for row in records if row["dataset_version"] == target] + context = [row for row in records if row["dataset_version"] != target] + sources = [] + for name, role, rows in ( + ("primary.jsonl", FileRole.RECORDS_PRIMARY, primary), + ("context.jsonl", FileRole.RECORDS_COMPARE, context), + ): + if rows: + path = tmp_path / name + _write_rows(path, rows) + sources.append(InputSource(role, path)) + if provenance: + path = tmp_path / "provenance.jsonl" + _write_rows(path, provenance) + sources.append(InputSource(FileRole.PROVENANCE_MANIFEST, path)) + if chronology: + path = tmp_path / "order.json" + versions = sorted({row["dataset_version"] for row in records} | {target}) + path.write_text(json.dumps({"version_order": versions}), encoding="utf-8") + sources.append(InputSource(FileRole.VERSION_ORDER, path)) + configuration = None + if strict: + configuration = {"strict_mode": True, "strict_warning_codes": ["W_PARENT_UNRESOLVED"]} + return validate_bundle(AuditBundle(tuple(sources)), configuration=configuration) + + +def _edges(graph): + return {(str(parent), str(child)) for child, parents in graph.parents_by_child.items() + for parent in parents} + + +def _evidence(graph, identity): + return graph.parent_evidence_by_record[_key(identity)] + + +def test_exact_graph_keeps_context_out_of_target_population(tmp_path): + validation = _validation(tmp_path, { + "v1::a": [], "v1::b": [], + "v2::c": ["v1::a", "v1::b"], "v2::d": ["v2::c"], + }) + graph = build_lineage_graph(validation, target_dataset_version="v2") + assert tuple(map(str, graph.node_keys)) == ("v1::a", "v1::b", "v2::c", "v2::d") + assert graph.scope.target_record_keys == (_key("v2::c"), _key("v2::d")) + assert (graph.scope.target_record_count, graph.scope.loaded_record_count, + graph.scope.context_record_count) == (2, 4, 2) + assert graph.scope.loaded_dataset_versions == ("v1", "v2") + assert _edges(graph) == {("v1::a", "v2::c"), ("v1::b", "v2::c"), ("v2::c", "v2::d")} + assert set(graph.parents_by_child) == set(graph.node_keys) == set(graph.children_by_parent) + assert {(str(parent), str(child)) for parent, children in graph.children_by_parent.items() + for child in children} == _edges(graph) + assert graph.parents_by_child[_key("v1::a")] == () + assert graph.children_by_parent[_key("v2::d")] == () + assert (graph.resource_usage.admitted_node_count, graph.resource_usage.admitted_edge_count) == (4, 3) + assert graph.resource_usage.stored_root_membership_count == 0 + assert graph.resource_usage.root_union_visit_count == 0 + assert graph.resource_usage.exhausted_limit is None + + +def test_aliases_preserve_original_reference_count_and_one_edge(tmp_path): + validation = _validation(tmp_path, {"v1::a": [], "v2::c": ["a", "v1::a", "a"]}) + graph = build_lineage_graph(validation, target_dataset_version="v2", limits=LineageLimits(max_edges=1)) + evidence = _evidence(graph, "v2::c") + assert _edges(graph) == {("v1::a", "v2::c")} + assert evidence.declared_reference_count == evidence.resolved_reference_count == 3 + assert evidence.result.references[0].source_references == ("a", "a", "v1::a") + assert graph.resource_usage.admitted_edge_count == 1 + + +@pytest.mark.parametrize("reference,error_code", [ + ("v1::missing", "W_PARENT_UNRESOLVED"), + ("p", "E_PARENT_AMBIGUOUS"), + ("v3::future", "E_PARENT_FUTURE_VERSION"), + (" malformed ", "E_PARENT_FORMAT"), + ("v2::c", "E_LINEAGE_CYCLE"), +]) +def test_invalid_sibling_never_erases_valid_reference_or_becomes_edge(tmp_path, reference, error_code): + validation = _validation(tmp_path, { + "v1::p": [], "v1::q": [], "v2::p": [], "v3::future": [], + "v2::c": [reference, "v1::q"], + }) + graph = build_lineage_graph(validation, target_dataset_version="v2") + evidence = _evidence(graph, "v2::c") + assert _edges(graph) == {("v1::q", "v2::c")} + assert evidence.declared_reference_count == 2 + assert evidence.resolved_reference_count == 1 + assert error_code in {message.code for message in graph.messages} + expected_self = (_key("v2::c"),) if reference == "v2::c" else () + assert graph.self_parent_record_keys == expected_self + assert evidence.invalid_self_reference_count == len(expected_self) + assert _key("v1::missing") not in graph.node_keys + + +def test_unknown_cross_version_chronology_cannot_certify_edge(tmp_path): + validation = _validation(tmp_path, {"v10::p": [], "v2::c": ["v10::p"]}, chronology=False) + graph = build_lineage_graph(validation, target_dataset_version="v2") + evidence = _evidence(graph, "v2::c") + assert evidence.result.references[0].temporal_status == "unavailable" + assert "W_VERSION_ORDER_MISSING" in {message.code for message in graph.messages} + assert _edges(graph) == set() + + +def test_absent_null_empty_and_missing_provenance_remain_distinct(tmp_path): + validation = _validation(tmp_path, { + "v2::absent": ABSENT, "v2::null": None, + "v2::empty": [], "v2::missing": MISSING_PROVENANCE, + }) + graph = build_lineage_graph(validation, target_dataset_version="v2") + for identity, state in (("v2::absent", "absent"), ("v2::null", "null"), ("v2::empty", "empty")): + evidence = _evidence(graph, identity) + assert evidence.provenance_available + assert evidence.result.declaration_state == state + assert evidence.declared_reference_count == 0 + missing = _evidence(graph, "v2::missing") + assert not missing.provenance_available + assert missing.result.declaration_state == "absent" + assert len(graph.node_keys) == graph.scope.target_record_count == 4 + assert not _edges(graph) + + +def test_missing_reference_strict_promotion_survives_graph_handoff(tmp_path): + declarations = {"v2::c": ["missing"]} + ordinary = build_lineage_graph(_validation(tmp_path / "ordinary", declarations), target_dataset_version="v2") + strict = build_lineage_graph(_validation(tmp_path / "strict", declarations, strict=True), target_dataset_version="v2") + assert [m.severity for m in ordinary.messages if m.code == "W_PARENT_UNRESOLVED"] == [ValidationSeverity.WARNING] + assert [m.severity for m in strict.messages if m.code == "W_PARENT_UNRESOLVED"] == [ValidationSeverity.ERROR] + assert ordinary.node_keys == strict.node_keys + assert _edges(ordinary) == _edges(strict) == set() + + +def test_cycle_fixture_preserves_graph_and_self_evidence_without_execution_claim(tmp_path): + validation = _validation(tmp_path, { + "v1::self": ["v1::self"], "v1::a": ["v1::b"], "v1::b": ["v1::a"], + "v2::child": ["v1::a"], "v2::safe": [], + }) + graph = build_lineage_graph(validation, target_dataset_version="v2") + assert _edges(graph) == {("v1::a", "v1::b"), ("v1::b", "v1::a"), ("v1::a", "v2::child")} + assert graph.self_parent_record_keys == (_key("v1::self"),) + assert len(graph.node_keys) == 5 + + +def test_graph_is_deterministic_under_rows_and_references_permutation(tmp_path): + declarations = {"v1::b": [], "v1::a": [], "v2::z": ["v1::b", "a", "v1::a"]} + left = build_lineage_graph(_validation(tmp_path / "left", declarations), target_dataset_version="v2") + right = build_lineage_graph(_validation(tmp_path / "right", declarations, reverse=True), target_dataset_version="v2") + assert left == right + + +def test_graph_mappings_are_defensive_immutable_snapshots(tmp_path): + graph = build_lineage_graph(_validation(tmp_path, {"v1::a": [], "v2::c": ["v1::a"]}), target_dataset_version="v2") + parents = dict(graph.parents_by_child) + children = dict(graph.children_by_parent) + evidence = dict(graph.parent_evidence_by_record) + copied = replace(graph, parents_by_child=parents, children_by_parent=children, + parent_evidence_by_record=evidence) + parents.clear() + children.clear() + evidence.clear() + assert copied == graph + for mapping in (graph.parents_by_child, graph.children_by_parent, graph.parent_evidence_by_record): + with pytest.raises(TypeError): + mapping[_key("v2::c")] = () + with pytest.raises(FrozenInstanceError): + graph.scope.target_record_count = 999 + with pytest.raises(FrozenInstanceError): + _evidence(graph, "v2::c").declared_reference_count = 0 + + +def test_graph_construction_does_not_reopen_input_or_use_network(tmp_path, monkeypatch): + validation = _validation(tmp_path, {"v1::a": [], "v2::c": ["v1::a", "missing"]}) + def forbidden(*args, **kwargs): + raise AssertionError("Graph construction must consume its in-memory validated handoff") + monkeypatch.setattr("builtins.open", forbidden) + monkeypatch.setattr(Path, "open", forbidden) + monkeypatch.setattr(socket, "socket", forbidden) + graph = build_lineage_graph(validation, target_dataset_version="v2") + assert _edges(graph) == {("v1::a", "v2::c")} + assert graph.messages + assert all(message.file_path is None and message.row_number is None + and message.line_number is None for message in graph.messages) + assert "Private fixture content" not in repr(graph) + assert str(tmp_path) not in repr(graph) + + +@pytest.mark.parametrize("field,value", [ + ("target_record_count", True), ("loaded_record_count", 1), + ("context_record_count", -1), ("target_record_keys", (_key("v1::a"),)), +]) +def test_forged_scope_cannot_replace_graph_population(tmp_path, field, value): + graph = build_lineage_graph(_validation(tmp_path, {"v1::a": [], "v2::c": ["v1::a"]}), target_dataset_version="v2") + with pytest.raises((CanonicalValidationError, TypeError, ValueError)): + replace(graph, scope=replace(graph.scope, **{field: value})) + + +@pytest.mark.parametrize("selection", [None, "missing", "v1"]) +def test_target_selection_rejects_implicit_or_context_population(tmp_path, selection): + validation = _validation(tmp_path, {"v1::a": [], "v2::c": ["v1::a"]}) + with pytest.raises(CanonicalValidationError): + build_lineage_graph(validation, target_dataset_version=selection) + + +@pytest.mark.parametrize("selection", [None, "v2"]) +def test_empty_selected_scope_preserves_context_and_explicit_version(tmp_path, selection): + validation = _validation(tmp_path, {"v1::a": []}, target="v1") + # The approved empty-selection fixture is an in-memory scope contract. + # Current input CLI still requires a nonempty primary table. Retain the + # validated canonical data, explicitly mark its sole version as context, + # and provide the empty version declaration without inventing a record. + order = resolve_version_order(("v1",), document={"version_order": ["v1", "v2"]}) + validation = replace( + validation, + records=tuple(replace(row, location=replace(row.location, file_role=FileRole.RECORDS_COMPARE)) + for row in validation.records), + version_order=order, + parent_validation=replace(validation.parent_validation, version_order=order), + ) + graph = build_lineage_graph(validation, target_dataset_version=selection) + assert graph.scope.target_dataset_version == selection + assert graph.scope.target_record_keys == () + assert graph.scope.target_record_count == 0 + assert graph.scope.loaded_record_count == graph.scope.context_record_count == 1 + assert graph.node_keys == (_key("v1::a"),) + assert not _edges(graph) + + +@pytest.mark.parametrize("mutation", ["missing_batch", "duplicate_node", "removed_node", "counter", "reference"]) +def test_invalid_typed_handoff_is_rejected(tmp_path, mutation): + validation = _validation(tmp_path, {"v1::a": [], "v2::c": ["v1::a"]}) + if mutation == "missing_batch": + validation = replace(validation, parent_validation=None) + elif mutation == "duplicate_node": + validation = replace(validation, records=validation.records + (validation.records[0],)) + elif mutation == "removed_node": + validation = replace(validation, records=validation.records[1:]) + else: + batch = validation.parent_validation + evidence = next(item for item in batch.assessments if item.child_key == _key("v2::c")) + if mutation == "counter": + forged = replace(evidence, resolved_reference_count=0) + else: + reference = evidence.result.references[0] + forged_reference = replace(reference, parent_key=_key("v2::c"), canonical_reference="v2::c") + forged = replace(evidence, result=replace(evidence.result, references=(forged_reference,))) + validation = replace(validation, parent_validation=replace(batch, assessments=tuple( + forged if item.child_key == evidence.child_key else item for item in batch.assessments))) + with pytest.raises(CanonicalValidationError): + build_lineage_graph(validation, target_dataset_version="v2") + + +@pytest.mark.parametrize("field,value", [ + ("max_nodes", True), ("max_edges", False), ("max_nodes", 0), ("max_edges", -1), + ("max_root_memberships", None), ("max_root_union_visits", 1.5), + ("max_edges", float("inf")), +]) +def test_resource_limit_values_require_positive_exact_integers(field, value): + with pytest.raises((CanonicalValidationError, TypeError, ValueError)): + LineageLimits(**{field: value}) + + +@pytest.mark.parametrize("exhausted,limit,admitted,attempted", [ + ("max_nodes", 2, 2, 3), ("max_edges", 1, 1, 2), +]) +def test_resource_exhaustion_stops_at_next_admission(tmp_path, exhausted, limit, admitted, attempted): + validation = _validation(tmp_path, {"v1::a": [], "v1::b": [], "v2::c": ["v1::a", "v1::b"]}) + limits = LineageLimits(**{exhausted: limit}) + with pytest.raises(LineageResourceLimitError) as error: + build_lineage_graph(validation, target_dataset_version="v2", limits=limits) + usage = error.value.resource_usage + assert usage.exhausted_limit == exhausted + assert usage.attempted_value == attempted + assert (usage.admitted_node_count if exhausted == "max_nodes" else usage.admitted_edge_count) == admitted + assert usage.limits == limits + assert error.value.reason_code == "LINEAGE_RESOURCE_LIMIT_EXCEEDED" + assert error.value.code.value == "E_LINEAGE_RESOURCE_LIMIT_EXCEEDED" + + +def test_resource_limits_allow_exact_boundary_without_truncation(tmp_path): + validation = _validation(tmp_path, {"v1::a": [], "v1::b": [], "v2::c": ["v1::a", "v1::b"]}) + graph = build_lineage_graph(validation, target_dataset_version="v2", limits=LineageLimits(max_nodes=3, max_edges=2)) + assert len(graph.node_keys) == 3 and len(_edges(graph)) == 2 + assert graph.resource_usage.exhausted_limit is None + assert graph.resource_usage.attempted_value is None + + +def test_direct_graph_handoff_cannot_drop_selected_version_records(tmp_path): + graph = build_lineage_graph(_validation(tmp_path, { + "v1::a": [], "v2::c": ["v1::a"], "v2::d": [], + }), target_dataset_version="v2") + # Counts can be internally consistent while silently excluding a target. + forged_scope = replace(graph.scope, target_record_keys=(_key("v2::c"),), + target_record_count=1, context_record_count=2) + with pytest.raises(CanonicalValidationError): + replace(graph, scope=forged_scope) + + +def test_direct_graph_handoff_rejects_counter_disagreement_with_reference_evidence(tmp_path): + graph = build_lineage_graph(_validation(tmp_path, { + "v1::a": [], "v2::c": ["v1::a"], + }), target_dataset_version="v2") + evidence = dict(graph.parent_evidence_by_record) + evidence[_key("v2::c")] = replace(evidence[_key("v2::c")], resolved_reference_count=0) + with pytest.raises(CanonicalValidationError): + replace(graph, parent_evidence_by_record=evidence) diff --git a/tests/unit/test_parent_batch.py b/tests/unit/test_parent_batch.py new file mode 100644 index 0000000..42764db --- /dev/null +++ b/tests/unit/test_parent_batch.py @@ -0,0 +1,188 @@ +"""Direct parent-retention behavior, shared interpretation and boundary tests.""" + +from dataclasses import FrozenInstanceError, replace +from types import MappingProxyType + +import pytest + +from recursive_integrity_toolkit.errors import CanonicalValidationError, ErrorCode, WarningCode +from recursive_integrity_toolkit.io.normalization import normalize_row +from recursive_integrity_toolkit.io import validation +from recursive_integrity_toolkit.models import ( + ParentResolutionStatus, RecordKey, ValidationSeverity, +) + + +_ABSENT = object() + + +def row(identifier, parents=_ABSENT, *, version="v1", grounding="no"): + values = {"dataset_version": version, "record_id": identifier, + "source_type": "unknown", "provenance_confidence": "confirmed", + "external_grounding": grounding} + if parents is not _ABSENT: + values["parent_ids"] = parents + return normalize_row(values, kind="provenance") + + +def batch(rows, **options): + return validation.resolve_parent_batch(tuple(item.record_key for item in rows), rows, **options) + + +def test_parent_batch_preserves_declaration_states_and_missing_provenance(): + rows = (row("absent"), row("null", None), row("empty", []), row("child", ["v1::empty"])) + missing = RecordKey("v1", "missing") + result = validation.resolve_parent_batch(tuple(item.record_key for item in rows) + (missing,), rows) + by_key = {item.child_key.record_id: item for item in result.assessments} + assert {key: item.result.declaration_state for key, item in by_key.items()} == { + "absent": "absent", "null": "null", "empty": "empty", "child": "declared", "missing": "absent"} + assert by_key["missing"].provenance_available is False + assert all(by_key[key].provenance_available for key in ("absent", "null", "empty", "child")) + assert sum(item.declared_reference_count for item in result.assessments) == 1 + assert sum(item.resolved_reference_count for item in result.assessments) == 1 + + +def test_parent_batch_retains_all_failures_valid_siblings_and_alias_counts(): + rows = (row("p", [], grounding="yes"), row("p", [], version="v2", grounding="yes"), + row("future", [], version="v3"), + row("child", ["v1::p", "v1::p", "v1::missing", "p", "v3::future", + "v2::child", "child", "private malformed reference "], version="v2")) + order = validation.resolve_version_order(("v1", "v2", "v3"), invocation_order=("v1", "v2", "v3")) + result = batch(rows, version_order=order) + evidence = next(item for item in result.assessments if item.child_key.record_id == "child") + assert evidence.declared_reference_count == 8 + assert evidence.resolved_reference_count == 2 + assert evidence.invalid_self_reference_count == 2 + assert {item.code for item in evidence.messages} >= { + ErrorCode.PARENT_AMBIGUOUS.value, ErrorCode.PARENT_FUTURE_VERSION.value, + ErrorCode.LINEAGE_CYCLE.value, ErrorCode.PARENT_FORMAT.value, + WarningCode.PARENT_UNRESOLVED.value, WarningCode.PARENT_DUPLICATE_REFERENCE.value} + assert tuple(ref.parent_key for ref in evidence.result.references) == ( + RecordKey("v1", "missing"), RecordKey("v1", "p")) + assert all("private malformed" not in message.message for message in result.messages) + assert "private malformed" not in repr(result) + + +@pytest.mark.parametrize("parents,declared,retained", [ + ("private scalar", None, 0), ({"private": "value"}, None, 0), + (["v1::p", 17, None], 3, 1), ([None], 1, 0), +]) +def test_parent_batch_malformed_entries_do_not_erase_known_counts(parents, declared, retained): + parent, child = row("p", []), row("child", []) + values = dict(child.values) + values["parent_ids"] = parents + child = replace(child, values=MappingProxyType(values)) + result = batch((parent, child)) + evidence = result.assessments[0] + assert evidence.child_key.record_id == "child" + assert evidence.declared_reference_count == declared + assert evidence.resolved_reference_count == retained + assert (evidence.result is None) == (declared is None) + assert any(message.code == ErrorCode.PARENT_FORMAT.value for message in evidence.messages) + if type(parents) is list: + parents.append("v1::new") + assert evidence.declared_reference_count == declared + + +def test_parent_batch_uses_one_lookup_and_matches_existing_successful_resolver(monkeypatch): + rows = (row("a", [], grounding="yes"), row("b", ["a", "v1::a"]), row("c", ["b", "missing"])) + real = validation._parent_lookup + calls = [] + + def tracked(keys, order): + calls.append(keys) + return real(keys, order) + + monkeypatch.setattr(validation, "_parent_lookup", tracked) + retained = batch(rows) + assert len(calls) == 1 + for item in rows: + previous = validation.resolve_parent_references(item.record_key, item.values["parent_ids"], + retained.record_keys) + current = next(value for value in retained.assessments if value.child_key == item.record_key) + assert current.result == previous + + +def test_parent_batch_missing_chronology_preserves_identity_resolution(): + rows = (row("root", [], grounding="yes"), row("child", ["v1::root"], version="v2")) + retained = batch(rows) + child = retained.assessments[1] + assert child.resolved_reference_count == 1 + assert child.result.references[0].resolution_status is ParentResolutionStatus.RESOLVED + assert child.result.references[0].temporal_status == "unavailable" + + +def test_parent_batch_missing_parent_strict_promotion_does_not_change_resolution(): + rows = (row("a", ["absent"]),) + warning = batch(rows) + strict = batch(rows, strict_mode=True, strict_warning_codes=(WarningCode.PARENT_UNRESOLVED.value,)) + assert warning.assessments[0].result.references == strict.assessments[0].result.references + assert warning.messages[0].severity is ValidationSeverity.WARNING + assert strict.messages[0].severity is ValidationSeverity.ERROR + + +def test_parent_batch_order_and_input_permutation_do_not_change_retained_evidence(): + rows = (row("a", []), row("b", ["a", "v1::a", "missing"])) + changed = (replace(rows[1], values=MappingProxyType({**rows[1].values, + "parent_ids": ("missing", "v1::a", "a")})), rows[0]) + assert batch(rows) == batch(changed) + + +def test_parent_batch_rejects_duplicate_and_forged_identity_evidence(): + valid = row("a", []) + with pytest.raises(CanonicalValidationError) as failure: + validation.resolve_parent_batch((valid.record_key, valid.record_key), (valid,)) + assert failure.value.code is ErrorCode.RECORD_DUPLICATE_ID + with pytest.raises(CanonicalValidationError) as failure: + validation.resolve_parent_batch((valid.record_key,), (valid, valid)) + assert failure.value.code is ErrorCode.PROVENANCE_DUPLICATE_ROW + with pytest.raises(CanonicalValidationError): + validation.resolve_parent_batch((valid.record_key,), (replace(valid, record_key=RecordKey("v1", "forged")),)) + with pytest.raises(CanonicalValidationError) as failure: + validation.resolve_parent_batch((RecordKey("v1", "other"),), (valid,)) + assert failure.value.code is ErrorCode.PROVENANCE_UNMATCHED_ROW + + +def test_parent_batch_rejects_forged_resolved_order_and_declaration_state(): + rows = (row("a", []), row("b", ["v1::a"], version="v2")) + order = validation.resolve_version_order(("v1", "v2"), invocation_order=("v1", "v2")) + with pytest.raises(CanonicalValidationError) as failure: + batch(rows, version_order=replace(order, order=("v2", "v1"))) + assert failure.value.code is ErrorCode.VERSION_ORDER_CONFLICT + forged = replace(rows[1], field_states=MappingProxyType({"parent_ids": "absent"})) + with pytest.raises(CanonicalValidationError): + batch((rows[0], forged), version_order=order) + + +def test_parent_batch_empty_scope_and_immutable_results(): + empty = validation.resolve_parent_batch((), None) + assert empty.record_keys == empty.assessments == empty.messages == () + assert empty.version_order.loaded_versions == () + result = batch((row("a", []),)) + with pytest.raises(FrozenInstanceError): + result.assessments[0].resolved_reference_count = 2 + with pytest.raises(TypeError): + result.version_order.declarations["version_order"] = ("v2",) + for value in (True, -1, None): + with pytest.raises(ValueError): + replace(result.assessments[0], resolved_reference_count=value) + + +def test_parent_batch_empty_scope_preserves_explicit_declared_version_inventory(): + declared = validation.resolve_version_order(("v1", "v2"), invocation_order=("v1", "v2")) + order = replace(declared, loaded_versions=()) + result = validation.resolve_parent_batch((), None, version_order=order) + assert result.version_order == order + assert result.version_order.order == ("v1", "v2") + with pytest.raises(CanonicalValidationError): + validation.resolve_parent_batch((), None, version_order=replace(order, order=("v2", "v1"))) + + +def test_parent_batch_missing_required_provenance_stays_an_error(): + row_value = validation.assess_provenance_row({"dataset_version": "v1", "record_id": "a", "parent_ids": []}) + result = validation.resolve_parent_batch((row_value.record_key,), (row_value,)) + assert result.assessments[0].provenance_available is True + assert result.assessments[0].declared_reference_count == 0 + assert {message.field for message in result.messages} == { + "source_type", "provenance_confidence", "external_grounding"} + assert all(message.severity is ValidationSeverity.ERROR for message in result.messages) From 11192a3c353292f11d85eb3725c8edc73a71967b Mon Sep 17 00:00:00 2001 From: DavidWallstructurallaw Date: Tue, 22 Sep 2026 20:35:44 -0700 Subject: [PATCH 03/12] Implement Phase 5 Step 3 cycles topology and depth --- .github/workflows/ci.yml | 5 +- PHASE_5_DECISIONS.md | 19 +- PHASE_5_STEP_3.md | 88 ++++ docs/lineage_contract.md | 21 +- docs/release_process.md | 18 +- scripts/release_check.py | 15 +- .../lineage/cycles.py | 413 ++++++++++++++- .../integration/test_current_verification.py | 2 +- tests/integration/test_phase4_cli.py | 16 +- tests/unit/test_T6_cycles.py | 492 +++++++++++++++++- 10 files changed, 1042 insertions(+), 47 deletions(-) create mode 100644 PHASE_5_STEP_3.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1da35f4..25d87cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,13 +50,14 @@ jobs: with: python-version: "3.12" - run: python -m pip install ".[test]" - - name: Check Step 2 parent graph and affected boundaries + - name: Check Step 3 cycles and depth with affected boundaries run: | set -euo pipefail python scripts/check_spec_consistency.py python scripts/check_traceability.py python scripts/release_check.py - python -m pytest -p no:cacheprovider -q tests/unit/test_parent_batch.py tests/unit/test_lineage_graph.py tests/unit/test_PR007_version_order.py tests/unit/test_PR008_parent_resolution.py tests/unit/test_PR009_generation.py tests/unit/test_lineage_fixture_inputs.py tests/unit/test_phase3_contracts.py tests/integration/test_phase3_metric_pipeline.py tests/unit/test_PR016_determinism.py tests/integration/test_current_verification.py tests/integration/test_ci_workflows.py tests/integration/test_no_algorithms.py tests/integration/test_repository_structure.py + python -m pytest -p no:cacheprovider -q tests/unit/test_T6_cycles.py tests/unit/test_parent_batch.py tests/unit/test_lineage_graph.py tests/unit/test_PR007_version_order.py tests/unit/test_PR008_parent_resolution.py tests/unit/test_PR009_generation.py tests/unit/test_lineage_fixture_inputs.py tests/unit/test_phase3_contracts.py tests/integration/test_phase3_metric_pipeline.py tests/unit/test_PR016_determinism.py tests/integration/test_current_verification.py tests/integration/test_ci_workflows.py tests/integration/test_no_algorithms.py tests/integration/test_repository_structure.py + python -m pytest -p no:cacheprovider -q tests/integration/test_phase4_cli.py::test_phase4_step7_validate_blocks_every_calculation_and_content_reader tests/integration/test_phase4_cli.py::test_phase4_step8_pair_does_not_activate_other_calculation_families core: if: github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate' diff --git a/PHASE_5_DECISIONS.md b/PHASE_5_DECISIONS.md index 8fe4fa6..71b83e9 100644 --- a/PHASE_5_DECISIONS.md +++ b/PHASE_5_DECISIONS.md @@ -1,6 +1,6 @@ # Phase 5 Decisions and Current Authorization -Status: **PLAN APPROVED; STEPS 1-2 COMPLETE; PHASE INCOMPLETE**. +Status: **PLAN APPROVED; STEPS 1-3 COMPLETE; PHASE INCOMPLETE**. On 2026-09-23 UTC the Theory Owner instructed: `批准，Phase 5 step 1开始`. @@ -136,3 +136,20 @@ full environment matrix or performance measurement is authorized by this step. Stop after Step 2; subsequent execution needs its own instruction. Existing authorization to synchronize this branch is retained. Merge, tag and publication remain separate actions. + +## Step 3 execution instruction + +On 2026-09-23 UTC the Theory Owner instructed `Phase 5 Step 3 继续`. +This authorizes iterative strongly connected component detection, affected +descendants, bounded cycle witnesses, topological ordering and structural depth. +Work starts from Step 2 commit +`e40543ff5ed3d2793a41bf58d7e4831bd2a9bfe9` on `phase5-lineage`. + +Runtime implementation is confined to the existing `lineage/cycles.py` owner. +The existing T6 test module becomes a direct behavior suite; input/graph and +generation neighbors remain the affected gate. The single current source check +advances to the completed Step 2 tree and opens only this implementation module. +No additional historical dispatcher or source-binding registry is introduced. +Root propagation, ancestry metrics, schema/report integration and CLI lineage +dispatch remain subsequent steps. Stop after Step 3; merge, tag and publication +remain outside this instruction. diff --git a/PHASE_5_STEP_3.md b/PHASE_5_STEP_3.md new file mode 100644 index 0000000..e6d84f9 --- /dev/null +++ b/PHASE_5_STEP_3.md @@ -0,0 +1,88 @@ +# Phase 5 Step 3 Completion + +Date: 2026-09-23 UTC. Status: **STEP 3 COMPLETE; PHASE 5 INCOMPLETE**. + +The Theory Owner instructed `Phase 5 Step 3 继续`. Work starts from +`e40543ff5ed3d2793a41bf58d7e4831bd2a9bfe9` on `phase5-lineage`. +This step covers cycle detection, topology and structural depth. It does not +authorize later steps, a merge, tag or release. + +## Implementation + +`lineage.cycles.analyze_cycles` consumes and revalidates the immutable Step 2 +graph. Iterative strongly connected component detection counts cyclic components +once, including explicit self-parent evidence retained outside accepted adjacency. +Affected records include members and descendants along parent-to-child edges. +Disconnected loaded cycles remain errors even when target paths are unaffected. + +Canonical traversal makes component ordering, witnesses, unaffected topology and +depth deterministic. Component detail is limited to 100 rows. A witness is a +closed path of actual edges, with explicit self-reference evidence permitted for +a self-loop. Witnesses longer than 64 edges are omitted with a reason. Internal +membership and affected sets remain complete for the next computation stage. + +Structural depth is zero for an explicitly parentless record and one plus the +maximum parent depth only when all required paths are valid and complete. +Unknown grounding alone leaves structural depth available. Missing provenance, +absent/null declarations, invalid or unresolved parents, unavailable chronology +and cycle-affected paths produce explicit unavailable reasons. Whole-target depth +is unavailable if any target is unresolved or the target is empty; the resolved +subset maximum is separate. Generation semantics are unchanged. + +Results use frozen data and detached read-only mappings. Diagnostics retain only +safe codes, severity and canonical identities. Analysis performs no file/network +access and is not dispatched by current validation, CLI audit or comparison. + +## Verification + +| Check | Actual result | +|---|---| +| Final focused regression command in `ci.yml` | 481 passed in 4.48 seconds | +| Separate focused CLI validation/comparison isolation command | 2 passed in 0.59 seconds | +| Direct T6 cases included in the 481-test run | 43 passed | +| Current specification, traceability and source-scope checks | PASS | +| Workflow YAML parsing and diff formatting | PASS | +| Independent implementation and oracle review | PASS; no remaining blocker | + +The focused gate covers T6 behavior, parent graph/retention, reference resolution, +generation, input-only metric boundaries and current source/workflow checks. +The 43 T6 cases include the frozen cycle and depth fixtures, an independent +reachability oracle over 64 three-node directed graphs, a 1,500-node chain, +64/65-edge witness boundaries and 101 cyclic components. These bounded cases +establish direct behavior; they are not substitutes for the Step 9 measurements. +No raw test-count target was used. + +Independent review found typed graph handoffs could retain a resolved reference +with no parent key, hide a self-reference outside retained self evidence, or +mislabel same-version chronology. Step 3 rejects these inconsistent inputs before +producing analytical results. Regression cases cover each failure. Fatal input +diagnostics also remain errors. Missing unrelated provenance metadata does not +erase complete structural depth. + +The first separate CLI isolation run had one pass and one failure. The existing +test imported dependent modules while their helpers were already patched, leaving +a temporary blocking function cached in another module. Importing all relevant +modules before applying patches fixes the test-order dependency. The final two +cases pass and explicitly block graph/cycle/ancestry dispatch. No product CLI +code changed, and no failed run is counted as success. + +Commands match the two focused invocations in `.github/workflows/ci.yml`; the +local regression also produced JUnit output. The local environment used Python +3.12.14 and pytest 9.1.1. No full matrix or release-level verification was run. + +The existing source-scope check advances its single comparison commit to the +completed Step 2 and permits only `lineage/cycles.py`. The remaining 58 product +files and exact file inventory stay protected. Sixteen frozen specifications and +seven packaged resource copies remain unchanged. No new registry, historical +source migration or evidence framework is introduced. + +## Handoff + +Package version remains `0.1.0.dev3`; report schema remains `1.0`. This direct API +does not compute external roots, ancestry metrics or lineage bounds, and does +not integrate lineage into reports or CLI dispatch. The full environment matrix, +release builds, optional Parquet profile and actual 100k workloads remain their +designated candidate/performance gates. Hosted CI success is not claimed. + +Step 4 can consume cycle membership, affected records and unaffected topology to +propagate complete external-root sets after its separate execution instruction. diff --git a/docs/lineage_contract.md b/docs/lineage_contract.md index 3760ff0..3e700e8 100644 --- a/docs/lineage_contract.md +++ b/docs/lineage_contract.md @@ -9,9 +9,10 @@ definitions/specifications remain unchanged. This document defines interfaces for Steps 2-8. Step 2 implements retained batch parent evidence and `build_lineage_graph` with immutable scope, adjacency and -node/edge limits. Cycle, depth, root, metric, report and CLI lineage interfaces -remain later implementation targets. The package remains at `0.1.0.dev3` and the -executable report schema at `1.0`. +node/edge limits. Step 3 adds `analyze_cycles` for iterative cycle detection, +unaffected topology and structural depth. Root, metric, report and CLI lineage +interfaces remain later implementation targets. The package remains at +`0.1.0.dev3` and the executable report schema at `1.0`. ## 1. Scope and reference semantics @@ -149,6 +150,20 @@ while the result identifies which computation stages actually completed. ## 3. Cycles and structural depth +The direct Step 3 entry is `lineage.cycles.analyze_cycles(graph: LineageGraph)`. +It revalidates the typed graph and returns immutable `CycleAnalysis` data without +calling loaders, generation, root propagation, metrics or rendering. Full SCC +membership, affected keys, unaffected topology and per-record depth assessments +are internal computation inputs for subsequent ancestry work. They retain all +loaded nodes. `component_details` holds at most 100 canonical component rows; +omitted component counts remain explicit. The general report detail envelope +and identity protection in section 5 belong to Step 7. + +Cycle status explicitly covers accepted edges and retained self-reference +evidence. Structural input reasons and validation errors remain separate, so an +acyclic accepted subgraph does not certify rejected declarations. A detected +cycle adds `E_LINEAGE_CYCLE` even when its component is disconnected from targets. + Cycle analysis uses accepted resolved edges plus separately retained explicit self-parent evidence. The self-edge remains invalid ancestry evidence while permitting a truthful self-cycle diagnostic. Rejected ambiguous/future/malformed diff --git a/docs/release_process.md b/docs/release_process.md index e51ce18..6d1a299 100644 --- a/docs/release_process.md +++ b/docs/release_process.md @@ -2,9 +2,9 @@ ## Current Phase 5 boundary -Phase 5 Step 2 is approved. Runtime remains at `0.1.0.dev3`, with executable -report schema 1.0. Parent-graph construction is the current implementation scope; -cycle, depth, ancestry and report integration remain subsequent steps. +Phase 5 Step 3 is approved. Runtime remains at `0.1.0.dev3`, with executable +report schema 1.0. Cycle detection, topology and structural depth are the current +implementation scope; ancestry and report integration remain subsequent steps. Stable v0.1 publication, a tag and main merge require separate authorization. @@ -18,11 +18,11 @@ python scripts/release_check.py Historical `--phase` and `--step` dispatch is retired. Earlier source forms, migrations and receipts remain recoverable from Git and archived phase records. -The accepted Phase 4 commit supplies the existing historical authority; no new -Phase 5 hash registry is needed. During Step 2 the existing source check permits -the approved models, errors, input-validation and lineage-graph modules, retaining -the exact product file inventory and rejecting unrelated runtime, schema, -package-metadata and canonical Hero changes. Sixteen frozen +The existing source check advances its single reference to the completed Step 2 +commit; no new Phase 5 hash registry is needed. During Step 3 it permits changes +only to the existing cycle module, retaining the exact product file inventory and +rejecting unrelated runtime, schema, package-metadata and canonical Hero changes. +The frozen Phase 0 specification hashes are unchanged. Sixteen frozen specifications and seven packaged resource copies remain protected. ## Risk-sensitive gates @@ -46,7 +46,7 @@ Dispatch `ci.yml` with `gate=candidate` for a stable candidate. The workflow run Compatibility cells run complete canonical regression with `--ignore=tests/performance`. The designated performance job runs `tests/performance` once, avoiding nine repetitions of the costly 100k report -workload. The Phase 5 sparse-lineage test is still a placeholder in Step 2; +workload. The Phase 5 sparse-lineage test is still a placeholder in Step 3; it becomes an executable graph benchmark in Step 9. No placeholder is counted as a successful lineage measurement. diff --git a/scripts/release_check.py b/scripts/release_check.py index bf01fe6..69f88d6 100644 --- a/scripts/release_check.py +++ b/scripts/release_check.py @@ -1,6 +1,6 @@ """Current source and candidate checks for Recursive Integrity Toolkit. -Step 2 opens the approved parent-graph implementation scope. Historical dispatch, source-body +Step 3 opens the approved cycle, topology and depth implementation scope. Historical dispatch, source-body migrations and phase registries are recoverable from the accepted Git commit. Installed checks below retain their existing product, privacy and package cases. """ @@ -24,13 +24,10 @@ import zipfile ROOT = Path(__file__).resolve().parents[1] -ACCEPTED_COMMIT = "1db3b1a460c233242ff37fbe45b8fac74d6101ef" +ACCEPTED_COMMIT = "e40543ff5ed3d2793a41bf58d7e4831bd2a9bfe9" PROTECTED_PREFIXES = ("src/", "schemas/", "examples/hero/") CURRENT_IMPLEMENTATION_PATHS = frozenset({ - "src/recursive_integrity_toolkit/models.py", - "src/recursive_integrity_toolkit/errors.py", - "src/recursive_integrity_toolkit/io/validation.py", - "src/recursive_integrity_toolkit/lineage/graph.py", + "src/recursive_integrity_toolkit/lineage/cycles.py", }) PARQUET_CASES = {"test_PR002_parquet_real_roundtrip", "test_PR002_parquet_real_row_limit", "test_PR002_parquet_real_invalid_file"} HERO_NAMES = {"records_v1.csv", "records_v2.csv", "provenance.csv", "config.json", "version_order.json", "EXPECTED_OUTPUTS.md"} @@ -60,7 +57,7 @@ def _regular_file(root: Path, relative: str) -> Path: def verify_source_scope(root: Path = ROOT) -> dict: - """Reject product changes outside the currently authorized parent-graph scope. + """Reject product changes outside the currently authorized cycle/depth scope. Later authorized implementation updates this single current boundary. It does not append another historical dispatcher or grant authority through metadata. @@ -70,7 +67,7 @@ def verify_source_scope(root: Path = ROOT) -> dict: for name, raw in expected.items(): current = _regular_file(root, name).read_bytes() if name not in CURRENT_IMPLEMENTATION_PATHS and current != raw: - raise ValueError(f"Unauthorized product mutation outside Step 2 scope: {name}") + raise ValueError(f"Unauthorized product mutation outside Step 3 scope: {name}") for prefix in PROTECTED_PREFIXES: paths = list((root / prefix).rglob("*")) if (root / prefix).is_symlink() or any(path.is_symlink() for path in paths): @@ -78,7 +75,7 @@ def verify_source_scope(root: Path = ROOT) -> dict: actual = {path.relative_to(root).as_posix() for path in paths if path.is_file() and "__pycache__" not in path.parts} if actual != {name for name in expected if name.startswith(prefix)}: - raise ValueError(f"Unauthorized product file-set mutation during Step 2: {prefix}") + raise ValueError(f"Unauthorized product file-set mutation during Step 3: {prefix}") return {"protected_product_files": len(expected) - len(CURRENT_IMPLEMENTATION_PATHS), "authorized_implementation_files": len(CURRENT_IMPLEMENTATION_PATHS), "product_file_inventory": "unchanged"} diff --git a/src/recursive_integrity_toolkit/lineage/cycles.py b/src/recursive_integrity_toolkit/lineage/cycles.py index 1f383ca..2346c20 100644 --- a/src/recursive_integrity_toolkit/lineage/cycles.py +++ b/src/recursive_integrity_toolkit/lineage/cycles.py @@ -1,20 +1,405 @@ -"""Own future cycle detection and affected-node reporting for record-ancestry graphs. +"""Analyze structural cycles, unaffected topology and complete structural depth. -Owner IDs: - T6 +Owner IDs: T6, PR-009 structural depth; Phase 5 Step 3. -Future inputs: - Future resolved record-ancestry graph. +Iterative traversals consume an immutable, revalidated parent graph. Cyclic +strongly connected components include separately retained invalid self-parents. +Complete internal identity sets support later ancestry work; presentation rows +are capped independently. No roots, scientific metrics, rendering or I/O occur. -Future outputs: - Future graph-validity status, cycle records, and affected-node sets. +Current phase status: + Phase 5 Step 3 cycle detection, descendant propagation, topology and depth. + External-root analysis and report integration remain later steps. +""" +from __future__ import annotations -Assumptions: - Detected cycles block ancestry interpretation and are not silently repaired. +from collections import deque +from dataclasses import dataclass, replace +from types import MappingProxyType +from typing import Mapping -Limits: - No traversal, topological ordering, cycle detection, or repair logic is implemented here. +from ..errors import ErrorCode +from ..models import ParentResolutionStatus, RecordKey, ValidationMessage, ValidationSeverity +from .graph import LineageGraph, LineageScope, _integer, _invalid, _key, _keys, _messages -Current phase status: - Phase 1 package scaffold only. Import-safe. No analytical or data-processing logic. -""" + +_DETAIL_LIMIT = 100 +_WITNESS_EDGE_LIMIT = 64 +_DEPTH_REASONS = frozenset({ + "MISSING_PROVENANCE", "PARENT_DECLARATION_UNAVAILABLE", "INVALID_PARENT_REFERENCE", + "UNRESOLVED_PARENT_REFERENCE", "VERSION_ORDER_UNAVAILABLE", "CYCLE_AFFECTED", + "INCOMPLETE_PARENT_DEPTH", +}) + + +@dataclass(frozen=True, slots=True) +class DepthAssessment: + """Complete structural depth, or explicit structural failure reasons.""" + lineage_depth: int | None + reason_codes: tuple[str, ...] = () + + def __post_init__(self) -> None: + if (type(self.reason_codes) is not tuple + or any(type(code) is not str or code not in _DEPTH_REASONS for code in self.reason_codes) + or self.reason_codes != tuple(sorted(set(self.reason_codes)))): + raise _invalid("structural depth requires canonical known reason codes") + if self.lineage_depth is None: + if not self.reason_codes: + raise _invalid("unavailable structural depth requires a reason") + else: + _integer(self.lineage_depth) + if self.reason_codes: + raise _invalid("complete structural depth cannot carry failure reasons") + + +@dataclass(frozen=True, slots=True) +class CycleComponent: + """Bounded component detail, independent of its complete internal membership.""" + component_index: int + member_count: int + target_member_count: int + witness_record_keys: tuple[RecordKey, ...] | None + witness_edge_count: int | None + witness_reason: str | None + + def __post_init__(self) -> None: + _integer(self.component_index, positive=True) + _integer(self.member_count, positive=True) + _integer(self.target_member_count) + if self.target_member_count > self.member_count: + raise _invalid("target cycle members exceed their component") + witness = self.witness_record_keys + if witness is None: + if self.witness_edge_count is not None or self.witness_reason != "diagnostic_limit": + raise _invalid("an omitted cycle witness requires its diagnostic limit reason") + else: + if type(witness) is not tuple or not 2 <= len(witness) <= _WITNESS_EDGE_LIMIT + 1: + raise _invalid("cycle witnesses require bounded immutable key sequences") + for key in witness: + _key(key) + _integer(self.witness_edge_count, positive=True) + if (witness[0] != witness[-1] or self.witness_edge_count != len(witness) - 1 + or self.witness_reason is not None): + raise _invalid("cycle witness must be closed with its exact edge count") + + +@dataclass(frozen=True, slots=True) +class CycleAnalysis: + """Full internal observations with separately bounded component diagnostics. + + Complete components/member/affected keys support later ancestry. Step 7 will + wrap identity-bearing report collections in the approved bounded/privacy + detail shape. Witness work already covers only the first 100 components. + """ + scope: LineageScope + cyclic_components: tuple[tuple[RecordKey, ...], ...] + cycle_member_record_keys: tuple[RecordKey, ...] + affected_record_keys: tuple[RecordKey, ...] + topological_order: tuple[RecordKey, ...] + depths_by_record: Mapping[RecordKey, DepthAssessment] + component_details: tuple[CycleComponent, ...] + lineage_depth: int | None + maximum_resolved_target_depth: int | None + depth_resolved_record_count: int + messages: tuple[ValidationMessage, ...] + + def __post_init__(self) -> None: + if type(self.scope) is not LineageScope: + raise _invalid("cycle analysis requires a typed scope") + scope = replace(self.scope) + if type(self.depths_by_record) not in (dict, MappingProxyType): + raise _invalid("cycle depth assessments require a mapping snapshot") + nodes = _keys(tuple(sorted(self.depths_by_record))) + if (len(nodes) != scope.loaded_record_count + or tuple(key for key in nodes if key.dataset_version == scope.target_dataset_version) != scope.target_record_keys + or tuple(sorted({key.dataset_version for key in nodes})) != scope.loaded_dataset_versions): + raise _invalid("cycle depth coverage disagrees with the loaded scope") + depths = {} + for key in nodes: + assessment = self.depths_by_record[key] + if type(assessment) is not DepthAssessment: + raise _invalid("cycle depths require typed assessments") + depths[key] = replace(assessment) + if type(self.cyclic_components) is not tuple: + raise _invalid("cyclic components require immutable tuples") + components = tuple(_keys(component) for component in self.cyclic_components) + if any(not component for component in components) or components != tuple(sorted(components)): + raise _invalid("cyclic components require canonical nonempty membership") + members = _keys(self.cycle_member_record_keys) + flattened = tuple(sorted(key for component in components for key in component)) + if flattened != members: + raise _invalid("cyclic components must have disjoint, complete member coverage") + affected = _keys(self.affected_record_keys) + affected_set = set(affected) + if not set(members) <= affected_set <= set(nodes): + raise _invalid("cycle affected records must include all members within loaded scope") + if (type(self.topological_order) is not tuple + or any(type(key) is not RecordKey for key in self.topological_order) + or len(set(self.topological_order)) != len(self.topological_order) + or set(self.topological_order) != set(nodes) - affected_set): + raise _invalid("cycle topology must cover exactly the unaffected records") + if any(depths[key].lineage_depth is not None or "CYCLE_AFFECTED" not in depths[key].reason_codes for key in affected): + raise _invalid("cycle affected records cannot have complete structural depth") + if any("CYCLE_AFFECTED" in depths[key].reason_codes for key in nodes if key not in affected_set): + raise _invalid("cycle depth reasons disagree with the affected-record set") + if (type(self.component_details) is not tuple + or len(self.component_details) != min(len(components), _DETAIL_LIMIT)): + raise _invalid("cycle component details must observe the diagnostic cap") + target_set = set(scope.target_record_keys) + details = [] + for index, detail in enumerate(self.component_details, 1): + if type(detail) is not CycleComponent: + raise _invalid("cycle component details require typed rows") + detail = replace(detail) + component = components[index - 1] + if (detail.component_index != index or detail.member_count != len(component) + or detail.target_member_count != sum(key in target_set for key in component) + or (detail.witness_record_keys is not None and not set(detail.witness_record_keys) <= set(component))): + raise _invalid("cycle component detail disagrees with its complete membership") + details.append(detail) + complete = [depths[key].lineage_depth for key in scope.target_record_keys + if depths[key].lineage_depth is not None] + maximum = max(complete, default=None) + whole = maximum if len(complete) == scope.target_record_count else None + _integer(self.depth_resolved_record_count) + for value in (self.lineage_depth, self.maximum_resolved_target_depth): + if value is not None: + _integer(value) + if (self.depth_resolved_record_count != len(complete) + or self.maximum_resolved_target_depth != maximum or self.lineage_depth != whole): + raise _invalid("cycle depth summaries disagree with the complete target subset") + messages = _messages(self.messages) + if components and not any(message.code == ErrorCode.LINEAGE_CYCLE.value + and message.severity is ValidationSeverity.ERROR for message in messages): + raise _invalid("detected cycles require an explicit lineage error") + object.__setattr__(self, "scope", scope) + object.__setattr__(self, "depths_by_record", MappingProxyType(depths)) + object.__setattr__(self, "component_details", tuple(details)) + object.__setattr__(self, "messages", messages) + + @property + def detected(self) -> bool: + return bool(self.cyclic_components) + + @property + def cycle_count(self) -> int: + return len(self.cyclic_components) + + @property + def counting_method(self) -> str: + return "cyclic_strongly_connected_components" + + @property + def cycle_status(self) -> str: + return "cyclic" if self.detected else "acyclic" + + @property + def cycle_status_scope(self) -> str: + return "accepted_edges_and_explicit_self_references" + + @property + def cycle_member_count(self) -> int: + return len(self.cycle_member_record_keys) + + @property + def target_cycle_member_count(self) -> int: + targets = set(self.scope.target_record_keys) + return sum(key in targets for key in self.cycle_member_record_keys) + + @property + def affected_record_count(self) -> int: + return len(self.affected_record_keys) + + @property + def target_affected_record_count(self) -> int: + targets = set(self.scope.target_record_keys) + return sum(key in targets for key in self.affected_record_keys) + + @property + def omitted_component_count(self) -> int: + return self.cycle_count - len(self.component_details) + + @property + def depth_reason_codes(self) -> tuple[str, ...]: + if not self.scope.target_record_count: + return ("EMPTY_TARGET_SCOPE",) + return () if self.lineage_depth is not None else ("INCOMPLETE_TARGET_DEPTH",) + + @property + def input_reason_codes(self) -> tuple[str, ...]: + return tuple(sorted({code for assessment in self.depths_by_record.values() + for code in assessment.reason_codes + if code not in ("CYCLE_AFFECTED", "INCOMPLETE_PARENT_DEPTH")})) + + @property + def has_errors(self) -> bool: + return any(message.severity in (ValidationSeverity.ERROR, ValidationSeverity.FATAL) + for message in self.messages) + + +def _cyclic_components(graph: LineageGraph) -> tuple[tuple[RecordKey, ...], ...]: + """Iterative Kosaraju traversal, retaining only cyclic components.""" + visited, finished = set(), [] + for start in graph.node_keys: + if start in visited: + continue + visited.add(start) + stack = [(start, iter(graph.children_by_parent[start]))] + while stack: + node, neighbors = stack[-1] + neighbor = next(neighbors, None) + if neighbor is None: + finished.append(node) + stack.pop() + elif neighbor not in visited: + visited.add(neighbor) + stack.append((neighbor, iter(graph.children_by_parent[neighbor]))) + visited.clear() + self_parents = set(graph.self_parent_record_keys) + components = [] + for start in reversed(finished): + if start in visited: + continue + component, pending = [], [start] + visited.add(start) + while pending: + node = pending.pop() + component.append(node) + for parent in graph.parents_by_child[node]: + if parent not in visited: + visited.add(parent) + pending.append(parent) + if len(component) > 1 or start in self_parents: + components.append(tuple(sorted(component))) + return tuple(sorted(components)) + + +def _witness(graph: LineageGraph, component: tuple[RecordKey, ...], + self_parents: set[RecordKey]) -> tuple[RecordKey, ...] | None: + """Canonical DFS with at most 64 active nodes; a missed witness stays omitted.""" + membership = set(component) + + def neighbors(node: RecordKey): + needs_self = node in self_parents + for child in graph.children_by_parent[node]: + if needs_self and child > node: + yield node + needs_self = False + if child in membership: + yield child + if needs_self: + yield node + + start = component[0] + path, positions, visited = [start], {start: 0}, {start} + stack = [neighbors(start)] + while stack: + child = next(stack[-1], None) + if child is None: + stack.pop() + positions.pop(path.pop()) + elif child in positions: + return tuple(path[positions[child]:] + [child]) + elif child not in visited: + if len(path) == _WITNESS_EDGE_LIMIT: + return None + visited.add(child) + positions[child] = len(path) + path.append(child) + stack.append(neighbors(child)) + return None + + +def _local_depth_reasons(graph: LineageGraph, key: RecordKey) -> set[str]: + """Inspect structural evidence only; grounding and unrelated fields are inert.""" + evidence = graph.parent_evidence_by_record[key] + reasons = set() + if not evidence.provenance_available: + reasons.add("MISSING_PROVENANCE") + result = evidence.result + if result is None: + reasons.add("INVALID_PARENT_REFERENCE") + return reasons + if result.declaration_state in ("absent", "null"): + reasons.add("PARENT_DECLARATION_UNAVAILABLE") + represented = sum(len(ref.source_references) for ref in result.references) + if (evidence.invalid_self_reference_count + or evidence.declared_reference_count != represented + or (result.declaration_state == "declared" and not represented)): + reasons.add("INVALID_PARENT_REFERENCE") + for reference in result.references: + if reference.resolution_status is ParentResolutionStatus.RESOLVED and reference.parent_key is None: + raise _invalid("resolved lineage references require a canonical parent identity") + parent = reference.parent_key + if parent == key: + raise _invalid("self-parent references require separately retained invalid evidence") + if parent is not None: + same_version = parent.dataset_version == key.dataset_version + if (reference.temporal_status == "same_version") != same_version: + raise _invalid("lineage reference chronology disagrees with its canonical identities") + if reference.resolution_status is not ParentResolutionStatus.RESOLVED: + reasons.add("UNRESOLVED_PARENT_REFERENCE") + if reference.temporal_status == "unavailable": + reasons.add("VERSION_ORDER_UNAVAILABLE") + return reasons + + +def analyze_cycles(graph: LineageGraph) -> CycleAnalysis: + """Compute exact loaded-graph SCCs and target depth without recursive calls. + + Traversals are O(V + E), apart from canonical sorting and constructor + validation. Witness search is bounded and cannot weaken an SCC finding. + ``cycle_status`` describes accepted edges plus explicit self-reference + evidence; ``input_reason_codes`` discloses structural input incompleteness. + """ + if type(graph) is not LineageGraph: + raise _invalid("cycle analysis requires a typed lineage graph") + graph = replace(graph) + components = _cyclic_components(graph) + members = tuple(sorted(key for component in components for key in component)) + affected, pending = set(members), deque(members) + while pending: + for child in graph.children_by_parent[pending.popleft()]: + if child not in affected: + affected.add(child) + pending.append(child) + unaffected = tuple(key for key in graph.node_keys if key not in affected) + indegree = {key: len(graph.parents_by_child[key]) for key in unaffected} + pending = deque(key for key in unaffected if not indegree[key]) + topology = [] + local_reasons = {key: _local_depth_reasons(graph, key) for key in graph.node_keys} + depths = {key: DepthAssessment(None, tuple(sorted(local_reasons[key] | {"CYCLE_AFFECTED"}))) + for key in graph.node_keys if key in affected} + while pending: + key = pending.popleft() + topology.append(key) + reasons = local_reasons[key] + parent_depths = [depths[parent].lineage_depth for parent in graph.parents_by_child[key]] + if any(value is None for value in parent_depths): + reasons.add("INCOMPLETE_PARENT_DEPTH") + value = None if reasons else 1 + max(parent_depths, default=-1) + depths[key] = DepthAssessment(value, tuple(sorted(reasons))) + for child in graph.children_by_parent[key]: + if child not in affected: + indegree[child] -= 1 + if not indegree[child]: + pending.append(child) + if len(topology) != len(unaffected): + raise _invalid("cycle topology did not cover every unaffected record") + targets, self_parents = set(graph.scope.target_record_keys), set(graph.self_parent_record_keys) + details = [] + for index, component in enumerate(components[:_DETAIL_LIMIT], 1): + witness = _witness(graph, component, self_parents) + details.append(CycleComponent(index, len(component), sum(key in targets for key in component), + witness, None if witness is None else len(witness) - 1, + "diagnostic_limit" if witness is None else None)) + complete = [depths[key].lineage_depth for key in graph.scope.target_record_keys + if depths[key].lineage_depth is not None] + maximum = max(complete, default=None) + whole = maximum if len(complete) == graph.scope.target_record_count else None + messages = graph.messages + if components: + messages += (ValidationMessage(ErrorCode.LINEAGE_CYCLE.value, ValidationSeverity.ERROR, + "A cyclic component was detected in the loaded lineage graph."),) + return CycleAnalysis(graph.scope, components, members, tuple(sorted(affected)), tuple(topology), + depths, tuple(details), whole, maximum, len(complete), messages) diff --git a/tests/integration/test_current_verification.py b/tests/integration/test_current_verification.py index 0bcc4bb..6a9309e 100644 --- a/tests/integration/test_current_verification.py +++ b/tests/integration/test_current_verification.py @@ -27,7 +27,7 @@ def protected_copy(repo_root, tmp_path): @pytest.mark.parametrize("mutation", ["formula", "new_module", "deleted_module"]) def test_current_scope_rejects_unauthorized_product_changes(current_tools, protected_copy, mutation): - """Parent-graph implementation does not authorize unrelated product changes.""" + """Cycle/depth implementation does not authorize unrelated product changes.""" verify = current_tools["verify_source_scope"] verify(protected_copy) target = protected_copy / "src/recursive_integrity_toolkit/metrics/diversity.py" diff --git a/tests/integration/test_phase4_cli.py b/tests/integration/test_phase4_cli.py index 8de3968..83332d6 100644 --- a/tests/integration/test_phase4_cli.py +++ b/tests/integration/test_phase4_cli.py @@ -202,10 +202,13 @@ def test_phase4_step7_validate_blocks_every_calculation_and_content_reader(tmp_p def denied(*args, **kwargs): touched.append(True) raise AssertionError("calculation or content-reference reader executed") - for name in ("metrics.diversity", "metrics.provenance", "metrics.bounds", "metrics.duplicates", "metrics.tail", - "metrics.resampling", "representations.base", "representations.field", "representations.content_hash", - "representations.compatibility"): - module = importlib.import_module("recursive_integrity_toolkit." + name) + names = ("metrics.diversity", "metrics.provenance", "metrics.bounds", "metrics.duplicates", "metrics.tail", + "metrics.resampling", "representations.base", "representations.field", "representations.content_hash", + "representations.compatibility", "lineage.graph", "lineage.cycles", "lineage.ancestry") + # Import dependencies before patching so late imports cannot retain a stub + # after monkeypatch restores the original owner module. + modules = [importlib.import_module("recursive_integrity_toolkit." + name) for name in names] + for module in modules: for key, value in vars(module).copy().items(): if inspect.isfunction(value) and value.__module__ == module.__name__: monkeypatch.setattr(module, key, denied) @@ -602,8 +605,9 @@ def test_phase4_step8_pair_does_not_activate_other_calculation_families(tmp_path def denied(*args, **kwargs): calls.append(True) raise AssertionError("unopened operation") - for name in ("metrics.resampling", "lineage.graph", "lineage.ancestry", "lineage.cycles"): - module = importlib.import_module("recursive_integrity_toolkit." + name) + names = ("metrics.resampling", "lineage.graph", "lineage.ancestry", "lineage.cycles") + modules = [importlib.import_module("recursive_integrity_toolkit." + name) for name in names] + for module in modules: for key, value in vars(module).copy().items(): if inspect.isfunction(value): monkeypatch.setattr(module, key, denied) for owner, name in ((socket, "socket"), (socket, "getaddrinfo"), (urllib.request, "urlopen"), (loaders, "load_content_reference")): diff --git a/tests/unit/test_T6_cycles.py b/tests/unit/test_T6_cycles.py index 98fc95d..9ae49a2 100644 --- a/tests/unit/test_T6_cycles.py +++ b/tests/unit/test_T6_cycles.py @@ -1,4 +1,492 @@ -"""Reserved for direct Phase 5 cycle behavior tests. +"""T6 cycle and structural-depth behavior over the retained parent graph. -Step 1 records independently authored fixtures before implementation. +Frozen fixture expectations predate implementation. A small reachability oracle +checks SCC behavior independently; scale and release gates remain separate. """ + +from dataclasses import FrozenInstanceError, replace +from itertools import permutations +import json +from pathlib import Path +import socket +from types import MappingProxyType + +import pytest + +from recursive_integrity_toolkit.errors import CanonicalValidationError +from recursive_integrity_toolkit.io.normalization import normalize_row +from recursive_integrity_toolkit.io.validation import ( + assess_provenance_row, resolve_parent_batch, resolve_version_order, validate_bundle, + validate_generation_declarations, +) +from recursive_integrity_toolkit.lineage import cycles +from recursive_integrity_toolkit.lineage.graph import ( + LineageGraph, LineageLimits, LineageResourceUsage, LineageScope, +) +from recursive_integrity_toolkit.models import ( + AuditBundle, FileRole, InputSource, ParentResolutionStatus, RecordKey, + ValidationMessage, ValidationSeverity, +) + + +_ABSENT = object() +_NO_PROVENANCE = object() +_FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" + + +def _key(identity): + return RecordKey.parse(identity) + + +def _graph_from_rows(keys, provenance, *, target="v2", chronology=True, order=None): + """Use real normalization/reference semantics without generation traversal.""" + nodes = tuple(sorted(keys)) + versions = tuple(sorted({key.dataset_version for key in nodes})) + ordering = None + if chronology and nodes: + ordering = resolve_version_order(versions, invocation_order=tuple(order or versions)) + batch = resolve_parent_batch(nodes, tuple(provenance), version_order=ordering) + evidence = {entry.child_key: entry for entry in batch.assessments} + parents, children = {}, {node: [] for node in nodes} + for child in nodes: + result = evidence[child].result + parents[child] = tuple(sorted({ + ref.parent_key for ref in (() if result is None else result.references) + if ref.resolution_status is ParentResolutionStatus.RESOLVED + and ref.temporal_status in ("same_version", "earlier_version") + and ref.parent_key != child + })) + for parent in parents[child]: + children[parent].append(child) + targets = tuple(key for key in nodes if key.dataset_version == target) + return LineageGraph( + LineageScope(target, targets, len(targets), len(nodes), len(nodes) - len(targets), versions), + nodes, parents, {key: tuple(value) for key, value in children.items()}, evidence, + tuple(key for key in nodes if evidence[key].invalid_self_reference_count), + batch.messages, + LineageResourceUsage(len(nodes), sum(map(len, parents.values())), 0, 0, LineageLimits()), + ) + + +def _graph(declarations, *, target="v2", chronology=True, grounding=None, reverse=False): + rows = [] + items = list(declarations.items()) + if reverse: + items.reverse() + for identity, parents in items: + if parents is _NO_PROVENANCE: + continue + key = _key(identity) + row = { + "dataset_version": key.dataset_version, "record_id": key.record_id, + "source_type": "unknown", "provenance_confidence": "confirmed", + "external_grounding": (grounding or {}).get(identity, "no"), + } + if parents is not _ABSENT: + row["parent_ids"] = list(reversed(parents)) if reverse and isinstance(parents, list) else parents + rows.append(normalize_row(row, kind="provenance")) + return _graph_from_rows(tuple(_key(identity) for identity in declarations), rows, + target=target, chronology=chronology) + + +def _fixture_cases(folder): + return json.loads((_FIXTURES / folder / "cases.json").read_text(encoding="utf-8"))["cases"] + + +def _fixture_graph(case): + keys = tuple(RecordKey(row["dataset_version"], row["record_id"]) for row in case["records"]) + rows = tuple(normalize_row(row, kind="provenance") for row in case["provenance"]) + target_keys = case["target_record_keys"] + target = _key(target_keys[0]).dataset_version if target_keys else None + return _graph_from_rows(keys, rows, target=target, order=case["version_order"]) + + +def _depth(result, identity): + return result.depths_by_record[_key(identity)].lineage_depth + + +def _check_topology(result, graph): + order = result.topological_order + assert len(set(order)) == len(order) + assert set(order) == set(graph.node_keys) - set(result.affected_record_keys) + positions = {key: index for index, key in enumerate(order)} + for parent in order: + for child in graph.children_by_parent[parent]: + if child in positions: + assert positions[parent] < positions[child] + + +def _check_witnesses(result, graph): + for detail in result.component_details: + component = result.cyclic_components[detail.component_index - 1] + assert detail.member_count == len(component) + assert detail.target_member_count == len(set(component) & set(graph.scope.target_record_keys)) + witness = detail.witness_record_keys + if witness is None: + assert detail.witness_reason == "diagnostic_limit" + assert detail.witness_edge_count is None + continue + assert witness[0] == witness[-1] + assert detail.witness_edge_count == len(witness) - 1 + assert 1 <= detail.witness_edge_count <= 64 + assert detail.witness_reason is None + assert set(witness) <= set(component) + for parent, child in zip(witness, witness[1:]): + assert child in graph.children_by_parent[parent] or ( + parent == child and parent in graph.self_parent_record_keys + ) + + +@pytest.mark.parametrize("case", _fixture_cases("lineage_cycles"), ids=lambda case: case["case_id"]) +def test_frozen_cycle_oracles_and_unaffected_topology(case): + graph = _fixture_graph(case) + result = cycles.analyze_cycles(graph) + expected = case["expected_lineage"] + assert result.cyclic_components == tuple(tuple(map(_key, group)) for group in expected["cyclic_components"]) + assert result.cycle_member_record_keys == tuple(map(_key, expected["cycle_member_keys"])) + assert result.affected_record_keys == tuple(map(_key, expected["affected_record_keys"])) + assert result.detected is expected["cycle_detected"] + assert result.cycle_count == expected["cycle_count"] + assert result.counting_method == expected["cycle_count_method"] + assert result.cycle_status == "cyclic" + targets = set(graph.scope.target_record_keys) + assert result.cycle_member_count == len(result.cycle_member_record_keys) + assert result.target_cycle_member_count == len(targets & set(result.cycle_member_record_keys)) + assert result.affected_record_count == len(result.affected_record_keys) + assert result.target_affected_record_count == len(targets & set(result.affected_record_keys)) + assert result.has_errors + assert any(message.code == "E_LINEAGE_CYCLE" and message.severity is ValidationSeverity.ERROR + for message in result.messages) + for key in result.affected_record_keys: + assert result.depths_by_record[key].lineage_depth is None + assert "CYCLE_AFFECTED" in result.depths_by_record[key].reason_codes + if "lineage_depth" in expected: + assert result.lineage_depth == expected["lineage_depth"] + _check_topology(result, graph) + _check_witnesses(result, graph) + + +@pytest.mark.parametrize("case", _fixture_cases("lineage_complete"), ids=lambda case: case["case_id"]) +def test_frozen_complete_depth_oracles(case): + graph = _fixture_graph(case) + result = cycles.analyze_cycles(graph) + expected = case["expected_lineage"] + assert result.cycle_count == 0 + assert result.cycle_status == "acyclic" + for key, depth in expected.get("depth_by_record", {}).items(): + assert _depth(result, key) == depth + if "lineage_depth" in expected: + assert result.lineage_depth == expected["lineage_depth"] + if "carryover_depth" in expected: + assert _depth(result, "v2::carry") == expected["carryover_depth"] + assert _depth(result, "v2::diamond") == expected["diamond_depth"] + rows = tuple(normalize_row(row, kind="provenance") for row in case["provenance"]) + generation = validate_generation_declarations(graph.node_keys, rows, + version_order=resolve_version_order(graph.scope.loaded_dataset_versions, + invocation_order=tuple(case["version_order"]))) + carry = next(entry for entry in generation.assessments if entry.record_key == _key("v2::carry")) + assert carry.expected_generation == expected["carryover_generation"] == 0 + assert _depth(result, "v2::carry") == 1 + _check_topology(result, graph) + + +def test_depth_uses_longest_required_branch_and_deduplicates_diamond_paths(): + graph = _graph({ + "v2::root": [], "v2::a": ["root"], "v2::b": ["a"], + "v2::c": ["b"], "v2::short": ["root"], + "v2::join": ["short", "c", "v2::c"], "v2::last": ["join", "root"], + }) + result = cycles.analyze_cycles(graph) + expected = {"v2::root": 0, "v2::a": 1, "v2::b": 2, "v2::c": 3, + "v2::short": 1, "v2::join": 4, "v2::last": 5} + assert {_key(key): depth for key, depth in expected.items()} == { + key: value.lineage_depth for key, value in result.depths_by_record.items()} + assert result.lineage_depth == result.maximum_resolved_target_depth == 5 + assert result.depth_resolved_record_count == 7 + + +@pytest.mark.parametrize("parents,reason", [ + (_ABSENT, "PARENT_DECLARATION_UNAVAILABLE"), + (None, "PARENT_DECLARATION_UNAVAILABLE"), + (_NO_PROVENANCE, "MISSING_PROVENANCE"), + (["v2::missing"], "UNRESOLVED_PARENT_REFERENCE"), + ([" invalid "], "INVALID_PARENT_REFERENCE"), + (["v3::future"], "INVALID_PARENT_REFERENCE"), +]) +def test_incomplete_branch_prevents_whole_target_depth(parents, reason): + graph = _graph({"v2::safe": [], "v2::bad": parents, + "v2::join": ["safe", "bad"], "v2::child": ["join"], "v3::future": []}) + result = cycles.analyze_cycles(graph) + assert _depth(result, "v2::safe") == 0 + assert _depth(result, "v2::bad") is None + assert reason in result.depths_by_record[_key("v2::bad")].reason_codes + for key in ("v2::join", "v2::child"): + assert _depth(result, key) is None + assert "INCOMPLETE_PARENT_DEPTH" in result.depths_by_record[_key(key)].reason_codes + assert result.lineage_depth is None + assert result.maximum_resolved_target_depth == 0 + assert result.depth_resolved_record_count == 1 + assert not result.detected + assert result.cycle_status == "acyclic" + assert result.cycle_status_scope == "accepted_edges_and_explicit_self_references" + assert reason in result.input_reason_codes + assert result.depth_reason_codes == ("INCOMPLETE_TARGET_DEPTH",) + + +def test_ambiguous_parent_blocks_depth_without_creating_cycle(): + graph = _graph({"v1::p": [], "v2::p": [], "v2::child": ["p", "v1::p"]}) + result = cycles.analyze_cycles(graph) + assert _depth(result, "v2::child") is None + assert "INVALID_PARENT_REFERENCE" in result.depths_by_record[_key("v2::child")].reason_codes + assert result.cycle_count == 0 + assert result.has_errors + + +def test_malformed_parent_container_has_no_invented_zero_depth(): + key = _key("v2::bad") + row = normalize_row({"dataset_version": "v2", "record_id": "bad", + "source_type": "unknown", "provenance_confidence": "confirmed", + "external_grounding": "no", "parent_ids": []}, kind="provenance") + row = replace(row, values=MappingProxyType({**row.values, "parent_ids": "private scalar"})) + graph = _graph_from_rows((key,), (row,)) + result = cycles.analyze_cycles(graph) + assert _depth(result, "v2::bad") is None + assert "INVALID_PARENT_REFERENCE" in result.depths_by_record[key].reason_codes + assert result.depth_resolved_record_count == 0 + assert result.maximum_resolved_target_depth is None + assert result.lineage_depth is None + + +def test_unknown_grounding_preserves_structural_depth(): + graph = _graph({"v2::root": [], "v2::child": ["root"]}, + grounding={"v2::root": "unknown", "v2::child": "unknown"}) + result = cycles.analyze_cycles(graph) + assert _depth(result, "v2::root") == 0 + assert _depth(result, "v2::child") == result.lineage_depth == 1 + assert all(entry.reason_codes == () for entry in result.depths_by_record.values()) + + +def test_missing_nonstructural_metadata_does_not_invalidate_explicit_empty_depth(): + row = assess_provenance_row({"dataset_version": "v2", "record_id": "root", "parent_ids": []}) + graph = _graph_from_rows((row.record_key,), (row,)) + assert row.missing_required_fields + result = cycles.analyze_cycles(graph) + assert result.lineage_depth == 0 + assert result.depths_by_record[row.record_key].reason_codes == () + assert result.has_errors + + +def test_fatal_input_evidence_preserves_error_status_in_acyclic_graph(): + graph = _graph({"v2::root": []}) + graph = replace(graph, messages=(ValidationMessage( + "E_PARENT_FORMAT", ValidationSeverity.FATAL, "Private arbitrary diagnostic"),)) + result = cycles.analyze_cycles(graph) + assert result.cycle_status == "acyclic" + assert result.has_errors + assert result.messages[0].severity is ValidationSeverity.FATAL + assert "Private arbitrary diagnostic" not in repr(result) + + +def test_resolved_identity_without_chronology_cannot_establish_depth(): + graph = _graph({"v1::root": [], "v2::child": ["v1::root"]}, chronology=False) + assert graph.parent_evidence_by_record[_key("v2::child")].resolved_reference_count == 1 + result = cycles.analyze_cycles(graph) + assert _depth(result, "v1::root") == 0 + assert _depth(result, "v2::child") is None + assert "VERSION_ORDER_UNAVAILABLE" in result.depths_by_record[_key("v2::child")].reason_codes + assert result.maximum_resolved_target_depth is None + assert result.depth_resolved_record_count == 0 + assert result.lineage_depth is None + + +@pytest.mark.parametrize("size", [64, 65]) +def test_cycle_witness_exact_edge_limit_and_order(size): + keys = [f"v2::n{index:03}" for index in range(size)] + graph = _graph({key: [keys[(index - 1) % size]] for index, key in enumerate(keys)}) + result = cycles.analyze_cycles(graph) + detail = result.component_details[0] + assert result.cycle_count == 1 + assert result.cycle_member_count == result.affected_record_count == size + if size == 64: + assert detail.witness_record_keys == tuple(map(_key, keys + [keys[0]])) + assert detail.witness_edge_count == 64 + else: + assert detail.witness_record_keys is None + assert detail.witness_reason == "diagnostic_limit" + _check_witnesses(result, graph) + + +def test_self_cycle_witness_and_bounded_detail_do_not_truncate_analysis(): + declarations = {f"v2::self{index:03}": [f"v2::self{index:03}"] for index in range(101)} + result = cycles.analyze_cycles(_graph(declarations)) + assert result.cycle_count == result.cycle_member_count == result.affected_record_count == 101 + assert result.depth_resolved_record_count == 0 + assert len(result.component_details) == 100 + for index, detail in enumerate(result.component_details): + key = _key(f"v2::self{index:03}") + assert detail.component_index == index + 1 + assert detail.witness_record_keys == (key, key) + assert detail.witness_edge_count == 1 + assert result.topological_order == () + + +def test_cycle_depth_and_witness_results_ignore_input_permutations(): + declarations = {"v2::a": ["c", "b"], "v2::b": ["a", "c"], + "v2::c": ["a"], "v2::tail": ["c"], "v2::root": [], "v2::safe": ["root"]} + left = cycles.analyze_cycles(_graph(declarations)) + right = cycles.analyze_cycles(_graph(declarations, reverse=True)) + assert left == right + + +def test_iterative_chain_beyond_python_recursion_depth(): + count = 1500 + declarations = {f"v2::n{index:04}": [] if index == 0 else [f"v2::n{index - 1:04}"] + for index in range(count)} + graph = _graph(declarations) + result = cycles.analyze_cycles(graph) + assert not result.detected + assert result.lineage_depth == result.maximum_resolved_target_depth == count - 1 + assert result.depth_resolved_record_count == count + assert result.topological_order == graph.node_keys + assert tuple(item.lineage_depth for item in result.depths_by_record.values()) == tuple(range(count)) + + +def test_context_depth_is_excluded_from_target_maximum(): + graph = _graph({"v1::a": [], "v1::b": ["a"], "v1::c": ["b"], "v2::target": []}) + result = cycles.analyze_cycles(graph) + assert _depth(result, "v1::c") == 2 + assert result.lineage_depth == result.maximum_resolved_target_depth == 0 + assert result.depth_resolved_record_count == 1 + + +@pytest.mark.parametrize("target", [None, "v2"]) +def test_empty_target_never_borrows_context_depth(target): + result = cycles.analyze_cycles(_graph({"v1::root": [], "v1::child": ["root"]}, target=target)) + assert result.scope.target_record_count == 0 + assert result.lineage_depth is result.maximum_resolved_target_depth is None + assert result.depth_resolved_record_count == 0 + assert result.depth_reason_codes == ("EMPTY_TARGET_SCOPE",) + assert _depth(result, "v1::child") == 1 + + +def test_empty_graph_has_complete_empty_topology_and_unavailable_depth(): + result = cycles.analyze_cycles(_graph({}, target=None)) + assert result.cyclic_components == result.cycle_member_record_keys == result.affected_record_keys == () + assert result.topological_order == () + assert dict(result.depths_by_record) == {} + assert result.lineage_depth is result.maximum_resolved_target_depth is None + assert result.depth_resolved_record_count == 0 + assert result.cycle_status == "acyclic" + + +def test_result_is_a_defensive_immutable_snapshot(): + result = cycles.analyze_cycles(_graph({"v2::root": [], "v2::child": ["root"]})) + external = dict(result.depths_by_record) + copied = replace(result, depths_by_record=external) + external.clear() + assert copied == result + with pytest.raises(TypeError): + copied.depths_by_record[_key("v2::root")] = None + with pytest.raises(FrozenInstanceError): + copied.lineage_depth = 999 + with pytest.raises(FrozenInstanceError): + copied.depths_by_record[_key("v2::root")].lineage_depth = 999 + + +@pytest.mark.parametrize("mutation", ["type", "adjacency", "evidence", "scope", + "resolved_without_key", "resolved_self", + "false_earlier_version", "false_unavailable"]) +def test_forged_graph_handoff_is_rejected(mutation): + graph = _graph({"v2::root": [], "v2::child": ["root"]}) + if mutation == "type": + graph = {"node_keys": graph.node_keys} + elif mutation == "adjacency": + object.__setattr__(graph, "children_by_parent", MappingProxyType({key: () for key in graph.node_keys})) + elif mutation == "evidence": + evidence = dict(graph.parent_evidence_by_record) + child = _key("v2::child") + evidence[child] = replace(evidence[child], resolved_reference_count=0) + object.__setattr__(graph, "parent_evidence_by_record", evidence) + elif mutation == "scope": + object.__setattr__(graph.scope, "target_record_count", True) + else: + evidence = dict(graph.parent_evidence_by_record) + child = _key("v2::child") + entry = evidence[child] + reference = entry.result.references[0] + if mutation in ("false_earlier_version", "false_unavailable"): + reference = replace(reference, temporal_status=( + "earlier_version" if mutation == "false_earlier_version" else "unavailable")) + else: + parent = child if mutation == "resolved_self" else None + reference = replace(reference, parent_key=parent, + canonical_reference=None if parent is None else str(parent)) + evidence[child] = replace(entry, result=replace(entry.result, references=(reference,))) + if mutation == "false_earlier_version": + graph = replace(graph, parent_evidence_by_record=evidence) + else: + # These forms have no accepted adjacency. Rebuild a consistent + # outer graph to exercise semantic reference admission. + graph = replace(graph, parents_by_child={key: () for key in graph.node_keys}, + children_by_parent={key: () for key in graph.node_keys}, + parent_evidence_by_record=evidence, + resource_usage=replace(graph.resource_usage, admitted_edge_count=0)) + with pytest.raises(CanonicalValidationError): + cycles.analyze_cycles(graph) + + +def test_cycle_analysis_uses_no_file_or_network_io(monkeypatch): + graph = _graph({"v2::a": ["b"], "v2::b": ["a"], "v2::missing": ["unknown"]}) + def forbidden(*args, **kwargs): + raise AssertionError("Cycle analysis must consume only its immutable graph") + monkeypatch.setattr("builtins.open", forbidden) + monkeypatch.setattr(Path, "open", forbidden) + monkeypatch.setattr(socket, "socket", forbidden) + result = cycles.analyze_cycles(graph) + assert result.cycle_count == 1 + assert all(message.file_path is message.row_number is message.line_number is None + for message in result.messages) + + +def test_input_validation_never_dispatches_cycle_analysis(tmp_path, monkeypatch): + records = tmp_path / "records.jsonl" + provenance = tmp_path / "provenance.jsonl" + records.write_text(json.dumps({"dataset_version": "v2", "record_id": "a", "content": "private"}) + "\n") + provenance.write_text(json.dumps({"dataset_version": "v2", "record_id": "a", "source_type": "unknown", + "provenance_confidence": "confirmed", "external_grounding": "no", "parent_ids": []}) + "\n") + def forbidden(*args, **kwargs): + raise AssertionError("Input validation cannot dispatch cycle analysis") + monkeypatch.setattr(cycles, "analyze_cycles", forbidden) + result = validate_bundle(AuditBundle((InputSource(FileRole.RECORDS_PRIMARY, records), + InputSource(FileRole.PROVENANCE_MANIFEST, provenance)))) + assert result.parent_validation is not None + assert not result.has_errors + + +def test_all_three_node_directed_graphs_match_independent_reachability_oracle(): + """64 small graphs detect SCC splitting, descendant direction and lost edges.""" + keys = tuple(_key(f"v2::{name}") for name in "abc") + possible_edges = tuple(permutations(range(3), 2)) + for mask in range(1 << len(possible_edges)): + edges = {edge for index, edge in enumerate(possible_edges) if mask & (1 << index)} + reach = [[(start, end) in edges for end in range(3)] for start in range(3)] + for via in range(3): + for start in range(3): + for end in range(3): + reach[start][end] |= reach[start][via] and reach[via][end] + components, seen = [], set() + for start in range(3): + if start not in seen and reach[start][start]: + members = tuple(index for index in range(3) if reach[start][index] and reach[index][start]) + components.append(tuple(keys[index] for index in members)) + seen.update(members) + affected = {index for member in seen for index in range(3) if index == member or reach[member][index]} + declarations = {str(key): [str(keys[parent]) for parent in range(3) if (parent, child) in edges] + for child, key in enumerate(keys)} + graph = _graph(declarations) + result = cycles.analyze_cycles(graph) + assert result.cyclic_components == tuple(components), mask + assert result.cycle_member_record_keys == tuple(keys[index] for index in sorted(seen)), mask + assert result.affected_record_keys == tuple(keys[index] for index in sorted(affected)), mask + _check_topology(result, graph) + _check_witnesses(result, graph) From 2450100d08355d8cb1e0be931345f44572861e4a Mon Sep 17 00:00:00 2001 From: DavidWallstructurallaw Date: Tue, 22 Sep 2026 21:03:31 -0700 Subject: [PATCH 04/12] Implement Phase 5 Step 4 root resolution and coverage --- .github/workflows/ci.yml | 4 +- PHASE_5_DECISIONS.md | 25 +- PHASE_5_STEP_4.md | 106 ++++ docs/lineage_contract.md | 26 +- docs/release_process.md | 15 +- scripts/release_check.py | 12 +- .../lineage/ancestry.py | 379 ++++++++++++- .../integration/test_current_verification.py | 2 +- tests/unit/test_T4_ancestry.py | 530 +++++++++++++++++- tests/unit/test_lineage_root_resources.py | 221 ++++++++ 10 files changed, 1285 insertions(+), 35 deletions(-) create mode 100644 PHASE_5_STEP_4.md create mode 100644 tests/unit/test_lineage_root_resources.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25d87cb..f4a2c13 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,13 +50,13 @@ jobs: with: python-version: "3.12" - run: python -m pip install ".[test]" - - name: Check Step 3 cycles and depth with affected boundaries + - name: Check Step 4 roots and coverage with affected boundaries run: | set -euo pipefail python scripts/check_spec_consistency.py python scripts/check_traceability.py python scripts/release_check.py - python -m pytest -p no:cacheprovider -q tests/unit/test_T6_cycles.py tests/unit/test_parent_batch.py tests/unit/test_lineage_graph.py tests/unit/test_PR007_version_order.py tests/unit/test_PR008_parent_resolution.py tests/unit/test_PR009_generation.py tests/unit/test_lineage_fixture_inputs.py tests/unit/test_phase3_contracts.py tests/integration/test_phase3_metric_pipeline.py tests/unit/test_PR016_determinism.py tests/integration/test_current_verification.py tests/integration/test_ci_workflows.py tests/integration/test_no_algorithms.py tests/integration/test_repository_structure.py + python -m pytest -p no:cacheprovider -q tests/unit/test_T4_ancestry.py tests/unit/test_lineage_root_resources.py tests/unit/test_T6_cycles.py tests/unit/test_parent_batch.py tests/unit/test_lineage_graph.py tests/unit/test_PR007_version_order.py tests/unit/test_PR008_parent_resolution.py tests/unit/test_PR009_generation.py tests/unit/test_lineage_fixture_inputs.py tests/unit/test_phase3_contracts.py tests/integration/test_phase3_metric_pipeline.py tests/unit/test_PR016_determinism.py tests/integration/test_current_verification.py tests/integration/test_ci_workflows.py tests/integration/test_no_algorithms.py tests/integration/test_repository_structure.py python -m pytest -p no:cacheprovider -q tests/integration/test_phase4_cli.py::test_phase4_step7_validate_blocks_every_calculation_and_content_reader tests/integration/test_phase4_cli.py::test_phase4_step8_pair_does_not_activate_other_calculation_families core: diff --git a/PHASE_5_DECISIONS.md b/PHASE_5_DECISIONS.md index 71b83e9..3be45d4 100644 --- a/PHASE_5_DECISIONS.md +++ b/PHASE_5_DECISIONS.md @@ -1,6 +1,6 @@ # Phase 5 Decisions and Current Authorization -Status: **PLAN APPROVED; STEPS 1-3 COMPLETE; PHASE INCOMPLETE**. +Status: **PLAN APPROVED; STEPS 1-4 COMPLETE; PHASE INCOMPLETE**. On 2026-09-23 UTC the Theory Owner instructed: `批准，Phase 5 step 1开始`. @@ -153,3 +153,26 @@ No additional historical dispatcher or source-binding registry is introduced. Root propagation, ancestry metrics, schema/report integration and CLI lineage dispatch remain subsequent steps. Stop after Step 3; merge, tag and publication remain outside this instruction. + +## Step 4 execution instruction + +On 2026-09-23 UTC the Theory Owner instructed `Phase 5 Step 4 继续`. +This authorizes external-root anchors, carryover inheritance, complete root +unions, G/C/U classification and coverage, including the approved root resource +guards. Work starts from Step 3 commit +`11192a3c353292f11d85eb3725c8edc73a71967b` on `phase5-lineage`. + +Runtime changes are confined to `lineage/ancestry.py`. Direct T4 and root-resource +tests exercise the new behavior; graph/cycle, provenance and input-only neighbors +retain their relevant checks. The existing source boundary advances its single +comparison commit to Step 3 and opens only the ancestry module. No new registry +or historical migration is added. Incidence, concentration, bounds, shared-root +proxy, report/schema integration and CLI lineage dispatch remain later steps. + +The conflict rule follows frozen `DEFINITIONS_AND_UNITS.md` section 5.8 and +`DATA_AND_PROVENANCE_SPEC.md` section 9.12: unresolved grounding/ancestry conflicts +block root classification. A `batch_id` or `timestamp` difference between record +and provenance namespaces alone does not establish such a conflict. Original +metadata is preserved; no new grounding inference is made from labels or URIs. + +Stop after Step 4; merge, tag and publication remain outside this instruction. diff --git a/PHASE_5_STEP_4.md b/PHASE_5_STEP_4.md new file mode 100644 index 0000000..4676156 --- /dev/null +++ b/PHASE_5_STEP_4.md @@ -0,0 +1,106 @@ +# Phase 5 Step 4 Completion + +Date: 2026-09-23 UTC. Status: **COMPLETE; PHASE INCOMPLETE**. + +The Theory Owner instructed `Phase 5 Step 4 继续`. Work starts from +`11192a3c353292f11d85eb3725c8edc73a71967b` on `phase5-lineage`. +This step implements external-root resolution and coverage. It does not +authorize incidence/concentration, bounds/proxy, report or CLI integration, +subsequent steps, a merge, tag or release. + +## Delivered behavior + +`lineage.ancestry.analyze_lineage` consumes the retained validation handoff, +reuses the existing graph and cycle owners, and revalidates provenance metadata. +An explicitly parentless grounded-yes record supports itself. A grounded-no +parentless record has a complete empty root set. Missing or unknown evidence is +represented by a null root set with reasons, preserving the distinction between +unknown ancestry and known closure. + +A grounded-yes single-parent carryover inherits its parent's complete root set, +including an empty set. Repeated aliases still mean one canonical parent. +Other grounded-with-parent forms remain unresolved under the approved rule. +Grounding-no descendants union unique roots only when every required branch is +complete. Cycles, invalid/unresolved parents, missing required provenance, +unknown grounding and undeclared boundaries propagate unresolved ancestry. +Source labels, evidence URIs and record content never mint roots. + +Only targets enter G/C/U classification and coverage denominators. Context nodes +support propagation without entering N. Original reference multiplicity remains +separate from deduplicated adjacency and root sets. Known identity resolution +with unavailable chronology can retain reference coverage while ancestry remains +unavailable. Malformed declarations with unknown cardinality do not become zero +declared references. Structural depth remains independent of root uncertainty. + +The conflict interpretation follows frozen definitions: blocking grounding or +ancestry contradictions are distinct from `batch_id` or `timestamp` differences +between metadata namespaces. Those metadata differences alone do not erase valid +roots, and their original values are not overwritten. + +## Resource boundaries + +Propagation processes loaded nodes in deterministic dependency order, choosing +canonical ready nodes and canonical parent/root order. Logical memberships are +charged before each new stored `(record, root)` association. Every candidate +parent-root visit is charged, including duplicates already seen from another +parent. An anchor self-membership costs one membership and zero union visits. +Unknown branches do not contribute partial observed-root sets as exact ancestry. + +Root-stage exhaustion yields a failed result with no records, G/C/U partition or +root coverage. Previously completed graph/cycle/depth observations and immediate +reference coverage remain available. Existing graph node/edge admission failures +continue to raise their typed error before an analytical result exists. Limits +describe defined work units, not process RSS or already-loaded input memory. + +The current implementation processes loaded context as well as targets; resource +usage includes context associations actually stored. Returned ancestry records +contain targets only. Result containers are immutable and exclude raw content, +locations and provenance URI values. + +## Verification + +Local verification used Python 3.12.14 with the core dependency profile. + +| Check | Result | +|---|---| +| Combined Step 4 direct and affected regression gate | 541 passed in 5.17 seconds | +| Additional provenance coverage/source-share neighbors | 203 passed in 0.51 seconds | +| Separate validation/pair CLI isolation checks | 2 passed in 0.48 seconds | +| Specification consistency, traceability and source scope | PASS | +| Workflow YAML parsing and whitespace check | PASS | +| Independent implementation and test review | PASS; no outstanding blockers | + +The combined gate includes 48 ancestry cases and 12 resource cases. Direct +checks cover all twelve frozen root/classification fixtures and Hero exact root +sets, separate unknown/missing branches, conservative carryover rules, original +reference multiplicity, context exclusion from target denominators, strict and +fatal outcomes, immutable results and input-only/no-I/O guarantees. Resource +checks cover exact membership and candidate-visit boundaries, duplicate roots +versus duplicate aliases, canonical ordering, failure retention and a 1,200-node +chain without recursion. Existing cycle, graph, parent, generation, metric and +current boundary checks remain in the same focused selection. + +The initial direct-test draft had two fixture-construction failures: an incorrect +provenance-match field name and invalid empty content rejected before lineage. +Both were corrected in the tests without changing frozen inputs. Independent +review also found a missing fatal-status branch and opportunities to construct +inconsistent result statuses or boolean coverage ratios. These were fixed with +direct regression cases. Completed cycle diagnostics remain preserved, and +complete root identities must belong to the loaded scope. + +The existing source check advances its single comparison commit to completed +Step 3 and opens only `lineage/ancestry.py`. The other 58 product files, exact +file inventory, 16 frozen specifications and seven packaged resource copies +remain protected. No new registry, historical migration or evidence framework +is added. + +## Handoff + +Package version remains `0.1.0.dev3`; report schema remains `1.0`. Current report +assembly and ordinary CLI flows continue their existing behavior. Step 5 may +add root incidence and fractional concentration from complete target root sets +after its separate execution instruction. + +No full regression matrix, release build, clean install, optional Parquet profile +or 100k performance measurement is claimed here. Those remain their designated +candidate/performance gates. Hosted CI success is not claimed. diff --git a/docs/lineage_contract.md b/docs/lineage_contract.md index 3e700e8..9393346 100644 --- a/docs/lineage_contract.md +++ b/docs/lineage_contract.md @@ -10,8 +10,10 @@ definitions/specifications remain unchanged. This document defines interfaces for Steps 2-8. Step 2 implements retained batch parent evidence and `build_lineage_graph` with immutable scope, adjacency and node/edge limits. Step 3 adds `analyze_cycles` for iterative cycle detection, -unaffected topology and structural depth. Root, metric, report and CLI lineage -interfaces remain later implementation targets. The package remains at +unaffected topology and structural depth. Step 4 adds `analyze_lineage` for +external-root resolution, G/C/U counts and coverage. Incidence, concentration, +bounds, report and CLI lineage interfaces remain later implementation targets. +The package remains at `0.1.0.dev3` and the executable report schema at `1.0`. ## 1. Scope and reference semantics @@ -88,6 +90,19 @@ records. Selection is explicit, revalidated and independent of ordinary metric eligibility. `validate_bundle` remains input-only and never calls this function. CLI orchestration calls it only after an explicit lineage request. +Step 4 implements this direct API for root resolution and coverage. Its +`LineageAnalysisResult.records` contains target records in canonical order; +context roots are intermediate computation data and do not enter target +denominators. The existing `ExecutionStatus` vocabulary applies. Incidence, +fractional mass, concentration, bounds and proxy fields remain later steps. + +Graph admission failures retain the existing `LineageResourceLimitError` before +a valid analytical result can be created. A root-membership or union-work +failure after graph/cycle analysis returns a failed result with root records, +G/C/U counts and root coverage unavailable. Completed cycle/depth observations, +original-reference coverage and exact consumed-work counters survive. No partial +root traversal is converted into an exact target partition. + Step 2 supplies the lower-level input-only handoff in `models.py`: `ParentRecordValidation` retains provenance presence, an optional `ParentValidationResult`, declared/resolved/self-reference counts and diagnostics; @@ -206,6 +221,13 @@ Strict ancestry still excludes self. Explicit parentless grounding-no produces a known-empty set. Grounding-unknown, missing required provenance, conflicting declarations or an undeclared boundary produces an unresolved set. +Conflicts here concern grounding or required ancestry, as defined in frozen +`DEFINITIONS_AND_UNITS.md` section 5.8. Record/provenance `batch_id` or `timestamp` +differences remain separate metadata observations under +`DATA_AND_PROVENANCE_SPEC.md` section 9.12; they do not alone invalidate roots. +Provenance-specific declarations retain their manifest authority. Labels, URI +spelling and metadata disagreements do not introduce inferred grounding values. + A grounding-yes carryover with exactly one deduplicated resolved canonical parent inherits the parent's set. Any unresolved additional declaration blocks it. Other grounded-with-parent forms, including multi-parent grounded carryover, diff --git a/docs/release_process.md b/docs/release_process.md index 6d1a299..08ce9c6 100644 --- a/docs/release_process.md +++ b/docs/release_process.md @@ -2,9 +2,10 @@ ## Current Phase 5 boundary -Phase 5 Step 3 is approved. Runtime remains at `0.1.0.dev3`, with executable -report schema 1.0. Cycle detection, topology and structural depth are the current -implementation scope; ancestry and report integration remain subsequent steps. +Phase 5 Step 4 is approved. Runtime remains at `0.1.0.dev3`, with executable +report schema 1.0. External-root resolution, G/C/U classification and coverage are +the current implementation scope; concentration, bounds and report integration +remain subsequent steps. Stable v0.1 publication, a tag and main merge require separate authorization. @@ -18,9 +19,9 @@ python scripts/release_check.py Historical `--phase` and `--step` dispatch is retired. Earlier source forms, migrations and receipts remain recoverable from Git and archived phase records. -The existing source check advances its single reference to the completed Step 2 -commit; no new Phase 5 hash registry is needed. During Step 3 it permits changes -only to the existing cycle module, retaining the exact product file inventory and +The existing source check advances its single reference to the completed Step 3 +commit; no new Phase 5 hash registry is needed. During Step 4 it permits changes +only to the existing ancestry module, retaining the exact product file inventory and rejecting unrelated runtime, schema, package-metadata and canonical Hero changes. The frozen Phase 0 specification hashes are unchanged. Sixteen frozen specifications and seven packaged resource copies remain protected. @@ -46,7 +47,7 @@ Dispatch `ci.yml` with `gate=candidate` for a stable candidate. The workflow run Compatibility cells run complete canonical regression with `--ignore=tests/performance`. The designated performance job runs `tests/performance` once, avoiding nine repetitions of the costly 100k report -workload. The Phase 5 sparse-lineage test is still a placeholder in Step 3; +workload. The Phase 5 sparse-lineage test is still a placeholder in Step 4; it becomes an executable graph benchmark in Step 9. No placeholder is counted as a successful lineage measurement. diff --git a/scripts/release_check.py b/scripts/release_check.py index 69f88d6..2931d48 100644 --- a/scripts/release_check.py +++ b/scripts/release_check.py @@ -1,6 +1,6 @@ """Current source and candidate checks for Recursive Integrity Toolkit. -Step 3 opens the approved cycle, topology and depth implementation scope. Historical dispatch, source-body +Step 4 opens the approved root resolution and coverage implementation scope. Historical dispatch, source-body migrations and phase registries are recoverable from the accepted Git commit. Installed checks below retain their existing product, privacy and package cases. """ @@ -24,10 +24,10 @@ import zipfile ROOT = Path(__file__).resolve().parents[1] -ACCEPTED_COMMIT = "e40543ff5ed3d2793a41bf58d7e4831bd2a9bfe9" +ACCEPTED_COMMIT = "11192a3c353292f11d85eb3725c8edc73a71967b" PROTECTED_PREFIXES = ("src/", "schemas/", "examples/hero/") CURRENT_IMPLEMENTATION_PATHS = frozenset({ - "src/recursive_integrity_toolkit/lineage/cycles.py", + "src/recursive_integrity_toolkit/lineage/ancestry.py", }) PARQUET_CASES = {"test_PR002_parquet_real_roundtrip", "test_PR002_parquet_real_row_limit", "test_PR002_parquet_real_invalid_file"} HERO_NAMES = {"records_v1.csv", "records_v2.csv", "provenance.csv", "config.json", "version_order.json", "EXPECTED_OUTPUTS.md"} @@ -57,7 +57,7 @@ def _regular_file(root: Path, relative: str) -> Path: def verify_source_scope(root: Path = ROOT) -> dict: - """Reject product changes outside the currently authorized cycle/depth scope. + """Reject product changes outside the currently authorized ancestry scope. Later authorized implementation updates this single current boundary. It does not append another historical dispatcher or grant authority through metadata. @@ -67,7 +67,7 @@ def verify_source_scope(root: Path = ROOT) -> dict: for name, raw in expected.items(): current = _regular_file(root, name).read_bytes() if name not in CURRENT_IMPLEMENTATION_PATHS and current != raw: - raise ValueError(f"Unauthorized product mutation outside Step 3 scope: {name}") + raise ValueError(f"Unauthorized product mutation outside Step 4 scope: {name}") for prefix in PROTECTED_PREFIXES: paths = list((root / prefix).rglob("*")) if (root / prefix).is_symlink() or any(path.is_symlink() for path in paths): @@ -75,7 +75,7 @@ def verify_source_scope(root: Path = ROOT) -> dict: actual = {path.relative_to(root).as_posix() for path in paths if path.is_file() and "__pycache__" not in path.parts} if actual != {name for name in expected if name.startswith(prefix)}: - raise ValueError(f"Unauthorized product file-set mutation during Step 3: {prefix}") + raise ValueError(f"Unauthorized product file-set mutation during Step 4: {prefix}") return {"protected_product_files": len(expected) - len(CURRENT_IMPLEMENTATION_PATHS), "authorized_implementation_files": len(CURRENT_IMPLEMENTATION_PATHS), "product_file_inventory": "unchanged"} diff --git a/src/recursive_integrity_toolkit/lineage/ancestry.py b/src/recursive_integrity_toolkit/lineage/ancestry.py index f423e84..66ed8a9 100644 --- a/src/recursive_integrity_toolkit/lineage/ancestry.py +++ b/src/recursive_integrity_toolkit/lineage/ancestry.py @@ -1,20 +1,371 @@ -"""Own future external-root tracing, ancestry incidence, fractional root mass, and concentration. +"""Resolve exact external roots and target ancestry coverage. -Owner IDs: - T4 +Owner IDs: T4; Phase 5 Step 4. -Future inputs: - Future validated acyclic lineage graph and explicit external-root classifications. +Validated explicit grounding supplies anchors. Iterative dependency scheduling +unions complete root sets; unknown branches never supply exact roots. Logical +membership and candidate-visit budgets bound propagation before each work unit. +The allocation is topological and makes no causal or scientific quality claim. -Future outputs: - Future root sets, incidence, ancestry HHI, effective roots, and lineage coverage. +Current phase status: + Phase 5 Step 4 root resolution, G/C/U classification and target coverage. + Incidence, fractional mass, concentration, bounds and reports remain later + steps. No file or network I/O, generation calculation or implicit invocation. +""" +from __future__ import annotations -Assumptions: - Ancestry concentration is topological and does not establish causal contribution. +from dataclasses import dataclass, replace +from heapq import heapify, heappop, heappush +from types import MappingProxyType +from typing import Literal -Limits: - No root tracing, allocation, concentration, coverage, or graph traversal is implemented here. +from ..errors import ErrorCode, LineageResourceLimitError +from ..io.validation import join_provenance +from ..models import BundleValidationResult, RecordKey, ValidationMessage, ValidationSeverity +from ..result import ExecutionStatus +from .cycles import CycleAnalysis, DepthAssessment, analyze_cycles, _local_depth_reasons +from .graph import ( + LineageGraph, LineageLimits, LineageResourceUsage, LineageScope, + _integer, _invalid, _key, _messages, build_lineage_graph, +) -Current phase status: - Phase 1 package scaffold only. Import-safe. No analytical or data-processing logic. -""" + +_ROOT_REASONS = frozenset({ + "MISSING_PROVENANCE", "MISSING_REQUIRED_PROVENANCE", "UNKNOWN_GROUNDING", + "PARENT_DECLARATION_UNAVAILABLE", "INVALID_PARENT_REFERENCE", + "UNRESOLVED_PARENT_REFERENCE", "VERSION_ORDER_UNAVAILABLE", "CYCLE_AFFECTED", + "INCOMPLETE_PARENT_ANCESTRY", "GROUNDED_PARENT_RULE_UNSUPPORTED", +}) +_EXECUTION_REASONS = frozenset({ + "INVALID_LINEAGE_INPUT", "UNRESOLVED_ANCESTRY", "LINEAGE_RESOURCE_LIMIT_EXCEEDED", +}) + + +def _reasons(values: object, allowed: frozenset[str]) -> tuple[str, ...]: + if (type(values) is not tuple + or any(type(value) is not str or value not in allowed for value in values) + or values != tuple(sorted(set(values)))): + raise _invalid("ancestry reasons require canonical known codes") + return values + + +def _coverage(value: object, expected: float | None) -> bool: + return value is None if expected is None else type(value) is float and value == expected + + +def _execution(messages: tuple[ValidationMessage, ...], complete: int, unresolved: int): + if any(message.severity is ValidationSeverity.FATAL for message in messages): + return ExecutionStatus.FAILED, ("INVALID_LINEAGE_INPUT",) + if any(message.severity is ValidationSeverity.ERROR for message in messages): + return (ExecutionStatus.PARTIAL if complete else ExecutionStatus.FAILED), ("INVALID_LINEAGE_INPUT",) + if unresolved: + return ExecutionStatus.PARTIAL, ("UNRESOLVED_ANCESTRY",) + return ExecutionStatus.COMPLETED, () + + +@dataclass(frozen=True, slots=True) +class RecordAncestry: + """Exact complete roots, including known-empty, or explicit uncertainty.""" + record_key: RecordKey + classification: Literal["grounded", "closed", "unresolved"] + external_root_keys: frozenset[RecordKey] | None + reason_codes: tuple[str, ...] + lineage_depth: int | None + depth_reason_codes: tuple[str, ...] + + def __post_init__(self) -> None: + _key(self.record_key) + if type(self.classification) is not str: + raise _invalid("ancestry classification requires a literal status") + reasons = _reasons(self.reason_codes, _ROOT_REASONS) + roots = self.external_root_keys + if roots is None: + if self.classification != "unresolved" or not reasons: + raise _invalid("unresolved ancestry requires null roots and explicit reasons") + else: + if type(roots) is not frozenset: + raise _invalid("complete ancestry requires an immutable root set") + for root in roots: + _key(root) + if reasons or self.classification != ("grounded" if roots else "closed"): + raise _invalid("ancestry classification disagrees with its complete root set") + if self.lineage_depth is None: + raise _invalid("complete ancestry requires complete structural paths") + DepthAssessment(self.lineage_depth, self.depth_reason_codes) + + +@dataclass(frozen=True, slots=True) +class LineageAnalysisResult: + """Step 4 targets and completed structural observations. + + Resource-aborted root propagation supplies no partition or record rows. + Earlier graph/depth/reference observations survive. Node/edge admission + failures precede this result and retain the graph typed exception contract. + """ + scope: LineageScope + execution_status: ExecutionStatus + execution_reason_codes: tuple[str, ...] + records: tuple[RecordAncestry, ...] | None + cycles: CycleAnalysis + grounded_record_count: int | None + closed_record_count: int | None + unresolved_record_count: int | None + records_with_resolved_external_ancestry: int | None + resolved_lineage_coverage: float | None + external_ancestry_coverage: float | None + ancestry_coverage_reason_codes: tuple[str, ...] + declared_parent_reference_count: int | None + resolved_parent_reference_count: int | None + unresolved_parent_reference_count: int | None + resolved_parent_edge_coverage: float | None + no_declared_parents: bool + reference_coverage_reason_codes: tuple[str, ...] + resource_usage: LineageResourceUsage + messages: tuple[ValidationMessage, ...] + + def __post_init__(self) -> None: + if type(self.scope) is not LineageScope or type(self.cycles) is not CycleAnalysis: + raise _invalid("ancestry analysis requires typed scope and cycle observations") + scope = replace(self.scope) + cycles = replace(self.cycles) + messages = _messages(self.messages) + if cycles.scope != scope: + raise _invalid("ancestry and structural scopes disagree") + if not set(cycles.messages) <= set(messages): + raise _invalid("ancestry must preserve completed structural diagnostics") + if type(self.resource_usage) is not LineageResourceUsage: + raise _invalid("ancestry analysis requires typed resource usage") + usage = replace(self.resource_usage) + if usage.admitted_node_count != scope.loaded_record_count: + raise _invalid("ancestry resource usage disagrees with the loaded scope") + if type(self.execution_status) is not ExecutionStatus: + raise _invalid("ancestry analysis requires a typed execution status") + _reasons(self.execution_reason_codes, _EXECUTION_REASONS) + counts = (self.grounded_record_count, self.closed_record_count, + self.unresolved_record_count, self.records_with_resolved_external_ancestry) + if self.records is None: + if (self.execution_status is not ExecutionStatus.FAILED + or self.execution_reason_codes != ("LINEAGE_RESOURCE_LIMIT_EXCEEDED",) + or usage.exhausted_limit not in ("max_root_memberships", "max_root_union_visits") + or any(value is not None for value in counts) + or self.resolved_lineage_coverage is not None + or self.external_ancestry_coverage is not None + or self.ancestry_coverage_reason_codes != ("LINEAGE_RESOURCE_LIMIT_EXCEEDED",) + or not any(message.code == ErrorCode.LINEAGE_RESOURCE_LIMIT_EXCEEDED.value + and message.severity in (ValidationSeverity.ERROR, ValidationSeverity.FATAL) + for message in messages)): + raise _invalid("aborted ancestry cannot expose a partial partition as exact") + else: + if type(self.records) is not tuple or usage.exhausted_limit is not None: + raise _invalid("complete ancestry rows require an immutable completed partition") + for record in self.records: + if type(record) is not RecordAncestry: + raise _invalid("ancestry records require typed assessments") + replace(record) + depth = cycles.depths_by_record.get(record.record_key) + if depth != DepthAssessment(record.lineage_depth, record.depth_reason_codes): + raise _invalid("ancestry depth disagrees with structural observations") + if record.external_root_keys is not None and any( + root not in cycles.depths_by_record for root in record.external_root_keys): + raise _invalid("external roots require loaded canonical identities") + if tuple(record.record_key for record in self.records) != scope.target_record_keys: + raise _invalid("ancestry records must cover the entire canonical target scope") + for count in counts: + _integer(count) + ground = sum(record.classification == "grounded" for record in self.records) + closed = sum(record.classification == "closed" for record in self.records) + unresolved = len(self.records) - ground - closed + if counts != (ground, closed, unresolved, ground + closed): + raise _invalid("ancestry counts disagree with the complete target partition") + total = scope.target_record_count + if (not _coverage(self.resolved_lineage_coverage, (ground + closed) / total if total else None) + or not _coverage(self.external_ancestry_coverage, ground / total if total else None) + or self.ancestry_coverage_reason_codes != (() if total else ("EMPTY_TARGET_SCOPE",))): + raise _invalid("ancestry coverage must use the full explicit target denominator") + if (self.execution_status, self.execution_reason_codes) != _execution(messages, ground + closed, unresolved): + raise _invalid("ancestry execution status disagrees with its target partition and diagnostics") + references = (self.declared_parent_reference_count, self.resolved_parent_reference_count, + self.unresolved_parent_reference_count) + if type(self.no_declared_parents) is not bool: + raise _invalid("ancestry declaration status requires an explicit boolean") + if self.declared_parent_reference_count is None: + if (any(value is not None for value in references) + or self.resolved_parent_edge_coverage is not None or self.no_declared_parents + or self.reference_coverage_reason_codes != ("PARENT_REFERENCE_COUNT_UNAVAILABLE",)): + raise _invalid("unknown reference cardinality cannot imply zero declared parents") + else: + for count in references: + _integer(count) + declared, resolved, unresolved = references + if (declared != resolved + unresolved or self.no_declared_parents != (declared == 0) + or not _coverage(self.resolved_parent_edge_coverage, resolved / declared if declared else 1.0) + or self.reference_coverage_reason_codes != ()): + raise _invalid("ancestry reference coverage disagrees with original declarations") + object.__setattr__(self, "scope", scope) + object.__setattr__(self, "cycles", cycles) + object.__setattr__(self, "resource_usage", usage) + object.__setattr__(self, "messages", messages) + + @property + def lineage_depth(self) -> int | None: + return self.cycles.lineage_depth + + @property + def maximum_resolved_target_depth(self) -> int | None: + return self.cycles.maximum_resolved_target_depth + + @property + def depth_resolved_record_count(self) -> int: + return self.cycles.depth_resolved_record_count + + +class _RootBudget: + """Charge each logical unit before performing it; never expose partial roots.""" + def __init__(self, usage: LineageResourceUsage) -> None: + self.initial = usage + self.memberships = 0 + self.visits = 0 + + def usage(self, exhausted: str | None = None) -> LineageResourceUsage: + attempted = None if exhausted is None else getattr(self.initial.limits, exhausted) + 1 + return replace(self.initial, stored_root_membership_count=self.memberships, + root_union_visit_count=self.visits, exhausted_limit=exhausted, + attempted_value=attempted) + + def membership(self) -> None: + if self.memberships == self.initial.limits.max_root_memberships: + raise LineageResourceLimitError(self.usage("max_root_memberships")) + self.memberships += 1 + + def visit(self) -> None: + if self.visits == self.initial.limits.max_root_union_visits: + raise LineageResourceLimitError(self.usage("max_root_union_visits")) + self.visits += 1 + + +def _metadata(validation: BundleValidationResult, graph: LineageGraph): + """Reassess original metadata while the shared graph owns parent syntax. + + Excluding parent_ids from this metadata-only join preserves the graph's + per-record malformed declaration failures. No declaration is reinterpreted. + """ + if not graph.node_keys: + return {}, () + provenance = None if validation.provenance is None else tuple( + replace(row, values=MappingProxyType({name: value for name, value in row.values.items() + if name != "parent_ids"})) + for row in validation.provenance) + promoted = validation.parent_validation.promoted_warning_codes + joined = join_provenance(validation.records, provenance, strict_mode=bool(promoted), + strict_warning_codes=promoted) + return {match.record_key: match for match in joined.matches}, joined.messages + + +def _resolve_roots(graph: LineageGraph, cycles: CycleAnalysis, metadata, budget: _RootBudget): + affected = set(cycles.affected_record_keys) + roots: dict[RecordKey, frozenset[RecordKey] | None] = {} + reasons: dict[RecordKey, tuple[str, ...]] = {} + for key in cycles.affected_record_keys: + roots[key] = None + reasons[key] = tuple(sorted(_local_depth_reasons(graph, key) | {"CYCLE_AFFECTED"})) + indegree = {key: len(graph.parents_by_child[key]) for key in graph.node_keys if key not in affected} + pending = [key for key, count in indegree.items() if count == 0] + heapify(pending) + while pending: + key = heappop(pending) + parents = graph.parents_by_child[key] + failures = _local_depth_reasons(graph, key) + row = metadata[key].provenance + if row is None: + failures.add("MISSING_PROVENANCE") + else: + if not row.required_fields_valid: + failures.add("MISSING_REQUIRED_PROVENANCE") + grounding = row.values.get("external_grounding") + if grounding == "unknown": + failures.add("UNKNOWN_GROUNDING") + if (grounding == "yes" and parents + and (len(parents) != 1 or row.values.get("transformation") != "carryover")): + failures.add("GROUNDED_PARENT_RULE_UNSUPPORTED") + for parent in parents: + if roots[parent] is None: + failures.add("INCOMPLETE_PARENT_ANCESTRY") + failures.update(reasons[parent]) + if failures: + roots[key] = None + reasons[key] = tuple(sorted(failures)) + elif not parents: + if row.values["external_grounding"] == "yes": + budget.membership() + roots[key] = frozenset((key,)) + else: + roots[key] = frozenset() + reasons[key] = () + else: + combined = set() + for parent in parents: + for root in sorted(roots[parent]): + budget.visit() + if root not in combined: + budget.membership() + combined.add(root) + roots[key] = frozenset(combined) + reasons[key] = () + for child in graph.children_by_parent[key]: + if child in indegree: + indegree[child] -= 1 + if indegree[child] == 0: + heappush(pending, child) + return roots, reasons + + +def analyze_lineage( + validation: BundleValidationResult, *, target_dataset_version: str | None, + limits: LineageLimits = LineageLimits(), +) -> LineageAnalysisResult: + """Compute explicit Step 4 ancestry, stopping dependent work on exhaustion. + + All loaded context nodes participate in resolution and resource accounting. + Only selected targets enter returned record rows, G/C/U and coverage. + Canonical dependency-ready nodes precede canonical parent and root visits. + """ + graph = build_lineage_graph(validation, target_dataset_version=target_dataset_version, limits=limits) + cycles = analyze_cycles(graph) + metadata, metadata_messages = _metadata(validation, graph) + messages = _messages(cycles.messages + metadata_messages) + target_evidence = tuple(graph.parent_evidence_by_record[key] for key in graph.scope.target_record_keys) + count_known = all(item.declared_reference_count is not None for item in target_evidence) + declared = sum(item.declared_reference_count for item in target_evidence) if count_known else None + resolved = sum(item.resolved_reference_count for item in target_evidence) if count_known else None + reference_fields = dict( + declared_parent_reference_count=declared, resolved_parent_reference_count=resolved, + unresolved_parent_reference_count=None if declared is None else declared - resolved, + resolved_parent_edge_coverage=None if declared is None else (resolved / declared if declared else 1.0), + no_declared_parents=declared == 0, + reference_coverage_reason_codes=() if count_known else ("PARENT_REFERENCE_COUNT_UNAVAILABLE",), + ) + budget = _RootBudget(graph.resource_usage) + try: + roots, reasons = _resolve_roots(graph, cycles, metadata, budget) + except LineageResourceLimitError as error: + messages += (ValidationMessage(ErrorCode.LINEAGE_RESOURCE_LIMIT_EXCEEDED.value, + ValidationSeverity.ERROR, "Lineage root propagation exhausted its resource budget."),) + return LineageAnalysisResult( + graph.scope, ExecutionStatus.FAILED, (error.reason_code,), None, cycles, + None, None, None, None, None, None, (error.reason_code,), + **reference_fields, resource_usage=error.resource_usage, messages=messages) + records = tuple(RecordAncestry( + key, "unresolved" if roots[key] is None else "grounded" if roots[key] else "closed", + roots[key], reasons[key], cycles.depths_by_record[key].lineage_depth, + cycles.depths_by_record[key].reason_codes) for key in graph.scope.target_record_keys) + ground = sum(record.classification == "grounded" for record in records) + closed = sum(record.classification == "closed" for record in records) + unresolved = len(records) - ground - closed + status, execution_reasons = _execution(messages, ground + closed, unresolved) + total = graph.scope.target_record_count + return LineageAnalysisResult( + graph.scope, status, execution_reasons, records, cycles, + ground, closed, unresolved, ground + closed, + (ground + closed) / total if total else None, ground / total if total else None, + () if total else ("EMPTY_TARGET_SCOPE",), + **reference_fields, resource_usage=budget.usage(), messages=messages) diff --git a/tests/integration/test_current_verification.py b/tests/integration/test_current_verification.py index 6a9309e..1fa4ede 100644 --- a/tests/integration/test_current_verification.py +++ b/tests/integration/test_current_verification.py @@ -27,7 +27,7 @@ def protected_copy(repo_root, tmp_path): @pytest.mark.parametrize("mutation", ["formula", "new_module", "deleted_module"]) def test_current_scope_rejects_unauthorized_product_changes(current_tools, protected_copy, mutation): - """Cycle/depth implementation does not authorize unrelated product changes.""" + """Root/coverage implementation does not authorize unrelated product changes.""" verify = current_tools["verify_source_scope"] verify(protected_copy) target = protected_copy / "src/recursive_integrity_toolkit/metrics/diversity.py" diff --git a/tests/unit/test_T4_ancestry.py b/tests/unit/test_T4_ancestry.py index 1d75e02..798a77f 100644 --- a/tests/unit/test_T4_ancestry.py +++ b/tests/unit/test_T4_ancestry.py @@ -1,4 +1,530 @@ -"""Reserved for direct Phase 5 ancestry behavior tests. +"""T4 complete external roots, target classification and evidence boundaries. -Step 1 records independently authored fixtures before implementation. +Frozen fixture root sets and rational coverages predate implementation. These +tests make no incidence, concentration, bound, report or performance claim. """ + +from dataclasses import FrozenInstanceError, replace +from fractions import Fraction +import json +from pathlib import Path +import socket +from types import MappingProxyType + +import pytest + +from recursive_integrity_toolkit.config import RepresentationConfig +from recursive_integrity_toolkit.errors import CanonicalValidationError +from recursive_integrity_toolkit.io.validation import ( + assess_provenance_row, join_provenance, resolve_parent_batch, + resolve_version_order, validate_bundle, +) +from recursive_integrity_toolkit.lineage import ancestry +from recursive_integrity_toolkit.models import ( + AuditBundle, FileRole, InputSource, RecordKey, ValidationMessage, ValidationSeverity, +) +from recursive_integrity_toolkit.representations.field import assign_field_states +from recursive_integrity_toolkit.result import ExecutionStatus + + +_ABSENT = object() +_MISSING = object() +_ROOT = Path(__file__).resolve().parents[2] +_FIXTURES = _ROOT / "tests" / "fixtures" +_CASES = [ + case + for directory in ("lineage_complete", "lineage_multi_root", "lineage_cycles") + for case in json.loads((_FIXTURES / directory / "cases.json").read_text())["cases"] +] +_FIXTURE_REASONS = { + "unknown_grounding": "UNKNOWN_GROUNDING", + "undeclared_parent_boundary": "PARENT_DECLARATION_UNAVAILABLE", + "unsupported_grounded_parent_rule": "GROUNDED_PARENT_RULE_UNSUPPORTED", + "missing_parent": "UNRESOLVED_PARENT_REFERENCE", +} + + +def _key(identity): + return RecordKey.parse(identity) + + +def _load_rows(tmp_path, records, provenance, *, target="v2", order=None, strict=False): + """Construct real validated inputs; only empty selection needs a typed role.""" + tmp_path.mkdir(parents=True, exist_ok=True) + primary = [row for row in records if row["dataset_version"] == target] + context = [row for row in records if row["dataset_version"] != target] + empty_target = not primary + if empty_target: + primary, context = context, [] + sources = [] + for name, role, rows in ( + ("primary.jsonl", FileRole.RECORDS_PRIMARY, primary), + ("context.jsonl", FileRole.RECORDS_COMPARE, context), + ("provenance.jsonl", FileRole.PROVENANCE_MANIFEST, provenance), + ): + if rows: + path = tmp_path / name + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + sources.append(InputSource(role, path)) + if order is not None: + path = tmp_path / "order.json" + path.write_text(json.dumps({"version_order": order}), encoding="utf-8") + sources.append(InputSource(FileRole.VERSION_ORDER, path)) + config = {"strict_mode": True, "strict_warning_codes": ["W_PARENT_UNRESOLVED"]} if strict else None + result = validate_bundle(AuditBundle(tuple(sources)), configuration=config) + if empty_target: + # The API permits an empty selection over loaded context. Current CLI + # ingestion still requires a primary file, so change only role metadata. + result = replace(result, records=tuple( + replace(row, location=replace(row.location, file_role=FileRole.RECORDS_COMPARE)) + for row in result.records + )) + return result + + +def _validation(tmp_path, declarations, *, grounding=None, transformation=None, + target="v2", chronology=True, strict=False, reverse=False): + records, provenance = [], [] + for identity, parents in declarations.items(): + key = _key(identity) + identity_fields = {"dataset_version": key.dataset_version, "record_id": key.record_id} + records.append({**identity_fields, "content": "Private ancestry fixture"}) + if parents is _MISSING: + continue + grounding_value = (grounding or {}).get(identity, "no") + row = { + **identity_fields, "source_type": "human" if grounding_value == "yes" else "synthetic", + "provenance_confidence": "confirmed", "external_grounding": grounding_value, + "transformation": (transformation or {}).get(identity, "generate"), + } + if parents is not _ABSENT: + row["parent_ids"] = list(reversed(parents)) if reverse and isinstance(parents, list) else parents + provenance.append(row) + if reverse: + records.reverse() + provenance.reverse() + versions = sorted({row["dataset_version"] for row in records} | ({target} if target else set())) + return _load_rows(tmp_path, records, provenance, target=target, + order=versions if chronology else None, strict=strict) + + +def _fixture_validation(tmp_path, case): + targets = case["target_record_keys"] + target = _key(targets[0]).dataset_version if targets else None + return _load_rows(tmp_path, case["records"], case["provenance"], + target=target, order=case["version_order"]), target + + +def _by_key(result): + assert result.records is not None + return {str(item.record_key): item for item in result.records} + + +def _analyze(validation, target="v2"): + return ancestry.analyze_lineage(validation, target_dataset_version=target) + + +def _replace_provenance(validation, rows, *, rebuild_join=False): + keys = tuple(sorted(row.record_key for row in validation.records)) + batch = resolve_parent_batch(keys, rows, version_order=validation.version_order) + changes = {"provenance": rows, "parent_validation": batch, "generation": None} + if rebuild_join: + changes["provenance_join"] = join_provenance(validation.records, rows) + return replace(validation, **changes) + + +@pytest.mark.parametrize("case", _CASES, ids=lambda case: case["case_id"]) +def test_frozen_ancestry_oracles_cover_complete_sets_and_target_partition(case, tmp_path): + validation, target = _fixture_validation(tmp_path, case) + result = _analyze(validation, target) + expected = case["expected_lineage"] + counts = expected["counts"] + records = _by_key(result) + assert tuple(records) == tuple(case["target_record_keys"]) + assert result.scope.target_record_count == counts["target"] + assert result.scope.loaded_record_count == len(case["records"]) + assert result.scope.context_record_count == len(case["records"]) - counts["target"] + assert (result.grounded_record_count, result.closed_record_count, result.unresolved_record_count) == ( + counts["grounded"], counts["closed"], counts["unresolved"]) + assert result.records_with_resolved_external_ancestry == counts["grounded"] + counts["closed"] + assert sum((result.grounded_record_count, result.closed_record_count, result.unresolved_record_count)) == len(records) + actual_complete = {key: sorted(map(str, item.external_root_keys)) for key, item in records.items() + if item.external_root_keys is not None} + assert actual_complete == expected["complete_root_sets"] + for key, item in records.items(): + if key not in expected["complete_root_sets"]: + assert item.classification == "unresolved" + assert item.external_root_keys is None + assert item.reason_codes + else: + assert item.classification == ("grounded" if expected["complete_root_sets"][key] else "closed") + assert item.reason_codes == () + for reason in expected.get("unresolved_reasons", {}).get(key, ()): + assert _FIXTURE_REASONS[reason] in item.reason_codes + if key in expected.get("depth_by_record", {}): + assert item.lineage_depth == expected["depth_by_record"][key] + for fixture_name, result_name in ( + ("resolved_lineage_coverage", "resolved_lineage_coverage"), + ("external_ancestry_coverage", "external_ancestry_coverage"), + ("edge_coverage", "resolved_parent_edge_coverage"), + ): + if fixture_name in expected: + value = getattr(result, result_name) + if expected[fixture_name] is None: + assert value is None + else: + assert value == pytest.approx(float(Fraction(expected[fixture_name]))) + if "no_declared_parents" in expected: + assert result.no_declared_parents is expected["no_declared_parents"] + input_expected = case["expected_input"] + if "target_declared_references" in input_expected: + assert result.declared_parent_reference_count == input_expected["target_declared_references"] + assert result.resolved_parent_reference_count == input_expected["target_resolved_references"] + if expected.get("lineage_error") or counts["unresolved"]: + assert result.execution_status is ExecutionStatus.PARTIAL + else: + assert result.execution_status is ExecutionStatus.COMPLETED + assert result.cycles.detected is expected.get("cycle_detected", False) + + +def test_hero_all_targets_have_the_frozen_single_root_assignments(): + hero = _ROOT / "examples" / "hero" + expected = json.loads((_FIXTURES / "lineage_complete" / "hero_expected.json").read_text()) + validation = validate_bundle(AuditBundle(( + InputSource(FileRole.RECORDS_PRIMARY, hero / "records_v2.csv"), + InputSource(FileRole.RECORDS_COMPARE, hero / "records_v1.csv"), + InputSource(FileRole.PROVENANCE_MANIFEST, hero / "provenance.csv"), + InputSource(FileRole.VERSION_ORDER, hero / "version_order.json"), + ))) + result = _analyze(validation) + assert result.execution_status is ExecutionStatus.COMPLETED + assert result.scope.target_record_count == expected["target_records"] + assert result.scope.loaded_record_count == expected["loaded_records"] + assert (result.grounded_record_count, result.closed_record_count, result.unresolved_record_count) == (8, 0, 0) + expected_roots = { + "v2::v2_01": "v1::v1_01", "v2::v2_02": "v1::v1_02", + "v2::v2_03": "v1::v1_03", "v2::v2_04": "v1::v1_04", + "v2::v2_05": "v1::v1_01", "v2::v2_06": "v1::v1_02", + "v2::v2_07": "v1::v1_07", "v2::v2_08": "v1::v1_01", + } + assert {key: item.external_root_keys for key, item in _by_key(result).items()} == { + key: frozenset({_key(root)}) for key, root in expected_roots.items() + } + assert result.declared_parent_reference_count == expected["declared_target_references"] + assert result.resolved_parent_reference_count == expected["resolved_target_references"] + assert result.resolved_lineage_coverage == result.external_ancestry_coverage == 1.0 + + +def test_missing_provenance_unknown_and_undeclared_boundaries_never_become_empty_roots(tmp_path): + result = _analyze(_validation(tmp_path, { + "v2::absent": _ABSENT, "v2::null": None, "v2::missing": _MISSING, + "v2::unknown": [], "v2::closed": [], "v2::anchor": [], + }, grounding={"v2::unknown": "unknown", "v2::anchor": "yes"})) + records = _by_key(result) + for identity in ("v2::absent", "v2::null", "v2::missing", "v2::unknown"): + assert records[identity].external_root_keys is None + assert records["v2::closed"].external_root_keys == frozenset() + assert records["v2::anchor"].external_root_keys == frozenset({_key("v2::anchor")}) + assert "MISSING_PROVENANCE" in records["v2::missing"].reason_codes + assert records["v2::unknown"].lineage_depth == 0 + assert result.no_declared_parents is True + assert result.resolved_parent_edge_coverage == 1.0 + assert (result.grounded_record_count, result.closed_record_count, result.unresolved_record_count) == (1, 1, 4) + assert result.resolved_lineage_coverage == pytest.approx(1 / 3) + assert result.external_ancestry_coverage == pytest.approx(1 / 6) + + +@pytest.mark.parametrize("parent_grounding,expected_roots", [("yes", {"v1::a"}), ("no", set()), ("unknown", None)]) +def test_carryover_inherits_single_canonical_parent_including_known_empty(tmp_path, parent_grounding, expected_roots): + result = _analyze(_validation(tmp_path, {"v1::a": [], "v2::carry": ["a", "v1::a", "a"]}, + grounding={"v1::a": parent_grounding, "v2::carry": "yes"}, + transformation={"v2::carry": "carryover"})) + record = _by_key(result)["v2::carry"] + assert record.external_root_keys == (None if expected_roots is None else frozenset(map(_key, expected_roots))) + assert record.lineage_depth == 1 + assert result.declared_parent_reference_count == result.resolved_parent_reference_count == 3 + + +@pytest.mark.parametrize("transformation,parents", [ + ("generate", ["v1::a"]), ("carryover", ["v1::a", "v1::b"]), +]) +def test_unsupported_grounded_parent_rule_never_mints_new_root(tmp_path, transformation, parents): + result = _analyze(_validation(tmp_path, {"v1::a": [], "v1::b": [], "v2::child": parents}, + grounding={"v1::a": "yes", "v1::b": "yes", "v2::child": "yes"}, + transformation={"v2::child": transformation})) + child = _by_key(result)["v2::child"] + assert child.external_root_keys is None + assert "GROUNDED_PARENT_RULE_UNSUPPORTED" in child.reason_codes + assert child.lineage_depth == 1 + assert result.execution_status is ExecutionStatus.PARTIAL + + +def test_unresolved_branch_propagates_without_promoting_observed_roots(tmp_path): + result = _analyze(_validation(tmp_path, { + "v1::a": [], "v1::u": [], "v2::incomplete": ["v1::a", "v1::u", "v1::missing"], + "v2::next": ["v2::incomplete", "v1::a"], "v2::closed": [], + }, grounding={"v1::a": "yes", "v1::u": "unknown"})) + records = _by_key(result) + for key in ("v2::incomplete", "v2::next"): + assert records[key].external_root_keys is None + assert records[key].classification == "unresolved" + assert {"UNKNOWN_GROUNDING", "UNRESOLVED_PARENT_REFERENCE"} <= set(records[key].reason_codes) + assert (result.grounded_record_count, result.closed_record_count, result.unresolved_record_count) == (0, 1, 2) + + +def test_unknown_chronology_preserves_identity_coverage_and_blocks_exact_ancestry(tmp_path): + result = _analyze(_validation(tmp_path, {"v10::anchor": [], "v2::child": ["v10::anchor"]}, + grounding={"v10::anchor": "yes"}, chronology=False)) + record = _by_key(result)["v2::child"] + assert record.external_root_keys is None + assert "VERSION_ORDER_UNAVAILABLE" in record.reason_codes + assert record.lineage_depth is None + assert result.declared_parent_reference_count == result.resolved_parent_reference_count == 1 + assert result.resolved_parent_edge_coverage == 1.0 + assert result.cycles.cycle_count == 0 + assert result.execution_status is ExecutionStatus.PARTIAL + + +@pytest.mark.parametrize("invalid,error_code", [ + ("p", "E_PARENT_AMBIGUOUS"), ("v3::future", "E_PARENT_FUTURE_VERSION"), + (" malformed ", "E_PARENT_FORMAT"), ("v2::child", "E_LINEAGE_CYCLE"), +]) +def test_invalid_parent_with_valid_sibling_never_yields_partial_exact_root_set(tmp_path, invalid, error_code): + result = _analyze(_validation(tmp_path, { + "v1::a": [], "v1::p": [], "v2::p": [], "v3::future": [], + "v2::child": ["v1::a", invalid], + }, grounding={"v1::a": "yes"})) + assert _by_key(result)["v2::child"].external_root_keys is None + assert result.declared_parent_reference_count == 2 + assert result.resolved_parent_reference_count == 1 + assert result.unresolved_parent_reference_count == 1 + assert result.resolved_parent_edge_coverage == 0.5 + assert error_code in {message.code for message in result.messages} + assert result.execution_status is ExecutionStatus.PARTIAL # Independent target p is known closed. + + +@pytest.mark.parametrize("safe_target,expected_status", [(False, ExecutionStatus.FAILED), (True, ExecutionStatus.PARTIAL)]) +def test_strict_missing_parent_preserves_severity_and_usable_target_rule(tmp_path, safe_target, expected_status): + declarations = {"v2::child": ["missing"]} + if safe_target: + declarations["v2::closed"] = [] + ordinary = _analyze(_validation(tmp_path / "ordinary", declarations)) + strict = _analyze(_validation(tmp_path / "strict", declarations, strict=True)) + assert ordinary.execution_status is ExecutionStatus.PARTIAL + assert strict.execution_status is expected_status + assert ordinary.records == strict.records + assert strict.unresolved_record_count == 1 + assert strict.closed_record_count == int(safe_target) + assert [m.severity for m in ordinary.messages if m.code == "W_PARENT_UNRESOLVED"] == [ValidationSeverity.WARNING] + assert [m.severity for m in strict.messages if m.code == "W_PARENT_UNRESOLVED"] == [ValidationSeverity.ERROR] + + +def test_all_cycle_targets_fail_without_discarding_completed_partition(tmp_path): + result = _analyze(_validation(tmp_path, {"v2::a": ["v2::b"], "v2::b": ["v2::a"]})) + assert result.execution_status is ExecutionStatus.FAILED + assert (result.grounded_record_count, result.closed_record_count, result.unresolved_record_count) == (0, 0, 2) + assert result.records_with_resolved_external_ancestry == 0 + assert result.resolved_lineage_coverage == result.external_ancestry_coverage == 0.0 + assert result.resolved_parent_edge_coverage == 1.0 + assert all(item.external_root_keys is None and "CYCLE_AFFECTED" in item.reason_codes for item in result.records) + + +@pytest.mark.parametrize("selection", [None, "v2"]) +def test_empty_selected_population_keeps_context_out_of_ratios(tmp_path, selection): + validation = _validation(tmp_path, {"v1::anchor": []}, target=None, + grounding={"v1::anchor": "yes"}) + if selection is not None: + order = resolve_version_order(("v1",), document={"version_order": ["v1", "v2"]}) + validation = replace(validation, version_order=order, + parent_validation=replace(validation.parent_validation, version_order=order)) + result = _analyze(validation, selection) + assert result.execution_status is ExecutionStatus.COMPLETED + assert result.scope.target_dataset_version == selection + assert result.scope.target_record_count == 0 + assert result.scope.context_record_count == 1 + assert result.records == () + assert (result.grounded_record_count, result.closed_record_count, result.unresolved_record_count) == (0, 0, 0) + assert result.resolved_lineage_coverage is result.external_ancestry_coverage is None + assert "EMPTY_TARGET_SCOPE" in result.ancestry_coverage_reason_codes + assert result.no_declared_parents is True + assert result.resolved_parent_edge_coverage == 1.0 + + +def test_missing_required_provenance_is_unresolved_with_error_and_intact_depth(tmp_path): + validation = _validation(tmp_path, {"v2::incomplete": [], "v2::closed": []}, + grounding={"v2::incomplete": "yes"}) + rows = [] + for row in validation.provenance: + if str(row.record_key) == "v2::incomplete": + values = dict(row.values) + del values["source_type"] + rows.append(assess_provenance_row(values, location=row.location)) + else: + rows.append(row) + result = _analyze(_replace_provenance(validation, tuple(rows), rebuild_join=True)) + incomplete = _by_key(result)["v2::incomplete"] + assert incomplete.external_root_keys is None + assert "MISSING_REQUIRED_PROVENANCE" in incomplete.reason_codes + assert incomplete.lineage_depth == 0 + assert result.execution_status is ExecutionStatus.PARTIAL + assert any(m.code == "E_SCHEMA_REQUIRED_FIELD" and m.severity is ValidationSeverity.ERROR for m in result.messages) + + +def test_record_and_provenance_metadata_namespaces_do_not_invalidate_grounding(tmp_path): + records = [{"dataset_version": "v2", "record_id": "anchor", "content": "Fixture", "batch_id": "one"}] + provenance = [{"dataset_version": "v2", "record_id": "anchor", "batch_id": "two", + "source_type": "human", "provenance_confidence": "confirmed", + "external_grounding": "yes", "parent_ids": []}] + validation = _load_rows(tmp_path, records, provenance, order=["v2"]) + assert validation.provenance_join.matches[0].conflicting_fields == ("batch_id",) + result = _analyze(validation) + record = _by_key(result)["v2::anchor"] + assert record.external_root_keys == frozenset({_key("v2::anchor")}) + assert record.reason_codes == () + assert record.lineage_depth == 0 + assert result.execution_status is ExecutionStatus.COMPLETED + assert validation.records[0].values["batch_id"] == "one" + assert validation.provenance[0].values["batch_id"] == "two" + + +@pytest.mark.parametrize("malformed,expected_declared,expected_resolved", [ + ("not-a-parent-list", None, None), (("v1::a", 7), 2, 1), +]) +def test_malformed_parent_container_never_becomes_zero_declared_parents(tmp_path, malformed, expected_declared, expected_resolved): + validation = _validation(tmp_path, {"v1::a": [], "v2::child": ["v1::a"], "v2::closed": []}, + grounding={"v1::a": "yes"}) + rows = tuple(replace(row, values=MappingProxyType({**row.values, "parent_ids": malformed})) + if str(row.record_key) == "v2::child" else row for row in validation.provenance) + result = _analyze(_replace_provenance(validation, rows)) + assert _by_key(result)["v2::child"].external_root_keys is None + assert result.declared_parent_reference_count == expected_declared + assert result.resolved_parent_reference_count == expected_resolved + assert result.no_declared_parents is False + if expected_declared is None: + assert result.unresolved_parent_reference_count is None + assert result.resolved_parent_edge_coverage is None + assert "PARENT_REFERENCE_COUNT_UNAVAILABLE" in result.reference_coverage_reason_codes + else: + assert result.unresolved_parent_reference_count == 1 + assert result.resolved_parent_edge_coverage == 0.5 + assert result.execution_status is ExecutionStatus.PARTIAL + + +def test_representation_ineligible_records_remain_in_target_population(tmp_path): + records = [{"dataset_version": "v2", "record_id": name, "content": "Fixture", "topic": None} + for name in ("anchor", "closed")] + provenance = [{"dataset_version": "v2", "record_id": name, "source_type": "unknown", + "provenance_confidence": "confirmed", "external_grounding": grounding, "parent_ids": []} + for name, grounding in (("anchor", "yes"), ("closed", "no"))] + validation = _load_rows(tmp_path, records, provenance, order=["v2"]) + represented = assign_field_states(validation.records, dataset_versions=("v2",), scope_id="missing-topic", + config=RepresentationConfig("topic", "topic_field", "topic", "fixture-v1", "exclude")) + assert represented.excluded_count == 2 + assert represented.included_count == 0 + result = _analyze(validation) + assert result.scope.target_record_count == 2 + assert (result.grounded_record_count, result.closed_record_count, result.unresolved_record_count) == (1, 1, 0) + assert result.external_ancestry_coverage == 0.5 + + +def test_results_are_permutation_invariant_and_immutable(tmp_path): + declarations = {"v1::b": [], "v1::a": [], "v2::child": ["v1::b", "a", "v1::a"]} + grounding = {"v1::a": "yes", "v1::b": "yes"} + left = _analyze(_validation(tmp_path / "left", declarations, grounding=grounding)) + right = _analyze(_validation(tmp_path / "right", declarations, grounding=grounding, reverse=True)) + assert left == right + record = left.records[0] + assert type(record.external_root_keys) is frozenset + with pytest.raises(FrozenInstanceError): + left.grounded_record_count = 0 + with pytest.raises(FrozenInstanceError): + record.external_root_keys = frozenset() + with pytest.raises(AttributeError): + record.external_root_keys.add(_key("v1::new")) + with pytest.raises(TypeError): + left.cycles.depths_by_record[_key("v2::child")] = None + + +def test_analysis_consumes_in_memory_evidence_without_io_or_content_leakage(tmp_path, monkeypatch): + validation = _validation(tmp_path, {"v1::a": [], "v2::child": ["v1::a", "secret-missing-parent"]}, + grounding={"v1::a": "yes"}) + def forbidden(*args, **kwargs): + raise AssertionError("Lineage must consume the validated in-memory handoff") + monkeypatch.setattr("builtins.open", forbidden) + monkeypatch.setattr(Path, "open", forbidden) + monkeypatch.setattr(socket, "socket", forbidden) + monkeypatch.setattr("recursive_integrity_toolkit.io.validation.validate_bundle", forbidden) + result = _analyze(validation) + rendered = repr(result) + assert "Private ancestry fixture" not in rendered + assert "secret-missing-parent" not in rendered + assert str(tmp_path) not in rendered + assert all(message.file_path is None for message in result.messages) + + +def test_validation_remains_input_only_after_ancestry_is_implemented(tmp_path, monkeypatch): + def forbidden(*args, **kwargs): + raise AssertionError("Input validation must never dispatch ancestry") + monkeypatch.setattr(ancestry, "analyze_lineage", forbidden) + validation = _validation(tmp_path, {"v2::a": [], "v2::b": ["v2::a"]}) + assert validation.parent_validation is not None + + +@pytest.mark.parametrize("mutation", ["missing_batch", "duplicate_record", "changed_parent_counter"]) +def test_forged_validation_handoff_cannot_certify_ancestry(tmp_path, mutation): + validation = _validation(tmp_path, {"v1::a": [], "v2::child": ["v1::a"]}, + grounding={"v1::a": "yes"}) + if mutation == "missing_batch": + validation = replace(validation, parent_validation=None) + elif mutation == "duplicate_record": + validation = replace(validation, records=validation.records + (validation.records[0],)) + else: + batch = validation.parent_validation + assessments = tuple(replace(item, resolved_reference_count=0) + if str(item.child_key) == "v2::child" else item for item in batch.assessments) + validation = replace(validation, parent_validation=replace(batch, assessments=assessments)) + with pytest.raises(CanonicalValidationError): + _analyze(validation) + + +def test_fatal_input_diagnostic_cannot_be_hidden_by_complete_target_roots(tmp_path): + validation = _validation(tmp_path, {"v2::anchor": []}, grounding={"v2::anchor": "yes"}) + fatal = ValidationMessage("E_SCHEMA_TYPE", ValidationSeverity.FATAL, "Malformed declared input") + result = _analyze(replace(validation, validation_messages=validation.validation_messages + (fatal,))) + assert result.execution_status is ExecutionStatus.FAILED + assert result.execution_reason_codes == ("INVALID_LINEAGE_INPUT",) + assert result.grounded_record_count == 1 + assert _by_key(result)["v2::anchor"].external_root_keys == frozenset({_key("v2::anchor")}) + assert any(message.severity is ValidationSeverity.FATAL for message in result.messages) + + +def test_typed_result_rejects_promoting_unknown_partition_to_completed(tmp_path): + result = _analyze(_validation(tmp_path, {"v2::unknown": []}, grounding={"v2::unknown": "unknown"})) + with pytest.raises(CanonicalValidationError): + replace(result, execution_status=ExecutionStatus.COMPLETED, execution_reason_codes=()) + + +@pytest.mark.parametrize("field", ["resolved_lineage_coverage", "external_ancestry_coverage", "resolved_parent_edge_coverage"]) +def test_typed_result_rejects_boolean_coverage_equal_to_one(tmp_path, field): + result = _analyze(_validation(tmp_path, {"v2::anchor": []}, grounding={"v2::anchor": "yes"})) + with pytest.raises(CanonicalValidationError): + replace(result, **{field: True}) + + +def test_typed_result_rejects_unloaded_root_identity_and_incomplete_complete_record(tmp_path): + result = _analyze(_validation(tmp_path, {"v2::anchor": []}, grounding={"v2::anchor": "yes"})) + with pytest.raises(CanonicalValidationError): + replace(result.records[0], lineage_depth=None, depth_reason_codes=("PARENT_DECLARATION_UNAVAILABLE",)) + forged_record = replace(result.records[0], external_root_keys=frozenset({_key("v1::unloaded")})) + with pytest.raises(CanonicalValidationError): + replace(result, records=(forged_record,)) + + +def test_typed_result_preserves_disconnected_cycle_error(tmp_path): + result = _analyze(_validation(tmp_path, { + "v1::a": ["v1::b"], "v1::b": ["v1::a"], "v2::anchor": [], + }, grounding={"v2::anchor": "yes"})) + assert result.execution_status is ExecutionStatus.PARTIAL + with pytest.raises(CanonicalValidationError): + replace(result, messages=tuple(message for message in result.messages if message.code != "E_LINEAGE_CYCLE"), + execution_status=ExecutionStatus.COMPLETED, execution_reason_codes=()) diff --git a/tests/unit/test_lineage_root_resources.py b/tests/unit/test_lineage_root_resources.py new file mode 100644 index 0000000..50df9af --- /dev/null +++ b/tests/unit/test_lineage_root_resources.py @@ -0,0 +1,221 @@ +"""Direct work-budget behavior for complete external-root propagation. + +Fixtures use canonical normalization, joining and batch reference validation. +They omit generation traversal so the long chain isolates iterative ancestry. +""" + +from dataclasses import FrozenInstanceError + +import pytest + +from recursive_integrity_toolkit.errors import LineageResourceLimitError +from recursive_integrity_toolkit.io.normalization import normalize_row +from recursive_integrity_toolkit.io.validation import ( + join_provenance, resolve_parent_batch, resolve_version_order, +) +from recursive_integrity_toolkit.lineage.ancestry import analyze_lineage +from recursive_integrity_toolkit.lineage.graph import LineageLimits +from recursive_integrity_toolkit.models import BundleValidationResult, RecordKey, ValidationSeverity +from recursive_integrity_toolkit.observability.levels import classify_observability +from recursive_integrity_toolkit.result import ExecutionStatus + + +def _key(identity): + return RecordKey.parse(identity) + + +def _validation(declarations, *, grounding=None, transformation=None, reverse=False): + """Construct a current validated input handoff without legacy generation.""" + records, provenance = [], [] + items = list(declarations.items()) + if reverse: + items.reverse() + for identity, parents in items: + key = _key(identity) + fields = {"dataset_version": key.dataset_version, "record_id": key.record_id} + records.append(normalize_row({**fields, "content": "private input"}, kind="records")) + provenance.append(normalize_row({ + **fields, "source_type": "unknown", "provenance_confidence": "confirmed", + "external_grounding": (grounding or {}).get(identity, "no"), + "transformation": (transformation or {}).get(identity, "generate"), + "parent_ids": list(reversed(parents)) if reverse else parents, + }, kind="provenance")) + records, provenance = tuple(records), tuple(provenance) + keys = tuple(row.record_key for row in records) + versions = tuple(sorted({key.dataset_version for key in keys})) + order = resolve_version_order(versions, invocation_order=versions) + joined = join_provenance(records, provenance) + batch = resolve_parent_batch(keys, provenance, version_order=order) + observability = classify_observability(records, provenance=provenance, version_order=order) + return BundleValidationResult( + (), records, provenance, joined, order, None, observability, + (), (), tuple(joined.messages) + tuple(batch.messages), batch, + ) + + +def _analyze(declarations, *, grounding=None, transformation=None, reverse=False, **limits): + return analyze_lineage(_validation(declarations, grounding=grounding, + transformation=transformation, reverse=reverse), + target_dataset_version="v2", limits=LineageLimits(**limits)) + + +def _shared_roots(*, aliases=False): + """Two roots, one union, then a union with a duplicate candidate root.""" + return { + "v1::a": [], "v1::b": [], + "v2::m": ["a", "b", "v1::a", "a"] if aliases else ["a", "b"], + "v2::z": ["m", "a", "v2::m", "a"] if aliases else ["m", "a"], + } + + +def _assert_root_failure(result, *, name, memberships, visits, attempted): + assert result.execution_status is ExecutionStatus.FAILED + assert "LINEAGE_RESOURCE_LIMIT_EXCEEDED" in result.execution_reason_codes + assert any(message.code == "E_LINEAGE_RESOURCE_LIMIT_EXCEEDED" + and message.severity in (ValidationSeverity.ERROR, ValidationSeverity.FATAL) + for message in result.messages) + assert result.records is None + assert result.grounded_record_count is None + assert result.closed_record_count is None + assert result.unresolved_record_count is None + assert result.records_with_resolved_external_ancestry is None + assert result.resolved_lineage_coverage is None + assert result.external_ancestry_coverage is None + usage = result.resource_usage + assert usage.exhausted_limit == name + assert usage.stored_root_membership_count == memberships + assert usage.root_union_visit_count == visits + assert usage.attempted_value == attempted + # Graph and structural analysis finished before root propagation stopped. + assert result.cycles is not None + assert result.cycles.cycle_count == 0 + assert result.cycles.lineage_depth == 2 + assert result.resolved_parent_edge_coverage == 1.0 + + +def test_anchor_storage_and_known_empty_sets_have_distinct_work_costs(): + result = _analyze({"v2::anchor": [], "v2::closed": []}, + grounding={"v2::anchor": "yes"}, + max_root_memberships=1, max_root_union_visits=1) + usage = result.resource_usage + assert result.execution_status is ExecutionStatus.COMPLETED + assert usage.stored_root_membership_count == 1 + assert usage.root_union_visit_count == 0 + assert usage.exhausted_limit is None + assert usage.attempted_value is None + roots = {entry.record_key: entry.external_root_keys for entry in result.records} + assert roots[_key("v2::anchor")] == frozenset({_key("v2::anchor")}) + assert roots[_key("v2::closed")] == frozenset() + with pytest.raises(FrozenInstanceError): + usage.root_union_visit_count = 1 + + +def test_grounded_carryover_charges_logical_membership_even_if_set_is_shared(): + result = _analyze({"v2::anchor": [], "v2::carry": ["anchor"]}, + grounding={"v2::anchor": "yes", "v2::carry": "yes"}, + transformation={"v2::carry": "carryover"}, + max_root_memberships=2, max_root_union_visits=1) + assert result.execution_status is ExecutionStatus.COMPLETED + assert result.resource_usage.stored_root_membership_count == 2 + assert result.resource_usage.root_union_visit_count == 1 + assert all(entry.external_root_keys == frozenset({_key("v2::anchor")}) + for entry in result.records) + + +@pytest.mark.parametrize("aliases", [False, True]) +def test_root_union_counts_duplicate_candidates_but_deduplicates_alias_edges(aliases): + result = _analyze(_shared_roots(aliases=aliases), + grounding={"v1::a": "yes", "v1::b": "yes"}, + max_root_memberships=6, max_root_union_visits=5) + assert result.execution_status is ExecutionStatus.COMPLETED + usage = result.resource_usage + assert (usage.admitted_node_count, usage.admitted_edge_count) == (4, 4) + assert (usage.stored_root_membership_count, usage.root_union_visit_count) == (6, 5) + assert usage.exhausted_limit is None + assert usage.attempted_value is None + by_key = {entry.record_key: entry for entry in result.records} + expected = frozenset({_key("v1::a"), _key("v1::b")}) + assert result.scope.target_record_count == result.grounded_record_count == 2 + assert by_key[_key("v2::m")].external_root_keys == expected + assert by_key[_key("v2::z")].external_root_keys == expected + + +@pytest.mark.parametrize("membership_limit,expected_visits", [(4, 3), (5, 5)]) +def test_membership_limit_rejects_next_storage_without_returning_partial_partition( + membership_limit, expected_visits): + result = _analyze(_shared_roots(), grounding={"v1::a": "yes", "v1::b": "yes"}, + max_root_memberships=membership_limit, max_root_union_visits=5) + _assert_root_failure(result, name="max_root_memberships", memberships=membership_limit, + visits=expected_visits, attempted=membership_limit + 1) + + +def test_union_limit_rejects_next_candidate_before_visit_or_storage(): + result = _analyze(_shared_roots(), grounding={"v1::a": "yes", "v1::b": "yes"}, + max_root_memberships=6, max_root_union_visits=4) + _assert_root_failure(result, name="max_root_union_visits", memberships=5, + visits=4, attempted=5) + + +def test_unresolved_branches_never_consume_storage_for_observed_partial_roots(): + result = _analyze({ + "v2::anchor": [], "v2::unknown": [], + "v2::mixed": ["anchor", "unknown"], "v2::descendant": ["mixed"], + "v2::unsupported": ["anchor", "unknown"], + }, grounding={"v2::anchor": "yes", "v2::unknown": "unknown", "v2::unsupported": "yes"}, + max_root_memberships=1, max_root_union_visits=1) + assert result.execution_status is ExecutionStatus.PARTIAL + assert result.resource_usage.stored_root_membership_count == 1 + assert result.resource_usage.root_union_visit_count == 0 + assert result.resource_usage.exhausted_limit is None + by_key = {entry.record_key: entry for entry in result.records} + for name in ("unknown", "mixed", "descendant", "unsupported"): + assert by_key[_key("v2::" + name)].external_root_keys is None + assert (result.grounded_record_count, result.closed_record_count, + result.unresolved_record_count) == (1, 0, 4) + + +def test_budget_schedule_uses_canonical_ready_nodes_and_is_input_permutation_invariant(): + # z must run before a because a depends on z. Once z finishes, a is the + # canonical next ready node, ahead of the independent zz anchor. + declarations = {"v2::z": [], "v2::a": ["z"], "v2::b": ["a"], "v2::zz": []} + grounding = {"v2::z": "yes", "v2::zz": "yes"} + results = [_analyze(declarations, grounding=grounding, reverse=reverse, + max_root_memberships=2, max_root_union_visits=10) + for reverse in (False, True)] + for result in results: + _assert_root_failure(result, name="max_root_memberships", memberships=2, + visits=2, attempted=3) + assert results[0] == results[1] + + +def test_root_propagation_is_iterative_for_a_chain_beyond_python_recursion_depth(): + size = 1200 + declarations = {f"v2::n{index:04d}": [] if index == size - 1 else [f"n{index + 1:04d}"] + for index in range(size)} + anchor = f"v2::n{size - 1:04d}" + result = _analyze(declarations, grounding={anchor: "yes"}, + max_root_memberships=size, max_root_union_visits=size - 1) + assert result.execution_status is ExecutionStatus.COMPLETED + assert result.grounded_record_count == size + assert result.closed_record_count == result.unresolved_record_count == 0 + assert result.resource_usage.stored_root_membership_count == size + assert result.resource_usage.root_union_visit_count == size - 1 + assert result.cycles.lineage_depth == size - 1 + assert all(entry.external_root_keys == frozenset({_key(anchor)}) for entry in result.records) + + +@pytest.mark.parametrize("limits,limit_name,expected_nodes,expected_edges", [ + ({"max_nodes": 1}, "max_nodes", 1, 0), + ({"max_edges": 1}, "max_edges", 3, 1), +]) +def test_graph_admission_failure_keeps_existing_typed_error_before_root_work( + limits, limit_name, expected_nodes, expected_edges): + with pytest.raises(LineageResourceLimitError) as caught: + _analyze({"v2::a": [], "v2::b": ["a"], "v2::c": ["b"]}, + grounding={"v2::a": "yes"}, **limits) + usage = caught.value.resource_usage + assert caught.value.reason_code == "LINEAGE_RESOURCE_LIMIT_EXCEEDED" + assert usage.exhausted_limit == limit_name + assert (usage.admitted_node_count, usage.admitted_edge_count) == (expected_nodes, expected_edges) + assert usage.stored_root_membership_count == usage.root_union_visit_count == 0 + assert usage.attempted_value == 2 From d81966561502b94701712a3d4b4b7fd50783cba9 Mon Sep 17 00:00:00 2001 From: DavidWallstructurallaw Date: Tue, 22 Sep 2026 21:14:12 -0700 Subject: [PATCH 05/12] Implement Phase 5 Step 5 root incidence and concentration --- .github/workflows/ci.yml | 4 +- PHASE_5_DECISIONS.md | 21 +- PHASE_5_STEP_5.md | 100 ++++++ docs/lineage_contract.md | 28 +- docs/release_process.md | 12 +- scripts/release_check.py | 8 +- .../lineage/ancestry.py | 122 ++++++- .../integration/test_current_verification.py | 2 +- tests/unit/test_lineage_concentration.py | 312 ++++++++++++++++++ 9 files changed, 579 insertions(+), 30 deletions(-) create mode 100644 PHASE_5_STEP_5.md create mode 100644 tests/unit/test_lineage_concentration.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4a2c13..22c2b7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,13 +50,13 @@ jobs: with: python-version: "3.12" - run: python -m pip install ".[test]" - - name: Check Step 4 roots and coverage with affected boundaries + - name: Check Step 5 root concentration with affected boundaries run: | set -euo pipefail python scripts/check_spec_consistency.py python scripts/check_traceability.py python scripts/release_check.py - python -m pytest -p no:cacheprovider -q tests/unit/test_T4_ancestry.py tests/unit/test_lineage_root_resources.py tests/unit/test_T6_cycles.py tests/unit/test_parent_batch.py tests/unit/test_lineage_graph.py tests/unit/test_PR007_version_order.py tests/unit/test_PR008_parent_resolution.py tests/unit/test_PR009_generation.py tests/unit/test_lineage_fixture_inputs.py tests/unit/test_phase3_contracts.py tests/integration/test_phase3_metric_pipeline.py tests/unit/test_PR016_determinism.py tests/integration/test_current_verification.py tests/integration/test_ci_workflows.py tests/integration/test_no_algorithms.py tests/integration/test_repository_structure.py + python -m pytest -p no:cacheprovider -q tests/unit/test_lineage_concentration.py tests/unit/test_T4_ancestry.py tests/unit/test_lineage_root_resources.py tests/unit/test_T6_cycles.py tests/unit/test_parent_batch.py tests/unit/test_lineage_graph.py tests/unit/test_PR007_version_order.py tests/unit/test_PR008_parent_resolution.py tests/unit/test_PR009_generation.py tests/unit/test_lineage_fixture_inputs.py tests/unit/test_phase3_contracts.py tests/integration/test_phase3_metric_pipeline.py tests/unit/test_PR016_determinism.py tests/integration/test_current_verification.py tests/integration/test_ci_workflows.py tests/integration/test_no_algorithms.py tests/integration/test_repository_structure.py python -m pytest -p no:cacheprovider -q tests/integration/test_phase4_cli.py::test_phase4_step7_validate_blocks_every_calculation_and_content_reader tests/integration/test_phase4_cli.py::test_phase4_step8_pair_does_not_activate_other_calculation_families core: diff --git a/PHASE_5_DECISIONS.md b/PHASE_5_DECISIONS.md index 3be45d4..38d110c 100644 --- a/PHASE_5_DECISIONS.md +++ b/PHASE_5_DECISIONS.md @@ -1,6 +1,6 @@ # Phase 5 Decisions and Current Authorization -Status: **PLAN APPROVED; STEPS 1-4 COMPLETE; PHASE INCOMPLETE**. +Status: **PLAN APPROVED; STEPS 1-5 COMPLETE; PHASE INCOMPLETE**. On 2026-09-23 UTC the Theory Owner instructed: `批准，Phase 5 step 1开始`. @@ -176,3 +176,22 @@ and provenance namespaces alone does not establish such a conflict. Original metadata is preserved; no new grounding inference is made from labels or URIs. Stop after Step 4; merge, tag and publication remain outside this instruction. + +## Step 5 execution instruction + +On 2026-09-23 UTC the Theory Owner instructed `Phase 5 Step 5 继续`. +This authorizes typed root contributions, incidence with the full target +denominator, fractional root mass with the grounded target denominator, +ancestry HHI and effective external-root count. Work starts from Step 4 commit +`2450100d08355d8cb1e0be931345f44572861e4a` on `phase5-lineage`. + +Runtime changes remain confined to `lineage/ancestry.py`. Direct rational +oracles, deterministic invariance checks and affected ancestry/resource +neighbors verify the new calculations. The current source check advances its +single comparison commit to Step 4 and continues to open only the ancestry +module. No additional governance framework or historical migration is added. +Package version and executable report schema remain `0.1.0.dev3` and `1.0`. + +Stop after Step 5. Bounds, shared-root proxy, report/schema integration and CLI +lineage dispatch remain later steps. Existing branch synchronization approval +continues; merge, tag and publication remain separate actions. diff --git a/PHASE_5_STEP_5.md b/PHASE_5_STEP_5.md new file mode 100644 index 0000000..5375728 --- /dev/null +++ b/PHASE_5_STEP_5.md @@ -0,0 +1,100 @@ +# Phase 5 Step 5 Completion + +Date: 2026-09-23 UTC. Status: **COMPLETE; PHASE INCOMPLETE**. + +The Theory Owner instructed `Phase 5 Step 5 继续`. Work starts from +`2450100d08355d8cb1e0be931345f44572861e4a` on `phase5-lineage`. +This step implements root incidence, fractional mass, ancestry concentration +and effective external-root count in the existing ancestry owner. + +## Delivered behavior + +`analyze_lineage` extends its immutable result with typed `RootContribution` +rows and the complete distinct-root count. Each root's incidence counts complete +nonempty-root targets containing that root. Incidence shares divide by the full +target population N. Each grounded target contributes mass one, divided equally +among its unique roots. Root weights divide accumulated mass by G, the number +of targets with complete nonempty roots. HHI sums squared weights; effective +root count is its reciprocal. + +Context records supply ancestry without entering either denominator. Repeated +paths and aliases do not multiply a target's root contribution. Unresolved +targets supply no exact mass. Their presence marks conditional root metrics +partial, accompanied by existing G/N coverage. Complete known-empty targets +remain distinct from unresolved targets. + +Contributions retain all supporting roots and are ordered by decreasing +incidence with canonical-key ties. Numerical reduction is deterministic. The +future report detail limit does not truncate the calculation. The values +describe a topological allocation; they do not measure causal influence, +semantic error, independent information or universal source diversity. + +A completed G=0 partition returns an empty contribution tuple and root count +zero. HHI and effective roots are null with `NO_RESOLVED_EXTERNAL_ROOTS`. +Resource-aborted propagation returns null contributions and aggregate values +with `LINEAGE_RESOURCE_LIMIT_EXCEEDED`; independently completed structural and +reference observations retain their Step 4 behavior. + +## Verification + +Local verification used Python 3.12.14 with the core dependency profile. + +| Check | Result | +|---|---| +| Combined Step 5 direct and affected regression gate | 565 passed in 6.59 seconds | +| Separate validation/pair CLI isolation checks | 2 passed in 0.48 seconds | +| Specification consistency, traceability and source scope | PASS | +| Four workflow YAML files and whitespace check | PASS | +| Independent implementation and test review | PASS; no outstanding blockers | + +The combined gate includes 24 new concentration cases alongside the existing +ancestry, resource, graph, cycle, parent and dependency-neighbor cases. It covers +all twelve frozen ancestry fixtures and the Hero. Hero HHI/effective roots remain +`1/4` and `4`; the frozen mixed multi-root fixture remains `5/8` and `8/5`. +An additional unequal-support oracle with N=6, G=4, C=1 and U=1 produces HHI +`35/96` and effective roots `96/35` using exact rational expectations. + +Direct tests also exercise permutation, identity renaming, duplicate paths and +aliases, incidence-first ranking, a full 105-root distribution, unavailable +concentration, resource-aborted null distributions, and immutable typed +handoffs. The completed all-unresolved partition retains its empty observed +root distribution even when input errors make overall execution fail. +Independent review ran forty bounded rational/relabeling cases; the maximum +observed relative HHI rounding difference was approximately `3.16e-16`. +Its initial ad hoc `3e-16` tolerance was adjusted to `1e-15`; no product change +was required. The permanent new suite passed on its first execution. + +### Bounded mutation check + +One passing unequal-support test was rerun against three disposable source +variants. Each variant successfully imported, calculated and constructed its +result, then failed an independent assertion with exit code 1. Denominator +mutations changed the corresponding declared metadata as well as arithmetic, +so local row consistency alone could not reject them. + +| Seeded defect | Detection | +|---|---| +| Incidence denominator N replaced with G | Reported denominator 4 rejected against independent N=6. | +| Weight denominator G replaced with N | Reported denominator 6 rejected against independent G=4. | +| Effective count uses HHI without its reciprocal | `0.3645833333333333` rejected against `96/35`. | + +All three meaningful mutations were killed. No permanent mutation runner, +registry or additional execution layer was added. Broader adversarial and scale +work remains Step 9. + +The existing source check advances its single comparison commit to completed +Step 4 and continues to open only `lineage/ancestry.py`. The other 58 product +files, product inventory, 16 frozen specifications and seven packaged resource +copies remain protected. No new governance registry or historical migration +is introduced. + +## Handoff + +Package version remains `0.1.0.dev3`; executable report schema remains `1.0`. +Step 6 lineage bounds and shared-root proxy need their separate execution +instruction. Report and CLI integration remain Steps 7-8. + +No full compatibility matrix, release build, clean install, optional Parquet +profile or 100k performance measurement is claimed here. Their designated +candidate/performance gates remain unchanged. No merge, tag, publication or +hosted CI success is claimed. diff --git a/docs/lineage_contract.md b/docs/lineage_contract.md index 9393346..bba3d12 100644 --- a/docs/lineage_contract.md +++ b/docs/lineage_contract.md @@ -11,8 +11,9 @@ This document defines interfaces for Steps 2-8. Step 2 implements retained batch parent evidence and `build_lineage_graph` with immutable scope, adjacency and node/edge limits. Step 3 adds `analyze_cycles` for iterative cycle detection, unaffected topology and structural depth. Step 4 adds `analyze_lineage` for -external-root resolution, G/C/U counts and coverage. Incidence, concentration, -bounds, report and CLI lineage interfaces remain later implementation targets. +external-root resolution, G/C/U counts and coverage. Step 5 adds root incidence, +fractional mass and ancestry concentration. Bounds, report and CLI lineage +interfaces remain later implementation targets. The package remains at `0.1.0.dev3` and the executable report schema at `1.0`. @@ -90,11 +91,11 @@ records. Selection is explicit, revalidated and independent of ordinary metric eligibility. `validate_bundle` remains input-only and never calls this function. CLI orchestration calls it only after an explicit lineage request. -Step 4 implements this direct API for root resolution and coverage. Its +Steps 4-5 implement this direct API for root resolution, coverage and concentration. Its `LineageAnalysisResult.records` contains target records in canonical order; context roots are intermediate computation data and do not enter target -denominators. The existing `ExecutionStatus` vocabulary applies. Incidence, -fractional mass, concentration, bounds and proxy fields remain later steps. +denominators. The existing `ExecutionStatus` vocabulary applies. Bounds and proxy +fields remain later steps. Graph admission failures retain the existing `LineageResourceLimitError` before a valid analytical result can be created. A root-membership or union-work @@ -153,6 +154,23 @@ as a full transitive-closure table. finite. Independent fixture expectations use exact fractions where possible; deterministic numeric reduction uses canonical target/root order. +Step 5 returns all `root_contributions` in decreasing incidence order, with +canonical root-key ties. Its aggregate fields are +`distinct_external_root_count`, `ancestry_concentration_hhi` and +`effective_external_root_count`. These computations precede the Step 7 detail +limit; the direct result does not truncate the root distribution. + +Read-only `root_metrics_status` and `concentration_status` properties use the +existing `ReportStatus` vocabulary. A completed partition has available root +observations when U=0 and partial observations when U>0. A G-only concentration +uses that status when G>0; existing G/N coverage and contribution denominators +disclose its conditional population. With G=0, the contribution tuple is empty +and the distinct-root count is zero, while concentration is unavailable with +`concentration_reason_codes=("NO_RESOLVED_EXTERNAL_ROOTS",)`. A root-resource +abort instead makes contributions and all three aggregates null, with unavailable +statuses and `LINEAGE_RESOURCE_LIMIT_EXCEEDED`. These field states remain +separate from the overall execution status and its input diagnostics. + `LineageAnalysisResult` contains `scope: LineageScope`, `execution_status: ExecutionStatus`, `execution_reason_codes: tuple[str, ...]`, `records: tuple[RecordAncestry, ...] | None`, `cycles: CycleAnalysis | None`, diff --git a/docs/release_process.md b/docs/release_process.md index 08ce9c6..af89938 100644 --- a/docs/release_process.md +++ b/docs/release_process.md @@ -2,9 +2,9 @@ ## Current Phase 5 boundary -Phase 5 Step 4 is approved. Runtime remains at `0.1.0.dev3`, with executable -report schema 1.0. External-root resolution, G/C/U classification and coverage are -the current implementation scope; concentration, bounds and report integration +Phase 5 Step 5 is approved. Runtime remains at `0.1.0.dev3`, with executable +report schema 1.0. Root incidence, fractional mass and ancestry concentration are +the current implementation scope; bounds and report integration remain subsequent steps. Stable v0.1 publication, a tag and main merge require separate authorization. @@ -19,8 +19,8 @@ python scripts/release_check.py Historical `--phase` and `--step` dispatch is retired. Earlier source forms, migrations and receipts remain recoverable from Git and archived phase records. -The existing source check advances its single reference to the completed Step 3 -commit; no new Phase 5 hash registry is needed. During Step 4 it permits changes +The existing source check advances its single reference to the completed Step 4 +commit; no new Phase 5 hash registry is needed. During Step 5 it permits changes only to the existing ancestry module, retaining the exact product file inventory and rejecting unrelated runtime, schema, package-metadata and canonical Hero changes. The frozen Phase 0 specification hashes are unchanged. Sixteen frozen @@ -47,7 +47,7 @@ Dispatch `ci.yml` with `gate=candidate` for a stable candidate. The workflow run Compatibility cells run complete canonical regression with `--ignore=tests/performance`. The designated performance job runs `tests/performance` once, avoiding nine repetitions of the costly 100k report -workload. The Phase 5 sparse-lineage test is still a placeholder in Step 4; +workload. The Phase 5 sparse-lineage test is still a placeholder in Step 5; it becomes an executable graph benchmark in Step 9. No placeholder is counted as a successful lineage measurement. diff --git a/scripts/release_check.py b/scripts/release_check.py index 2931d48..7e0b173 100644 --- a/scripts/release_check.py +++ b/scripts/release_check.py @@ -1,6 +1,6 @@ """Current source and candidate checks for Recursive Integrity Toolkit. -Step 4 opens the approved root resolution and coverage implementation scope. Historical dispatch, source-body +Step 5 opens the approved root incidence and concentration implementation scope. Historical dispatch, source-body migrations and phase registries are recoverable from the accepted Git commit. Installed checks below retain their existing product, privacy and package cases. """ @@ -24,7 +24,7 @@ import zipfile ROOT = Path(__file__).resolve().parents[1] -ACCEPTED_COMMIT = "11192a3c353292f11d85eb3725c8edc73a71967b" +ACCEPTED_COMMIT = "2450100d08355d8cb1e0be931345f44572861e4a" PROTECTED_PREFIXES = ("src/", "schemas/", "examples/hero/") CURRENT_IMPLEMENTATION_PATHS = frozenset({ "src/recursive_integrity_toolkit/lineage/ancestry.py", @@ -67,7 +67,7 @@ def verify_source_scope(root: Path = ROOT) -> dict: for name, raw in expected.items(): current = _regular_file(root, name).read_bytes() if name not in CURRENT_IMPLEMENTATION_PATHS and current != raw: - raise ValueError(f"Unauthorized product mutation outside Step 4 scope: {name}") + raise ValueError(f"Unauthorized product mutation outside Step 5 scope: {name}") for prefix in PROTECTED_PREFIXES: paths = list((root / prefix).rglob("*")) if (root / prefix).is_symlink() or any(path.is_symlink() for path in paths): @@ -75,7 +75,7 @@ def verify_source_scope(root: Path = ROOT) -> dict: actual = {path.relative_to(root).as_posix() for path in paths if path.is_file() and "__pycache__" not in path.parts} if actual != {name for name in expected if name.startswith(prefix)}: - raise ValueError(f"Unauthorized product file-set mutation during Step 4: {prefix}") + raise ValueError(f"Unauthorized product file-set mutation during Step 5: {prefix}") return {"protected_product_files": len(expected) - len(CURRENT_IMPLEMENTATION_PATHS), "authorized_implementation_files": len(CURRENT_IMPLEMENTATION_PATHS), "product_file_inventory": "unchanged"} diff --git a/src/recursive_integrity_toolkit/lineage/ancestry.py b/src/recursive_integrity_toolkit/lineage/ancestry.py index 66ed8a9..4c1a8f8 100644 --- a/src/recursive_integrity_toolkit/lineage/ancestry.py +++ b/src/recursive_integrity_toolkit/lineage/ancestry.py @@ -1,6 +1,6 @@ -"""Resolve exact external roots and target ancestry coverage. +"""Resolve exact external roots, target coverage and root concentration. -Owner IDs: T4; Phase 5 Step 4. +Owner IDs: T4; Phase 5 Step 5. Validated explicit grounding supplies anchors. Iterative dependency scheduling unions complete root sets; unknown branches never supply exact roots. Logical @@ -8,21 +8,22 @@ The allocation is topological and makes no causal or scientific quality claim. Current phase status: - Phase 5 Step 4 root resolution, G/C/U classification and target coverage. - Incidence, fractional mass, concentration, bounds and reports remain later - steps. No file or network I/O, generation calculation or implicit invocation. + Phase 5 Step 5 adds target incidence, fractional mass and concentration. + Bounds and reports remain later steps. No file or network I/O, generation + calculation or implicit invocation. """ from __future__ import annotations from dataclasses import dataclass, replace from heapq import heapify, heappop, heappush +from math import fsum, isfinite from types import MappingProxyType from typing import Literal from ..errors import ErrorCode, LineageResourceLimitError from ..io.validation import join_provenance from ..models import BundleValidationResult, RecordKey, ValidationMessage, ValidationSeverity -from ..result import ExecutionStatus +from ..result import ExecutionStatus, ReportStatus from .cycles import CycleAnalysis, DepthAssessment, analyze_cycles, _local_depth_reasons from .graph import ( LineageGraph, LineageLimits, LineageResourceUsage, LineageScope, @@ -94,9 +95,66 @@ def __post_init__(self) -> None: DepthAssessment(self.lineage_depth, self.depth_reason_codes) +@dataclass(frozen=True, slots=True) +class RootContribution: + """One root's target incidence and equal-per-record fractional allocation.""" + record_key: RecordKey + incidence_count: int + incidence_share: float + incidence_denominator: int + fractional_mass: float + normalized_weight: float + weight_denominator: int + + def __post_init__(self) -> None: + _key(self.record_key) + for value in (self.incidence_count, self.incidence_denominator, self.weight_denominator): + _integer(value) + if not 1 <= self.incidence_count <= self.weight_denominator <= self.incidence_denominator: + raise _invalid("root incidence requires positive target and grounded denominators") + for value in (self.incidence_share, self.fractional_mass, self.normalized_weight): + if type(value) is not float or not isfinite(value) or value <= 0: + raise _invalid("root contributions require finite positive floating-point values") + try: + incidence_share = self.incidence_count / self.incidence_denominator + normalized_weight = self.fractional_mass / self.weight_denominator + except OverflowError: + raise _invalid("root contribution denominators exceed the supported numeric range") from None + if (self.fractional_mass > self.incidence_count + or self.incidence_share != incidence_share + or self.normalized_weight != normalized_weight): + raise _invalid("root contribution values disagree with their declared denominators") + + +def _root_metrics(records: tuple[RecordAncestry, ...], total: int, grounded: int): + """Aggregate complete target sets before any future display truncation. + + Input targets and each root visit use canonical identity order. fsum retains + small fractional contributions without order-sensitive running totals. The + temporary terms are bounded by the already admitted target memberships. + """ + terms: dict[RecordKey, list[float]] = {} + for record in records: + roots = record.external_root_keys + if roots: + mass = 1.0 / len(roots) + for root in sorted(roots): + terms.setdefault(root, []).append(mass) + contributions = [] + for root in sorted(terms): + incidence = len(terms[root]) + mass = fsum(terms[root]) + contributions.append(RootContribution( + root, incidence, incidence / total, total, mass, mass / grounded, grounded)) + hhi = fsum(row.normalized_weight ** 2 for row in contributions) if contributions else None + effective = 1.0 / hhi if hhi is not None else None + contributions.sort(key=lambda row: (-row.incidence_count, row.record_key)) + return tuple(contributions), len(contributions), hhi, effective + + @dataclass(frozen=True, slots=True) class LineageAnalysisResult: - """Step 4 targets and completed structural observations. + """Targets, complete root metrics and completed structural observations. Resource-aborted root propagation supplies no partition or record rows. Earlier graph/depth/reference observations survive. Node/edge admission @@ -120,6 +178,10 @@ class LineageAnalysisResult: resolved_parent_edge_coverage: float | None no_declared_parents: bool reference_coverage_reason_codes: tuple[str, ...] + root_contributions: tuple[RootContribution, ...] | None + distinct_external_root_count: int | None + ancestry_concentration_hhi: float | None + effective_external_root_count: float | None resource_usage: LineageResourceUsage messages: tuple[ValidationMessage, ...] @@ -143,6 +205,8 @@ def __post_init__(self) -> None: _reasons(self.execution_reason_codes, _EXECUTION_REASONS) counts = (self.grounded_record_count, self.closed_record_count, self.unresolved_record_count, self.records_with_resolved_external_ancestry) + root_metrics = (self.root_contributions, self.distinct_external_root_count, + self.ancestry_concentration_hhi, self.effective_external_root_count) if self.records is None: if (self.execution_status is not ExecutionStatus.FAILED or self.execution_reason_codes != ("LINEAGE_RESOURCE_LIMIT_EXCEEDED",) @@ -151,6 +215,7 @@ def __post_init__(self) -> None: or self.resolved_lineage_coverage is not None or self.external_ancestry_coverage is not None or self.ancestry_coverage_reason_codes != ("LINEAGE_RESOURCE_LIMIT_EXCEEDED",) + or any(value is not None for value in root_metrics) or not any(message.code == ErrorCode.LINEAGE_RESOURCE_LIMIT_EXCEEDED.value and message.severity in (ValidationSeverity.ERROR, ValidationSeverity.FATAL) for message in messages)): @@ -184,6 +249,20 @@ def __post_init__(self) -> None: raise _invalid("ancestry coverage must use the full explicit target denominator") if (self.execution_status, self.execution_reason_codes) != _execution(messages, ground + closed, unresolved): raise _invalid("ancestry execution status disagrees with its target partition and diagnostics") + if type(self.root_contributions) is not tuple: + raise _invalid("completed root metrics require immutable contribution rows") + for contribution in self.root_contributions: + if type(contribution) is not RootContribution: + raise _invalid("root contributions require typed values") + replace(contribution) + _integer(self.distinct_external_root_count) + expected_roots, expected_count, expected_hhi, expected_effective = _root_metrics( + self.records, total, ground) + if (self.root_contributions != expected_roots + or self.distinct_external_root_count != expected_count + or not _coverage(self.ancestry_concentration_hhi, expected_hhi) + or not _coverage(self.effective_external_root_count, expected_effective)): + raise _invalid("root metrics disagree with complete target root sets") references = (self.declared_parent_reference_count, self.resolved_parent_reference_count, self.unresolved_parent_reference_count) if type(self.no_declared_parents) is not bool: @@ -206,6 +285,22 @@ def __post_init__(self) -> None: object.__setattr__(self, "resource_usage", usage) object.__setattr__(self, "messages", messages) + @property + def root_metrics_status(self) -> ReportStatus: + if self.records is None: + return ReportStatus.UNAVAILABLE + return ReportStatus.PARTIAL if self.unresolved_record_count else ReportStatus.AVAILABLE + + @property + def concentration_status(self) -> ReportStatus: + return self.root_metrics_status if self.grounded_record_count else ReportStatus.UNAVAILABLE + + @property + def concentration_reason_codes(self) -> tuple[str, ...]: + if self.records is None: + return ("LINEAGE_RESOURCE_LIMIT_EXCEEDED",) + return () if self.grounded_record_count else ("NO_RESOLVED_EXTERNAL_ROOTS",) + @property def lineage_depth(self) -> int | None: return self.cycles.lineage_depth @@ -323,10 +418,10 @@ def analyze_lineage( validation: BundleValidationResult, *, target_dataset_version: str | None, limits: LineageLimits = LineageLimits(), ) -> LineageAnalysisResult: - """Compute explicit Step 4 ancestry, stopping dependent work on exhaustion. + """Compute explicit ancestry and root metrics, stopping on exhaustion. All loaded context nodes participate in resolution and resource accounting. - Only selected targets enter returned record rows, G/C/U and coverage. + Only selected targets enter returned rows, G/C/U, coverage and root metrics. Canonical dependency-ready nodes precede canonical parent and root visits. """ graph = build_lineage_graph(validation, target_dataset_version=target_dataset_version, limits=limits) @@ -353,7 +448,9 @@ def analyze_lineage( return LineageAnalysisResult( graph.scope, ExecutionStatus.FAILED, (error.reason_code,), None, cycles, None, None, None, None, None, None, (error.reason_code,), - **reference_fields, resource_usage=error.resource_usage, messages=messages) + **reference_fields, root_contributions=None, distinct_external_root_count=None, + ancestry_concentration_hhi=None, effective_external_root_count=None, + resource_usage=error.resource_usage, messages=messages) records = tuple(RecordAncestry( key, "unresolved" if roots[key] is None else "grounded" if roots[key] else "closed", roots[key], reasons[key], cycles.depths_by_record[key].lineage_depth, @@ -363,9 +460,12 @@ def analyze_lineage( unresolved = len(records) - ground - closed status, execution_reasons = _execution(messages, ground + closed, unresolved) total = graph.scope.target_record_count + contributions, distinct, hhi, effective = _root_metrics(records, total, ground) return LineageAnalysisResult( graph.scope, status, execution_reasons, records, cycles, ground, closed, unresolved, ground + closed, (ground + closed) / total if total else None, ground / total if total else None, () if total else ("EMPTY_TARGET_SCOPE",), - **reference_fields, resource_usage=budget.usage(), messages=messages) + **reference_fields, root_contributions=contributions, distinct_external_root_count=distinct, + ancestry_concentration_hhi=hhi, effective_external_root_count=effective, + resource_usage=budget.usage(), messages=messages) diff --git a/tests/integration/test_current_verification.py b/tests/integration/test_current_verification.py index 1fa4ede..a24fe2a 100644 --- a/tests/integration/test_current_verification.py +++ b/tests/integration/test_current_verification.py @@ -27,7 +27,7 @@ def protected_copy(repo_root, tmp_path): @pytest.mark.parametrize("mutation", ["formula", "new_module", "deleted_module"]) def test_current_scope_rejects_unauthorized_product_changes(current_tools, protected_copy, mutation): - """Root/coverage implementation does not authorize unrelated product changes.""" + """Root concentration does not authorize unrelated product changes.""" verify = current_tools["verify_source_scope"] verify(protected_copy) target = protected_copy / "src/recursive_integrity_toolkit/metrics/diversity.py" diff --git a/tests/unit/test_lineage_concentration.py b/tests/unit/test_lineage_concentration.py new file mode 100644 index 0000000..0b9908f --- /dev/null +++ b/tests/unit/test_lineage_concentration.py @@ -0,0 +1,312 @@ +"""Exact root allocation, target denominators and conditional concentration. + +Frozen acceptance values and independent rational arithmetic protect the +topological allocation convention. No report caps, bounds or proxy are applied. +""" + +from dataclasses import FrozenInstanceError, replace +from fractions import Fraction +import json +import math + +import pytest + +from recursive_integrity_toolkit.errors import CanonicalValidationError +from recursive_integrity_toolkit.io.validation import validate_bundle +from recursive_integrity_toolkit.lineage import ancestry +from recursive_integrity_toolkit.lineage.graph import LineageLimits +from recursive_integrity_toolkit.models import AuditBundle, FileRole, InputSource +from recursive_integrity_toolkit.result import ExecutionStatus, ReportStatus + +# Reuse input construction only; all allocation oracles below are independent +# of the product implementation and the existing root-resolution assertions. +from test_T4_ancestry import _CASES, _FIXTURES, _ROOT, _fixture_validation, _key, _validation + + +def _analyze(validation, target="v2", **limits): + return ancestry.analyze_lineage(validation, target_dataset_version=target, + limits=LineageLimits(**limits)) + + +def _assert_fraction(actual, expected): + assert type(actual) is float + assert math.isfinite(actual) + assert actual == pytest.approx(float(Fraction(expected)), rel=1e-13, abs=1e-15) + + +def _assert_allocation(result, complete_sets, *, total, grounded, unresolved): + """Count incidences directly from independently specified target root sets.""" + complete = [frozenset(roots) for roots in complete_sets.values() if roots] + assert len(complete) == grounded + roots = set().union(*complete) if complete else set() + incidence = {root: sum(root in support for support in complete) for root in roots} + mass = {root: sum((Fraction(1, len(support)) for support in complete if root in support), + Fraction()) for root in roots} + expected_order = sorted(roots, key=lambda root: (-incidence[root], _key(root))) + rows = result.root_contributions + assert type(rows) is tuple + assert [str(row.record_key) for row in rows] == expected_order + assert result.distinct_external_root_count == len(roots) + assert result.root_metrics_status is (ReportStatus.PARTIAL if unresolved else ReportStatus.AVAILABLE) + for row in rows: + root = str(row.record_key) + assert type(row) is ancestry.RootContribution + assert row.incidence_count == incidence[root] + assert row.incidence_denominator == total + assert row.weight_denominator == grounded + _assert_fraction(row.incidence_share, Fraction(incidence[root], total)) + _assert_fraction(row.fractional_mass, mass[root]) + _assert_fraction(row.normalized_weight, mass[root] / grounded) + if grounded: + hhi = sum(((mass[root] / grounded) ** 2 for root in roots), Fraction()) + _assert_fraction(result.ancestry_concentration_hhi, hhi) + _assert_fraction(result.effective_external_root_count, 1 / hhi) + assert result.concentration_status is result.root_metrics_status + assert result.concentration_reason_codes == () + assert math.fsum(row.fractional_mass for row in rows) == pytest.approx(grounded) + assert math.fsum(row.normalized_weight for row in rows) == pytest.approx(1) + else: + assert rows == () + assert result.ancestry_concentration_hhi is None + assert result.effective_external_root_count is None + assert result.concentration_status is ReportStatus.UNAVAILABLE + assert result.concentration_reason_codes == ("NO_RESOLVED_EXTERNAL_ROOTS",) + + +@pytest.mark.parametrize("case", _CASES, ids=lambda case: case["case_id"]) +def test_frozen_root_sets_and_rational_oracles_determine_complete_allocation(case, tmp_path): + validation, target = _fixture_validation(tmp_path, case) + result = _analyze(validation, target) + expected = case["expected_lineage"] + counts = expected["counts"] + _assert_allocation(result, expected["complete_root_sets"], total=counts["target"], + grounded=counts["grounded"], unresolved=counts["unresolved"]) + for source, field in (("ancestry_hhi", "ancestry_concentration_hhi"), + ("effective_external_root_count", "effective_external_root_count")): + if source in expected: + if expected[source] is None: + assert getattr(result, field) is None + else: + _assert_fraction(getattr(result, field), expected[source]) + if "root_order" in expected: + assert [str(row.record_key) for row in result.root_contributions] == expected["root_order"] + assert [row.incidence_count for row in result.root_contributions] == expected["incidence"] + for fixture_field, field in (("incidence_shares", "incidence_share"), + ("fractional_masses", "fractional_mass"), + ("normalized_weights", "normalized_weight")): + for row, value in zip(result.root_contributions, expected[fixture_field], strict=True): + _assert_fraction(getattr(row, field), value) + + +def test_hero_incidence_mass_and_concentration_match_preimplementation_oracle(): + hero = _ROOT / "examples" / "hero" + expected = json.loads((_FIXTURES / "lineage_complete" / "hero_expected.json").read_text()) + validation = validate_bundle(AuditBundle(( + InputSource(FileRole.RECORDS_PRIMARY, hero / "records_v2.csv"), + InputSource(FileRole.RECORDS_COMPARE, hero / "records_v1.csv"), + InputSource(FileRole.PROVENANCE_MANIFEST, hero / "provenance.csv"), + InputSource(FileRole.VERSION_ORDER, hero / "version_order.json"), + ))) + result = _analyze(validation) + rows = result.root_contributions + assert result.scope.target_record_count == 8 + assert result.scope.loaded_record_count == 16 + assert result.distinct_external_root_count == expected["distinct_external_roots"] + assert [str(row.record_key) for row in rows] == expected["root_order"] + assert [row.incidence_count for row in rows] == expected["incidence"] + for fixture_field, field in (("incidence_shares", "incidence_share"), + ("fractional_masses", "fractional_mass"), + ("normalized_weights", "normalized_weight")): + for row, value in zip(rows, expected[fixture_field], strict=True): + _assert_fraction(getattr(row, field), value) + assert all(row.incidence_denominator == row.weight_denominator == 8 for row in rows) + _assert_fraction(result.ancestry_concentration_hhi, expected["ancestry_hhi"]) + _assert_fraction(result.effective_external_root_count, expected["effective_external_root_count"]) + assert result.root_metrics_status is result.concentration_status is ReportStatus.AVAILABLE + + +def _unequal_supports(): + return { + "v1::a": [], "v1::b": [], "v1::c": [], "v1::unrelated": [], + "v2::t1": ["v1::a"], "v2::t2": ["v1::a", "v1::b"], + "v2::t3": ["v1::a", "v1::b", "v1::c"], "v2::t4": ["v1::b", "v1::c"], + "v2::closed": [], "v2::unknown": [], + } + + +def _grounding(): + return {"v1::a": "yes", "v1::b": "yes", "v1::c": "yes", + "v1::unrelated": "yes", "v2::unknown": "unknown"} + + +def test_unequal_supports_use_n_for_incidence_and_g_for_weights(tmp_path): + result = _analyze(_validation(tmp_path, _unequal_supports(), grounding=_grounding())) + assert result.scope.loaded_record_count == 10 + assert (result.grounded_record_count, result.closed_record_count, result.unresolved_record_count) == (4, 1, 1) + rows = result.root_contributions + assert [str(row.record_key) for row in rows] == ["v1::a", "v1::b", "v1::c"] + assert [row.incidence_count for row in rows] == [3, 3, 2] + for row, mass, weight, share in zip(rows, ("11/6", "4/3", "5/6"), + ("11/24", "1/3", "5/24"), ("1/2", "1/2", "1/3"), strict=True): + assert row.incidence_denominator == 6 + assert row.weight_denominator == 4 + _assert_fraction(row.fractional_mass, mass) + _assert_fraction(row.normalized_weight, weight) + _assert_fraction(row.incidence_share, share) + _assert_fraction(result.ancestry_concentration_hhi, "35/96") + _assert_fraction(result.effective_external_root_count, "96/35") + assert result.root_metrics_status is result.concentration_status is ReportStatus.PARTIAL + assert result.external_ancestry_coverage == 4 / 6 + + +def test_permutation_and_redundant_paths_cannot_multiply_fractional_mass(tmp_path): + declarations = _unequal_supports() + baseline = _analyze(_validation(tmp_path / "base", declarations, grounding=_grounding())) + reversed_input = _analyze(_validation(tmp_path / "reverse", declarations, + grounding=_grounding(), reverse=True)) + expanded = {**declarations, "v1::path_a": ["v1::a"], "v1::path_ab": ["v1::a", "v1::b"]} + expanded["v2::t2"] = ["a", "v1::a", "v1::b", "v1::path_a", "v1::path_ab", "a"] + redundant = _analyze(_validation(tmp_path / "redundant", expanded, grounding=_grounding())) + for result in (reversed_input, redundant): + assert result.root_contributions == baseline.root_contributions + assert result.distinct_external_root_count == baseline.distinct_external_root_count + assert result.ancestry_concentration_hhi == baseline.ancestry_concentration_hhi + assert result.effective_external_root_count == baseline.effective_external_root_count + assert redundant.declared_parent_reference_count > baseline.declared_parent_reference_count + + +def test_bijective_identity_rename_preserves_values_and_reorders_only_canonical_ties(tmp_path): + declarations, grounding = _unequal_supports(), _grounding() + baseline = _analyze(_validation(tmp_path / "base", declarations, grounding=grounding)) + renamed = {key: key.split("::")[0] + "::" + str(100 - index) + for index, key in enumerate(declarations)} + changed = {renamed[key]: [renamed[parent] for parent in parents] + for key, parents in declarations.items()} + result = _analyze(_validation(tmp_path / "renamed", changed, + grounding={renamed[key]: value for key, value in grounding.items()})) + expected = {renamed[str(row.record_key)]: row for row in baseline.root_contributions} + for row in result.root_contributions: + original = expected[str(row.record_key)] + assert row.incidence_count == original.incidence_count + assert row.fractional_mass == original.fractional_mass + assert row.normalized_weight == original.normalized_weight + assert row.incidence_share == original.incidence_share + assert tuple(result.root_contributions) == tuple(sorted(result.root_contributions, + key=lambda row: (-row.incidence_count, row.record_key))) + assert result.ancestry_concentration_hhi == pytest.approx(baseline.ancestry_concentration_hhi) + assert result.effective_external_root_count == pytest.approx(baseline.effective_external_root_count) + + +def test_ranking_uses_incidence_before_mass_and_includes_singletons(tmp_path): + result = _analyze(_validation(tmp_path, { + "v1::a": [], "v1::b": [], "v1::c": [], "v1::d": [], + "v2::one": ["v1::a"], "v2::two": ["v1::b", "v1::c", "v1::d"], + "v2::three": ["v1::b", "v1::c", "v1::d"], + }, grounding={f"v1::{name}": "yes" for name in "abcd"})) + assert [str(row.record_key) for row in result.root_contributions] == ["v1::b", "v1::c", "v1::d", "v1::a"] + assert [row.incidence_count for row in result.root_contributions] == [2, 2, 2, 1] + assert result.root_contributions[-1].fractional_mass == 1.0 + assert result.root_contributions[0].fractional_mass < 1.0 + + +def test_all_loaded_roots_are_computed_before_any_future_detail_limit(tmp_path): + roots = [f"v1::r{index:03d}" for index in range(105)] + declarations = {root: [] for root in roots} + declarations.update({"v2::wide": roots, "v2::single": [roots[-1]]}) + result = _analyze(_validation(tmp_path, declarations, grounding={root: "yes" for root in roots})) + _assert_allocation(result, {"v2::wide": roots, "v2::single": [roots[-1]]}, + total=2, grounded=2, unresolved=0) + assert len(result.root_contributions) == result.distinct_external_root_count == 105 + assert result.root_contributions[0].record_key == _key(roots[-1]) + _assert_fraction(result.ancestry_concentration_hhi, "9/35") + _assert_fraction(result.effective_external_root_count, "35/9") + + +def test_completed_all_unresolved_partition_retains_exact_empty_root_distribution(tmp_path): + result = _analyze(_validation(tmp_path, {"v2::self": ["self"]})) + assert result.execution_status is ExecutionStatus.FAILED + assert result.unresolved_record_count == 1 + _assert_allocation(result, {}, total=1, grounded=0, unresolved=1) + + +@pytest.mark.parametrize("limits", [{"max_root_memberships": 1}, {"max_root_union_visits": 1}]) +def test_resource_abort_cannot_emit_partial_exact_root_distribution(tmp_path, limits): + result = _analyze(_validation(tmp_path, { + "v1::a": [], "v1::b": [], "v2::one": ["v1::a", "v1::b"], + }, grounding={"v1::a": "yes", "v1::b": "yes"}), **limits) + assert result.execution_status is ExecutionStatus.FAILED + assert result.records is None + assert result.root_contributions is None + assert result.distinct_external_root_count is None + assert result.ancestry_concentration_hhi is None + assert result.effective_external_root_count is None + assert result.root_metrics_status is result.concentration_status is ReportStatus.UNAVAILABLE + assert result.concentration_reason_codes == ("LINEAGE_RESOURCE_LIMIT_EXCEEDED",) + with pytest.raises(CanonicalValidationError): + replace(result, root_contributions=(), distinct_external_root_count=0) + + +def _valid_contribution(): + return ancestry.RootContribution( + record_key=_key("v1::a"), incidence_count=2, incidence_share=0.5, + incidence_denominator=4, fractional_mass=1.5, normalized_weight=0.75, + weight_denominator=2, + ) + + +def test_root_contribution_is_immutable_and_rejects_invalid_units_or_ratios(): + contribution = _valid_contribution() + with pytest.raises(FrozenInstanceError): + contribution.fractional_mass = 1.0 + invalid = ( + {"record_key": "v1::a"}, {"incidence_count": True}, {"incidence_count": 0}, + {"incidence_count": 3}, {"incidence_denominator": True}, {"incidence_denominator": 0}, + {"incidence_denominator": 1}, {"weight_denominator": False}, {"weight_denominator": 0}, + {"weight_denominator": 5}, {"incidence_share": True}, {"incidence_share": 0.6}, + {"incidence_share": float("inf")}, {"fractional_mass": float("nan")}, + {"fractional_mass": float("inf")}, {"fractional_mass": 0.0}, {"fractional_mass": -1.0}, + {"fractional_mass": 3.0}, {"normalized_weight": True}, {"normalized_weight": 1.1}, + {"normalized_weight": float("nan")}, {"normalized_weight": 0.5}, + ) + for changes in invalid: + with pytest.raises(CanonicalValidationError): + replace(contribution, **changes) + + +def test_result_rejects_truncation_wrong_denominators_and_forged_concentration(tmp_path): + result = _analyze(_validation(tmp_path, _unequal_supports(), grounding=_grounding())) + rows = result.root_contributions + with pytest.raises(FrozenInstanceError): + result.root_contributions = () + # Each replacement below can be internally valid as an individual root, + # while contradicting the complete typed target population or root sets. + wrong_n = replace(rows[0], incidence_denominator=7, incidence_share=3 / 7) + wrong_g = replace(rows[0], weight_denominator=5, normalized_weight=rows[0].fractional_mass / 5) + wrong_mass = replace(rows[0], fractional_mass=1.0, normalized_weight=1 / 4) + absent_root = replace(rows[0], record_key=_key("v1::unrelated")) + invalid = ( + {"root_contributions": list(rows)}, {"root_contributions": None}, + {"root_contributions": rows[:-1]}, {"root_contributions": tuple(reversed(rows))}, + {"root_contributions": (wrong_n,) + rows[1:]}, + {"root_contributions": (wrong_g,) + rows[1:]}, + {"root_contributions": (wrong_mass,) + rows[1:]}, + {"root_contributions": (absent_root,) + rows[1:]}, + {"distinct_external_root_count": 2}, {"distinct_external_root_count": True}, + {"ancestry_concentration_hhi": None}, {"ancestry_concentration_hhi": 0.0}, + {"ancestry_concentration_hhi": True}, {"ancestry_concentration_hhi": float("nan")}, + {"effective_external_root_count": None}, {"effective_external_root_count": 3.0}, + {"effective_external_root_count": float("inf")}, + ) + for changes in invalid: + with pytest.raises(CanonicalValidationError): + replace(result, **changes) + + +def test_no_grounded_targets_cannot_be_given_zero_or_infinite_concentration(tmp_path): + result = _analyze(_validation(tmp_path, {"v2::closed": []})) + for changes in ( + {"ancestry_concentration_hhi": 0.0}, {"effective_external_root_count": float("inf")}, + {"root_contributions": None}, {"distinct_external_root_count": None}, + ): + with pytest.raises(CanonicalValidationError): + replace(result, **changes) From e2666515d98cfe41483b5c43345c642bab02ced0 Mon Sep 17 00:00:00 2001 From: DavidWallstructurallaw Date: Tue, 22 Sep 2026 21:28:46 -0700 Subject: [PATCH 06/12] Implement Phase 5 Step 6 lineage closure and shared-root proxy --- .github/workflows/ci.yml | 4 +- PHASE_5_DECISIONS.md | 25 +- PHASE_5_STEP_6.md | 98 ++++++ docs/lineage_contract.md | 41 ++- docs/release_process.md | 16 +- scripts/release_check.py | 11 +- .../lineage/ancestry.py | 123 ++++++- .../metrics/bounds.py | 138 +++++++- .../integration/test_current_verification.py | 2 +- tests/unit/test_lineage_bounds_proxy.py | 319 ++++++++++++++++++ 10 files changed, 742 insertions(+), 35 deletions(-) create mode 100644 PHASE_5_STEP_6.md create mode 100644 tests/unit/test_lineage_bounds_proxy.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22c2b7f..f4b6454 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,13 +50,13 @@ jobs: with: python-version: "3.12" - run: python -m pip install ".[test]" - - name: Check Step 5 root concentration with affected boundaries + - name: Check Step 6 lineage closure and proxy with affected boundaries run: | set -euo pipefail python scripts/check_spec_consistency.py python scripts/check_traceability.py python scripts/release_check.py - python -m pytest -p no:cacheprovider -q tests/unit/test_lineage_concentration.py tests/unit/test_T4_ancestry.py tests/unit/test_lineage_root_resources.py tests/unit/test_T6_cycles.py tests/unit/test_parent_batch.py tests/unit/test_lineage_graph.py tests/unit/test_PR007_version_order.py tests/unit/test_PR008_parent_resolution.py tests/unit/test_PR009_generation.py tests/unit/test_lineage_fixture_inputs.py tests/unit/test_phase3_contracts.py tests/integration/test_phase3_metric_pipeline.py tests/unit/test_PR016_determinism.py tests/integration/test_current_verification.py tests/integration/test_ci_workflows.py tests/integration/test_no_algorithms.py tests/integration/test_repository_structure.py + python -m pytest -p no:cacheprovider -q tests/unit/test_lineage_bounds_proxy.py tests/unit/test_T3_bounds.py tests/unit/test_lineage_concentration.py tests/unit/test_T4_ancestry.py tests/unit/test_lineage_root_resources.py tests/unit/test_T6_cycles.py tests/unit/test_parent_batch.py tests/unit/test_lineage_graph.py tests/unit/test_PR007_version_order.py tests/unit/test_PR008_parent_resolution.py tests/unit/test_PR009_generation.py tests/unit/test_lineage_fixture_inputs.py tests/unit/test_phase3_contracts.py tests/integration/test_phase3_metric_pipeline.py tests/unit/test_PR016_determinism.py tests/integration/test_current_verification.py tests/integration/test_ci_workflows.py tests/integration/test_no_algorithms.py tests/integration/test_repository_structure.py python -m pytest -p no:cacheprovider -q tests/integration/test_phase4_cli.py::test_phase4_step7_validate_blocks_every_calculation_and_content_reader tests/integration/test_phase4_cli.py::test_phase4_step8_pair_does_not_activate_other_calculation_families core: diff --git a/PHASE_5_DECISIONS.md b/PHASE_5_DECISIONS.md index 38d110c..b39f0b1 100644 --- a/PHASE_5_DECISIONS.md +++ b/PHASE_5_DECISIONS.md @@ -1,6 +1,6 @@ # Phase 5 Decisions and Current Authorization -Status: **PLAN APPROVED; STEPS 1-5 COMPLETE; PHASE INCOMPLETE**. +Status: **PLAN APPROVED; STEPS 1-6 COMPLETE; PHASE INCOMPLETE**. On 2026-09-23 UTC the Theory Owner instructed: `批准，Phase 5 step 1开始`. @@ -195,3 +195,26 @@ Package version and executable report schema remain `0.1.0.dev3` and `1.0`. Stop after Step 5. Bounds, shared-root proxy, report/schema integration and CLI lineage dispatch remain later steps. Existing branch synchronization approval continues; merge, tag and publication remain separate actions. + +## Step 6 execution instruction + +On 2026-09-23 UTC the Theory Owner instructed `ok 继续Phase 5 Step 6`. +This authorizes lineage closure bounds and the descriptive shared-ancestry +dependence proxy under P5-D07. Work starts from Step 5 commit +`d81966561502b94701712a3d4b4b7fd50783cba9` on `phase5-lineage`. + +Runtime changes are confined to `metrics/bounds.py` and `lineage/ancestry.py`. +Lineage bounds use the complete target G/C/U partition and preserve direct +bounds. The proxy uses exact root incidence with explicit coverage and no +calibrated risk interpretation. Its absence requires complete nonempty target +evidence; incomplete evidence without a witness remains unavailable. + +The current source check advances its single comparison commit to Step 5 and +opens these two existing owners. Direct oracle, uncertainty, input-boundary and +direct-bound regression checks cover the change; no additional governance +architecture is introduced. Package version and executable report schema stay +`0.1.0.dev3` and `1.0`. + +Stop after Step 6. Report/schema and CLI integration remain Steps 7-8. Existing +branch synchronization approval continues; merge, tag and publication remain +separate actions. diff --git a/PHASE_5_STEP_6.md b/PHASE_5_STEP_6.md new file mode 100644 index 0000000..363da28 --- /dev/null +++ b/PHASE_5_STEP_6.md @@ -0,0 +1,98 @@ +# Phase 5 Step 6 Completion + +Date: 2026-09-23 UTC. Status: **COMPLETE; PHASE INCOMPLETE**. + +The Theory Owner instructed `ok 继续Phase 5 Step 6`. Work starts from +`d81966561502b94701712a3d4b4b7fd50783cba9` on `phase5-lineage`. +This step implements lineage closure bounds and the descriptive shared-root +proxy using the completed ancestry result. + +## Delivered behavior + +`metrics.bounds.lineage_closure_exposure` returns the conservative interval +`[C/N, (C+U)/N]` with width `U/N`. Only selected targets enter the denominator. +The immutable result retains the source G/C/U partition, coverage, unresolved +reasons and input diagnostics. A complete all-unresolved partition yields +`[0,1]`; empty targets and resource-aborted propagation have unavailable values +with explicit reasons. Available bounds do not erase input errors or imply +complete ancestry. Existing direct-bound behavior is preserved. + +`lineage.ancestry.shared_ancestry_dependence` returns `present` only when an +exact root supports at least two targets. It discloses partial status when U>0. +`not_present` requires nonempty targets with fully resolved ancestry and no +shared root. Incomplete evidence without a witness, empty targets and resource +failure remain `unavailable` with `indeterminate` level. Duplicate paths and +aliases cannot manufacture target incidence. + +Both results retain one validated immutable ancestry source, with computed +properties instead of duplicate record/root tables or independently mutable +summary fields. The proxy exposes a real root contribution witness and coverage +references. Neither operation starts ingestion, graph analysis, report assembly +or file/network access. Direct-only bounds retain their lazy dependency boundary. +The outputs remain operational descriptions of lineage topology and conservative +exposure, with no risk calibration or causal/scientific certification. + +## Verification + +Local verification used Python 3.12.14 with the core dependency profile. + +| Check | Result | +|---|---| +| Combined Step 6 direct and affected regression gate | 664 passed in 5.50 seconds | +| Separate validation/pair CLI isolation checks | 2 passed in 0.47 seconds | +| Specification consistency, traceability and source scope | PASS | +| Four workflow YAML files and whitespace check | PASS | +| Independent implementation and test review | PASS; no outstanding blockers | + +The combined gate includes 25 new lineage-bound/proxy cases, all 74 unchanged +direct-bound cases, and existing ancestry, concentration, graph, cycle, resource, +metric-pipeline and input-boundary checks. All twelve frozen ancestry fixtures +retain their interval values. Hero lineage closure remains `[0,0]`, while +its direct closure remains `[1/2,1/2]` before and after explicit lineage calls. + +The direct tests distinguish exact incidence one from two, target incidence from +context/path/alias multiplicity, partial positive witnesses from incomplete +negative evidence, missing from unknown branches, complete all-unresolved +classification from resource-aborted classification, and empty target scope. +Masking grounded or closed evidence into unresolved evidence can only widen the +conservative interval. Invalid typed handoffs are rejected, frozen snapshots +retain input errors, and blocked loaders/graph functions confirm no repeated +graph execution or I/O. + +Independent review additionally checked nine bounded semantic cases with graph, +cycle and provenance dispatch blocked. An isolated process imported the bounds +module and executed both existing direct entry points while all lineage modules +remained unloaded. The review harness needed its source import path and empty +fixture role corrected; no product defect or corresponding source change was +required. The new permanent suite passed on its first execution. + +### Bounded mutation check + +Three passing selected tests were rerun against three disposable source +variants. Each variant imported and executed successfully, then failed actual +semantic assertions with exit code 1. + +| Seeded defect | Detection | +|---|---| +| Upper bound omits U | Expected `2/3`; observed `1/3`. | +| Shared-root threshold changes from `>=2` to `>2` | Exactly two target records lost their required `present` signal. | +| Absence drops the U=0 requirement | Incomplete evidence incorrectly returned `available` instead of `unavailable`. | + +All three mutations were killed. The unchanged variant passed all three selected +tests. No permanent mutation runner or registry was added. + +The current source check advances its single comparison commit to completed +Step 5 and opens only the existing ancestry and bounds modules. The other 57 +product files, exact product inventory, 16 frozen specifications and seven +packaged resource copies remain protected. No additional governance framework, +historical migration chain or permanent mutation mechanism is introduced. + +## Handoff + +Package version remains `0.1.0.dev3`; executable report schema remains `1.0`. +Step 7 report/schema integration requires its separate execution instruction. +Ordinary reports and CLI calls retain their current behavior. + +Full compatibility/release verification and 100k lineage performance measurement +remain their designated candidate/scale gates. No merge, tag, software +publication or hosted CI success is claimed here. diff --git a/docs/lineage_contract.md b/docs/lineage_contract.md index bba3d12..ccde67d 100644 --- a/docs/lineage_contract.md +++ b/docs/lineage_contract.md @@ -12,8 +12,9 @@ parent evidence and `build_lineage_graph` with immutable scope, adjacency and node/edge limits. Step 3 adds `analyze_cycles` for iterative cycle detection, unaffected topology and structural depth. Step 4 adds `analyze_lineage` for external-root resolution, G/C/U counts and coverage. Step 5 adds root incidence, -fractional mass and ancestry concentration. Bounds, report and CLI lineage -interfaces remain later implementation targets. +fractional mass and ancestry concentration. Step 6 adds explicit lineage closure +and shared-root proxy calls. Report and CLI lineage interfaces remain later +implementation targets. The package remains at `0.1.0.dev3` and the executable report schema at `1.0`. @@ -94,8 +95,9 @@ CLI orchestration calls it only after an explicit lineage request. Steps 4-5 implement this direct API for root resolution, coverage and concentration. Its `LineageAnalysisResult.records` contains target records in canonical order; context roots are intermediate computation data and do not enter target -denominators. The existing `ExecutionStatus` vocabulary applies. Bounds and proxy -fields remain later steps. +denominators. The existing `ExecutionStatus` vocabulary applies. Step 6 derives +bounds and the descriptive proxy through separate explicit calls over that +typed result; `analyze_lineage` does not invoke either automatically. Graph admission failures retain the existing `LineageResourceLimitError` before a valid analytical result can be created. A root-membership or union-work @@ -128,8 +130,8 @@ Ownership stays in existing modules: |---|---| | `lineage/graph.py` | `LineageGraph`: scope, sorted node keys, deduplicated immutable `parents_by_child` and `children_by_parent`, retained per-record reference/declaration evidence, invalid-self-reference evidence and validation messages. | | `lineage/cycles.py` | `CycleAnalysis`: cyclic components, member keys, affected descendant keys, topology for unaffected nodes and structural depth assessments. Iterative algorithms only. | -| `lineage/ancestry.py` | `RecordAncestry`, `RootContribution`, `LineageAnalysisResult`; complete root resolution, G/C/U partition, root incidence/mass and concentration. | -| `metrics/bounds.py` | Lineage closure envelope from the typed G/C/U counts, preserving the direct-bound branch. | +| `lineage/ancestry.py` | `RecordAncestry`, `RootContribution`, `LineageAnalysisResult`; complete root resolution, G/C/U partition, root incidence/mass and concentration. `SharedAncestryDependence` describes shared-root evidence. | +| `metrics/bounds.py` | `LineageClosureExposureBounds` from the typed G/C/U counts, preserving the direct-bound branch. | | `result.py`, `reports/assembly.py` | Validate/assemble supplied typed results, scope/status metadata and canonical schema 1.1 values. No graph traversal or metric recomputation. | | `reports/json_report.py`, `reports/markdown_report.py` | Render the same privacy-safe canonical result. No computation ownership. | @@ -319,6 +321,33 @@ biological relatedness, universal diversity, integrity or collapse. T4 retains its theory-guided operationalization label; T6 is an engineering graph-validity rule. Existing unavailable scientific conclusions remain explicit. +### Step 6 direct calculation interfaces + +`metrics.bounds.lineage_closure_exposure(result)` returns immutable +`LineageClosureExposureBounds`. Its only stored field, `source`, is a revalidated +`LineageAnalysisResult`, excluded from the wrapper representation. Scope, target +G/C/U counts, denominator, coverage, unresolved reasons and input diagnostics +remain accessible without copying another record or root table. `lower_bound`, +`upper_bound` and `interval_width` expose the three ratios as floats or null. + +The interval's `status` is `available` for a completed nonempty target partition, +including U>0 and the all-unresolved interval `[0,1]`. This status describes the +conservative interval. Input execution status, errors and unresolved reasons +retain their separate meaning. Empty scope yields unavailable values with +`EMPTY_TARGET_SCOPE`; an aborted root traversal yields unavailable values with +`LINEAGE_RESOURCE_LIMIT_EXCEEDED`. The original direct-bound functions preserve +their formulas, classifications and input-only dependency behavior. + +`lineage.ancestry.shared_ancestry_dependence(result)` returns immutable +`SharedAncestryDependence`, retaining one revalidated `source` and deriving the +status/level table above. A positive witness is the first already ranked +`RootContribution` with incidence at least two. The proxy exposes actual +incidence and coverage field references, target scope and source diagnostics. +It adds no calibrated severity, risk threshold or causal conclusion. Both calls +consume already computed ancestry and perform no graph traversal, ingestion, +report rendering or network/file access. Serialized schema integration remains +Step 7. + ## 5. Schema 1.1 placement, detail bounds and privacy Step 7 changes every Phase 5 output to report schema 1.1, including requests diff --git a/docs/release_process.md b/docs/release_process.md index af89938..611e586 100644 --- a/docs/release_process.md +++ b/docs/release_process.md @@ -2,10 +2,10 @@ ## Current Phase 5 boundary -Phase 5 Step 5 is approved. Runtime remains at `0.1.0.dev3`, with executable -report schema 1.0. Root incidence, fractional mass and ancestry concentration are -the current implementation scope; bounds and report integration -remain subsequent steps. +Phase 5 Step 6 is approved. Runtime remains at `0.1.0.dev3`, with executable +report schema 1.0. Lineage closure bounds and the descriptive shared-root proxy +are the current implementation scope; report and CLI integration remain +subsequent steps. Stable v0.1 publication, a tag and main merge require separate authorization. @@ -19,9 +19,9 @@ python scripts/release_check.py Historical `--phase` and `--step` dispatch is retired. Earlier source forms, migrations and receipts remain recoverable from Git and archived phase records. -The existing source check advances its single reference to the completed Step 4 -commit; no new Phase 5 hash registry is needed. During Step 5 it permits changes -only to the existing ancestry module, retaining the exact product file inventory and +The existing source check advances its single reference to the completed Step 5 +commit; no new Phase 5 hash registry is needed. During Step 6 it permits changes +only to the existing ancestry and bounds modules, retaining the exact product file inventory and rejecting unrelated runtime, schema, package-metadata and canonical Hero changes. The frozen Phase 0 specification hashes are unchanged. Sixteen frozen specifications and seven packaged resource copies remain protected. @@ -47,7 +47,7 @@ Dispatch `ci.yml` with `gate=candidate` for a stable candidate. The workflow run Compatibility cells run complete canonical regression with `--ignore=tests/performance`. The designated performance job runs `tests/performance` once, avoiding nine repetitions of the costly 100k report -workload. The Phase 5 sparse-lineage test is still a placeholder in Step 5; +workload. The Phase 5 sparse-lineage test is still a placeholder in Step 6; it becomes an executable graph benchmark in Step 9. No placeholder is counted as a successful lineage measurement. diff --git a/scripts/release_check.py b/scripts/release_check.py index 7e0b173..3d4f3d6 100644 --- a/scripts/release_check.py +++ b/scripts/release_check.py @@ -1,6 +1,6 @@ """Current source and candidate checks for Recursive Integrity Toolkit. -Step 5 opens the approved root incidence and concentration implementation scope. Historical dispatch, source-body +Step 6 opens the approved lineage closure and descriptive proxy scope. Historical dispatch, source-body migrations and phase registries are recoverable from the accepted Git commit. Installed checks below retain their existing product, privacy and package cases. """ @@ -24,10 +24,11 @@ import zipfile ROOT = Path(__file__).resolve().parents[1] -ACCEPTED_COMMIT = "2450100d08355d8cb1e0be931345f44572861e4a" +ACCEPTED_COMMIT = "d81966561502b94701712a3d4b4b7fd50783cba9" PROTECTED_PREFIXES = ("src/", "schemas/", "examples/hero/") CURRENT_IMPLEMENTATION_PATHS = frozenset({ "src/recursive_integrity_toolkit/lineage/ancestry.py", + "src/recursive_integrity_toolkit/metrics/bounds.py", }) PARQUET_CASES = {"test_PR002_parquet_real_roundtrip", "test_PR002_parquet_real_row_limit", "test_PR002_parquet_real_invalid_file"} HERO_NAMES = {"records_v1.csv", "records_v2.csv", "provenance.csv", "config.json", "version_order.json", "EXPECTED_OUTPUTS.md"} @@ -57,7 +58,7 @@ def _regular_file(root: Path, relative: str) -> Path: def verify_source_scope(root: Path = ROOT) -> dict: - """Reject product changes outside the currently authorized ancestry scope. + """Reject product changes outside the currently authorized lineage owners. Later authorized implementation updates this single current boundary. It does not append another historical dispatcher or grant authority through metadata. @@ -67,7 +68,7 @@ def verify_source_scope(root: Path = ROOT) -> dict: for name, raw in expected.items(): current = _regular_file(root, name).read_bytes() if name not in CURRENT_IMPLEMENTATION_PATHS and current != raw: - raise ValueError(f"Unauthorized product mutation outside Step 5 scope: {name}") + raise ValueError(f"Unauthorized product mutation outside Step 6 scope: {name}") for prefix in PROTECTED_PREFIXES: paths = list((root / prefix).rglob("*")) if (root / prefix).is_symlink() or any(path.is_symlink() for path in paths): @@ -75,7 +76,7 @@ def verify_source_scope(root: Path = ROOT) -> dict: actual = {path.relative_to(root).as_posix() for path in paths if path.is_file() and "__pycache__" not in path.parts} if actual != {name for name in expected if name.startswith(prefix)}: - raise ValueError(f"Unauthorized product file-set mutation during Step 5: {prefix}") + raise ValueError(f"Unauthorized product file-set mutation during Step 6: {prefix}") return {"protected_product_files": len(expected) - len(CURRENT_IMPLEMENTATION_PATHS), "authorized_implementation_files": len(CURRENT_IMPLEMENTATION_PATHS), "product_file_inventory": "unchanged"} diff --git a/src/recursive_integrity_toolkit/lineage/ancestry.py b/src/recursive_integrity_toolkit/lineage/ancestry.py index 4c1a8f8..9f51e0f 100644 --- a/src/recursive_integrity_toolkit/lineage/ancestry.py +++ b/src/recursive_integrity_toolkit/lineage/ancestry.py @@ -1,6 +1,6 @@ -"""Resolve exact external roots, target coverage and root concentration. +"""Resolve external roots, coverage, concentration and shared-root evidence. -Owner IDs: T4; Phase 5 Step 5. +Owner IDs: T4; Phase 5 Step 6. Validated explicit grounding supplies anchors. Iterative dependency scheduling unions complete root sets; unknown branches never supply exact roots. Logical @@ -8,13 +8,13 @@ The allocation is topological and makes no causal or scientific quality claim. Current phase status: - Phase 5 Step 5 adds target incidence, fractional mass and concentration. - Bounds and reports remain later steps. No file or network I/O, generation - calculation or implicit invocation. + Phase 5 Step 6 adds an explicitly requested descriptive shared-root proxy. + Reports remain a later step. No file or network I/O, generation calculation + or implicit invocation. """ from __future__ import annotations -from dataclasses import dataclass, replace +from dataclasses import dataclass, field, replace from heapq import heapify, heappop, heappush from math import fsum, isfinite from types import MappingProxyType @@ -314,6 +314,117 @@ def depth_resolved_record_count(self) -> int: return self.cycles.depth_resolved_record_count +@dataclass(frozen=True, slots=True) +class SharedAncestryDependence: + """Descriptive exact-root witness with its original coverage and errors. + + A single ranked contribution is sufficient to witness shared support. The + immutable source retains all evidence without copying the contribution list. + Unresolved targets can preserve a witnessed presence but cannot prove absence. + """ + source: LineageAnalysisResult = field(repr=False) + + def __post_init__(self) -> None: + if type(self.source) is not LineageAnalysisResult: + raise _invalid("shared ancestry requires a typed lineage analysis") + object.__setattr__(self, "source", replace(self.source)) + + @property + def witness(self) -> RootContribution | None: + roots = self.source.root_contributions + return roots[0] if roots and roots[0].incidence_count >= 2 else None + + @property + def status(self) -> ReportStatus: + if self.witness is not None: + return ReportStatus.PARTIAL if self.unresolved_record_count else ReportStatus.AVAILABLE + if self.source.records is not None and self.scope.target_record_count and not self.unresolved_record_count: + return ReportStatus.AVAILABLE + return ReportStatus.UNAVAILABLE + + @property + def level(self) -> Literal["present", "not_present", "indeterminate"]: + if self.witness is not None: + return "present" + return "not_present" if self.status is ReportStatus.AVAILABLE else "indeterminate" + + @property + def reason_codes(self) -> tuple[str, ...]: + if self.source.records is None: + return ("LINEAGE_RESOURCE_LIMIT_EXCEEDED",) + if not self.scope.target_record_count: + return ("EMPTY_TARGET_SCOPE",) + return ("UNRESOLVED_ANCESTRY",) if self.unresolved_record_count else () + + @property + def scope(self) -> LineageScope: + return self.source.scope + + @property + def grounded_record_count(self) -> int | None: + return self.source.grounded_record_count + + @property + def closed_record_count(self) -> int | None: + return self.source.closed_record_count + + @property + def unresolved_record_count(self) -> int | None: + return self.source.unresolved_record_count + + @property + def resolved_lineage_coverage(self) -> float | None: + return self.source.resolved_lineage_coverage + + @property + def external_ancestry_coverage(self) -> float | None: + return self.source.external_ancestry_coverage + + @property + def ancestry_coverage_reason_codes(self) -> tuple[str, ...]: + return self.source.ancestry_coverage_reason_codes + + @property + def unresolved_reason_codes(self) -> tuple[str, ...]: + return tuple(sorted({reason for record in self.source.records or () for reason in record.reason_codes})) + + @property + def input_execution_status(self) -> ExecutionStatus: + return self.source.execution_status + + @property + def input_execution_reason_codes(self) -> tuple[str, ...]: + return self.source.execution_reason_codes + + @property + def input_has_errors(self) -> bool: + return any(message.severity in (ValidationSeverity.ERROR, ValidationSeverity.FATAL) + for message in self.source.messages) + + @property + def validation_messages(self) -> tuple[ValidationMessage, ...]: + return self.source.messages + + @property + def evidence_fields(self) -> tuple[str, ...]: + return ("root_contributions.incidence_count", "resolved_lineage_coverage", "external_ancestry_coverage") + + @property + def operationalization_label(self) -> str: + return "theory_guided_operationalization" + + @property + def limitations(self) -> tuple[str, ...]: + return ("Descriptive shared-root topology under supplied metadata; no calibrated risk level.", + "Does not establish causal contribution, semantic error, independent information, " + "biological relatedness, universal integrity or collapse.") + + +def shared_ancestry_dependence(result: LineageAnalysisResult) -> SharedAncestryDependence: + """Explicit pure proxy request; never reload data or rerun graph analysis.""" + return SharedAncestryDependence(result) + + class _RootBudget: """Charge each logical unit before performing it; never expose partial roots.""" def __init__(self, usage: LineageResourceUsage) -> None: diff --git a/src/recursive_integrity_toolkit/metrics/bounds.py b/src/recursive_integrity_toolkit/metrics/bounds.py index 80661dd..1b0077b 100644 --- a/src/recursive_integrity_toolkit/metrics/bounds.py +++ b/src/recursive_integrity_toolkit/metrics/bounds.py @@ -1,11 +1,12 @@ -"""Compute the direct closure-exposure interval, without lineage inference. +"""Compute separately labeled direct and lineage closure-exposure intervals. Owner IDs: - T3: F-009, F-010; Definitions 5.3-5.5; Phase 3 Step 6. + T3: F-009, F-010; Definitions 5.3-5.5 and 5.9; Phase 5 Step 6. Inputs: Explicit integer counts and a single-version scope, or the accepted Step 5 - ProvenanceCompositionResult. No source files or parent references are read. + ProvenanceCompositionResult for direct bounds. Explicit LineageAnalysisResult + for lineage bounds. No source files or parent references are read. Outputs: Immutable lower, upper and width calculations, denominator, operationalization @@ -17,17 +18,19 @@ has not been independently verified by this calculation. Limits: - No lineage bounds, midpoint, confidence discount, risk threshold, universal - score, empirical truth checking, simulation, report or file/network access. + No ancestry inference, midpoint, confidence discount, risk threshold, + universal score, empirical truth checking, simulation, report or file/network access. A count envelope alone never certifies usable dataset provenance. Current phase status: - Phase 3 Step 6 direct bounds only. Pure explicit calls; import-safe. + Phase 5 Step 6 adds explicit lineage bounds. Direct behavior is unchanged. + Pure explicit calls; direct-only users do not import lineage modules. """ from __future__ import annotations from dataclasses import dataclass, field, replace from math import isclose +from typing import TYPE_CHECKING from ..errors import CanonicalValidationError, ErrorCode from ..models import ( @@ -41,6 +44,11 @@ ProvenanceCompositionResult, ) +if TYPE_CHECKING: + from ..lineage.ancestry import LineageAnalysisResult + from ..lineage.graph import LineageScope + from ..result import ExecutionStatus, ReportStatus + @dataclass(frozen=True, slots=True) class DirectClosureExposureBounds: @@ -338,3 +346,121 @@ def direct_closure_exposure(result: ProvenanceCompositionResult) -> DirectClosur grounding_field_coverage=grounding_cov, confidence_field_coverage=confidence_cov, confidence_status=confidence_status, confidence_counts=confidence_counts, input_has_errors=result.input_has_errors, validation_messages=messages) + + +@dataclass(frozen=True, slots=True) +class LineageClosureExposureBounds: + """Exact conservative envelope from a complete target G/C/U partition. + + Interval availability is separate from input validity and ancestry coverage. + A nonempty all-unresolved partition supplies [0, 1], retaining all errors. + An aborted partition or an empty population cannot supply numeric bounds. + """ + source: LineageAnalysisResult = field(repr=False) + + def __post_init__(self) -> None: + from ..lineage.ancestry import LineageAnalysisResult + if type(self.source) is not LineageAnalysisResult: + raise CanonicalValidationError( + ErrorCode.SCHEMA_TYPE, "lineage bounds require a typed lineage analysis", + field="lineage_closure_exposure") + object.__setattr__(self, "source", replace(self.source)) + + @property + def status(self) -> ReportStatus: + from ..result import ReportStatus + return ReportStatus.UNAVAILABLE if self.reason_codes else ReportStatus.AVAILABLE + + @property + def reason_codes(self) -> tuple[str, ...]: + if self.source.records is None: + return ("LINEAGE_RESOURCE_LIMIT_EXCEEDED",) + return () if self.denominator else ("EMPTY_TARGET_SCOPE",) + + @property + def lower_bound(self) -> float | None: + return None if self.reason_codes else self.closed_record_count / self.denominator + + @property + def upper_bound(self) -> float | None: + return None if self.reason_codes else (self.closed_record_count + self.unresolved_record_count) / self.denominator + + @property + def interval_width(self) -> float | None: + return None if self.reason_codes else self.unresolved_record_count / self.denominator + + @property + def scope(self) -> LineageScope: + return self.source.scope + + @property + def denominator(self) -> int: + return self.scope.target_record_count + + @property + def denominator_basis(self) -> str: + return "all_valid_records_in_selected_dataset_scope" + + @property + def grounded_record_count(self) -> int | None: + return self.source.grounded_record_count + + @property + def closed_record_count(self) -> int | None: + return self.source.closed_record_count + + @property + def unresolved_record_count(self) -> int | None: + return self.source.unresolved_record_count + + @property + def resolved_lineage_coverage(self) -> float | None: + return self.source.resolved_lineage_coverage + + @property + def external_ancestry_coverage(self) -> float | None: + return self.source.external_ancestry_coverage + + @property + def ancestry_coverage_reason_codes(self) -> tuple[str, ...]: + return self.source.ancestry_coverage_reason_codes + + @property + def unresolved_reason_codes(self) -> tuple[str, ...]: + return tuple(sorted({reason for record in self.source.records or () for reason in record.reason_codes})) + + @property + def input_execution_status(self) -> ExecutionStatus: + return self.source.execution_status + + @property + def input_execution_reason_codes(self) -> tuple[str, ...]: + return self.source.execution_reason_codes + + @property + def input_has_errors(self) -> bool: + return any(message.severity in (ValidationSeverity.ERROR, ValidationSeverity.FATAL) + for message in self.source.messages) + + @property + def validation_messages(self) -> tuple[ValidationMessage, ...]: + return self.source.messages + + @property + def classification_basis(self) -> str: + return "toolkit_operationalization" + + @property + def operationalization_label(self) -> str: + return "toolkit_operationalization" + + @property + def limitations(self) -> tuple[str, ...]: + return ("Lineage exposure is relative to the audited loop and supplied metadata.", + "Unresolved ancestry remains in the conservative interval; availability does not certify input validity.", + "No midpoint, calibrated risk threshold or causal claim.") + + +def lineage_closure_exposure(result: LineageAnalysisResult) -> LineageClosureExposureBounds: + """Explicit pure lineage interval request; keep direct bounds unchanged.""" + return LineageClosureExposureBounds(result) diff --git a/tests/integration/test_current_verification.py b/tests/integration/test_current_verification.py index a24fe2a..1f42f4e 100644 --- a/tests/integration/test_current_verification.py +++ b/tests/integration/test_current_verification.py @@ -27,7 +27,7 @@ def protected_copy(repo_root, tmp_path): @pytest.mark.parametrize("mutation", ["formula", "new_module", "deleted_module"]) def test_current_scope_rejects_unauthorized_product_changes(current_tools, protected_copy, mutation): - """Root concentration does not authorize unrelated product changes.""" + """Lineage closure and proxy do not authorize unrelated product changes.""" verify = current_tools["verify_source_scope"] verify(protected_copy) target = protected_copy / "src/recursive_integrity_toolkit/metrics/diversity.py" diff --git a/tests/unit/test_lineage_bounds_proxy.py b/tests/unit/test_lineage_bounds_proxy.py new file mode 100644 index 0000000..28eb4ba --- /dev/null +++ b/tests/unit/test_lineage_bounds_proxy.py @@ -0,0 +1,319 @@ +"""Lineage closure envelopes and descriptive shared-root evidence. + +Frozen acceptance values, exact fractions and masking protect the partition +semantics. Direct grounding keeps its separate calculation and denominator. +""" + +import builtins +from dataclasses import FrozenInstanceError, replace +from fractions import Fraction +import json +import math +from pathlib import Path +import socket + +import pytest + +from recursive_integrity_toolkit.errors import CanonicalValidationError +from recursive_integrity_toolkit.io.validation import join_provenance, validate_bundle +from recursive_integrity_toolkit.lineage import ancestry, cycles, graph +from recursive_integrity_toolkit.lineage.graph import LineageLimits +from recursive_integrity_toolkit.metrics import bounds +from recursive_integrity_toolkit.metrics.provenance import summarize_provenance +from recursive_integrity_toolkit.models import AuditBundle, CalculationScope, FileRole, InputSource +from recursive_integrity_toolkit.result import ExecutionStatus, ReportStatus + +# Reuse canonical input construction only. Expected intervals and witnesses +# below come from the frozen fixtures or independently authored root sets. +from test_T4_ancestry import ( + _CASES, _FIXTURES, _MISSING, _ROOT, _fixture_validation, _key, _validation, +) + + +def _analyze(validation, target="v2", **limits): + return ancestry.analyze_lineage(validation, target_dataset_version=target, + limits=LineageLimits(**limits)) + + +def _values(result): + return result.lower_bound, result.upper_bound, result.interval_width + + +def _assert_interval(result, expected): + if expected is None: + assert _values(result) == (None, None, None) + assert result.status is ReportStatus.UNAVAILABLE + assert result.reason_codes + else: + for actual, value in zip(_values(result), expected, strict=True): + assert type(actual) is float and math.isfinite(actual) + assert actual == pytest.approx(float(Fraction(value)), rel=1e-13, abs=1e-15) + assert result.status is ReportStatus.AVAILABLE + assert result.reason_codes == () + + +@pytest.mark.parametrize("case", _CASES, ids=lambda case: case["case_id"]) +def test_frozen_partition_bounds_and_shared_root_evidence(case, tmp_path): + validation, target = _fixture_validation(tmp_path, case) + analysis = _analyze(validation, target) + interval = bounds.lineage_closure_exposure(analysis) + proxy = ancestry.shared_ancestry_dependence(analysis) + expected = case["expected_lineage"] + counts = expected["counts"] + _assert_interval(interval, expected["lineage_bounds"]) + assert interval.denominator == counts["target"] + assert (interval.grounded_record_count, interval.closed_record_count, + interval.unresolved_record_count) == ( + counts["grounded"], counts["closed"], counts["unresolved"]) + assert interval.scope == proxy.scope == analysis.scope + assert interval.input_execution_status is proxy.input_execution_status is analysis.execution_status + assert interval.validation_messages == proxy.validation_messages == analysis.messages + roots = [set(values) for values in expected["complete_root_sets"].values()] + incidence = {root: sum(root in support for support in roots) + for support in roots for root in support} + witnesses = sorted((root for root, count in incidence.items() if count >= 2), + key=lambda root: (-incidence[root], _key(root))) + if witnesses: + assert proxy.level == "present" + assert proxy.status is (ReportStatus.PARTIAL if counts["unresolved"] else ReportStatus.AVAILABLE) + assert str(proxy.witness.record_key) == witnesses[0] + assert proxy.witness.incidence_count == incidence[witnesses[0]] + elif counts["target"] and not counts["unresolved"]: + assert proxy.level == "not_present" + assert proxy.status is ReportStatus.AVAILABLE + assert proxy.witness is None + else: + assert proxy.level == "indeterminate" + assert proxy.status is ReportStatus.UNAVAILABLE + assert proxy.witness is None + if "shared_ancestry_proxy" in expected: + assert proxy.level == expected["shared_ancestry_proxy"] + reason = (("EMPTY_TARGET_SCOPE",) if not counts["target"] else + ("UNRESOLVED_ANCESTRY",) if counts["unresolved"] else ()) + assert proxy.reason_codes == reason + if not counts["target"]: + assert interval.reason_codes == ("EMPTY_TARGET_SCOPE",) + + +def test_hero_lineage_zero_and_direct_half_remain_distinct(): + hero = _ROOT / "examples" / "hero" + expected = json.loads((_FIXTURES / "lineage_complete" / "hero_expected.json").read_text()) + validation = validate_bundle(AuditBundle(( + InputSource(FileRole.RECORDS_PRIMARY, hero / "records_v2.csv"), + InputSource(FileRole.RECORDS_COMPARE, hero / "records_v1.csv"), + InputSource(FileRole.PROVENANCE_MANIFEST, hero / "provenance.csv"), + InputSource(FileRole.VERSION_ORDER, hero / "version_order.json"), + ))) + joined = join_provenance(validation.records, validation.provenance, dataset_versions=("v2",)) + scope = CalculationScope(("v2",), joined.scope_record_keys, (), + joined.provenance_row_coverage.denominator_name, "hero-direct") + composition = summarize_provenance(joined, scope=scope) + direct_before = bounds.direct_closure_exposure(composition) + analysis = _analyze(validation) + interval = bounds.lineage_closure_exposure(analysis) + proxy = ancestry.shared_ancestry_dependence(analysis) + direct_after = bounds.direct_closure_exposure(composition) + _assert_interval(interval, expected["lineage_bounds"]) + assert tuple(getattr(direct_after, field).value for field in + ("lower_bound", "upper_bound", "interval_width")) == tuple( + float(Fraction(value)) for value in expected["direct_bounds"]) + assert direct_after == direct_before + assert interval.denominator == direct_after.denominator == 8 + assert interval.scope.loaded_record_count == 16 + assert interval.input_execution_status is ExecutionStatus.COMPLETED + assert proxy.status is ReportStatus.AVAILABLE and proxy.level == "present" + assert str(proxy.witness.record_key) == "v1::v1_01" + assert proxy.witness.incidence_count == 3 + + +@pytest.mark.parametrize("target_count", (1, 2)) +def test_shared_root_boundary_counts_targets_and_deduplicates_aliases(target_count, tmp_path): + declarations = { + "v1::root": [], "v1::context_a": ["v1::root"], + "v1::context_b": ["v1::root"], + **{f"v2::t{index}": ["root", "v1::root", "root"] for index in range(target_count)}, + } + analysis = _analyze(_validation(tmp_path, declarations, grounding={"v1::root": "yes"})) + proxy = ancestry.shared_ancestry_dependence(analysis) + assert analysis.root_contributions[0].incidence_count == target_count + assert proxy.status is ReportStatus.AVAILABLE + assert proxy.level == ("present" if target_count == 2 else "not_present") + assert (proxy.witness.incidence_count if proxy.witness else None) == (2 if target_count == 2 else None) + assert proxy.scope.target_record_count == target_count + assert proxy.scope.context_record_count == 3 + _assert_interval(bounds.lineage_closure_exposure(analysis), (0, 0, 0)) + + +@pytest.mark.parametrize("witness", (False, True)) +def test_incomplete_ancestry_requires_positive_witness_for_partial_proxy(witness, tmp_path): + declarations = {"v1::a": [], "v1::b": [], "v2::one": ["v1::a"], + "v2::two": ["v1::a" if witness else "v1::b"], "v2::u": []} + analysis = _analyze(_validation(tmp_path, declarations, grounding={ + "v1::a": "yes", "v1::b": "yes", "v2::u": "unknown"})) + interval = bounds.lineage_closure_exposure(analysis) + proxy = ancestry.shared_ancestry_dependence(analysis) + _assert_interval(interval, (0, "1/3", "1/3")) + assert proxy.status is (ReportStatus.PARTIAL if witness else ReportStatus.UNAVAILABLE) + assert proxy.level == ("present" if witness else "indeterminate") + assert proxy.reason_codes == ("UNRESOLVED_ANCESTRY",) + assert (proxy.witness is not None) is witness + assert interval.resolved_lineage_coverage == proxy.resolved_lineage_coverage == 2 / 3 + assert interval.external_ancestry_coverage == proxy.external_ancestry_coverage == 2 / 3 + assert interval.unresolved_reason_codes == ("UNKNOWN_GROUNDING",) + assert set(proxy.evidence_fields) == { + "root_contributions.incidence_count", "resolved_lineage_coverage", "external_ancestry_coverage"} + + +@pytest.mark.parametrize("failure", (False, True)) +def test_completed_all_unresolved_partition_has_conservative_numeric_bounds(failure, tmp_path): + validation = (_validation(tmp_path, {"v2::u": ["v1::missing"]}, strict=True) if failure else + _validation(tmp_path, {"v2::u": []}, grounding={"v2::u": "unknown"})) + analysis = _analyze(validation) + interval = bounds.lineage_closure_exposure(analysis) + proxy = ancestry.shared_ancestry_dependence(analysis) + assert analysis.execution_status is (ExecutionStatus.FAILED if failure else ExecutionStatus.PARTIAL) + _assert_interval(interval, (0, 1, 1)) + assert interval.unresolved_record_count == interval.denominator == 1 + assert interval.input_has_errors is failure + assert interval.input_execution_reason_codes == analysis.execution_reason_codes + assert interval.validation_messages == analysis.messages + assert interval.unresolved_reason_codes + assert proxy.status is ReportStatus.UNAVAILABLE and proxy.level == "indeterminate" + assert proxy.input_has_errors is failure + + +def test_missing_and_unknown_branches_exclude_observed_roots_without_erasing_valid_witness(tmp_path): + declarations = {"v1::root": [], "v1::unknown": [], "v1::missing_provenance": _MISSING, + "v2::g1": ["v1::root"], "v2::g2": ["v1::root"], "v2::closed": [], + "v2::u_unknown": ["v1::root", "v1::unknown"], + "v2::u_missing": ["v1::root", "v1::missing_provenance"]} + analysis = _analyze(_validation(tmp_path, declarations, grounding={ + "v1::root": "yes", "v1::unknown": "unknown"})) + interval = bounds.lineage_closure_exposure(analysis) + proxy = ancestry.shared_ancestry_dependence(analysis) + _assert_interval(interval, ("1/5", "3/5", "2/5")) + assert (interval.grounded_record_count, interval.closed_record_count, + interval.unresolved_record_count) == (2, 1, 2) + assert proxy.status is ReportStatus.PARTIAL and proxy.level == "present" + assert proxy.witness.incidence_count == 2 + assert proxy.resolved_lineage_coverage == 3 / 5 + assert proxy.external_ancestry_coverage == 2 / 5 + assert {"UNKNOWN_GROUNDING", "MISSING_PROVENANCE", "INCOMPLETE_PARENT_ANCESTRY"} <= set( + interval.unresolved_reason_codes) + + +def test_declared_empty_target_and_resource_abort_have_distinct_unavailable_reasons(tmp_path): + empty = _analyze(_validation(tmp_path / "empty", {"v1::a": []}, + grounding={"v1::a": "yes"}, target="v2")) + aborted = _analyze(_validation(tmp_path / "resource", {"v1::a": [], "v2::t": ["v1::a"]}, + grounding={"v1::a": "yes"}), max_root_memberships=1) + for source, reason, count in ((empty, "EMPTY_TARGET_SCOPE", 0), + (aborted, "LINEAGE_RESOURCE_LIMIT_EXCEEDED", None)): + interval = bounds.lineage_closure_exposure(source) + proxy = ancestry.shared_ancestry_dependence(source) + _assert_interval(interval, None) + assert interval.reason_codes == proxy.reason_codes == (reason,) + assert interval.scope.target_dataset_version == "v2" + assert interval.grounded_record_count == interval.closed_record_count == interval.unresolved_record_count == count + assert interval.resolved_lineage_coverage is interval.external_ancestry_coverage is None + assert proxy.status is ReportStatus.UNAVAILABLE and proxy.level == "indeterminate" + assert proxy.witness is None + assert interval.input_execution_status is source.execution_status + assert empty.execution_status is ExecutionStatus.COMPLETED + assert aborted.execution_status is ExecutionStatus.FAILED + + +def test_masking_complete_grounded_or_closed_evidence_only_widens_the_interval(tmp_path): + declarations = {"v2::grounded": [], "v2::closed": [], "v2::unknown": []} + originals = {"v2::grounded": "yes", "v2::unknown": "unknown"} + baseline = bounds.lineage_closure_exposure(_analyze(_validation( + tmp_path / "base", declarations, grounding=originals))) + _assert_interval(baseline, ("1/3", "2/3", "1/3")) + for name, expected in (("grounded", ("1/3", 1, "2/3")), ("closed", (0, "2/3", "2/3"))): + masked = bounds.lineage_closure_exposure(_analyze(_validation( + tmp_path / name, declarations, grounding={**originals, f"v2::{name}": "unknown"}))) + _assert_interval(masked, expected) + assert masked.lower_bound <= baseline.lower_bound <= baseline.upper_bound <= masked.upper_bound + assert masked.interval_width >= baseline.interval_width + assert masked.denominator == baseline.denominator + + +def _mixed_analysis(tmp_path): + return _analyze(_validation(tmp_path, {"v1::a": [], "v2::g1": ["v1::a"], + "v2::g2": ["v1::a"], "v2::closed": [], "v2::u": []}, + grounding={"v1::a": "yes", "v2::u": "unknown"})) + + +def test_bounds_and_proxy_are_frozen_source_snapshots_without_unsupported_scores(tmp_path): + source = _mixed_analysis(tmp_path) + interval = bounds.lineage_closure_exposure(source) + proxy = ancestry.shared_ancestry_dependence(source) + assert type(interval) is bounds.LineageClosureExposureBounds + assert type(proxy) is ancestry.SharedAncestryDependence + assert interval.denominator_basis == "all_valid_records_in_selected_dataset_scope" + for value in (interval, proxy): + assert value.source is not source and value.source == source + with pytest.raises(FrozenInstanceError): + value.source = source + for name in ("midpoint", "risk_score", "risk_threshold", "severity", "universal_score", + "causal_contribution", "independent_information", "biological_relatedness"): + assert not hasattr(value, name) + with pytest.raises(TypeError): + replace(interval, lower_bound=0.0) + with pytest.raises(TypeError): + replace(proxy, level="not_present") + object.__setattr__(source, "closed_record_count", 999) + object.__setattr__(source, "execution_status", ExecutionStatus.COMPLETED) + _assert_interval(interval, ("1/4", "1/2", "1/4")) + assert proxy.status is ReportStatus.PARTIAL and proxy.level == "present" + assert interval.input_execution_status is proxy.input_execution_status is ExecutionStatus.PARTIAL + + +def test_new_entrypoints_reject_wrong_types_and_forged_complete_handoffs(tmp_path): + source = _mixed_analysis(tmp_path) + + class Hostile: + def __getattr__(self, name): + raise AssertionError("foreign input must not be dispatched") + + for calculate in (bounds.lineage_closure_exposure, ancestry.shared_ancestry_dependence): + for invalid in (None, {}, Hostile()): + with pytest.raises(CanonicalValidationError): + calculate(invalid) + for field, value in (("closed_record_count", True), ("grounded_record_count", 4), + ("unresolved_record_count", 0), ("records", source.records[:-1]), + ("root_contributions", ()), ("resolved_lineage_coverage", 1.0), + ("execution_status", ExecutionStatus.COMPLETED)): + forged = replace(source) + object.__setattr__(forged, field, value) + with pytest.raises(CanonicalValidationError): + calculate(forged) + + +def test_pure_summaries_do_not_rerun_graph_or_access_files_network_or_raw_data(tmp_path, monkeypatch): + source = _mixed_analysis(tmp_path) + + def blocked(*args, **kwargs): + raise AssertionError("a typed summary must not perform I/O or repeat graph resolution") + + with monkeypatch.context() as patch: + patch.setattr(builtins, "open", blocked) + patch.setattr(Path, "open", blocked) + patch.setattr(socket, "socket", blocked) + patch.setattr(socket, "create_connection", blocked) + patch.setattr(graph, "build_lineage_graph", blocked) + patch.setattr(cycles, "analyze_cycles", blocked) + patch.setattr(ancestry, "build_lineage_graph", blocked) + patch.setattr(ancestry, "analyze_cycles", blocked) + patch.setattr(ancestry, "analyze_lineage", blocked) + patch.setattr(ancestry, "_resolve_roots", blocked) + patch.setattr(ancestry, "join_provenance", blocked) + interval = bounds.lineage_closure_exposure(source) + proxy = ancestry.shared_ancestry_dependence(source) + _assert_interval(interval, ("1/4", "1/2", "1/4")) + assert proxy.level == "present" + for value in (interval, proxy, interval.source, proxy.source): + assert "Private ancestry fixture" not in repr(value) + assert str(tmp_path) not in repr(value) + for name in ("content", "content_ref", "uri", "local_path", "provenance"): + assert not hasattr(value, name) From 267504e8fee4784f6d041e445a9e690c982e61a3 Mon Sep 17 00:00:00 2001 From: DavidWallstructurallaw Date: Wed, 23 Sep 2026 00:35:59 -0700 Subject: [PATCH 07/12] Implement Phase 5 Step 7 lineage reports and schema 1.1 --- .github/workflows/ci.yml | 4 +- PHASE_5_DECISIONS.md | 26 +- PHASE_5_STEP_7.md | 110 + README.md | 8 +- docs/lineage_contract.md | 40 +- docs/privacy.md | 29 +- docs/release_process.md | 24 +- docs/report_schema.md | 136 +- schemas/report.schema.json | 12188 +++++++++++----- scripts/release_check.py | 37 +- .../data/report.schema.json | 12188 +++++++++++----- .../lineage/ancestry.py | 74 +- .../reports/assembly.py | 374 +- .../reports/markdown_report.py | 4 + src/recursive_integrity_toolkit/result.py | 350 +- .../utils/logging.py | 1 + tests/golden/phase4_hero_redacted.json | 77 +- tests/golden/phase4_hero_redacted.md | 82 +- tests/golden/phase4_hero_report.json | 77 +- tests/golden/phase4_hero_report.md | 82 +- tests/golden/phase4_report_cases.md | 16 +- tests/golden/phase4_report_expected.json | 30 +- tests/golden/test_phase4_reports.py | 18 +- .../integration/test_current_verification.py | 17 +- tests/integration/test_hero_end_to_end.py | 2 +- .../test_partial_lineage_report.py | 12 +- .../test_partial_provenance_report.py | 2 +- tests/integration/test_phase4_cli.py | 4 +- .../test_phase5_lineage_privacy.py | 301 + .../test_phase5_lineage_reports.py | 437 + tests/performance/test_hero_runtime.py | 2 +- tests/performance/test_metadata_100k.py | 2 +- tests/unit/test_PR012_evidence_classes.py | 4 +- tests/unit/test_PR013_report_schema.py | 12 +- tests/unit/test_PR014_unavailable.py | 2 +- tests/unit/test_PR015_redaction.py | 2 +- tests/unit/test_PR018_language.py | 4 +- tests/unit/test_phase4_output_safety.py | 2 +- 38 files changed, 19300 insertions(+), 7480 deletions(-) create mode 100644 PHASE_5_STEP_7.md create mode 100644 tests/integration/test_phase5_lineage_privacy.py create mode 100644 tests/integration/test_phase5_lineage_reports.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4b6454..2d983a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,13 +50,13 @@ jobs: with: python-version: "3.12" - run: python -m pip install ".[test]" - - name: Check Step 6 lineage closure and proxy with affected boundaries + - name: Check Step 7 lineage reports, schema and privacy with affected boundaries run: | set -euo pipefail python scripts/check_spec_consistency.py python scripts/check_traceability.py python scripts/release_check.py - python -m pytest -p no:cacheprovider -q tests/unit/test_lineage_bounds_proxy.py tests/unit/test_T3_bounds.py tests/unit/test_lineage_concentration.py tests/unit/test_T4_ancestry.py tests/unit/test_lineage_root_resources.py tests/unit/test_T6_cycles.py tests/unit/test_parent_batch.py tests/unit/test_lineage_graph.py tests/unit/test_PR007_version_order.py tests/unit/test_PR008_parent_resolution.py tests/unit/test_PR009_generation.py tests/unit/test_lineage_fixture_inputs.py tests/unit/test_phase3_contracts.py tests/integration/test_phase3_metric_pipeline.py tests/unit/test_PR016_determinism.py tests/integration/test_current_verification.py tests/integration/test_ci_workflows.py tests/integration/test_no_algorithms.py tests/integration/test_repository_structure.py + python -m pytest -p no:cacheprovider -q tests/integration/test_phase5_lineage_reports.py tests/integration/test_phase5_lineage_privacy.py tests/unit/test_PR013_report_schema.py tests/unit/test_PR012_evidence_classes.py tests/unit/test_PR015_redaction.py tests/integration/test_partial_lineage_report.py tests/golden/test_phase4_reports.py tests/unit/test_lineage_bounds_proxy.py tests/unit/test_T3_bounds.py tests/unit/test_lineage_concentration.py tests/unit/test_T4_ancestry.py tests/unit/test_lineage_root_resources.py tests/unit/test_T6_cycles.py tests/unit/test_parent_batch.py tests/unit/test_lineage_graph.py tests/unit/test_PR007_version_order.py tests/unit/test_PR008_parent_resolution.py tests/unit/test_PR009_generation.py tests/unit/test_lineage_fixture_inputs.py tests/unit/test_phase3_contracts.py tests/integration/test_phase3_metric_pipeline.py tests/unit/test_PR016_determinism.py tests/integration/test_current_verification.py tests/integration/test_ci_workflows.py tests/integration/test_no_algorithms.py tests/integration/test_repository_structure.py python -m pytest -p no:cacheprovider -q tests/integration/test_phase4_cli.py::test_phase4_step7_validate_blocks_every_calculation_and_content_reader tests/integration/test_phase4_cli.py::test_phase4_step8_pair_does_not_activate_other_calculation_families core: diff --git a/PHASE_5_DECISIONS.md b/PHASE_5_DECISIONS.md index b39f0b1..11654a8 100644 --- a/PHASE_5_DECISIONS.md +++ b/PHASE_5_DECISIONS.md @@ -1,6 +1,6 @@ # Phase 5 Decisions and Current Authorization -Status: **PLAN APPROVED; STEPS 1-6 COMPLETE; PHASE INCOMPLETE**. +Status: **PLAN APPROVED; STEPS 1-7 COMPLETE; PHASE INCOMPLETE**. On 2026-09-23 UTC the Theory Owner instructed: `批准，Phase 5 step 1开始`. @@ -218,3 +218,27 @@ architecture is introduced. Package version and executable report schema stay Stop after Step 6. Report/schema and CLI integration remain Steps 7-8. Existing branch synchronization approval continues; merge, tag and publication remain separate actions. + +## Step 7 execution instruction + +On 2026-09-23 UTC the Theory Owner instructed `Phase 5 Step 7 开始`. +This authorizes schema 1.1, explicit lineage report assembly and identity-safe +JSON/Markdown output under P5-D08. Work starts from Step 6 commit +`e2666515d98cfe41483b5c43345c642bab02ced0` on `phase5-lineage`. + +Runtime changes are confined to the existing result/schema, report assembly, +Markdown renderer, safe diagnostics and ancestry owners. The ancestry change binds supplied +results to relevant validated declarations so that same-key stale results cannot +enter a new report; it performs no graph or metric recalculation. Both report +schema resources advance together. No new product module is introduced. + +Current report fixtures are consolidated to schema 1.1 with their independently +authored scientific values preserved. Schema 1.0 fixtures remain recoverable in +Git history. Direct semantic, malformed-wire, redaction and report compatibility +checks cover this public boundary. The single current source control advances +to Step 6 and opens only the seven changed product paths. No new migration registry +or meta-verification architecture is added. + +Package version stays `0.1.0.dev3`. Step 8 CLI flags, context-role ingestion and +dispatch remain separate work. Stop after Step 7. Existing branch synchronization +approval continues; merge, tag and publication remain separate actions. diff --git a/PHASE_5_STEP_7.md b/PHASE_5_STEP_7.md new file mode 100644 index 0000000..6f9e163 --- /dev/null +++ b/PHASE_5_STEP_7.md @@ -0,0 +1,110 @@ +# Phase 5 Step 7 Completion + +Date: 2026-09-23 UTC. Status: **COMPLETE; PHASE INCOMPLETE**. + +The Theory Owner instructed `Phase 5 Step 7 开始`. Work starts from +`e2666515d98cfe41483b5c43345c642bab02ced0` on `phase5-lineage`. +This step integrates explicitly computed lineage evidence with canonical reports, +schema 1.1, privacy protection and the existing JSON/Markdown renderers. + +## Delivered behavior + +`assemble_report` accepts explicit `lineage`, `lineage_bounds` and +`shared_ancestry` results. Bounds and proxy must refer to the supplied ancestry +result. Assembly does not execute ingestion, graph traversal, root propagation +or metric kernels. A private input digest binds the result to relevant validated +declarations, roles, chronology and diagnostic classifications. Same record keys +with changed grounding or parent declarations cannot silently reuse old results. +Raw content, URIs, paths and notes are excluded from that digest, and the digest +is never serialized. This is a product handoff check, not a new evidence registry. + +All reports now declare schema `1.1`, retaining the twelve top-level sections, +equal capability mirrors and existing non-lineage values and scopes. Reports +expose loaded graph scope separately from the target population, cycle members +and affected descendants, depth, resource usage, G/C/U, coverage, roots, incidence, +fractional mass, concentration, lineage bounds and the descriptive shared-root +signal. Hero retains G=8, C=U=0, five roots, HHI=1/4, four effective roots and +lineage bounds `[0,0]`; its independent direct bounds remain `[1/2,1/2]`. + +Lineage defaults to `not_requested`. Completed, partial and failed execution +remain distinct from input eligibility and individual field availability. +Pregraph failure cannot claim an acyclic graph; a root-budget failure retains +already completed cycle, depth and reference observations without inventing root +results. Empty and unresolved populations retain explicit unavailable reasons. + +Five identity-bearing detail collections are bounded to 100 rows. Cycle witnesses +contain at most 64 edges and 65 keys. Full counts, root metrics and denominators +are calculated before display truncation. Existing scoped HMAC protection covers +new versions, roots, cycle witnesses and record identities. Omit mode removes +whole detail collections with explicit counts and reasons; it retains prior +diagnostic-limit reasons. JSON and Markdown share the validated safe view. +Resource-limit diagnostics now have a fixed readable message and remediation. + +## Verification + +Local checks used Python 3.12.14 and the core dependency profile without PyArrow. + +| Check | Result | +|---|---| +| New report semantics and integration cases | 29 passed | +| New privacy and identity-boundary cases | 14 passed | +| Combined affected regression selection | 1,583 passed; four setup-related failures resolved below | +| Installed Hero and source-boundary failure rechecks | 4 passed in 1.58 seconds | +| Current verification and CI checks after metadata fix | 41 passed | +| Wheel module/resource identity and schema mirror | PASS; all 47 current files match | +| Specification consistency and module boundaries | PASS | +| Four workflow YAML files and whitespace check | PASS | +| Independent implementation, schema and privacy review | PASS; no remaining blockers | + +The combined gate covered report contracts, both validators, malformed wire +values, privacy, CLI isolation, safe publication, golden reports, optional +dependency absence, no-network behavior, lineage algorithms and affected input +neighbors. It also ran the bounded Hero runtime checks. The 43 new cases include +same-key stale handoffs, no-recalculation guards, 100-row detail limits, 64/65-edge +witness boundaries, malicious identifiers and unknown diagnostic text. Independent +review found and corrected a post-HMAC reason-order validation error; all four +supported privacy modes now preserve source ordering without requiring alias +lexical ordering. Unsupported standard/hash and standard/omit remain rejected. + +The initial legacy check found a registered resource-limit code missing from safe +diagnostic templates. Its existing behavior test detected the omission; the +template and privacy whitelist were corrected. Initial golden candidate generation +also required the source import path. The installed golden first lacked a wheel, +then correctly rejected a wheel built before the final schema-validator edit. +Rebuilding and reinstalling the current wheel resolved that mismatch. + +Three source-boundary checks encountered setuptools metadata produced by the +local build. The existing control now excludes only the exact external package +metadata prefix `src/recursive_integrity_toolkit.egg-info/`, after its symlink +check. Product bytes and package inventory stay protected. Existing unauthorized +mutation tests run with that metadata present; similarly named and nested +directories remain rejected. The four setup failures passed targeted rechecks; +the subsequent boundary gate passed all 41 cases. Together, 1,589 distinct cases +in the affected selection and these two added boundary cases have passed. + +No full OS/Python/dependency matrix, 100k lineage measurement or release-candidate +execution was repeated. Those remain their designated candidate/scale gates. + +## Verification consolidation + +Current report goldens and independently authored expected status/version fields +advance to schema 1.1. Before promotion, both standard and redacted candidates +were compared with the previous fixtures: all non-lineage numerical metrics, +scopes, proxy signals, simulations, diagnostics and metadata recommendations were +unchanged. Current literal fixtures now carry package `0.1.0.dev3`, removing the +old package-version substitution adapter. Original numerical oracles and the six +canonical Hero files remain unchanged. Prior fixture forms and authority records +remain recoverable from Git history. + +The single source boundary advances to completed Step 6 and permits the seven +changed product files. The remaining 52 product files, exact package inventory, +16 frozen specifications and seven resource copies remain protected. No new +historical migration chain, approval registry or evidence-of-evidence mechanism +is introduced. + +## Handoff + +Package version stays `0.1.0.dev3`; report schema is now `1.1`. Strict schema-1.0 +readers must explicitly adopt 1.1. Step 8 remains the separately authorized CLI +lineage flag, context-input role and dispatch work. No merge, tag, release +publication or hosted CI success is claimed here. diff --git a/README.md b/README.md index 504d70d..664e861 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,11 @@ A local-first research toolkit for examining recursive closure exposure in synthetic-data and recursive-data pipelines under explicit representations and assumptions. -## Current milestone: Phase 4 reports and CLI +## Current milestone: Phase 5 lineage report integration Development version `0.1.0.dev3` provides local input validation, JSON and Markdown audit reports, privacy controls, one explicitly ordered dataset comparison and a packaged Hero example. The mathematical core includes literal topic/label and exact record-form representations, exact duplicates, support/diversity, provenance composition, direct closure-exposure bounds, tail ranking and explicitly invoked closed-resampling scenarios. Actual acceptance and execution evidence are recorded in `PHASE_4_COMPLETION.md`, `PHASE_4_VALIDATION_REPORT.md` and `PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md`. -General lineage/ancestry remains Phase 5 work; longitudinal orchestration remains Phase 6A work; external reopening and experiment orchestration remain Phase 6B work. The CLI does not execute simulations. HTML output is deferred. No stable release, main merge or registry publication is implied by this development milestone. +Phase 5 Steps 1-7 add explicit Python APIs for validated graphs, cycles, depth, external ancestry, root concentration, lineage closure bounds and descriptive shared-root evidence. Explicitly supplied lineage results integrate with JSON and Markdown under report schema 1.1. CLI lineage dispatch remains Step 8 work. Longitudinal orchestration remains Phase 6A work; external reopening and experiment orchestration remain Phase 6B work. The CLI does not execute simulations. HTML output is deferred. No stable release, main merge or registry publication is implied by this development milestone. ## Install and run @@ -26,9 +26,9 @@ After installation, the following command works in any local working directory w rit example --out ./hero-workspace ``` -It copies six packaged Hero files into `hero-workspace/inputs/` and writes `hero-workspace/reports/report.json` and `report.md`. It downloads nothing. The packaged `EXPECTED_OUTPUTS.md` also contains future lineage targets; the actual Phase 4 report explicitly defers those calculations. +It copies six packaged Hero files into `hero-workspace/inputs/` and writes `hero-workspace/reports/report.json` and `report.md`. It downloads nothing. The packaged `EXPECTED_OUTPUTS.md` also contains lineage targets; this CLI invocation does not request lineage calculations. -The report compares v1 with v2: support **8 to 5**, retention **5/8**, diversity **7/8 to 3/4**, and missing topic states `battery`, `lizard`, `turtle`. Later-version human and synthetic shares are each **1/2**, and direct closure exposure is **[1/2, 1/2]**. Input observability is Level 4, lineage execution is deferred and simulations are empty. +The report compares v1 with v2: support **8 to 5**, retention **5/8**, diversity **7/8 to 3/4**, and missing topic states `battery`, `lizard`, `turtle`. Later-version human and synthetic shares are each **1/2**, and direct closure exposure is **[1/2, 1/2]**. Input observability is Level 4, lineage execution is `not_requested` and simulations are empty. Use the extracted files for these independent commands, each with a fresh output destination: diff --git a/docs/lineage_contract.md b/docs/lineage_contract.md index ccde67d..dcef31d 100644 --- a/docs/lineage_contract.md +++ b/docs/lineage_contract.md @@ -13,10 +13,11 @@ node/edge limits. Step 3 adds `analyze_cycles` for iterative cycle detection, unaffected topology and structural depth. Step 4 adds `analyze_lineage` for external-root resolution, G/C/U counts and coverage. Step 5 adds root incidence, fractional mass and ancestry concentration. Step 6 adds explicit lineage closure -and shared-root proxy calls. Report and CLI lineage interfaces remain later -implementation targets. +and shared-root proxy calls. Step 7 integrates explicitly supplied lineage +results into schema 1.1, canonical assembly and privacy-safe JSON/Markdown. +CLI lineage invocation and context input options remain Step 8 targets. The package remains at -`0.1.0.dev3` and the executable report schema at `1.0`. +`0.1.0.dev3`; the executable report schema is `1.1`. ## 1. Scope and reference semantics @@ -345,17 +346,34 @@ status/level table above. A positive witness is the first already ranked incidence and coverage field references, target scope and source diagnostics. It adds no calibrated severity, risk threshold or causal conclusion. Both calls consume already computed ancestry and perform no graph traversal, ingestion, -report rendering or network/file access. Serialized schema integration remains -Step 7. +report rendering or network/file access. Step 7 serializes these explicitly +supplied results without invoking either calculation automatically. ## 5. Schema 1.1 placement, detail bounds and privacy -Step 7 changes every Phase 5 output to report schema 1.1, including requests +Step 7 changes every current output to report schema 1.1, including requests without lineage. Strict 1.0 readers must adopt the new schema. Preserve the twelve top-level sections, existing envelopes and all non-lineage numerical meanings. Root and packaged report schemas must agree. No migration-loader framework is introduced. +The implemented `assemble_report` keywords are `lineage`, `lineage_bounds` and +`shared_ancestry`. Bounds and proxy handoffs require their matching explicit +lineage source. Assembly checks exact target/context scope and a private +computation-owner input signature against retained declarations, policy and +diagnostics, rejecting stale same-identity results. This consistency check does +not authenticate a hostile Python caller. It does not rerun graph traversal or +the root/concentration kernel, and the private signature is never reported. + +Without a lineage result, execution is `not_requested` and immediate-validation +observations retain their limited meanings. An explicit existing +`FamilyFailure(CapabilityKey.LINEAGE, ...)` can report a failure before graph +admission; it supplies no completed graph or root observations. Root-resource +failures with a typed result retain completed graph/depth/reference observations +and consumed-work counters. A separate family failure cannot contradict a +supplied lineage result. Step 8 owns CLI dispatch and graph-admission exception +orchestration. + New observed fields are `observed_facts.lineage.graph_scope` (the public scope counts/version inventory), `cycle_analysis`, `depth_summary`, `unresolved_record_details`, and `resource_usage`. Existing declared/resolved/ @@ -405,15 +423,17 @@ around components provides the privacy omission shape for the entire row list. `maximum_resolved_target_depth` and the explicit target count. No arbitrary message, URI, path or content payload is admitted into these shapes. -In standard/hash views, identities use the existing protection transformation -appropriate to the selected privacy and record-ID modes. In redacted/hash, +Standard mode preserves declared structural identities and record IDs. +Redacted mode supports record-ID `hash` (default), `omit` and explicit `preserve`; +record-ID overrides require redacted mode. In redacted/hash, dataset IDs, root keys, witness keys and dynamic identities use the existing scoped HMAC mechanism consistently; sort/rank before privacy transformation. In record-ID omit mode, each identity-bearing collection uses its whole-list omission variant. Never remove a required `record_key` while retaining an invalid row. Aggregate values, counts, severity and execution status remain visible. -Existing options for preserving only the declared record-ID field retain their -scope. Both renderers consume the same schema-valid safe result. +Redacted/preserve retains only declared record-ID fields while dataset versions +and other protected identities still use their existing transformation. Both +renderers consume the same schema-valid safe result. ## 6. Resource and invocation interfaces diff --git a/docs/privacy.md b/docs/privacy.md index 55d21d9..61f0c3c 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -1,6 +1,6 @@ # Privacy and Local Input Boundaries -Status: Phase 4 development milestone (`0.1.0.dev3`). `PRIVACY_AND_DATA_HANDLING.md` remains authoritative. +Status: Phase 5 Step 7 report/privacy milestone (`0.1.0.dev3`, report schema `1.1`). `PRIVACY_AND_DATA_HANDLING.md` remains authoritative. Package import does not contact a network, open user audit files, require optional PyArrow or start a service. Runtime operates on explicitly supplied local files and declarations. No telemetry, background worker, plugin, cloud client, database, model download or LLM service exists. CI/package installation acquire dependencies separately; CI uses synthetic fixtures only. @@ -64,7 +64,7 @@ Scopes, state IDs, source declarations, weights, content digests and original di ## Explicit safe report views -These APIs implement approved P4-D05 and P4-D08, with PR-015 privacy and +These APIs implement approved P4-D05, P4-D08 and P5-D08, with PR-015 privacy and PR-016 reproducibility ownership. `assemble_report` continues to assemble standard internal evidence. Its `CanonicalReport` validates structure and evidence semantics; validation alone is not a privacy transformation. After calculation and assembly, @@ -91,6 +91,8 @@ denominators, evidence classes, availability and error severity remain unchanged Pseudonyms do not reorder parallel simulation arrays. Record-ID modes are `hash` (the redacted default), `preserve` and `omit`. +Standard mode preserves record IDs; explicit record-ID overrides require +redacted mode. An explicit `preserve` applies only to declared record-ID fields, never to a matching string embedded in a diagnostic, path or other narrative. `omit` removes record-level linkage while retaining aggregate counts and disclosure of exclusion @@ -98,6 +100,29 @@ reasons. An omitted duplicate-group identity list uses the existing `redacted_identity_details` contract. These omissions describe a privacy choice, not missing input evidence, and do not turn an available metric into unavailable. +Lineage adds root identities, target and loaded version labels, cycle witnesses, +cycle-member and affected-record lists, and unresolved-record details to the +same protection boundary. Redacted/hash uses existing dataset/record identity +domains consistently across roots, witnesses, scopes and diagnostics. Ranking +and the 100-row presentation cap are applied before identity transformation. +The complete root distribution determines concentration before this cap. + +In redacted/omit, each identity-bearing lineage collection is omitted as a whole: +`items: null`, `returned_count: 0`, `omitted_count: total_count` and +`detail_status: "omitted"`. Its reasons include `redacted_identity_details` and +retain `diagnostic_limit` if the original collection was already capped. No row +survives with a required identity removed. Counts, resource usage, numerical +values, evidence classes and execution/error status remain visible. Explicit +redacted/preserve retains only declared record-ID fields; dataset versions and +other protected labels remain pseudonymized. + +The private lineage input signature binds retained declarations to the supplied +calculation result so stale same-key evidence cannot silently enter a report. +It is an internal consistency check, excluded from report serialization, and +provides no authenticity or anonymity guarantee. Assembly and rendering do not +rerun ancestry calculations. CLI lineage flags and context-input orchestration +remain Step 8 work; current ordinary reports already use schema 1.1. + Identifier protection uses domain-separated HMAC-SHA-256 over unambiguous canonical encodings. A fresh secret gives run-scoped consistency and separates otherwise identical runs. An explicitly supplied local secret file permits diff --git a/docs/release_process.md b/docs/release_process.md index 611e586..0f54bc4 100644 --- a/docs/release_process.md +++ b/docs/release_process.md @@ -2,10 +2,10 @@ ## Current Phase 5 boundary -Phase 5 Step 6 is approved. Runtime remains at `0.1.0.dev3`, with executable -report schema 1.0. Lineage closure bounds and the descriptive shared-root proxy -are the current implementation scope; report and CLI integration remain -subsequent steps. +Phase 5 Step 7 is approved. Runtime remains at `0.1.0.dev3`, with executable +report schema 1.1 for every report. Typed lineage assembly, bounded diagnostics, +schema semantics and privacy integration are the current implementation scope. +CLI execution remains Step 8. Stable v0.1 publication, a tag and main merge require separate authorization. @@ -19,12 +19,16 @@ python scripts/release_check.py Historical `--phase` and `--step` dispatch is retired. Earlier source forms, migrations and receipts remain recoverable from Git and archived phase records. -The existing source check advances its single reference to the completed Step 5 -commit; no new Phase 5 hash registry is needed. During Step 6 it permits changes -only to the existing ancestry and bounds modules, retaining the exact product file inventory and -rejecting unrelated runtime, schema, package-metadata and canonical Hero changes. +The existing source check advances its single reference to the completed Step 6 +commit; no new Phase 5 hash registry is needed. During Step 7 it permits changes +only to the ancestry input handoff, canonical result, assembly, Markdown renderer, +the existing safe logging templates and the two report schema copies. It retains the exact product file inventory and +rejects unrelated runtime, package-metadata and canonical Hero changes. The frozen Phase 0 specification hashes are unchanged. Sixteen frozen -specifications and seven packaged resource copies remain protected. +specifications and seven packaged resource copies remain protected. The source +inventory excludes only the generated `src/recursive_integrity_toolkit.egg-info/` +metadata directory; similarly named or nested paths are not excluded, and +symlink rejection and shipped-package inventory checks remain in force. ## Risk-sensitive gates @@ -47,7 +51,7 @@ Dispatch `ci.yml` with `gate=candidate` for a stable candidate. The workflow run Compatibility cells run complete canonical regression with `--ignore=tests/performance`. The designated performance job runs `tests/performance` once, avoiding nine repetitions of the costly 100k report -workload. The Phase 5 sparse-lineage test is still a placeholder in Step 6; +workload. The Phase 5 sparse-lineage test is still a placeholder in Step 7; it becomes an executable graph benchmark in Step 9. No placeholder is counted as a successful lineage measurement. diff --git a/docs/report_schema.md b/docs/report_schema.md index 1dab399..5530592 100644 --- a/docs/report_schema.md +++ b/docs/report_schema.md @@ -1,6 +1,6 @@ # Report Schema -Status: Phase 4 development report contract (`0.1.0.dev3`). Schema version: `1.0`. +Status: Phase 5 Step 7 development report contract (`0.1.0.dev3`). Schema version: `1.1`. The authoritative public shape is `schemas/report.schema.json`, Draft 2020-12. Runtime validation is implemented in `result.py` using the standard library. `report_schema()` returns a detached copy of the same declarative contract. No runtime schema package, filesystem lookup or remote resolver is used. The `$schema` and `$id` identifiers are descriptive; all references are local `$defs` references. @@ -10,11 +10,13 @@ The installed CLI publishes `report.json` and `report.md` from the same validate privacy view. `audit` computes explicitly requested supported families; `validate` keeps analytical sections empty; `example` runs the packaged explicit Hero pair. Default `simulations` is `{}`. Existing Python scenario results may be supplied -explicitly to assembly, but CLI simulation execution is unsupported. General -lineage, ancestry, broader longitudinal orchestration and HTML remain deferred. +explicitly to assembly, but CLI simulation execution is unsupported. Python +callers can supply completed lineage, lineage bounds and shared-root proxy +results explicitly. CLI lineage flags and context inputs remain Step 8 work; +broader longitudinal orchestration and HTML remain deferred. Capability eligibility and execution are separate. An input Level 4 report can -retain deferred lineage, and a completed explicit pair has partial longitudinal +retain `not_requested` lineage, and a completed explicit pair has partial longitudinal execution because additional change families remain deferred. This alone does not create an error. Unavailable values retain null plus their required reasons, while actual zero and empty measured sets remain values. @@ -26,7 +28,7 @@ retains input identities when privacy permits them. Warning codes, counts, capability effects and representative locations are preserved. Distinct contexts are retained, and only equal diagnostic entries are deduplicated in encounter order. This bounded representation prevents repeated full-dataset identity copies -without changing schema version, analytical values or unknown-state meaning. +without changing analytical values or unknown-state meaning. `run.duration_seconds` for CLI runs measures input/calculation work before report assembly and publication. It must not be treated as the complete report runtime. @@ -74,7 +76,7 @@ Every section is present in empty, validation-only and error-only reports. Empty Input capability status is `available`, `partial`, `unavailable` or `experimental`. Execution status is independently `completed`, `partial`, `not_requested`, `deferred` or `failed`. Executed work names its operations in `execution_scope`; all noncompleted statuses carry `execution_reason_codes`. A nonempty matrix contains exactly the seven approved capability keys. Its copy at `observability.capabilities` is structurally equal on input and shares the same immutable object in the canonical result. A mismatch is rejected. -`observability` may be empty before assessment; otherwise all six fields are required. Maximum level 0–5 must match its exact registered label. Level 4 input eligibility can coexist with lineage execution deferred. Phase 4 cannot claim completed or partially executed Phase 5 lineage analysis. Existing declared/resolved edge observations and the explicitly named earlier-version ordering certificate retain their limited Phase 2 provenance. +`observability` may be empty before assessment; otherwise all six fields are required. Maximum level 0–5 must match its exact registered label. Level 4 input eligibility can coexist with lineage execution `not_requested`. Supplied lineage results retain their actual `completed`, `partial` or `failed` execution status. Without an explicit result, existing declared/resolved edge observations and the explicitly named earlier-version ordering certificate retain their limited validation provenance. ## Analytical field registry @@ -102,8 +104,13 @@ Input capability status is `available`, `partial`, `unavailable` or `experimenta | `observed_facts.lineage.declared_parent_edge_count` | integer | `edges` | `observed_fact` | `PR-008` | `PR-008.declared_parent_edge_count` | 3 | Null only with unavailable reasons | | `observed_facts.lineage.resolved_parent_edge_count` | integer | `edges` | `observed_fact` | `PR-008` | `PR-008.resolved_parent_edge_count` | 3 | Null only with unavailable reasons | | `observed_facts.lineage.unresolved_parent_edge_count` | integer | `edges` | `observed_fact` | `PR-008` | `PR-008.unresolved_parent_edge_count` | 3 | Null only with unavailable reasons | -| `observed_facts.lineage.cycle_status` | enum | `status` | `observed_fact` | `T6` | `T6.graph_cycle_check` | 3 | Unavailable only: Phase 5 deferred | +| `observed_facts.lineage.cycle_status` | enum | `status` | `observed_fact` | `T6` | `T6.graph_cycle_check` | 3 | Null when graph analysis is unavailable | | `observed_facts.lineage.ordering_certificate` | ordering_certificate | `certificate` | `observed_fact` | `PR-008` | `PR-008.earlier_version_certificate` | 3 | Null only with unavailable reasons | +| `observed_facts.lineage.graph_scope` | lineage_scope | `scope` | `observed_fact` | `T4` | `T4.graph_scope` | 3 | Explicit target and loaded scope | +| `observed_facts.lineage.cycle_analysis` | cycle_analysis | `cycles` | `observed_fact` | `T6` | `T6.cyclic_strongly_connected_components` | 3 | Null only with unavailable reasons | +| `observed_facts.lineage.depth_summary` | depth_summary | `edges` | `observed_fact` | `PR-009` | `PR-009.resolved_depth_summary` | 3 | Resolved subset retained | +| `observed_facts.lineage.unresolved_record_details` | unresolved_record_details | `records` | `observed_fact` | `T4` | `T4.unresolved_record_details` | 3 | Null when target partition is unavailable | +| `observed_facts.lineage.resource_usage` | lineage_resource_usage | `work_units` | `observed_fact` | `PR-015` | `PR-015.lineage_resource_usage` | 3 | Exact consumed work and explicit limits | | `observed_facts.state_counts.by_version.*` | state_count[] | `records` | `observed_fact` | `T1` | `T1.state_counts` | 1 | Null only with unavailable reasons | | `derived_metrics.support.by_version.*.support_size` | integer | `states` | `derived_metric` | `T1` | `F-002` | 1 | Null only with unavailable reasons | | `derived_metrics.support.by_version.*.weighted_support_size` | integer | `states` | `derived_metric` | `T1` | `F-002` | 1 | Null only with unavailable reasons | @@ -139,20 +146,24 @@ Input capability status is `available`, `partial`, `unavailable` or `experimenta | `derived_metrics.closure_exposure.direct.lower_bound` | ratio | `ratio` | `derived_metric` | `T3` | `F-009` | 2 | Null only with unavailable reasons | | `derived_metrics.closure_exposure.direct.upper_bound` | ratio | `ratio` | `derived_metric` | `T3` | `F-010` | 2 | Null only with unavailable reasons | | `derived_metrics.closure_exposure.direct.interval_width` | ratio | `ratio` | `derived_metric` | `T3` | `T3.upper_minus_lower` | 2 | Null only with unavailable reasons | -| `derived_metrics.closure_exposure.lineage.lower_bound` | ratio | `ratio` | `derived_metric` | `T3` | `T3.lineage_closure` | 3 | Unavailable only: Phase 5 deferred | -| `derived_metrics.closure_exposure.lineage.upper_bound` | ratio | `ratio` | `derived_metric` | `T3` | `T3.lineage_closure` | 3 | Unavailable only: Phase 5 deferred | -| `derived_metrics.closure_exposure.lineage.interval_width` | ratio | `ratio` | `derived_metric` | `T3` | `T3.lineage_closure` | 3 | Unavailable only: Phase 5 deferred | +| `derived_metrics.closure_exposure.lineage.lower_bound` | ratio | `ratio` | `derived_metric` | `T3` | `T3.lineage_closure` | 3 | Null for empty or unavailable partition | +| `derived_metrics.closure_exposure.lineage.upper_bound` | ratio | `ratio` | `derived_metric` | `T3` | `T3.lineage_closure` | 3 | Null for empty or unavailable partition | +| `derived_metrics.closure_exposure.lineage.interval_width` | ratio | `ratio` | `derived_metric` | `T3` | `T3.lineage_closure` | 3 | Null for empty or unavailable partition | +| `derived_metrics.lineage.grounded_record_count` | integer | `records` | `derived_metric` | `T4` | `T4.grounded_record_count` | 3 | Null for unavailable partition | +| `derived_metrics.lineage.closed_record_count` | integer | `records` | `derived_metric` | `T4` | `T4.closed_record_count` | 3 | Null for unavailable partition | +| `derived_metrics.lineage.unresolved_record_count` | integer | `records` | `derived_metric` | `T4` | `T4.unresolved_record_count` | 3 | Null for unavailable partition | +| `derived_metrics.lineage.records_with_resolved_external_ancestry` | integer | `records` | `derived_metric` | `T4` | `T4.records_with_resolved_external_ancestry` | 3 | Null for unavailable partition | | `derived_metrics.lineage.resolved_parent_edge_coverage` | ratio | `ratio` | `derived_metric` | `PR-008` | `PR-008.resolved_edges_over_declared` | 3 | Null only with unavailable reasons | -| `derived_metrics.lineage.resolved_lineage_coverage` | ratio | `ratio` | `derived_metric` | `T4` | `T4.resolved_records_over_scope` | 3 | Unavailable only: Phase 5 deferred | -| `derived_metrics.lineage.external_ancestry_coverage` | ratio | `ratio` | `derived_metric` | `T4` | `T4.external_roots_over_scope` | 3 | Unavailable only: Phase 5 deferred | -| `derived_metrics.lineage.distinct_external_root_count` | integer | `roots` | `derived_metric` | `T4` | `T4.root_set_union` | 3 | Unavailable only: Phase 5 deferred | -| `derived_metrics.lineage.top_shared_ancestors` | ancestor_incidence[] | `records` | `derived_metric` | `T4` | `T4.incidence_ranking` | 3 | Unavailable only: Phase 5 deferred | -| `derived_metrics.lineage.ancestry_concentration_hhi` | ratio | `ratio` | `derived_metric` | `T4` | `F-012` | 3 | Unavailable only: Phase 5 deferred | -| `derived_metrics.lineage.effective_external_root_count` | number | `roots` | `derived_metric` | `T4` | `F-013` | 3 | Unavailable only: Phase 5 deferred | -| `derived_metrics.lineage.lineage_depth` | integer | `edges` | `derived_metric` | `PR-009` | `PR-009.maximum_resolved_parent_depth` | 3 | Unavailable only: Phase 5 deferred | +| `derived_metrics.lineage.resolved_lineage_coverage` | ratio | `ratio` | `derived_metric` | `T4` | `T4.resolved_records_over_scope` | 3 | Null for empty or unavailable partition | +| `derived_metrics.lineage.external_ancestry_coverage` | ratio | `ratio` | `derived_metric` | `T4` | `T4.external_roots_over_scope` | 3 | Null for empty or unavailable partition | +| `derived_metrics.lineage.distinct_external_root_count` | integer | `roots` | `derived_metric` | `T4` | `T4.root_set_union` | 3 | Complete-root subset with explicit coverage | +| `derived_metrics.lineage.top_shared_ancestors` | root_contribution_details | `records` | `derived_metric` | `T4` | `T4.incidence_ranking` | 3 | Null for unavailable root analysis | +| `derived_metrics.lineage.ancestry_concentration_hhi` | ratio | `ratio` | `derived_metric` | `T4` | `F-012` | 3 | Null without resolved external roots | +| `derived_metrics.lineage.effective_external_root_count` | number | `roots` | `derived_metric` | `T4` | `F-013` | 3 | Null without resolved external roots | +| `derived_metrics.lineage.lineage_depth` | integer | `edges` | `derived_metric` | `PR-009` | `PR-009.maximum_resolved_parent_depth` | 3 | Null for incomplete whole-target depth | | `proxy_signals.support_contraction` | proxy | `signal` | `proxy_signal` | `T1` | `T1.support_contraction` | 4 | No scalar value; typed class fields | | `proxy_signals.tail_fragility` | proxy | `signal` | `proxy_signal` | `T2` | `T2.tail_fragility` | 1 | No scalar value; typed class fields | -| `proxy_signals.shared_ancestry_dependence` | proxy | `signal` | `proxy_signal` | `T4` | `T4.shared_ancestry_dependence` | 3 | Unavailable only: Phase 5 deferred | +| `proxy_signals.shared_ancestry_dependence` | proxy | `signal` | `proxy_signal` | `T4` | `T4.shared_ancestry_dependence` | 3 | Presence may retain partial coverage; absence requires complete ancestry | | `proxy_signals.provenance_uncertainty` | proxy | `signal` | `proxy_signal` | `T3` | `T3.provenance_uncertainty` | 2 | No scalar value; typed class fields | | `simulations.closed_resampling` | scenario | `scenario` | `simulation` | `T1` | `T1.closed_resampling` | 5 | No scalar value; typed class fields | | `simulations.tail_extinction` | scenario | `scenario` | `simulation` | `T2` | `T2.tail_extinction` | 1 | No scalar value; typed class fields | @@ -188,7 +199,7 @@ The following tables enumerate non-envelope public fields and containers. A `$de | `run` | object | required | PR-016 | Not nullable | | `run.run_id` | string | required | PR-016 | Not nullable | | `run.toolkit_version` | string | required | PR-016 | Not nullable | -| `run.report_schema_version` | constant "1.0" | required | PR-013 | Not nullable | +| `run.report_schema_version` | constant "1.1" | required | PR-013 | Not nullable | | `run.started_at` | string or null | required | PR-016 | Explicitly not applicable/unavailable as described above | | `run.completed_at` | string or null | required | PR-016 | Explicitly not applicable/unavailable as described above | | `run.duration_seconds` | number or null | required | PR-016 | Explicitly not applicable/unavailable as described above | @@ -323,6 +334,11 @@ The following tables enumerate non-envelope public fields and containers. A `$de | `observed_facts.lineage.unresolved_parent_edge_count` | object | optional | Enclosing registered owner | Not nullable | | `observed_facts.lineage.cycle_status` | object | optional | Enclosing registered owner | Not nullable | | `observed_facts.lineage.ordering_certificate` | object | optional | Enclosing registered owner | Not nullable | +| `observed_facts.lineage.graph_scope` | object | optional | T4 | Not nullable | +| `observed_facts.lineage.cycle_analysis` | object | optional | T6 | Not nullable | +| `observed_facts.lineage.depth_summary` | object | optional | PR-009 | Not nullable | +| `observed_facts.lineage.unresolved_record_details` | object | optional | T4 | Not nullable | +| `observed_facts.lineage.resource_usage` | object | optional | PR-015 | Not nullable | | `observed_facts.state_counts` | object | optional | Enclosing registered owner | Not nullable | | `observed_facts.state_counts.by_version` | object | optional | Enclosing registered owner | Not nullable | | `observed_facts.state_counts.by_version.*` | object | required | Enclosing registered owner | Not nullable | @@ -387,6 +403,10 @@ The following tables enumerate non-envelope public fields and containers. A `$de | `derived_metrics.closure_exposure.lineage.upper_bound` | object | optional | T3 | Not nullable | | `derived_metrics.closure_exposure.lineage.interval_width` | object | optional | T3 | Not nullable | | `derived_metrics.lineage` | object | optional | Enclosing registered owner | Not nullable | +| `derived_metrics.lineage.grounded_record_count` | object | optional | T4 | Not nullable | +| `derived_metrics.lineage.closed_record_count` | object | optional | T4 | Not nullable | +| `derived_metrics.lineage.unresolved_record_count` | object | optional | T4 | Not nullable | +| `derived_metrics.lineage.records_with_resolved_external_ancestry` | object | optional | T4 | Not nullable | | `derived_metrics.lineage.resolved_parent_edge_coverage` | object | optional | Enclosing registered owner | Not nullable | | `derived_metrics.lineage.resolved_lineage_coverage` | object | optional | Enclosing registered owner | Not nullable | | `derived_metrics.lineage.external_ancestry_coverage` | object | optional | Enclosing registered owner | Not nullable | @@ -413,7 +433,7 @@ The following tables enumerate non-envelope public fields and containers. A `$de | `simulations` | object | required | Enclosing registered owner | Not nullable | | `simulations.closed_resampling` | object | optional | Enclosing registered owner | Not nullable | | `simulations.tail_extinction` | object | optional | Enclosing registered owner | Not nullable | -| `simulations.external_reopening` | forbidden in Phase 4 | optional | Enclosing registered owner | Not nullable | +| `simulations.external_reopening` | forbidden in the current schema | optional | Enclosing registered owner | Not nullable | ### `unavailable_conclusions` @@ -741,7 +761,7 @@ All emitted scenario evidence remains `simulation` and `experimental`. The commo `simulations.closed_resampling` retains direct `expected_diversity`, `sampled_paths`, `support_trajectories` and `extinction_events` paths. Analytic expectation requires the supplied expectation vector, initial diversity, contraction factor and underflow steps; its vector length is horizon plus one. Sampled paths require seed, replicate count, NumPy/RNG identity and scheduling metadata. Path count must equal replicate count. Optional support trajectories and events may only preserve evidence already supplied by the accepted owner; their presence does not authorize the adapter or renderer to calculate them. -`simulations.external_reopening` is a registered future field, explicitly rejected by this Phase 4 schema and constructor. `external_reference_loss` has no approved public trace and is unregistered. Neither is executed. The lack of implementation belongs in capability execution reasons and unavailable conclusions. +`simulations.external_reopening` is a registered future field, explicitly rejected by the current schema and constructor. `external_reference_loss` has no approved public trace and is unregistered. Neither is executed. The lack of implementation belongs in capability execution reasons and unavailable conclusions. ## Path reconciliation and unchanged meaning @@ -753,7 +773,7 @@ All emitted scenario evidence remains `simulation` and `experimental`. The commo | Existing `TailSelectionResult.tail_membership` | Public `derived_metrics.tail.tail_states`, plus `tail_rule` and typed selection metadata | Frozen planned public path; explicit adapter mapping, no numerical change | | Existing duplicate group object | `observed_facts.content.exact_duplicate_groups` | Frozen PR-006 planned public path | | Reporting operations-applied description | `inputs.schema_mapping.operations` with the exact accepted operation enum | Frozen PR-003 planned path and Phase 2 dispatcher | -| Old `cycle_detected` and cycle-status detail sketches | Registered `observed_facts.lineage.cycle_status` is an unavailable envelope | Registry §32 and P4-D01; graph operations remain Phase 5 | +| Old `cycle_detected` and cycle-status detail sketches | Registered `observed_facts.lineage.cycle_status` plus `cycle_analysis` | P5-D08 and the approved lineage contract; explicit completed graph observations | | Reporting simplified proxy example uses `status: present` | `level: present`, independent availability status | Full reporting §9.3/§16.6 | | Older privacy enum includes debug | Public Phase 4 privacy mode is standard or redacted | Approved Phase 4 Step 4 boundary | @@ -909,8 +929,14 @@ The following additional fields are part of the registered envelopes, with the o | `derived_metrics.lineage.external_ancestry_coverage` | `method` | string | optional | | `derived_metrics.lineage.distinct_external_root_count` | `method` | string | optional | | `derived_metrics.lineage.top_shared_ancestors` | `method` | string | optional | -| `derived_metrics.lineage.top_shared_ancestors` | `value[].record_key` | record_key | required | -| `derived_metrics.lineage.top_shared_ancestors` | `value[].incidence_count` | integer | required | +| `derived_metrics.lineage.top_shared_ancestors` | `value.items[].record_key` | record_key | required on visible rows | +| `derived_metrics.lineage.top_shared_ancestors` | `value.items[].incidence_count` | integer | required on visible rows | +| `derived_metrics.lineage.top_shared_ancestors` | `value.items[].incidence_share` | ratio | required on visible rows | +| `derived_metrics.lineage.top_shared_ancestors` | `value.items[].incidence_denominator` | integer | required on visible rows | +| `derived_metrics.lineage.top_shared_ancestors` | `value.items[].fractional_mass` | number | required on visible rows | +| `derived_metrics.lineage.top_shared_ancestors` | `value.items[].normalized_weight` | ratio | required on visible rows | +| `derived_metrics.lineage.top_shared_ancestors` | `value.items[].weight_denominator` | integer | required on visible rows | +| `derived_metrics.lineage.resolved_parent_edge_coverage` | `no_declared_parents` | boolean | required for executed lineage | | `derived_metrics.lineage.ancestry_concentration_hhi` | `method` | string | optional | | `derived_metrics.lineage.effective_external_root_count` | `method` | string | optional | | `derived_metrics.lineage.lineage_depth` | `method` | string | optional | @@ -970,7 +996,7 @@ Schema validation is separate from evidence assembly, privacy selection, renderi ## Explicit evidence assembly -`recursive_integrity_toolkit.reports.assembly` retains report schema version `1.0`, the twelve sections, their order and registered field ownership. `weighted_source_type_masses` is a category-to-mass object (`category_mass_map`). +`recursive_integrity_toolkit.reports.assembly` emits report schema version `1.1`, retaining the twelve sections, their order and registered field ownership. Strict schema-1.0 readers must explicitly adopt 1.1; no migration loader is supplied. `weighted_source_type_masses` is a category-to-mass object (`category_mass_map`). ### Public Python interface @@ -985,6 +1011,9 @@ def assemble_report( tail: TailSelectionResult | None = None, comparison: SupportComparison | None = None, closure: DirectClosureExposureBounds | None = None, + lineage: LineageAnalysisResult | None = None, + lineage_bounds: LineageClosureExposureBounds | None = None, + shared_ancestry: SharedAncestryDependence | None = None, expected_diversity: ExpectedDiversityResult | None = None, resampling: ResamplingSimulation | None = None, extinction: tuple[ExtinctionProbabilityResult, ...] = (), @@ -1019,6 +1048,9 @@ Assembly validates result types, finite values, metadata ownership, accepted met | `tail` | `TailSelectionResult` | The declared rule, its parameters, supplied membership, support/share and rarity ranking | | `comparison` | `SupportComparison` | Supplied pair support delta, retention, loss/added counts, original support differences and diversity delta with explicit comparison basis | | `closure` | `DirectClosureExposureBounds` | Supplied lower bound, upper bound and interval width, with operationalization, coverage and separate confidence disclosure | +| `lineage` | `LineageAnalysisResult` | Supplied target/context scope, graph/cycle/depth observations, original-reference coverage, G/C/U partition, root metrics, bounded details, resource usage and execution status | +| `lineage_bounds` | `LineageClosureExposureBounds` | Explicit lineage closure interval from the same lineage source; direct bounds retain their own slot | +| `shared_ancestry` | `SharedAncestryDependence` | Explicit descriptive shared-root signal, independent availability, coverage and limitations | | `expected_diversity` | `ExpectedDiversityResult` | Experimental analytic closed-resampling expectation and supplied scenario metadata | | `resampling` | `ResamplingSimulation` | Experimental sampled paths, support trajectories, extinction events and supplied scenario metadata | | `extinction` | Tuple of `ExtinctionProbabilityResult` | Explicitly supplied selected-state one-step extinction marginals under one shared scenario basis | @@ -1026,6 +1058,41 @@ Assembly validates result types, finite values, metadata ownership, accepted met Empirical frequencies remain derived metrics. Explicit supplied probabilities are observed declarations under `observed_facts.supplied_state_probabilities.by_version`, with an explicit probability basis and no invented record-count denominator. Scenario probabilities remain simulations. Product metadata, capability decisions, warnings, errors and recommendations retain their product ownership without an additional scientific evidence class. +### Lineage scope, bounded details and source handoffs + +The complete field shapes are fixed in [the lineage contract](lineage_contract.md#5-schema-11-placement-detail-bounds-and-privacy). +Graph scope, cycle observations and resource usage describe all loaded records; +root/depth population metrics and original-reference counts describe only the +selected primary target. Context records support ancestry without changing N. +G/C/U counts cover all targets, independently of representation eligibility. +G-only concentration and root observations retain partial status and explicit +G/N coverage when U is positive. A complete conservative interval can remain +available even when its source execution is partial or failed for invalid input. + +Each of the five identity-bearing collections uses `items`, `total_count`, +`returned_count`, `omitted_count`, `limit: 100`, `detail_status` and +`omission_reasons`: top roots, unresolved records, cycle components, cycle +members and affected records. Visible detail is complete or truncated with +`diagnostic_limit`; whole-list privacy omission uses null items and +`redacted_identity_details`, retaining an existing limit reason. Counts always +satisfy `total_count = returned_count + omitted_count`. Component witnesses +contain at most 64 edges and 65 closed-path keys. Details preserve the owner's +rank/order, and their cap never changes full-distribution metrics or execution +status. + +Assembly binds the selected target and complete loaded identities to the bundle. +A private input signature additionally checks relevant retained declarations, +field presence, policy and diagnostics so changed grounding under identical +record keys cannot reuse an old result. Bounds and proxy results must match the +explicit lineage source. The signature remains internal and provides no +authenticity guarantee. These checks do not rerun graph or metric kernels. + +A typed root-resource failure retains completed graph/depth/reference results, +resource counters and diagnostics, while root counts, concentration and intervals +remain unavailable. A pre-result graph-admission error can use the existing +lineage `FamilyFailure`; it supplies no completed graph observations. Supplying +both a lineage result and a separate lineage family failure is rejected. + ### Scope, coverage and representation Empirical result scopes must use identities and versions retained in the validated bundle. Included and excluded identities, scope identifiers, denominator meanings and representation descriptors remain attached to their fields. Representation exclusions do not shrink input inventory, provenance denominators or unrelated result families. A missing representation permits input/provenance reporting and an explicit metadata recommendation; assembly does not select a topic field, hash representation or missing-state sentinel. @@ -1063,7 +1130,7 @@ The schema has one scalar/table slot per version and weighting mode, plus single Scenario model/version, assumptions, supplied/effective distributions, normalization disclosure, numerical policy, horizon, sample size, seed, RNG identity, replicate schedule, paths and underflow disclosures remain attached to the applicable result. Analytic expectations, sampled paths and analytic one-step probabilities retain their distinct method labels. Mathematical scopes may differ from the bundle's empirical scope without upgrading its observability assessment. -No scenario runs during assembly. Omitted scenario arguments leave `simulations` empty. A supplied extinction marginal alone does not select a tail or create a tail-fragility signal. External reopening remains rejected by the Phase 4 schema, and external-reference loss remains unregistered. Supplied scenarios cannot establish an empirical intervention effect or a calibrated production-failure forecast. +No scenario runs during assembly. Omitted scenario arguments leave `simulations` empty. A supplied extinction marginal alone does not select a tail or create a tail-fragility signal. External reopening remains rejected by the current schema, and external-reference loss remains unregistered. Supplied scenarios cannot establish an empirical intervention effect or a calibrated production-failure forecast. ### Input eligibility, execution and failures @@ -1074,7 +1141,7 @@ The seven capability input statuses, original reason codes, requirements and inp | `ingestion` | The retained bundle-validation operation is identified explicitly; retained ingestion errors produce partial/failed execution according to surviving input evidence. | | `content_diagnostics` | Names supplied distribution, duplicate and tail operations. With no supplied result or family error it is `not_requested`; supplied operations are `completed` unless retained family errors make execution `partial`. An error with no supplied operation is `failed`. | | `provenance` | Names supplied provenance-composition/direct-bound operations under the same completed/partial/failed rules. Independent validation coverage remains available even when no calculation is requested. | -| `lineage` | Always `deferred` in Phase 4, including Level 4 input eligibility. Retained immediate-parent validation errors remain errors without asserting graph execution. | +| `lineage` | Defaults to `not_requested`; an explicit source result retains its `completed`, `partial` or `failed` execution. A bound family failure before an analytical result reports `failed`. Immediate-parent validation alone never claims graph execution. | | `dataset_longitudinal` | A supplied pair records its limited support/diversity operation with `partial` execution and an explicit deferred-change-families reason. Without a supplied pair it is `not_requested`, or `failed` when an explicitly bound failure exists. | | `model_longitudinal` | `deferred`, with the missing implementation/evidence boundary retained. | | `intervention_simulation` | Names supplied analytic expectation, sampled path or extinction-marginal operations; no supplied scenario means `not_requested`. Explicitly bound failures remain failed/partial execution. | @@ -1096,12 +1163,13 @@ The current deterministic signals are restricted to these existing bases. Compar | `support_contraction` | A usable supplied explicit-pair support delta is negative. Basis fields cite the delta and supplied support differences/comparison details. | Restricted to the pair's selected scope, representation and declared common state meaning; no production failure or model-performance claim. | | `tail_fragility` | A usable supplied tail-support count is positive under the supplied explicit tail-selection rule. | Declared tail membership, with no hidden threshold, importance claim or calibrated production forecast. | | `provenance_uncertainty` | A supplied usable direct interval width is positive, or a cited usable row/required-field/grounding coverage is below one. | Cites actual fields with matching scopes and preserves their distinct denominator meanings; completeness does not establish truth or source independence. | +| `shared_ancestry_dependence` | An explicitly supplied proxy reports a complete external root with incidence at least two. An absent witness supports `not_present` only when nonempty target ancestry is complete. | Partial coverage can support `present`; incomplete evidence without a witness remains `indeterminate` and unavailable. No calibrated risk or causal conclusion. | -Zero/positive support delta and an empty selected tail can produce `not_present` under their own usable basis. Missing, unavailable or zero-denominator evidence is not converted into zero or a negative finding. When a signal has no usable basis it is omitted. Each emitted proxy retains basis paths, deterministic trigger text, scope, representation where applicable, coverage and limitations. No `shared_ancestry_dependence` proxy is generated in this phase. +Zero/positive support delta and an empty selected tail can produce `not_present` under their own usable basis. Missing, unavailable or zero-denominator evidence is not converted into zero or a negative finding. The existing non-lineage signals are omitted without a usable basis; an explicitly supplied shared-ancestry proxy retains its unavailable state and reasons. Each emitted proxy retains basis paths, deterministic trigger text, scope, representation where applicable, coverage and limitations. An omitted shared-ancestry argument does not request that proxy automatically. -The required unavailable-conclusion safeguards include model-performance decline, causal ancestor effect, universal integrity and universal collapse prediction. Applicable additional entries explain production failure, complete-pipeline closure, deferred lineage analysis/closure/external ancestry and empirical intervention effects. Each uses its registered owner and method, unavailable evidence class/status, specific reasons, blocking evidence, required next metadata, related capability and theory/product limit. Missing input, invalid evidence, incompatible declarations, deferred implementation and conclusions outside product scope remain distinguishable. Unavailable does not establish that a conclusion is false. +The required unavailable-conclusion safeguards include model-performance decline, causal ancestor effect, universal integrity and universal collapse prediction. Applicable additional entries explain production failure, complete-pipeline closure, unrequested or failed lineage analysis, incomplete ancestry coverage, unsupplied or unavailable lineage intervals, and empirical intervention effects. Each uses its registered owner and method, unavailable evidence class/status, specific reasons, blocking evidence, required next metadata, related capability and theory/product limit. Missing input, invalid evidence, incompatible declarations, deferred implementation and conclusions outside product scope remain distinguishable. Unavailable does not establish that a conclusion is false. -Lineage validation observations remain bounded by their source method. Retained reference-entry counts preserve the classifier's multiplicity and denominator. The earlier-version ordering certificate is described as its sufficient declared-order certificate; it is not reported as general graph-cycle traversal. Root counts, HHI, effective roots, lineage depth, graph traversal, lineage closure and ancestry proxies remain deferred with explicit disclosures. +Lineage validation observations remain bounded by their source method. Without an explicit analysis, retained reference-entry counts preserve the classifier's multiplicity and denominator, including null coverage when no reference exists. Executed lineage uses original target-reference multiplicity and coverage 1.0 for an exact zero total, with `no_declared_parents: true`. The earlier-version ordering certificate remains its limited declared-order observation. Supplied graph, root, concentration, depth, bounds and proxy results retain their independent scopes, availability and calculation owners. Recommendations are deterministic requests for evidence: missing matching provenance rows, usable required fields, unresolved external grounding, an explicitly missing representation, missing chronology, unresolved composite parent references and versioned model outcomes. Their priorities follow uncertainty reduction, the next available evidence requirement, error resolution and optional enrichment. Recommendations refer to the missing fields or retained classifier reasons; the absence of a single top-level representation summary alone does not assert that representation was never declared. @@ -1130,6 +1198,7 @@ error severity are preserved. Input-inventory and normalized configuration hashe remain linkable reproducibility metadata; raw per-record content digests are protected when they serve as state or group identities. +Standard mode preserves record IDs and does not accept record-ID overrides. In redacted mode, record IDs default to `hash`; `preserve` affects only the declared record-ID field, and `omit` removes identity-bearing lists. Duplicate groups with omitted identities retain `record_count` and the frozen @@ -1139,6 +1208,13 @@ reasons remain disclosed with their protected scope association under `run.identifier_protection.limitations`. Per-record linkage is intentionally omitted. Redaction does not introduce an unavailable analytical conclusion. +Lineage roots, witnesses, unresolved records and loaded/target version labels +use the same identifier domains as existing record keys and scopes. Each +identity-bearing lineage detail collection uses whole-list omission in +redacted/omit, preserving totals and aggregate analytical values. The +100-item cap and canonical ranking precede hashing. JSON and Markdown render +the same schema-valid safe result with identical metric values and omissions. + The existing optional `run.identifier_protection` object records `HMAC-SHA-256`, `run` or `cross_run` stability, record-ID mode and limitations. Fresh secrets separate runs; an explicitly supplied checked local secret file diff --git a/schemas/report.schema.json b/schemas/report.schema.json index c3fce78..3f52118 100644 --- a/schemas/report.schema.json +++ b/schemas/report.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://recursive-integrity-toolkit.example/schemas/report.schema.json", - "title": "Recursive Integrity Toolkit canonical report 1.0", - "description": "Phase 4 Step 2 public contract. Product metadata and five analytical evidence classes remain separate.", + "title": "Recursive Integrity Toolkit canonical report 1.1", + "description": "Phase 5 public contract with bounded lineage evidence. Product metadata and five analytical evidence classes remain separate.", "type": "object", "properties": { "run": { @@ -19,7 +19,7 @@ "pattern": "^[^\\u0000]*$" }, "report_schema_version": { - "const": "1.0" + "const": "1.1" }, "started_at": { "anyOf": [ @@ -6752,7 +6752,12 @@ "const": "observed_fact" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "T6.graph_cycle_check" @@ -7011,7 +7016,7 @@ } ] }, - "ordering_certificate": { + "graph_scope": { "type": "object", "properties": { "value": { @@ -7019,10 +7024,34 @@ { "type": "object", "properties": { - "method": { - "const": "declared_earlier_version_order" + "target_dataset_version": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] }, - "version_order": { + "target_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "loaded_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "context_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "loaded_dataset_versions": { "type": "array", "items": { "type": "string", @@ -7031,15 +7060,14 @@ }, "minItems": 0, "uniqueItems": true - }, - "all_resolved_edges_follow_order": { - "type": "boolean" } }, "required": [ - "method", - "version_order", - "all_resolved_edges_follow_order" + "target_dataset_version", + "target_record_count", + "loaded_record_count", + "context_record_count", + "loaded_dataset_versions" ], "additionalProperties": false }, @@ -7049,7 +7077,7 @@ ] }, "unit": { - "const": "certificate" + "const": "scope" }, "evidence_class": { "const": "observed_fact" @@ -7063,11 +7091,11 @@ ] }, "method_id": { - "const": "PR-008.earlier_version_certificate" + "const": "T4.graph_scope" }, "owner_ids": { "const": [ - "PR-008" + "T4" ] }, "theory_map_ids": { @@ -7310,10 +7338,34 @@ "value": { "type": "object", "properties": { - "method": { - "const": "declared_earlier_version_order" + "target_dataset_version": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] }, - "version_order": { + "target_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "loaded_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "context_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "loaded_dataset_versions": { "type": "array", "items": { "type": "string", @@ -7322,15 +7374,14 @@ }, "minItems": 0, "uniqueItems": true - }, - "all_resolved_edges_follow_order": { - "type": "boolean" } }, "required": [ - "method", - "version_order", - "all_resolved_edges_follow_order" + "target_dataset_version", + "target_record_count", + "loaded_record_count", + "context_record_count", + "loaded_dataset_versions" ], "additionalProperties": false } @@ -7338,71 +7389,3357 @@ } } ] - } - }, - "required": [], - "additionalProperties": false - }, - "state_counts": { - "type": "object", - "properties": { - "by_version": { + }, + "cycle_analysis": { "type": "object", - "propertyNames": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "additionalProperties": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "array", - "items": { + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "detected": { + "type": "boolean" + }, + "cycle_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "counting_method": { + "const": "cyclic_strongly_connected_components" + }, + "cycle_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "target_cycle_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "affected_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "target_affected_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "components": { "type": "object", "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_index": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "member_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "target_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "witness_record_keys": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 2, + "maxItems": 65 + }, + { + "type": "null" + } + ] + }, + "witness_edge_count": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 64 + }, + { + "type": "null" + } + ] + }, + "witness_reason": { + "anyOf": [ + { + "const": "diagnostic_limit" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "component_index", + "member_count", + "target_member_count", + "witness_record_keys", + "witness_edge_count", + "witness_reason" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] }, - "state_count": { + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { "type": "integer", "minimum": 0, "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true } }, "required": [ - "state_id", - "state_count" + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" ], - "additionalProperties": false + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_index": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "member_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "target_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "witness_record_keys": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 2, + "maxItems": 65 + }, + { + "type": "null" + } + ] + }, + "witness_edge_count": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 64 + }, + { + "type": "null" + } + ] + }, + "witness_reason": { + "anyOf": [ + { + "const": "diagnostic_limit" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "component_index", + "member_count", + "target_member_count", + "witness_record_keys", + "witness_edge_count", + "witness_reason" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] }, - "minItems": 0 - }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "records" - }, - "evidence_class": { - "const": "observed_fact" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "T1.state_counts" + "cycle_member_record_keys": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + }, + "affected_record_keys": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + } + }, + "required": [ + "detected", + "cycle_count", + "counting_method", + "cycle_member_count", + "target_cycle_member_count", + "affected_record_count", + "target_affected_record_count", + "components", + "cycle_member_record_keys", + "affected_record_keys" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "cycles" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T6.cyclic_strongly_connected_components" + }, + "owner_ids": { + "const": [ + "T6" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "detected": { + "type": "boolean" + }, + "cycle_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "counting_method": { + "const": "cyclic_strongly_connected_components" + }, + "cycle_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "target_cycle_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "affected_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "target_affected_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "components": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_index": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "member_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "target_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "witness_record_keys": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 2, + "maxItems": 65 + }, + { + "type": "null" + } + ] + }, + "witness_edge_count": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 64 + }, + { + "type": "null" + } + ] + }, + "witness_reason": { + "anyOf": [ + { + "const": "diagnostic_limit" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "component_index", + "member_count", + "target_member_count", + "witness_record_keys", + "witness_edge_count", + "witness_reason" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_index": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "member_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "target_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "witness_record_keys": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 2, + "maxItems": 65 + }, + { + "type": "null" + } + ] + }, + "witness_edge_count": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 64 + }, + { + "type": "null" + } + ] + }, + "witness_reason": { + "anyOf": [ + { + "const": "diagnostic_limit" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "component_index", + "member_count", + "target_member_count", + "witness_record_keys", + "witness_edge_count", + "witness_reason" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + }, + "cycle_member_record_keys": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + }, + "affected_record_keys": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + } + }, + "required": [ + "detected", + "cycle_count", + "counting_method", + "cycle_member_count", + "target_cycle_member_count", + "affected_record_count", + "target_affected_record_count", + "components", + "cycle_member_record_keys", + "affected_record_keys" + ], + "additionalProperties": false + } + } + } + } + ] + }, + "depth_summary": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "depth_resolved_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "maximum_resolved_target_depth": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "target_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "depth_resolved_record_count", + "maximum_resolved_target_depth", + "target_record_count" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "edges" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "PR-009.resolved_depth_summary" + }, + "owner_ids": { + "const": [ + "PR-009" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "depth_resolved_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "maximum_resolved_target_depth": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "target_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "depth_resolved_record_count", + "maximum_resolved_target_depth", + "target_record_count" + ], + "additionalProperties": false + } + } + } + } + ] + }, + "unresolved_record_details": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "record_key", + "reason_codes" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "record_key", + "reason_codes" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "records" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T4.unresolved_record_details" + }, + "owner_ids": { + "const": [ + "T4" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "record_key", + "reason_codes" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "record_key", + "reason_codes" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + } + } + } + } + ] + }, + "resource_usage": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "admitted_node_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "admitted_edge_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "stored_root_membership_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "root_union_visit_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limits": { + "type": "object", + "properties": { + "max_nodes": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_edges": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_root_memberships": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_root_union_visits": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "max_nodes", + "max_edges", + "max_root_memberships", + "max_root_union_visits" + ], + "additionalProperties": false + }, + "exhausted_limit": { + "anyOf": [ + { + "type": "string", + "enum": [ + "max_nodes", + "max_edges", + "max_root_memberships", + "max_root_union_visits" + ] + }, + { + "type": "null" + } + ] + }, + "attempted_value": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "admitted_node_count", + "admitted_edge_count", + "stored_root_membership_count", + "root_union_visit_count", + "limits", + "exhausted_limit", + "attempted_value" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "work_units" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "PR-015.lineage_resource_usage" + }, + "owner_ids": { + "const": [ + "PR-015" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "admitted_node_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "admitted_edge_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "stored_root_membership_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "root_union_visit_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limits": { + "type": "object", + "properties": { + "max_nodes": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_edges": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_root_memberships": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_root_union_visits": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "max_nodes", + "max_edges", + "max_root_memberships", + "max_root_union_visits" + ], + "additionalProperties": false + }, + "exhausted_limit": { + "anyOf": [ + { + "type": "string", + "enum": [ + "max_nodes", + "max_edges", + "max_root_memberships", + "max_root_union_visits" + ] + }, + { + "type": "null" + } + ] + }, + "attempted_value": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "admitted_node_count", + "admitted_edge_count", + "stored_root_membership_count", + "root_union_visit_count", + "limits", + "exhausted_limit", + "attempted_value" + ], + "additionalProperties": false + } + } + } + } + ] + }, + "ordering_certificate": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "method": { + "const": "declared_earlier_version_order" + }, + "version_order": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "all_resolved_edges_follow_order": { + "type": "boolean" + } + }, + "required": [ + "method", + "version_order", + "all_resolved_edges_follow_order" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "certificate" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "PR-008.earlier_version_certificate" + }, + "owner_ids": { + "const": [ + "PR-008" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "method": { + "const": "declared_earlier_version_order" + }, + "version_order": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "all_resolved_edges_follow_order": { + "type": "boolean" + } + }, + "required": [ + "method", + "version_order", + "all_resolved_edges_follow_order" + ], + "additionalProperties": false + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + }, + "state_counts": { + "type": "object", + "properties": { + "by_version": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "additionalProperties": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "state_id", + "state_count" + ], + "additionalProperties": false + }, + "minItems": 0 + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "records" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T1.state_counts" }, "owner_ids": { "const": [ @@ -8533,134 +11870,445 @@ "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + } + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + } + }, + "support_delta": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "integer", + "minimum": -1.7976931348623157e+308, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "states" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "F-005" + }, + "owner_ids": { + "const": [ + "T1" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true + "minItems": 1 }, "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minItems": 1 } - }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } - }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } + } + }, + "else": { + "properties": { + "value": { + "type": "integer", + "minimum": -1.7976931348623157e+308, + "maximum": 1.7976931348623157e+308 } - ] + } } }, - "required": [], - "additionalProperties": false - } + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] }, - "support_delta": { + "support_retention_ratio": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "integer", - "minimum": -1.7976931348623157e+308, - "maximum": 1.7976931348623157e+308 + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, { "type": "null" @@ -8668,7 +12316,7 @@ ] }, "unit": { - "const": "states" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -8682,7 +12330,7 @@ ] }, "method_id": { - "const": "F-005" + "const": "F-006" }, "owner_ids": { "const": [ @@ -8927,9 +12575,17 @@ "else": { "properties": { "value": { - "type": "integer", - "minimum": -1.7976931348623157e+308, - "maximum": 1.7976931348623157e+308 + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] } } } @@ -8955,23 +12611,15 @@ } ] }, - "support_retention_ratio": { + "support_loss_count": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 }, { "type": "null" @@ -8979,7 +12627,7 @@ ] }, "unit": { - "const": "ratio" + "const": "states" }, "evidence_class": { "const": "derived_metric" @@ -8993,7 +12641,7 @@ ] }, "method_id": { - "const": "F-006" + "const": "T1.support_set_difference" }, "owner_ids": { "const": [ @@ -9238,17 +12886,9 @@ "else": { "properties": { "value": { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 } } } @@ -9274,7 +12914,7 @@ } ] }, - "support_loss_count": { + "support_added_count": { "type": "object", "properties": { "value": { @@ -9577,15 +13217,20 @@ } ] }, - "support_added_count": { + "extinct_states": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true }, { "type": "null" @@ -9593,7 +13238,7 @@ ] }, "unit": { - "const": "states" + "const": "set_of_states" }, "evidence_class": { "const": "derived_metric" @@ -9607,7 +13252,7 @@ ] }, "method_id": { - "const": "T1.support_set_difference" + "const": "T1.earlier_minus_later" }, "owner_ids": { "const": [ @@ -9852,9 +13497,14 @@ "else": { "properties": { "value": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true } } } @@ -9880,7 +13530,7 @@ } ] }, - "extinct_states": { + "added_states": { "type": "object", "properties": { "value": { @@ -9915,7 +13565,7 @@ ] }, "method_id": { - "const": "T1.earlier_minus_later" + "const": "T1.later_minus_earlier" }, "owner_ids": { "const": [ @@ -10193,7 +13843,7 @@ } ] }, - "added_states": { + "retained_states": { "type": "object", "properties": { "value": { @@ -10228,7 +13878,7 @@ ] }, "method_id": { - "const": "T1.later_minus_earlier" + "const": "T1.support_intersection" }, "owner_ids": { "const": [ @@ -10476,50 +14126,276 @@ "type": "array", "items": { "type": "string", - "minLength": 0, + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + } + } + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] + }, + "comparison_details": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "earlier_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "later_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "version_order": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 2, + "uniqueItems": true + }, + "version_order_source": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "earlier_state_semantics": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "later_state_semantics": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "harmonized_state_semantics": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "compatibility_method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "earlier_representation": { + "$ref": "#/$defs/representation" + }, + "later_representation": { + "$ref": "#/$defs/representation" + }, + "harmonized_representation": { + "$ref": "#/$defs/representation" + }, + "original_earlier_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "original_later_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "harmonized_earlier_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "harmonized_later_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "retention_denominator": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "retention_denominator_basis": { + "type": "string", + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } - } - ] - }, - "retained_states": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "state_mapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "target_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "source_state", + "target_state" + ], + "additionalProperties": false + }, + "minItems": 0 + }, + "mapping_effect": { + "type": "array", + "items": { + "type": "object", + "properties": { + "dataset_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "original_support_size": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "harmonized_support_size": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "dataset_version", + "original_support_size", + "harmonized_support_size" + ], + "additionalProperties": false + }, + "minItems": 0 + }, + "collision_groups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "target_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "source_states": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 2, + "uniqueItems": true + } + }, + "required": [ + "target_state", + "source_states" + ], + "additionalProperties": false + }, + "minItems": 0 + } }, - "minItems": 0, - "uniqueItems": true + "required": [ + "earlier_version", + "later_version", + "version_order", + "version_order_source", + "earlier_state_semantics", + "later_state_semantics", + "harmonized_state_semantics", + "compatibility_method", + "earlier_representation", + "later_representation", + "harmonized_representation", + "original_earlier_support", + "original_later_support", + "harmonized_earlier_support", + "harmonized_later_support", + "retention_denominator", + "retention_denominator_basis", + "state_mapping", + "mapping_effect", + "collision_groups" + ], + "additionalProperties": false }, { "type": "null" @@ -10527,7 +14403,7 @@ ] }, "unit": { - "const": "set_of_states" + "const": "comparison" }, "evidence_class": { "const": "derived_metric" @@ -10541,7 +14417,7 @@ ] }, "method_id": { - "const": "T1.support_intersection" + "const": "T1.explicit_pair_basis" }, "owner_ids": { "const": [ @@ -10786,822 +14662,602 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } - } - ] - }, - "comparison_details": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "object", - "properties": { - "earlier_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "later_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "version_order": { - "type": "array", - "items": { + "type": "object", + "properties": { + "earlier_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "later_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "version_order": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 2, + "uniqueItems": true + }, + "version_order_source": { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 2, - "uniqueItems": true - }, - "version_order_source": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "earlier_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "later_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "harmonized_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "compatibility_method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "earlier_representation": { - "$ref": "#/$defs/representation" - }, - "later_representation": { - "$ref": "#/$defs/representation" - }, - "harmonized_representation": { - "$ref": "#/$defs/representation" - }, - "original_earlier_support": { - "type": "array", - "items": { + "earlier_state_semantics": { "type": "string", - "minLength": 0, + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - }, - "original_later_support": { - "type": "array", - "items": { + "later_state_semantics": { "type": "string", - "minLength": 0, + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - }, - "harmonized_earlier_support": { - "type": "array", - "items": { + "harmonized_state_semantics": { "type": "string", - "minLength": 0, + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - }, - "harmonized_later_support": { - "type": "array", - "items": { + "compatibility_method": { "type": "string", - "minLength": 0, + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - }, - "retention_denominator": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "earlier_representation": { + "$ref": "#/$defs/representation" + }, + "later_representation": { + "$ref": "#/$defs/representation" + }, + "harmonized_representation": { + "$ref": "#/$defs/representation" + }, + "original_earlier_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" }, - { - "type": "null" - } - ] - }, - "retention_denominator_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "state_mapping": { - "type": "array", - "items": { - "type": "object", - "properties": { - "source_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "target_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - } + "minItems": 0, + "uniqueItems": true + }, + "original_later_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" }, - "required": [ - "source_state", - "target_state" - ], - "additionalProperties": false + "minItems": 0, + "uniqueItems": true }, - "minItems": 0 - }, - "mapping_effect": { - "type": "array", - "items": { - "type": "object", - "properties": { - "dataset_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "original_support_size": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - { - "type": "null" - } - ] + "harmonized_earlier_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "harmonized_later_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "retention_denominator": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 }, - "harmonized_support_size": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - { - "type": "null" - } - ] + { + "type": "null" } + ] + }, + "retention_denominator_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "state_mapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "target_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "source_state", + "target_state" + ], + "additionalProperties": false + }, + "minItems": 0 + }, + "mapping_effect": { + "type": "array", + "items": { + "type": "object", + "properties": { + "dataset_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "original_support_size": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "harmonized_support_size": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "dataset_version", + "original_support_size", + "harmonized_support_size" + ], + "additionalProperties": false }, - "required": [ - "dataset_version", - "original_support_size", - "harmonized_support_size" - ], - "additionalProperties": false + "minItems": 0 }, - "minItems": 0 - }, - "collision_groups": { - "type": "array", - "items": { - "type": "object", - "properties": { - "target_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "source_states": { - "type": "array", - "items": { + "collision_groups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "target_state": { "type": "string", "minLength": 0, "pattern": "^[^\\u0000]*$" }, - "minItems": 2, - "uniqueItems": true - } - }, - "required": [ - "target_state", - "source_states" - ], - "additionalProperties": false - }, - "minItems": 0 - } - }, - "required": [ - "earlier_version", - "later_version", - "version_order", - "version_order_source", - "earlier_state_semantics", - "later_state_semantics", - "harmonized_state_semantics", - "compatibility_method", - "earlier_representation", - "later_representation", - "harmonized_representation", - "original_earlier_support", - "original_later_support", - "harmonized_earlier_support", - "harmonized_later_support", - "retention_denominator", - "retention_denominator_basis", - "state_mapping", - "mapping_effect", - "collision_groups" - ], - "additionalProperties": false - }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "comparison" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "T1.explicit_pair_basis" - }, - "owner_ids": { - "const": [ - "T1" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" - } - ] - }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "denominator_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" + "source_states": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 2, + "uniqueItems": true + } + }, + "required": [ + "target_state", + "source_states" + ], + "additionalProperties": false + }, + "minItems": 0 + } + }, + "required": [ + "earlier_version", + "later_version", + "version_order", + "version_order_source", + "earlier_state_semantics", + "later_state_semantics", + "harmonized_state_semantics", + "compatibility_method", + "earlier_representation", + "later_representation", + "harmonized_representation", + "original_earlier_support", + "original_later_support", + "harmonized_earlier_support", + "harmonized_later_support", + "retention_denominator", + "retention_denominator_basis", + "state_mapping", + "mapping_effect", + "collision_groups" + ], + "additionalProperties": false + } } - ] - }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" + } }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - } - }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ { "if": { "properties": { "status": { - "const": "unavailable" + "enum": [ + "available", + "partial" + ] } } }, "then": { "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 + "representation": { + "$ref": "#/$defs/representation" } } - }, - "else": { + } + } + ] + } + }, + "required": [], + "additionalProperties": false + }, + "diversity": { + "type": "object", + "properties": { + "by_version": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "additionalProperties": { + "type": "object", + "properties": { + "gini_simpson_diversity": { + "type": "object", "properties": { "value": { - "type": "object", - "properties": { - "earlier_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "later_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "version_order": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 2, - "uniqueItems": true - }, - "version_order_source": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "earlier_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "later_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + { + "type": "null" + } + ] + }, + "unit": { + "const": "dimensionless" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "F-003" + }, + "owner_ids": { + "const": [ + "T1" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" }, - "harmonized_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "compatibility_method": { + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "earlier_representation": { - "$ref": "#/$defs/representation" - }, - "later_representation": { - "$ref": "#/$defs/representation" - }, - "harmonized_representation": { - "$ref": "#/$defs/representation" - }, - "original_earlier_support": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "original_later_support": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "harmonized_earlier_support": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "harmonized_later_support": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "retention_denominator": { - "anyOf": [ + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "retention_denominator_basis": { + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "state_mapping": { - "type": "array", - "items": { - "type": "object", - "properties": { - "source_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "target_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - } - }, - "required": [ - "source_state", - "target_state" - ], - "additionalProperties": false + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, - "minItems": 0 - }, - "mapping_effect": { - "type": "array", - "items": { - "type": "object", - "properties": { - "dataset_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "original_support_size": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - { - "type": "null" - } - ] - }, - "harmonized_support_size": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - { - "type": "null" - } - ] - } - }, - "required": [ - "dataset_version", - "original_support_size", - "harmonized_support_size" - ], - "additionalProperties": false + "reason_codes": { + "minItems": 1 }, - "minItems": 0 - }, - "collision_groups": { - "type": "array", - "items": { - "type": "object", - "properties": { - "target_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 }, - "source_states": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 2, - "uniqueItems": true + { + "exclusiveMaximum": 0 } - }, - "required": [ - "target_state", - "source_states" - ], - "additionalProperties": false - }, - "minItems": 0 + ] + } + } + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } } }, - "required": [ - "earlier_version", - "later_version", - "version_order", - "version_order_source", - "earlier_state_semantics", - "later_state_semantics", - "harmonized_state_semantics", - "compatibility_method", - "earlier_representation", - "later_representation", - "harmonized_representation", - "original_earlier_support", - "original_later_support", - "harmonized_earlier_support", - "harmonized_later_support", - "retention_denominator", - "retention_denominator_basis", - "state_mapping", - "mapping_effect", - "collision_groups" - ], - "additionalProperties": false - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } } - } + ] }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } - } - ] - } - }, - "required": [], - "additionalProperties": false - }, - "diversity": { - "type": "object", - "properties": { - "by_version": { - "type": "object", - "propertyNames": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "additionalProperties": { - "type": "object", - "properties": { - "gini_simpson_diversity": { + "simpson_concentration": { "type": "object", "properties": { "value": { @@ -11639,7 +15295,7 @@ ] }, "method_id": { - "const": "F-003" + "const": "F-004" }, "owner_ids": { "const": [ @@ -11920,7 +15576,7 @@ } ] }, - "simpson_concentration": { + "weighted_gini_simpson_diversity": { "type": "object", "properties": { "value": { @@ -11958,7 +15614,7 @@ ] }, "method_id": { - "const": "F-004" + "const": "F-003" }, "owner_ids": { "const": [ @@ -12239,7 +15895,7 @@ } ] }, - "weighted_gini_simpson_diversity": { + "weighted_simpson_concentration": { "type": "object", "properties": { "value": { @@ -12277,7 +15933,7 @@ ] }, "method_id": { - "const": "F-003" + "const": "F-004" }, "owner_ids": { "const": [ @@ -12558,23 +16214,42 @@ } ] }, - "weighted_simpson_concentration": { + "state_frequencies": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_frequency": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - { - "exclusiveMaximum": 0 - } - ] + "required": [ + "state_id", + "state_frequency" + ], + "additionalProperties": false + }, + "minItems": 0 }, { "type": "null" @@ -12582,7 +16257,7 @@ ] }, "unit": { - "const": "dimensionless" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -12596,7 +16271,7 @@ ] }, "method_id": { - "const": "F-004" + "const": "F-001" }, "owner_ids": { "const": [ @@ -12841,17 +16516,36 @@ "else": { "properties": { "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_frequency": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - { - "exclusiveMaximum": 0 - } - ] + "required": [ + "state_id", + "state_frequency" + ], + "additionalProperties": false + }, + "minItems": 0 } } } @@ -12877,7 +16571,7 @@ } ] }, - "state_frequencies": { + "weighted_state_frequencies": { "type": "object", "properties": { "value": { @@ -13234,7 +16928,7 @@ } ] }, - "weighted_state_frequencies": { + "weighted_state_masses": { "type": "object", "properties": { "value": { @@ -13249,10 +16943,10 @@ "minLength": 0, "pattern": "^[^\\u0000]*$" }, - "state_frequency": { + "state_mass": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -13265,7 +16959,7 @@ }, "required": [ "state_id", - "state_frequency" + "state_mass" ], "additionalProperties": false }, @@ -13277,7 +16971,7 @@ ] }, "unit": { - "const": "ratio" + "const": "user_declared_weight_mass" }, "evidence_class": { "const": "derived_metric" @@ -13291,7 +16985,7 @@ ] }, "method_id": { - "const": "F-001" + "const": "T1.state_weight_mass" }, "owner_ids": { "const": [ @@ -13545,10 +17239,10 @@ "minLength": 0, "pattern": "^[^\\u0000]*$" }, - "state_frequency": { + "state_mass": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -13561,7 +17255,7 @@ }, "required": [ "state_id", - "state_frequency" + "state_mass" ], "additionalProperties": false }, @@ -13591,42 +17285,107 @@ } ] }, - "weighted_state_masses": { + "distribution_basis": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "array", - "items": { - "type": "object", - "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "state_mass": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - } + "type": "object", + "properties": { + "input_basis": { + "type": "string", + "enum": [ + "empirical_assignments", + "explicit_counts_divided_by_included_records", + "weighted_record_mass", + "explicit_probability_vector" + ] }, - "required": [ - "state_id", - "state_mass" - ], - "additionalProperties": false + "analyzed_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "frequency_denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "supplied_probability_total": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "probability_residual": { + "anyOf": [ + { + "type": "number", + "minimum": -1.7976931348623157e+308, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "numerical_policy": { + "$ref": "#/$defs/numerical_policy" + } }, - "minItems": 0 + "required": [ + "input_basis", + "analyzed_record_count", + "frequency_denominator", + "denominator_basis", + "supplied_probability_total", + "probability_residual", + "numerical_policy" + ], + "additionalProperties": false }, { "type": "null" @@ -13634,7 +17393,7 @@ ] }, "unit": { - "const": "user_declared_weight_mass" + "const": "basis" }, "evidence_class": { "const": "derived_metric" @@ -13648,7 +17407,7 @@ ] }, "method_id": { - "const": "T1.state_weight_mass" + "const": "T1.validated_distribution_basis" }, "owner_ids": { "const": [ @@ -13893,36 +17652,101 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "object", - "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "state_mass": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - } + "type": "object", + "properties": { + "input_basis": { + "type": "string", + "enum": [ + "empirical_assignments", + "explicit_counts_divided_by_included_records", + "weighted_record_mass", + "explicit_probability_vector" + ] + }, + "analyzed_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "frequency_denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "supplied_probability_total": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "probability_residual": { + "anyOf": [ + { + "type": "number", + "minimum": -1.7976931348623157e+308, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] }, - "required": [ - "state_id", - "state_mass" - ], - "additionalProperties": false + "numerical_policy": { + "$ref": "#/$defs/numerical_policy" + } }, - "minItems": 0 + "required": [ + "input_basis", + "analyzed_record_count", + "frequency_denominator", + "denominator_basis", + "supplied_probability_total", + "probability_residual", + "numerical_policy" + ], + "additionalProperties": false } } } @@ -13947,507 +17771,649 @@ } } ] - }, - "distribution_basis": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "object", - "properties": { - "input_basis": { - "type": "string", - "enum": [ - "empirical_assignments", - "explicit_counts_divided_by_included_records", - "weighted_record_mass", - "explicit_probability_vector" - ] - }, - "analyzed_record_count": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - "frequency_denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "denominator_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "supplied_probability_total": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "probability_residual": { - "anyOf": [ - { - "type": "number", - "minimum": -1.7976931348623157e+308, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "numerical_policy": { - "$ref": "#/$defs/numerical_policy" - } - }, - "required": [ - "input_basis", - "analyzed_record_count", - "frequency_denominator", - "denominator_basis", - "supplied_probability_total", - "probability_residual", - "numerical_policy" - ], - "additionalProperties": false - }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "basis" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "T1.validated_distribution_basis" - }, - "owner_ids": { - "const": [ - "T1" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] + } + }, + "required": [], + "additionalProperties": false + } + }, + "gini_simpson_diversity_delta": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": -1, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "dimensionless" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "F-018" + }, + "owner_ids": { + "const": [ + "T1" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, - "coverage_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" - } - ] + "reason_codes": { + "minItems": 1 }, - "denominator": { - "anyOf": [ + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": -1, + "maximum": 1, + "oneOf": [ { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] - }, - "denominator_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" - } + } + } + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" ] - }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + }, + "tail": { + "type": "object", + "properties": { + "tail_support_size": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } + { + "type": "null" + } + ] + }, + "unit": { + "const": "states" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T2.declared_tail_rule" + }, + "owner_ids": { + "const": [ + "T2" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 - } - } + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "else": { - "properties": { - "value": { - "type": "object", - "properties": { - "input_basis": { - "type": "string", - "enum": [ - "empirical_assignments", - "explicit_counts_divided_by_included_records", - "weighted_record_mass", - "explicit_probability_vector" - ] - }, - "analyzed_record_count": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - "frequency_denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "denominator_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "supplied_probability_total": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "probability_residual": { - "anyOf": [ - { - "type": "number", - "minimum": -1.7976931348623157e+308, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "numerical_policy": { - "$ref": "#/$defs/numerical_policy" - } - }, - "required": [ - "input_basis", - "analyzed_record_count", - "frequency_denominator", - "denominator_basis", - "supplied_probability_total", - "probability_residual", - "numerical_policy" - ], - "additionalProperties": false - } - } + { + "exclusiveMaximum": 0 } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 } - ] + } + }, + "else": { + "properties": { + "value": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + } } }, - "required": [], - "additionalProperties": false - } + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] }, - "gini_simpson_diversity_delta": { + "tail_record_share": { "type": "object", "properties": { "value": { "anyOf": [ { "type": "number", - "minimum": -1, + "minimum": 0, "maximum": 1, "oneOf": [ { @@ -14464,7 +18430,7 @@ ] }, "unit": { - "const": "dimensionless" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -14478,11 +18444,11 @@ ] }, "method_id": { - "const": "F-018" + "const": "T2.tail_record_share" }, "owner_ids": { "const": [ - "T1" + "T2" ] }, "theory_map_ids": { @@ -14724,7 +18690,7 @@ "properties": { "value": { "type": "number", - "minimum": -1, + "minimum": 0, "maximum": 1, "oneOf": [ { @@ -14758,23 +18724,59 @@ } } ] - } - }, - "required": [], - "additionalProperties": false - }, - "tail": { - "type": "object", - "properties": { - "tail_support_size": { + }, + "rarity_ranking": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "state_frequency": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "rarity_rank": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "in_tail": { + "type": "boolean" + } + }, + "required": [ + "state_id", + "state_count", + "state_frequency", + "rarity_rank", + "in_tail" + ], + "additionalProperties": false + }, + "minItems": 0 }, { "type": "null" @@ -14782,7 +18784,7 @@ ] }, "unit": { - "const": "states" + "const": "ordinal_rank" }, "evidence_class": { "const": "derived_metric" @@ -14796,7 +18798,7 @@ ] }, "method_id": { - "const": "T2.declared_tail_rule" + "const": "T2.frequency_count_unicode_order" }, "owner_ids": { "const": [ @@ -15041,9 +19043,52 @@ "else": { "properties": { "value": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "state_frequency": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "rarity_rank": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "in_tail": { + "type": "boolean" + } + }, + "required": [ + "state_id", + "state_count", + "state_frequency", + "rarity_rank", + "in_tail" + ], + "additionalProperties": false + }, + "minItems": 0 } } } @@ -15069,23 +19114,20 @@ } ] }, - "tail_record_share": { + "tail_states": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true }, { "type": "null" @@ -15093,7 +19135,7 @@ ] }, "unit": { - "const": "ratio" + "const": "set_of_states" }, "evidence_class": { "const": "derived_metric" @@ -15107,7 +19149,7 @@ ] }, "method_id": { - "const": "T2.tail_record_share" + "const": "T2.declared_tail_rule" }, "owner_ids": { "const": [ @@ -15351,18 +19393,15 @@ }, "else": { "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "value": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true } } } @@ -15388,58 +19427,107 @@ } ] }, - "rarity_ranking": { + "tail_rule": { + "type": "string", + "enum": [ + "singleton_count", + "count_at_or_below", + "frequency_at_or_below", + "state_list" + ] + }, + "selection": { + "$ref": "#/$defs/tail_selection" + } + }, + "required": [], + "additionalProperties": false + }, + "provenance": { + "type": "object", + "properties": { + "source_type_shares": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "array", - "items": { - "type": "object", - "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "state_count": { - "type": "integer", - "minimum": 1, - "maximum": 1.7976931348623157e+308 - }, - "state_frequency": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "rarity_rank": { - "type": "integer", - "minimum": 1, - "maximum": 1.7976931348623157e+308 - }, - "in_tail": { - "type": "boolean" - } + "type": "object", + "properties": { + "human": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "required": [ - "state_id", - "state_count", - "state_frequency", - "rarity_rank", - "in_tail" - ], - "additionalProperties": false + "synthetic": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "mixed": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "sensor": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "unknown": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - "minItems": 0 + "required": [ + "human", + "synthetic", + "mixed", + "sensor", + "unknown" + ], + "additionalProperties": false }, { "type": "null" @@ -15447,7 +19535,7 @@ ] }, "unit": { - "const": "ordinal_rank" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -15461,11 +19549,11 @@ ] }, "method_id": { - "const": "T2.frequency_count_unicode_order" + "const": "F-007" }, "owner_ids": { "const": [ - "T2" + "PR-005" ] }, "theory_map_ids": { @@ -15706,91 +19794,170 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "object", - "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "state_count": { - "type": "integer", - "minimum": 1, - "maximum": 1.7976931348623157e+308 - }, - "state_frequency": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "rarity_rank": { - "type": "integer", - "minimum": 1, - "maximum": 1.7976931348623157e+308 - }, - "in_tail": { - "type": "boolean" - } + "type": "object", + "properties": { + "human": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "required": [ - "state_id", - "state_count", - "state_frequency", - "rarity_rank", - "in_tail" - ], - "additionalProperties": false + "synthetic": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "mixed": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "sensor": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "unknown": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - "minItems": 0 - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" + "required": [ + "human", + "synthetic", + "mixed", + "sensor", + "unknown" + ], + "additionalProperties": false } } } } ] }, - "tail_states": { + "weighted_source_type_shares": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "type": "object", + "properties": { + "human": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "synthetic": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "mixed": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "sensor": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "unknown": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - "minItems": 0, - "uniqueItems": true + "required": [ + "human", + "synthetic", + "mixed", + "sensor", + "unknown" + ], + "additionalProperties": false }, { "type": "null" @@ -15798,7 +19965,7 @@ ] }, "unit": { - "const": "set_of_states" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -15812,11 +19979,11 @@ ] }, "method_id": { - "const": "T2.declared_tail_rule" + "const": "F-007" }, "owner_ids": { "const": [ - "T2" + "PR-005" ] }, "theory_map_ids": { @@ -16057,59 +20224,89 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "type": "object", + "properties": { + "human": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "synthetic": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "mixed": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "sensor": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "unknown": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - "minItems": 0, - "uniqueItems": true - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" + "required": [ + "human", + "synthetic", + "mixed", + "sensor", + "unknown" + ], + "additionalProperties": false } } } } ] }, - "tail_rule": { - "type": "string", - "enum": [ - "singleton_count", - "count_at_or_below", - "frequency_at_or_below", - "state_list" - ] - }, - "selection": { - "$ref": "#/$defs/tail_selection" - } - }, - "required": [], - "additionalProperties": false - }, - "provenance": { - "type": "object", - "properties": { - "source_type_shares": { + "weighted_source_type_masses": { "type": "object", "properties": { "value": { @@ -16120,7 +20317,7 @@ "human": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16133,7 +20330,7 @@ "synthetic": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16146,7 +20343,7 @@ "mixed": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16159,7 +20356,7 @@ "sensor": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16172,7 +20369,7 @@ "unknown": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16198,7 +20395,7 @@ ] }, "unit": { - "const": "ratio" + "const": "user_declared_weight_mass" }, "evidence_class": { "const": "derived_metric" @@ -16212,7 +20409,7 @@ ] }, "method_id": { - "const": "F-007" + "const": "PR-005.source_weight_mass" }, "owner_ids": { "const": [ @@ -16462,7 +20659,7 @@ "human": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16475,7 +20672,7 @@ "synthetic": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16488,7 +20685,7 @@ "mixed": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16501,7 +20698,7 @@ "sensor": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16514,7 +20711,7 @@ "unknown": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16539,88 +20736,23 @@ } ] }, - "weighted_source_type_shares": { + "total_weight": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "object", - "properties": { - "human": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "synthetic": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "mixed": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "sensor": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "unknown": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + { + "exclusiveMaximum": 0 } - }, - "required": [ - "human", - "synthetic", - "mixed", - "sensor", - "unknown" - ], - "additionalProperties": false + ] }, { "type": "null" @@ -16628,7 +20760,7 @@ ] }, "unit": { - "const": "ratio" + "const": "user_declared_weight_mass" }, "evidence_class": { "const": "derived_metric" @@ -16642,7 +20774,7 @@ ] }, "method_id": { - "const": "F-007" + "const": "PR-005.total_weight" }, "owner_ids": { "const": [ @@ -16876,181 +21008,51 @@ "value": { "type": "null" }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "object", - "properties": { - "human": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "synthetic": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "mixed": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "sensor": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "unknown": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - } - }, - "required": [ - "human", - "synthetic", - "mixed", - "sensor", - "unknown" - ], - "additionalProperties": false - } - } - } - } - ] - }, - "weighted_source_type_masses": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "object", - "properties": { - "human": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "synthetic": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "mixed": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "sensor": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } + } + ] + }, + "missing_provenance_weight": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "unknown": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + { + "exclusiveMaximum": 0 } - }, - "required": [ - "human", - "synthetic", - "mixed", - "sensor", - "unknown" - ], - "additionalProperties": false + ] }, { "type": "null" @@ -17072,7 +21074,7 @@ ] }, "method_id": { - "const": "PR-005.source_weight_mass" + "const": "PR-005.missing_provenance_weight" }, "owner_ids": { "const": [ @@ -17317,89 +21319,24 @@ "else": { "properties": { "value": { - "type": "object", - "properties": { - "human": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "synthetic": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "mixed": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "sensor": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "unknown": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + { + "exclusiveMaximum": 0 } - }, - "required": [ - "human", - "synthetic", - "mixed", - "sensor", - "unknown" - ], - "additionalProperties": false + ] } } } } ] }, - "total_weight": { + "missing_provenance_share": { "type": "object", "properties": { "value": { @@ -17407,7 +21344,7 @@ { "type": "number", "minimum": 0, - "maximum": 1.7976931348623157e+308, + "maximum": 1, "oneOf": [ { "minimum": 0 @@ -17423,7 +21360,7 @@ ] }, "unit": { - "const": "user_declared_weight_mass" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -17437,11 +21374,11 @@ ] }, "method_id": { - "const": "PR-005.total_weight" + "const": "PR-004.one_minus_row_coverage" }, "owner_ids": { "const": [ - "PR-005" + "PR-004" ] }, "theory_map_ids": { @@ -17684,7 +21621,7 @@ "value": { "type": "number", "minimum": 0, - "maximum": 1.7976931348623157e+308, + "maximum": 1, "oneOf": [ { "minimum": 0 @@ -17699,7 +21636,7 @@ } ] }, - "missing_provenance_weight": { + "weighted_missing_provenance_share": { "type": "object", "properties": { "value": { @@ -17707,7 +21644,7 @@ { "type": "number", "minimum": 0, - "maximum": 1.7976931348623157e+308, + "maximum": 1, "oneOf": [ { "minimum": 0 @@ -17723,7 +21660,7 @@ ] }, "unit": { - "const": "user_declared_weight_mass" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -17737,7 +21674,7 @@ ] }, "method_id": { - "const": "PR-005.missing_provenance_weight" + "const": "PR-005.missing_provenance_weight_share" }, "owner_ids": { "const": [ @@ -17984,7 +21921,7 @@ "value": { "type": "number", "minimum": 0, - "maximum": 1.7976931348623157e+308, + "maximum": 1, "oneOf": [ { "minimum": 0 @@ -17993,623 +21930,623 @@ "exclusiveMaximum": 0 } ] - } - } - } - } - ] - }, - "missing_provenance_share": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + }, + "closure_exposure": { + "type": "object", + "properties": { + "direct": { + "type": "object", + "properties": { + "lower_bound": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "F-009" + }, + "owner_ids": { + "const": [ + "T3" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ { - "minimum": 0 + "$ref": "#/$defs/representation" }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "ratio" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "PR-004.one_minus_row_coverage" - }, - "owner_ids": { - "const": [ - "PR-004" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + "coverage_reason": { + "anyOf": [ { - "minimum": 0 + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, { - "exclusiveMaximum": 0 + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ - { + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ { - "type": "null" + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } } ] }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ + "upper_bound": { + "type": "object", + "properties": { + "value": { + "anyOf": [ { - "minimum": 0 + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "denominator_reason": { - "anyOf": [ - { + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "enum": [ + "available", + "partial", + "unavailable" + ] }, - { - "type": "null" - } - ] - }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - } - }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } - }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 + "method_id": { + "const": "F-010" + }, + "owner_ids": { + "const": [ + "T3" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" ] - } - } - } - } - ] - }, - "weighted_missing_provenance_share": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ { - "minimum": 0 + "$ref": "#/$defs/representation" }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "ratio" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "PR-005.missing_provenance_weight_share" - }, - "owner_ids": { - "const": [ - "PR-005" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + "coverage": { + "anyOf": [ { - "minimum": 0 + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] }, - { - "type": "null" - } - ] - }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ { - "minimum": 0 + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "denominator_reason": { - "anyOf": [ - { + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - { - "type": "null" - } - ] - }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - } - }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" } }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 + "then": { + "properties": { + "value": { + "type": "null" }, - { - "exclusiveMaximum": 0 + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 } - ] + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } } } - } - } - ] - } - }, - "required": [], - "additionalProperties": false - }, - "closure_exposure": { - "type": "object", - "properties": { - "direct": { - "type": "object", - "properties": { - "lower_bound": { + ] + }, + "interval_width": { "type": "object", "properties": { "value": { @@ -18647,7 +22584,7 @@ ] }, "method_id": { - "const": "F-009" + "const": "T3.upper_minus_lower" }, "owner_ids": { "const": [ @@ -18909,7 +22846,20 @@ } ] }, - "upper_bound": { + "classification_basis": { + "const": "toolkit_operationalization" + }, + "confidence_disclosure": { + "const": "provenance_confidence_is_separate_and_does_not_discount_grounding" + } + }, + "required": [], + "additionalProperties": false + }, + "lineage": { + "type": "object", + "properties": { + "lower_bound": { "type": "object", "properties": { "value": { @@ -18947,7 +22897,7 @@ ] }, "method_id": { - "const": "F-010" + "const": "T3.lineage_closure" }, "owner_ids": { "const": [ @@ -19209,7 +23159,7 @@ } ] }, - "interval_width": { + "upper_bound": { "type": "object", "properties": { "value": { @@ -19247,7 +23197,7 @@ ] }, "method_id": { - "const": "T3.upper_minus_lower" + "const": "T3.lineage_closure" }, "owner_ids": { "const": [ @@ -19509,20 +23459,7 @@ } ] }, - "classification_basis": { - "const": "toolkit_operationalization" - }, - "confidence_disclosure": { - "const": "provenance_confidence_is_separate_and_does_not_discount_grounding" - } - }, - "required": [], - "additionalProperties": false - }, - "lineage": { - "type": "object", - "properties": { - "lower_bound": { + "interval_width": { "type": "object", "properties": { "value": { @@ -19552,7 +23489,12 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "T3.lineage_closure" @@ -19626,813 +23568,1682 @@ "representation": { "anyOf": [ { - "$ref": "#/$defs/representation" + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [], + "additionalProperties": false + }, + "lineage": { + "type": "object", + "properties": { + "resolved_parent_edge_coverage": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] + { + "type": "null" + } + ] + }, + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "PR-008.resolved_edges_over_declared" + }, + "owner_ids": { + "const": [ + "PR-008" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" }, - "coverage_reason": { - "anyOf": [ + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "denominator_reason": { - "anyOf": [ + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "no_declared_parents": { + "type": "boolean" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "reason_codes": { + "minItems": 1 }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "required_evidence": { + "minItems": 1 + } } }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } - }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + { + "exclusiveMaximum": 0 } - } + ] } } + } + } + ] + }, + "resolved_lineage_coverage": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T4.resolved_records_over_scope" + }, + "owner_ids": { + "const": [ + "T4" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } ] }, - "upper_bound": { - "type": "object", - "properties": { - "value": { - "anyOf": [ + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "unit": { - "const": "ratio" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "const": "unavailable" - }, - "method_id": { - "const": "T3.lineage_closure" + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "owner_ids": { - "const": [ - "T3" + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } ] }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" - } - ] - }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "denominator_reason": { - "anyOf": [ + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } + } + ] + }, + "external_ancestry_coverage": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true + { + "type": "null" + } + ] + }, + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T4.external_roots_over_scope" + }, + "owner_ids": { + "const": [ + "T4" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" }, - "weighting": { - "$ref": "#/$defs/weighting" + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "input_basis": { + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "redaction": { - "$ref": "#/$defs/redaction" + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "method": { + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } + } + ] + }, + "distinct_external_root_count": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" } + ] + }, + "unit": { + "const": "roots" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T4.root_set_union" + }, + "owner_ids": { + "const": [ + "T4" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } - }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - } - } - } + "$ref": "#/$defs/representation" + }, + { + "type": "null" } ] }, - "interval_width": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "ratio" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "const": "unavailable" - }, - "method_id": { - "const": "T3.lineage_closure" - }, - "owner_ids": { - "const": [ - "T3" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "denominator_reason": { - "anyOf": [ + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + { + "type": "null" } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + } + } + } + ] + }, + "top_shared_ancestors": { + "type": "object", + "properties": { + "value": { + "anyOf": [ { - "if": { - "properties": { - "status": { - "const": "unavailable" - } + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "incidence_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "incidence_share": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "incidence_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "fractional_mass": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "normalized_weight": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "weight_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "record_key", + "incidence_count", + "incidence_share", + "incidence_denominator", + "fractional_mass", + "normalized_weight", + "weight_denominator" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true } }, - "then": { - "properties": { - "value": { - "type": "null" + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } }, - "reason_codes": { - "minItems": 1 + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } }, - "required_evidence": { - "minItems": 1 + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "incidence_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "incidence_share": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "incidence_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "fractional_mass": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "normalized_weight": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "weight_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "record_key", + "incidence_count", + "incidence_share", + "incidence_denominator", + "fractional_mass", + "normalized_weight", + "weight_denominator" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 }, - { - "exclusiveMaximum": 0 + "omission_reasons": { + "maxItems": 0 } - ] + } } - } - } - } - ] - } - }, - "required": [], - "additionalProperties": false - } - }, - "required": [], - "additionalProperties": false - }, - "lineage": { - "type": "object", - "properties": { - "resolved_parent_edge_coverage": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 }, { - "exclusiveMaximum": 0 + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } } ] }, @@ -20442,7 +25253,7 @@ ] }, "unit": { - "const": "ratio" + "const": "records" }, "evidence_class": { "const": "derived_metric" @@ -20456,11 +25267,11 @@ ] }, "method_id": { - "const": "PR-008.resolved_edges_over_declared" + "const": "T4.incidence_ranking" }, "owner_ids": { "const": [ - "PR-008" + "T4" ] }, "theory_map_ids": { @@ -20701,15 +25512,287 @@ "else": { "properties": { "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "incidence_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "incidence_share": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "incidence_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "fractional_mass": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "normalized_weight": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "weight_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "record_key", + "incidence_count", + "incidence_share", + "incidence_denominator", + "fractional_mass", + "normalized_weight", + "weight_denominator" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ { - "minimum": 0 + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "incidence_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "incidence_share": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "incidence_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "fractional_mass": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "normalized_weight": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "weight_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "record_key", + "incidence_count", + "incidence_share", + "incidence_denominator", + "fractional_mass", + "normalized_weight", + "weight_denominator" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } }, { - "exclusiveMaximum": 0 + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } } ] } @@ -20718,23 +25801,15 @@ } ] }, - "resolved_lineage_coverage": { + "grounded_record_count": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 }, { "type": "null" @@ -20742,16 +25817,21 @@ ] }, "unit": { - "const": "ratio" + "const": "records" }, "evidence_class": { "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { - "const": "T4.resolved_records_over_scope" + "const": "T4.grounded_record_count" }, "owner_ids": { "const": [ @@ -20996,40 +26076,24 @@ "else": { "properties": { "value": { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 } } } } ] }, - "external_ancestry_coverage": { + "closed_record_count": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 }, { "type": "null" @@ -21037,16 +26101,21 @@ ] }, "unit": { - "const": "ratio" + "const": "records" }, "evidence_class": { "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { - "const": "T4.external_roots_over_scope" + "const": "T4.closed_record_count" }, "owner_ids": { "const": [ @@ -21291,24 +26360,16 @@ "else": { "properties": { "value": { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 } } } } ] }, - "distinct_external_root_count": { + "unresolved_record_count": { "type": "object", "properties": { "value": { @@ -21324,16 +26385,21 @@ ] }, "unit": { - "const": "roots" + "const": "records" }, "evidence_class": { "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { - "const": "T4.root_set_union" + "const": "T4.unresolved_record_count" }, "owner_ids": { "const": [ @@ -21587,32 +26653,15 @@ } ] }, - "top_shared_ancestors": { + "records_with_resolved_external_ancestry": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "array", - "items": { - "type": "object", - "properties": { - "record_key": { - "$ref": "#/$defs/record_key" - }, - "incidence_count": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - } - }, - "required": [ - "record_key", - "incidence_count" - ], - "additionalProperties": false - }, - "minItems": 0 + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 }, { "type": "null" @@ -21626,10 +26675,15 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { - "const": "T4.incidence_ranking" + "const": "T4.records_with_resolved_external_ancestry" }, "owner_ids": { "const": [ @@ -21874,26 +26928,9 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "object", - "properties": { - "record_key": { - "$ref": "#/$defs/record_key" - }, - "incidence_count": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - } - }, - "required": [ - "record_key", - "incidence_count" - ], - "additionalProperties": false - }, - "minItems": 0 + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 } } } @@ -21930,7 +26967,12 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "F-012" @@ -22225,7 +27267,12 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "F-013" @@ -22512,7 +27559,12 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "PR-009.maximum_resolved_parent_depth" @@ -23342,7 +28394,12 @@ "const": "proxy_signal" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "T4.shared_ancestry_dependence" @@ -23514,7 +28571,7 @@ "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 1, + "minItems": 0, "uniqueItems": true }, "required_evidence": { @@ -23524,14 +28581,19 @@ "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 1, + "minItems": 0, "uniqueItems": true }, "signal": { "const": "shared_ancestry_dependence" }, "level": { - "const": "indeterminate" + "type": "string", + "enum": [ + "present", + "not_present", + "indeterminate" + ] }, "basis_fields": { "type": "array", diff --git a/scripts/release_check.py b/scripts/release_check.py index 3d4f3d6..5428bf0 100644 --- a/scripts/release_check.py +++ b/scripts/release_check.py @@ -1,6 +1,6 @@ """Current source and candidate checks for Recursive Integrity Toolkit. -Step 6 opens the approved lineage closure and descriptive proxy scope. Historical dispatch, source-body +Step 7 opens the approved lineage schema, assembly and privacy scope. Historical dispatch, source-body migrations and phase registries are recoverable from the accepted Git commit. Installed checks below retain their existing product, privacy and package cases. """ @@ -24,11 +24,16 @@ import zipfile ROOT = Path(__file__).resolve().parents[1] -ACCEPTED_COMMIT = "d81966561502b94701712a3d4b4b7fd50783cba9" +ACCEPTED_COMMIT = "e2666515d98cfe41483b5c43345c642bab02ced0" PROTECTED_PREFIXES = ("src/", "schemas/", "examples/hero/") CURRENT_IMPLEMENTATION_PATHS = frozenset({ "src/recursive_integrity_toolkit/lineage/ancestry.py", - "src/recursive_integrity_toolkit/metrics/bounds.py", + "src/recursive_integrity_toolkit/reports/assembly.py", + "src/recursive_integrity_toolkit/reports/markdown_report.py", + "src/recursive_integrity_toolkit/utils/logging.py", + "src/recursive_integrity_toolkit/result.py", + "schemas/report.schema.json", + "src/recursive_integrity_toolkit/data/report.schema.json", }) PARQUET_CASES = {"test_PR002_parquet_real_roundtrip", "test_PR002_parquet_real_row_limit", "test_PR002_parquet_real_invalid_file"} HERO_NAMES = {"records_v1.csv", "records_v2.csv", "provenance.csv", "config.json", "version_order.json", "EXPECTED_OUTPUTS.md"} @@ -68,15 +73,19 @@ def verify_source_scope(root: Path = ROOT) -> dict: for name, raw in expected.items(): current = _regular_file(root, name).read_bytes() if name not in CURRENT_IMPLEMENTATION_PATHS and current != raw: - raise ValueError(f"Unauthorized product mutation outside Step 6 scope: {name}") + raise ValueError(f"Unauthorized product mutation outside Step 7 scope: {name}") for prefix in PROTECTED_PREFIXES: paths = list((root / prefix).rglob("*")) if (root / prefix).is_symlink() or any(path.is_symlink() for path in paths): raise ValueError(f"Aliased protected content: {prefix}") actual = {path.relative_to(root).as_posix() for path in paths - if path.is_file() and "__pycache__" not in path.parts} + if path.is_file() and "__pycache__" not in path.parts + # setuptools writes this metadata during normal wheel installs. + # It is outside the package; all product paths stay protected. + and not path.relative_to(root).as_posix().startswith( + "src/recursive_integrity_toolkit.egg-info/")} if actual != {name for name in expected if name.startswith(prefix)}: - raise ValueError(f"Unauthorized product file-set mutation during Step 6: {prefix}") + raise ValueError(f"Unauthorized product file-set mutation during Step 7: {prefix}") return {"protected_product_files": len(expected) - len(CURRENT_IMPLEMENTATION_PATHS), "authorized_implementation_files": len(CURRENT_IMPLEMENTATION_PATHS), "product_file_inventory": "unchanged"} @@ -396,7 +405,7 @@ def reject_schema_files(event, args): payload = { 'run': { 'run_id': 'installed-empty-case', 'toolkit_version': '0.1.0.dev2', - 'report_schema_version': '1.0', 'started_at': '2026-09-19T00:00:00+00:00', + 'report_schema_version': '1.1', 'started_at': '2026-09-19T00:00:00+00:00', 'completed_at': '2026-09-19T00:00:00+00:00', 'duration_seconds': 0, 'python_version': '3.12', 'platform': 'isolated-installed-smoke', 'command': 'rit validate', 'config_hash': '0123456789abcdef' * 4, @@ -571,7 +580,7 @@ def deny_effects(event, arguments): ) run = { 'run_id': 'installed-typed-empty-case', 'toolkit_version': '0.1.0.dev2', - 'report_schema_version': '1.0', 'started_at': None, 'completed_at': None, + 'report_schema_version': '1.1', 'started_at': None, 'completed_at': None, 'duration_seconds': None, 'python_version': None, 'platform': None, 'command': None, 'config_hash': None, 'random_seed': None, 'strict_mode': False, 'redacted_mode': False, 'network_call_count': 0, @@ -598,7 +607,7 @@ def deny_effects(event, arguments): assert payload['inputs']['scope']['record_count'] == 0 assert payload['inputs']['artifacts'] == [] and payload['inputs']['file_hashes'] == [] assert payload['simulations'] == {} and payload['errors'] == [] -assert payload['capabilities']['lineage']['execution_status'] == 'deferred' +assert payload['capabilities']['lineage']['execution_status'] == 'not_requested' assert report.sections['observability']['capabilities'] is report.sections['capabilities'] assert {'model_performance_decline', 'causal_ancestor_effect', 'universal_integrity', 'universal_collapse_prediction'} <= {item['conclusion'] for item in payload['unavailable_conclusions']} @@ -757,7 +766,7 @@ def deny_effects(event, arguments): ) run = { 'run_id': 'installed-typed-empty-case', 'toolkit_version': '0.1.0.dev2', - 'report_schema_version': '1.0', 'started_at': None, 'completed_at': None, + 'report_schema_version': '1.1', 'started_at': None, 'completed_at': None, 'duration_seconds': None, 'python_version': None, 'platform': None, 'command': None, 'config_hash': None, 'random_seed': None, 'strict_mode': False, 'redacted_mode': False, 'network_call_count': 0, @@ -784,7 +793,7 @@ def deny_effects(event, arguments): assert payload['inputs']['scope']['record_count'] == 0 assert payload['inputs']['artifacts'] == [] and payload['inputs']['file_hashes'] == [] assert payload['simulations'] == {} and payload['errors'] == [] -assert payload['capabilities']['lineage']['execution_status'] == 'deferred' +assert payload['capabilities']['lineage']['execution_status'] == 'not_requested' assert report.sections['observability']['capabilities'] is report.sections['capabilities'] assert {'model_performance_decline', 'causal_ancestor_effect', 'universal_integrity', 'universal_collapse_prediction'} <= {item['conclusion'] for item in payload['unavailable_conclusions']} @@ -1003,7 +1012,7 @@ def find_spec(self, fullname, path=None, target=None): views.append((report, view, copy.deepcopy(view.to_dict()))) assert views[0][2]['run']['run_status'] == 'partial' assert views[0][2]['simulations']['closed_resampling']['status'] == 'experimental' -assert views[0][2]['capabilities']['lineage']['execution_status'] == 'deferred' +assert views[0][2]['capabilities']['lineage']['execution_status'] == 'not_requested' # After this point even an already imported owner cannot execute a function. def denied(*args, **kwargs): raise AssertionError('installed renderer attempted file, network or process access') @@ -1083,7 +1092,7 @@ def smoke_installed_publication(wheel: Path) -> None: bindir = work / "venv" / ("Scripts" if os.name == "nt" else "bin") python = bindir / ("python.exe" if os.name == "nt" else "python") subprocess.run([str(python), "-m", "pip", "install", "--no-index", "--no-deps", str(wheel.resolve())], cwd=work, check=True) - program = 'import importlib.abc, json, os, socket, sys\nfrom pathlib import Path\nclass BlockOptional(importlib.abc.MetaPathFinder):\n def find_spec(self, fullname, path=None, target=None):\n if fullname.split(".")[0] in ("numpy", "pandas", "pyarrow"):\n raise AssertionError("optional or analytical dependency imported")\nsys.meta_path.insert(0, BlockOptional())\ndef phase4_step6_view(mode="standard", label="publication-case"):\n from recursive_integrity_toolkit.reports.assembly import privacy_view\n from recursive_integrity_toolkit.result import CanonicalReport\n from recursive_integrity_toolkit.utils.hashing import IdentifierProtection\n\n payload = {\n "run": {\n "run_id": label, "toolkit_version": "0.1.0.dev2", "report_schema_version": "1.0",\n "started_at": "2026-09-20T00:00:00+00:00", "completed_at": "2026-09-20T00:00:00.500000+00:00",\n "duration_seconds": 0.5, "python_version": "3.12.14", "platform": "independent-test-platform",\n "command": "rit validate", "config_hash": "0123456789abcdef" * 4,\n "random_seed": None, "strict_mode": False, "redacted_mode": False,\n "network_call_count": 0, "deterministic": True, "privacy_mode": "standard",\n "run_status": "complete", "null_reasons": {"random_seed": "No stochastic scenario was requested."},\n },\n "inputs": {}, "observability": {}, "capabilities": {}, "observed_facts": {},\n "derived_metrics": {}, "proxy_signals": {}, "simulations": {},\n "unavailable_conclusions": [], "recommended_next_metadata": [], "warnings": [], "errors": [],\n }\n return privacy_view(CanonicalReport.from_dict(payload), mode=mode,\n protection=IdentifierProtection.create(secret=b"publication-test-only-key-32byte!"))\nfrom recursive_integrity_toolkit.utils.paths import publish_reports, PublicationResult\nfrom recursive_integrity_toolkit.utils.logging import format_publication_diagnostic\nfrom recursive_integrity_toolkit.utils import paths\nfrom recursive_integrity_toolkit.reports.json_report import render_json\nfrom recursive_integrity_toolkit.reports.markdown_report import render_markdown\nimport recursive_integrity_toolkit as package\nassert not Path(package.__file__).resolve().is_relative_to(Path(sys.argv[2]).resolve())\nwork=Path(sys.argv[1]); source=work/\'input.txt\'; source.write_bytes(b\'PRIVATE_INSTALLED_SOURCE\')\nview=phase4_step6_view(\'redacted\')\nexpected={\'report.json\':render_json(view).encode(), \'report.md\':render_markdown(view).encode()}\ndef deny(*args, **kwargs):\n raise AssertionError(\'network forbidden\')\nsocket.socket=deny; socket.getaddrinfo=deny\nactual_open=os.open\ndef no_input(path,*args,**kwargs):\n assert Path(path)!=source\n return actual_open(path,*args,**kwargs)\nos.open=no_input\ndef no_analysis(frame,event,arg):\n if event==\'call\':\n name=frame.f_globals.get(\'__name__\',\'\')\n assert not name.startswith((\'recursive_integrity_toolkit.io.\',\'recursive_integrity_toolkit.metrics.\',\'recursive_integrity_toolkit.representations.\'))\nsys.setprofile(no_analysis)\ntry:\n result=publish_reports(view,work/\'out\',input_paths=(source,))\n assert result==PublicationResult(\'complete\',None,0,(\'report.json\',\'report.md\'))\n assert publish_reports(view,work/\'out\',input_paths=(source,)).code==\'E_OUTPUT_EXISTS\'\n actual_publish=paths._output_publish_one\n def fail_second(src,dst):\n if dst.name==\'report.md\':raise OSError(\'PRIVATE_INSTALLED_FAILURE\')\n return actual_publish(src,dst)\n paths._output_publish_one=fail_second\n failed=publish_reports(view,work/\'failure\',input_paths=(source,))\n assert failed.status==\'failed\' and failed.code==\'E_OUTPUT_IO\' and failed.temporary_cleanup_complete\n assert \'PRIVATE_\' not in format_publication_diagnostic(failed)\nfinally:\n sys.setprofile(None)\nassert {p.name:p.read_bytes() for p in (work/\'out\').iterdir()}==expected\nassert list((work/\'failure\').iterdir())==[]\nassert source.read_bytes()==b\'PRIVATE_INSTALLED_SOURCE\'\nprint(\'installed Step 6: exact safe JSON/Markdown bytes, no-overwrite, partial failure cleanup, safe diagnostics, unchanged input, blocked network/analysis: PASS\')\n' + program = 'import importlib.abc, json, os, socket, sys\nfrom pathlib import Path\nclass BlockOptional(importlib.abc.MetaPathFinder):\n def find_spec(self, fullname, path=None, target=None):\n if fullname.split(".")[0] in ("numpy", "pandas", "pyarrow"):\n raise AssertionError("optional or analytical dependency imported")\nsys.meta_path.insert(0, BlockOptional())\ndef phase4_step6_view(mode="standard", label="publication-case"):\n from recursive_integrity_toolkit.reports.assembly import privacy_view\n from recursive_integrity_toolkit.result import CanonicalReport\n from recursive_integrity_toolkit.utils.hashing import IdentifierProtection\n\n payload = {\n "run": {\n "run_id": label, "toolkit_version": "0.1.0.dev2", "report_schema_version": "1.1",\n "started_at": "2026-09-20T00:00:00+00:00", "completed_at": "2026-09-20T00:00:00.500000+00:00",\n "duration_seconds": 0.5, "python_version": "3.12.14", "platform": "independent-test-platform",\n "command": "rit validate", "config_hash": "0123456789abcdef" * 4,\n "random_seed": None, "strict_mode": False, "redacted_mode": False,\n "network_call_count": 0, "deterministic": True, "privacy_mode": "standard",\n "run_status": "complete", "null_reasons": {"random_seed": "No stochastic scenario was requested."},\n },\n "inputs": {}, "observability": {}, "capabilities": {}, "observed_facts": {},\n "derived_metrics": {}, "proxy_signals": {}, "simulations": {},\n "unavailable_conclusions": [], "recommended_next_metadata": [], "warnings": [], "errors": [],\n }\n return privacy_view(CanonicalReport.from_dict(payload), mode=mode,\n protection=IdentifierProtection.create(secret=b"publication-test-only-key-32byte!"))\nfrom recursive_integrity_toolkit.utils.paths import publish_reports, PublicationResult\nfrom recursive_integrity_toolkit.utils.logging import format_publication_diagnostic\nfrom recursive_integrity_toolkit.utils import paths\nfrom recursive_integrity_toolkit.reports.json_report import render_json\nfrom recursive_integrity_toolkit.reports.markdown_report import render_markdown\nimport recursive_integrity_toolkit as package\nassert not Path(package.__file__).resolve().is_relative_to(Path(sys.argv[2]).resolve())\nwork=Path(sys.argv[1]); source=work/\'input.txt\'; source.write_bytes(b\'PRIVATE_INSTALLED_SOURCE\')\nview=phase4_step6_view(\'redacted\')\nexpected={\'report.json\':render_json(view).encode(), \'report.md\':render_markdown(view).encode()}\ndef deny(*args, **kwargs):\n raise AssertionError(\'network forbidden\')\nsocket.socket=deny; socket.getaddrinfo=deny\nactual_open=os.open\ndef no_input(path,*args,**kwargs):\n assert Path(path)!=source\n return actual_open(path,*args,**kwargs)\nos.open=no_input\ndef no_analysis(frame,event,arg):\n if event==\'call\':\n name=frame.f_globals.get(\'__name__\',\'\')\n assert not name.startswith((\'recursive_integrity_toolkit.io.\',\'recursive_integrity_toolkit.metrics.\',\'recursive_integrity_toolkit.representations.\'))\nsys.setprofile(no_analysis)\ntry:\n result=publish_reports(view,work/\'out\',input_paths=(source,))\n assert result==PublicationResult(\'complete\',None,0,(\'report.json\',\'report.md\'))\n assert publish_reports(view,work/\'out\',input_paths=(source,)).code==\'E_OUTPUT_EXISTS\'\n actual_publish=paths._output_publish_one\n def fail_second(src,dst):\n if dst.name==\'report.md\':raise OSError(\'PRIVATE_INSTALLED_FAILURE\')\n return actual_publish(src,dst)\n paths._output_publish_one=fail_second\n failed=publish_reports(view,work/\'failure\',input_paths=(source,))\n assert failed.status==\'failed\' and failed.code==\'E_OUTPUT_IO\' and failed.temporary_cleanup_complete\n assert \'PRIVATE_\' not in format_publication_diagnostic(failed)\nfinally:\n sys.setprofile(None)\nassert {p.name:p.read_bytes() for p in (work/\'out\').iterdir()}==expected\nassert list((work/\'failure\').iterdir())==[]\nassert source.read_bytes()==b\'PRIVATE_INSTALLED_SOURCE\'\nprint(\'installed Step 6: exact safe JSON/Markdown bytes, no-overwrite, partial failure cleanup, safe diagnostics, unchanged input, blocked network/analysis: PASS\')\n' subprocess.run([str(python), "-I", "-c", program, str(work), str(ROOT)], cwd=work, check=True) @@ -1135,7 +1144,7 @@ def deny(*args,**kwargs):raise AssertionError('network forbidden') assert report['observed_facts']['provenance'][name]['value']==1.0 assert report['observed_facts']['provenance'][name]['scope']['dataset_versions']==['v2'] assert [report['derived_metrics']['closure_exposure']['direct'][n]['value'] for n in ('lower_bound','upper_bound','interval_width')]==[0.5,0.5,0] -assert report['capabilities']['lineage']['execution_status']=='deferred' +assert report['capabilities']['lineage']['execution_status']=='not_requested' assert report['capabilities']['dataset_longitudinal']['execution_status']=='partial' assert report['simulations']=={} and report['errors']==[] assert {'model_performance_decline','causal_ancestor_effect','universal_integrity','universal_collapse_prediction'} <= {x['conclusion'] for x in report['unavailable_conclusions']} diff --git a/src/recursive_integrity_toolkit/data/report.schema.json b/src/recursive_integrity_toolkit/data/report.schema.json index c3fce78..3f52118 100644 --- a/src/recursive_integrity_toolkit/data/report.schema.json +++ b/src/recursive_integrity_toolkit/data/report.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://recursive-integrity-toolkit.example/schemas/report.schema.json", - "title": "Recursive Integrity Toolkit canonical report 1.0", - "description": "Phase 4 Step 2 public contract. Product metadata and five analytical evidence classes remain separate.", + "title": "Recursive Integrity Toolkit canonical report 1.1", + "description": "Phase 5 public contract with bounded lineage evidence. Product metadata and five analytical evidence classes remain separate.", "type": "object", "properties": { "run": { @@ -19,7 +19,7 @@ "pattern": "^[^\\u0000]*$" }, "report_schema_version": { - "const": "1.0" + "const": "1.1" }, "started_at": { "anyOf": [ @@ -6752,7 +6752,12 @@ "const": "observed_fact" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "T6.graph_cycle_check" @@ -7011,7 +7016,7 @@ } ] }, - "ordering_certificate": { + "graph_scope": { "type": "object", "properties": { "value": { @@ -7019,10 +7024,34 @@ { "type": "object", "properties": { - "method": { - "const": "declared_earlier_version_order" + "target_dataset_version": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] }, - "version_order": { + "target_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "loaded_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "context_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "loaded_dataset_versions": { "type": "array", "items": { "type": "string", @@ -7031,15 +7060,14 @@ }, "minItems": 0, "uniqueItems": true - }, - "all_resolved_edges_follow_order": { - "type": "boolean" } }, "required": [ - "method", - "version_order", - "all_resolved_edges_follow_order" + "target_dataset_version", + "target_record_count", + "loaded_record_count", + "context_record_count", + "loaded_dataset_versions" ], "additionalProperties": false }, @@ -7049,7 +7077,7 @@ ] }, "unit": { - "const": "certificate" + "const": "scope" }, "evidence_class": { "const": "observed_fact" @@ -7063,11 +7091,11 @@ ] }, "method_id": { - "const": "PR-008.earlier_version_certificate" + "const": "T4.graph_scope" }, "owner_ids": { "const": [ - "PR-008" + "T4" ] }, "theory_map_ids": { @@ -7310,10 +7338,34 @@ "value": { "type": "object", "properties": { - "method": { - "const": "declared_earlier_version_order" + "target_dataset_version": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] }, - "version_order": { + "target_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "loaded_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "context_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "loaded_dataset_versions": { "type": "array", "items": { "type": "string", @@ -7322,15 +7374,14 @@ }, "minItems": 0, "uniqueItems": true - }, - "all_resolved_edges_follow_order": { - "type": "boolean" } }, "required": [ - "method", - "version_order", - "all_resolved_edges_follow_order" + "target_dataset_version", + "target_record_count", + "loaded_record_count", + "context_record_count", + "loaded_dataset_versions" ], "additionalProperties": false } @@ -7338,71 +7389,3357 @@ } } ] - } - }, - "required": [], - "additionalProperties": false - }, - "state_counts": { - "type": "object", - "properties": { - "by_version": { + }, + "cycle_analysis": { "type": "object", - "propertyNames": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "additionalProperties": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "array", - "items": { + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "detected": { + "type": "boolean" + }, + "cycle_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "counting_method": { + "const": "cyclic_strongly_connected_components" + }, + "cycle_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "target_cycle_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "affected_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "target_affected_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "components": { "type": "object", "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_index": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "member_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "target_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "witness_record_keys": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 2, + "maxItems": 65 + }, + { + "type": "null" + } + ] + }, + "witness_edge_count": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 64 + }, + { + "type": "null" + } + ] + }, + "witness_reason": { + "anyOf": [ + { + "const": "diagnostic_limit" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "component_index", + "member_count", + "target_member_count", + "witness_record_keys", + "witness_edge_count", + "witness_reason" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] }, - "state_count": { + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { "type": "integer", "minimum": 0, "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true } }, "required": [ - "state_id", - "state_count" + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" ], - "additionalProperties": false + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_index": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "member_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "target_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "witness_record_keys": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 2, + "maxItems": 65 + }, + { + "type": "null" + } + ] + }, + "witness_edge_count": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 64 + }, + { + "type": "null" + } + ] + }, + "witness_reason": { + "anyOf": [ + { + "const": "diagnostic_limit" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "component_index", + "member_count", + "target_member_count", + "witness_record_keys", + "witness_edge_count", + "witness_reason" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] }, - "minItems": 0 - }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "records" - }, - "evidence_class": { - "const": "observed_fact" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "T1.state_counts" + "cycle_member_record_keys": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + }, + "affected_record_keys": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + } + }, + "required": [ + "detected", + "cycle_count", + "counting_method", + "cycle_member_count", + "target_cycle_member_count", + "affected_record_count", + "target_affected_record_count", + "components", + "cycle_member_record_keys", + "affected_record_keys" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "cycles" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T6.cyclic_strongly_connected_components" + }, + "owner_ids": { + "const": [ + "T6" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "detected": { + "type": "boolean" + }, + "cycle_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "counting_method": { + "const": "cyclic_strongly_connected_components" + }, + "cycle_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "target_cycle_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "affected_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "target_affected_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "components": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_index": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "member_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "target_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "witness_record_keys": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 2, + "maxItems": 65 + }, + { + "type": "null" + } + ] + }, + "witness_edge_count": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 64 + }, + { + "type": "null" + } + ] + }, + "witness_reason": { + "anyOf": [ + { + "const": "diagnostic_limit" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "component_index", + "member_count", + "target_member_count", + "witness_record_keys", + "witness_edge_count", + "witness_reason" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_index": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "member_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "target_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "witness_record_keys": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 2, + "maxItems": 65 + }, + { + "type": "null" + } + ] + }, + "witness_edge_count": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 64 + }, + { + "type": "null" + } + ] + }, + "witness_reason": { + "anyOf": [ + { + "const": "diagnostic_limit" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "component_index", + "member_count", + "target_member_count", + "witness_record_keys", + "witness_edge_count", + "witness_reason" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + }, + "cycle_member_record_keys": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + }, + "affected_record_keys": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + } + }, + "required": [ + "detected", + "cycle_count", + "counting_method", + "cycle_member_count", + "target_cycle_member_count", + "affected_record_count", + "target_affected_record_count", + "components", + "cycle_member_record_keys", + "affected_record_keys" + ], + "additionalProperties": false + } + } + } + } + ] + }, + "depth_summary": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "depth_resolved_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "maximum_resolved_target_depth": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "target_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "depth_resolved_record_count", + "maximum_resolved_target_depth", + "target_record_count" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "edges" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "PR-009.resolved_depth_summary" + }, + "owner_ids": { + "const": [ + "PR-009" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "depth_resolved_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "maximum_resolved_target_depth": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "target_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "depth_resolved_record_count", + "maximum_resolved_target_depth", + "target_record_count" + ], + "additionalProperties": false + } + } + } + } + ] + }, + "unresolved_record_details": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "record_key", + "reason_codes" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "record_key", + "reason_codes" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "records" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T4.unresolved_record_details" + }, + "owner_ids": { + "const": [ + "T4" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "record_key", + "reason_codes" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "record_key", + "reason_codes" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + } + } + } + } + ] + }, + "resource_usage": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "admitted_node_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "admitted_edge_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "stored_root_membership_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "root_union_visit_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limits": { + "type": "object", + "properties": { + "max_nodes": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_edges": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_root_memberships": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_root_union_visits": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "max_nodes", + "max_edges", + "max_root_memberships", + "max_root_union_visits" + ], + "additionalProperties": false + }, + "exhausted_limit": { + "anyOf": [ + { + "type": "string", + "enum": [ + "max_nodes", + "max_edges", + "max_root_memberships", + "max_root_union_visits" + ] + }, + { + "type": "null" + } + ] + }, + "attempted_value": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "admitted_node_count", + "admitted_edge_count", + "stored_root_membership_count", + "root_union_visit_count", + "limits", + "exhausted_limit", + "attempted_value" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "work_units" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "PR-015.lineage_resource_usage" + }, + "owner_ids": { + "const": [ + "PR-015" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "admitted_node_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "admitted_edge_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "stored_root_membership_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "root_union_visit_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limits": { + "type": "object", + "properties": { + "max_nodes": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_edges": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_root_memberships": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_root_union_visits": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "max_nodes", + "max_edges", + "max_root_memberships", + "max_root_union_visits" + ], + "additionalProperties": false + }, + "exhausted_limit": { + "anyOf": [ + { + "type": "string", + "enum": [ + "max_nodes", + "max_edges", + "max_root_memberships", + "max_root_union_visits" + ] + }, + { + "type": "null" + } + ] + }, + "attempted_value": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "admitted_node_count", + "admitted_edge_count", + "stored_root_membership_count", + "root_union_visit_count", + "limits", + "exhausted_limit", + "attempted_value" + ], + "additionalProperties": false + } + } + } + } + ] + }, + "ordering_certificate": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "method": { + "const": "declared_earlier_version_order" + }, + "version_order": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "all_resolved_edges_follow_order": { + "type": "boolean" + } + }, + "required": [ + "method", + "version_order", + "all_resolved_edges_follow_order" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "certificate" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "PR-008.earlier_version_certificate" + }, + "owner_ids": { + "const": [ + "PR-008" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "method": { + "const": "declared_earlier_version_order" + }, + "version_order": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "all_resolved_edges_follow_order": { + "type": "boolean" + } + }, + "required": [ + "method", + "version_order", + "all_resolved_edges_follow_order" + ], + "additionalProperties": false + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + }, + "state_counts": { + "type": "object", + "properties": { + "by_version": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "additionalProperties": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "state_id", + "state_count" + ], + "additionalProperties": false + }, + "minItems": 0 + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "records" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T1.state_counts" }, "owner_ids": { "const": [ @@ -8533,134 +11870,445 @@ "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + } + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + } + }, + "support_delta": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "integer", + "minimum": -1.7976931348623157e+308, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "states" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "F-005" + }, + "owner_ids": { + "const": [ + "T1" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true + "minItems": 1 }, "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minItems": 1 } - }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } - }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } + } + }, + "else": { + "properties": { + "value": { + "type": "integer", + "minimum": -1.7976931348623157e+308, + "maximum": 1.7976931348623157e+308 } - ] + } } }, - "required": [], - "additionalProperties": false - } + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] }, - "support_delta": { + "support_retention_ratio": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "integer", - "minimum": -1.7976931348623157e+308, - "maximum": 1.7976931348623157e+308 + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, { "type": "null" @@ -8668,7 +12316,7 @@ ] }, "unit": { - "const": "states" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -8682,7 +12330,7 @@ ] }, "method_id": { - "const": "F-005" + "const": "F-006" }, "owner_ids": { "const": [ @@ -8927,9 +12575,17 @@ "else": { "properties": { "value": { - "type": "integer", - "minimum": -1.7976931348623157e+308, - "maximum": 1.7976931348623157e+308 + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] } } } @@ -8955,23 +12611,15 @@ } ] }, - "support_retention_ratio": { + "support_loss_count": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 }, { "type": "null" @@ -8979,7 +12627,7 @@ ] }, "unit": { - "const": "ratio" + "const": "states" }, "evidence_class": { "const": "derived_metric" @@ -8993,7 +12641,7 @@ ] }, "method_id": { - "const": "F-006" + "const": "T1.support_set_difference" }, "owner_ids": { "const": [ @@ -9238,17 +12886,9 @@ "else": { "properties": { "value": { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 } } } @@ -9274,7 +12914,7 @@ } ] }, - "support_loss_count": { + "support_added_count": { "type": "object", "properties": { "value": { @@ -9577,15 +13217,20 @@ } ] }, - "support_added_count": { + "extinct_states": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true }, { "type": "null" @@ -9593,7 +13238,7 @@ ] }, "unit": { - "const": "states" + "const": "set_of_states" }, "evidence_class": { "const": "derived_metric" @@ -9607,7 +13252,7 @@ ] }, "method_id": { - "const": "T1.support_set_difference" + "const": "T1.earlier_minus_later" }, "owner_ids": { "const": [ @@ -9852,9 +13497,14 @@ "else": { "properties": { "value": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true } } } @@ -9880,7 +13530,7 @@ } ] }, - "extinct_states": { + "added_states": { "type": "object", "properties": { "value": { @@ -9915,7 +13565,7 @@ ] }, "method_id": { - "const": "T1.earlier_minus_later" + "const": "T1.later_minus_earlier" }, "owner_ids": { "const": [ @@ -10193,7 +13843,7 @@ } ] }, - "added_states": { + "retained_states": { "type": "object", "properties": { "value": { @@ -10228,7 +13878,7 @@ ] }, "method_id": { - "const": "T1.later_minus_earlier" + "const": "T1.support_intersection" }, "owner_ids": { "const": [ @@ -10476,50 +14126,276 @@ "type": "array", "items": { "type": "string", - "minLength": 0, + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + } + } + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] + }, + "comparison_details": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "earlier_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "later_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "version_order": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 2, + "uniqueItems": true + }, + "version_order_source": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "earlier_state_semantics": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "later_state_semantics": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "harmonized_state_semantics": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "compatibility_method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "earlier_representation": { + "$ref": "#/$defs/representation" + }, + "later_representation": { + "$ref": "#/$defs/representation" + }, + "harmonized_representation": { + "$ref": "#/$defs/representation" + }, + "original_earlier_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "original_later_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "harmonized_earlier_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "harmonized_later_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "retention_denominator": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "retention_denominator_basis": { + "type": "string", + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } - } - ] - }, - "retained_states": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "state_mapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "target_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "source_state", + "target_state" + ], + "additionalProperties": false + }, + "minItems": 0 + }, + "mapping_effect": { + "type": "array", + "items": { + "type": "object", + "properties": { + "dataset_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "original_support_size": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "harmonized_support_size": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "dataset_version", + "original_support_size", + "harmonized_support_size" + ], + "additionalProperties": false + }, + "minItems": 0 + }, + "collision_groups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "target_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "source_states": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 2, + "uniqueItems": true + } + }, + "required": [ + "target_state", + "source_states" + ], + "additionalProperties": false + }, + "minItems": 0 + } }, - "minItems": 0, - "uniqueItems": true + "required": [ + "earlier_version", + "later_version", + "version_order", + "version_order_source", + "earlier_state_semantics", + "later_state_semantics", + "harmonized_state_semantics", + "compatibility_method", + "earlier_representation", + "later_representation", + "harmonized_representation", + "original_earlier_support", + "original_later_support", + "harmonized_earlier_support", + "harmonized_later_support", + "retention_denominator", + "retention_denominator_basis", + "state_mapping", + "mapping_effect", + "collision_groups" + ], + "additionalProperties": false }, { "type": "null" @@ -10527,7 +14403,7 @@ ] }, "unit": { - "const": "set_of_states" + "const": "comparison" }, "evidence_class": { "const": "derived_metric" @@ -10541,7 +14417,7 @@ ] }, "method_id": { - "const": "T1.support_intersection" + "const": "T1.explicit_pair_basis" }, "owner_ids": { "const": [ @@ -10786,822 +14662,602 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } - } - ] - }, - "comparison_details": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "object", - "properties": { - "earlier_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "later_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "version_order": { - "type": "array", - "items": { + "type": "object", + "properties": { + "earlier_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "later_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "version_order": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 2, + "uniqueItems": true + }, + "version_order_source": { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 2, - "uniqueItems": true - }, - "version_order_source": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "earlier_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "later_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "harmonized_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "compatibility_method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "earlier_representation": { - "$ref": "#/$defs/representation" - }, - "later_representation": { - "$ref": "#/$defs/representation" - }, - "harmonized_representation": { - "$ref": "#/$defs/representation" - }, - "original_earlier_support": { - "type": "array", - "items": { + "earlier_state_semantics": { "type": "string", - "minLength": 0, + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - }, - "original_later_support": { - "type": "array", - "items": { + "later_state_semantics": { "type": "string", - "minLength": 0, + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - }, - "harmonized_earlier_support": { - "type": "array", - "items": { + "harmonized_state_semantics": { "type": "string", - "minLength": 0, + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - }, - "harmonized_later_support": { - "type": "array", - "items": { + "compatibility_method": { "type": "string", - "minLength": 0, + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - }, - "retention_denominator": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "earlier_representation": { + "$ref": "#/$defs/representation" + }, + "later_representation": { + "$ref": "#/$defs/representation" + }, + "harmonized_representation": { + "$ref": "#/$defs/representation" + }, + "original_earlier_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" }, - { - "type": "null" - } - ] - }, - "retention_denominator_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "state_mapping": { - "type": "array", - "items": { - "type": "object", - "properties": { - "source_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "target_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - } + "minItems": 0, + "uniqueItems": true + }, + "original_later_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" }, - "required": [ - "source_state", - "target_state" - ], - "additionalProperties": false + "minItems": 0, + "uniqueItems": true }, - "minItems": 0 - }, - "mapping_effect": { - "type": "array", - "items": { - "type": "object", - "properties": { - "dataset_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "original_support_size": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - { - "type": "null" - } - ] + "harmonized_earlier_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "harmonized_later_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "retention_denominator": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 }, - "harmonized_support_size": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - { - "type": "null" - } - ] + { + "type": "null" } + ] + }, + "retention_denominator_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "state_mapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "target_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "source_state", + "target_state" + ], + "additionalProperties": false + }, + "minItems": 0 + }, + "mapping_effect": { + "type": "array", + "items": { + "type": "object", + "properties": { + "dataset_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "original_support_size": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "harmonized_support_size": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "dataset_version", + "original_support_size", + "harmonized_support_size" + ], + "additionalProperties": false }, - "required": [ - "dataset_version", - "original_support_size", - "harmonized_support_size" - ], - "additionalProperties": false + "minItems": 0 }, - "minItems": 0 - }, - "collision_groups": { - "type": "array", - "items": { - "type": "object", - "properties": { - "target_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "source_states": { - "type": "array", - "items": { + "collision_groups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "target_state": { "type": "string", "minLength": 0, "pattern": "^[^\\u0000]*$" }, - "minItems": 2, - "uniqueItems": true - } - }, - "required": [ - "target_state", - "source_states" - ], - "additionalProperties": false - }, - "minItems": 0 - } - }, - "required": [ - "earlier_version", - "later_version", - "version_order", - "version_order_source", - "earlier_state_semantics", - "later_state_semantics", - "harmonized_state_semantics", - "compatibility_method", - "earlier_representation", - "later_representation", - "harmonized_representation", - "original_earlier_support", - "original_later_support", - "harmonized_earlier_support", - "harmonized_later_support", - "retention_denominator", - "retention_denominator_basis", - "state_mapping", - "mapping_effect", - "collision_groups" - ], - "additionalProperties": false - }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "comparison" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "T1.explicit_pair_basis" - }, - "owner_ids": { - "const": [ - "T1" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" - } - ] - }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "denominator_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" + "source_states": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 2, + "uniqueItems": true + } + }, + "required": [ + "target_state", + "source_states" + ], + "additionalProperties": false + }, + "minItems": 0 + } + }, + "required": [ + "earlier_version", + "later_version", + "version_order", + "version_order_source", + "earlier_state_semantics", + "later_state_semantics", + "harmonized_state_semantics", + "compatibility_method", + "earlier_representation", + "later_representation", + "harmonized_representation", + "original_earlier_support", + "original_later_support", + "harmonized_earlier_support", + "harmonized_later_support", + "retention_denominator", + "retention_denominator_basis", + "state_mapping", + "mapping_effect", + "collision_groups" + ], + "additionalProperties": false + } } - ] - }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" + } }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - } - }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ { "if": { "properties": { "status": { - "const": "unavailable" + "enum": [ + "available", + "partial" + ] } } }, "then": { "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 + "representation": { + "$ref": "#/$defs/representation" } } - }, - "else": { + } + } + ] + } + }, + "required": [], + "additionalProperties": false + }, + "diversity": { + "type": "object", + "properties": { + "by_version": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "additionalProperties": { + "type": "object", + "properties": { + "gini_simpson_diversity": { + "type": "object", "properties": { "value": { - "type": "object", - "properties": { - "earlier_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "later_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "version_order": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 2, - "uniqueItems": true - }, - "version_order_source": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "earlier_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "later_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + { + "type": "null" + } + ] + }, + "unit": { + "const": "dimensionless" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "F-003" + }, + "owner_ids": { + "const": [ + "T1" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" }, - "harmonized_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "compatibility_method": { + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "earlier_representation": { - "$ref": "#/$defs/representation" - }, - "later_representation": { - "$ref": "#/$defs/representation" - }, - "harmonized_representation": { - "$ref": "#/$defs/representation" - }, - "original_earlier_support": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "original_later_support": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "harmonized_earlier_support": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "harmonized_later_support": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "retention_denominator": { - "anyOf": [ + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "retention_denominator_basis": { + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "state_mapping": { - "type": "array", - "items": { - "type": "object", - "properties": { - "source_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "target_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - } - }, - "required": [ - "source_state", - "target_state" - ], - "additionalProperties": false + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, - "minItems": 0 - }, - "mapping_effect": { - "type": "array", - "items": { - "type": "object", - "properties": { - "dataset_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "original_support_size": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - { - "type": "null" - } - ] - }, - "harmonized_support_size": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - { - "type": "null" - } - ] - } - }, - "required": [ - "dataset_version", - "original_support_size", - "harmonized_support_size" - ], - "additionalProperties": false + "reason_codes": { + "minItems": 1 }, - "minItems": 0 - }, - "collision_groups": { - "type": "array", - "items": { - "type": "object", - "properties": { - "target_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 }, - "source_states": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 2, - "uniqueItems": true + { + "exclusiveMaximum": 0 } - }, - "required": [ - "target_state", - "source_states" - ], - "additionalProperties": false - }, - "minItems": 0 + ] + } + } + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } } }, - "required": [ - "earlier_version", - "later_version", - "version_order", - "version_order_source", - "earlier_state_semantics", - "later_state_semantics", - "harmonized_state_semantics", - "compatibility_method", - "earlier_representation", - "later_representation", - "harmonized_representation", - "original_earlier_support", - "original_later_support", - "harmonized_earlier_support", - "harmonized_later_support", - "retention_denominator", - "retention_denominator_basis", - "state_mapping", - "mapping_effect", - "collision_groups" - ], - "additionalProperties": false - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } } - } + ] }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } - } - ] - } - }, - "required": [], - "additionalProperties": false - }, - "diversity": { - "type": "object", - "properties": { - "by_version": { - "type": "object", - "propertyNames": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "additionalProperties": { - "type": "object", - "properties": { - "gini_simpson_diversity": { + "simpson_concentration": { "type": "object", "properties": { "value": { @@ -11639,7 +15295,7 @@ ] }, "method_id": { - "const": "F-003" + "const": "F-004" }, "owner_ids": { "const": [ @@ -11920,7 +15576,7 @@ } ] }, - "simpson_concentration": { + "weighted_gini_simpson_diversity": { "type": "object", "properties": { "value": { @@ -11958,7 +15614,7 @@ ] }, "method_id": { - "const": "F-004" + "const": "F-003" }, "owner_ids": { "const": [ @@ -12239,7 +15895,7 @@ } ] }, - "weighted_gini_simpson_diversity": { + "weighted_simpson_concentration": { "type": "object", "properties": { "value": { @@ -12277,7 +15933,7 @@ ] }, "method_id": { - "const": "F-003" + "const": "F-004" }, "owner_ids": { "const": [ @@ -12558,23 +16214,42 @@ } ] }, - "weighted_simpson_concentration": { + "state_frequencies": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_frequency": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - { - "exclusiveMaximum": 0 - } - ] + "required": [ + "state_id", + "state_frequency" + ], + "additionalProperties": false + }, + "minItems": 0 }, { "type": "null" @@ -12582,7 +16257,7 @@ ] }, "unit": { - "const": "dimensionless" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -12596,7 +16271,7 @@ ] }, "method_id": { - "const": "F-004" + "const": "F-001" }, "owner_ids": { "const": [ @@ -12841,17 +16516,36 @@ "else": { "properties": { "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_frequency": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - { - "exclusiveMaximum": 0 - } - ] + "required": [ + "state_id", + "state_frequency" + ], + "additionalProperties": false + }, + "minItems": 0 } } } @@ -12877,7 +16571,7 @@ } ] }, - "state_frequencies": { + "weighted_state_frequencies": { "type": "object", "properties": { "value": { @@ -13234,7 +16928,7 @@ } ] }, - "weighted_state_frequencies": { + "weighted_state_masses": { "type": "object", "properties": { "value": { @@ -13249,10 +16943,10 @@ "minLength": 0, "pattern": "^[^\\u0000]*$" }, - "state_frequency": { + "state_mass": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -13265,7 +16959,7 @@ }, "required": [ "state_id", - "state_frequency" + "state_mass" ], "additionalProperties": false }, @@ -13277,7 +16971,7 @@ ] }, "unit": { - "const": "ratio" + "const": "user_declared_weight_mass" }, "evidence_class": { "const": "derived_metric" @@ -13291,7 +16985,7 @@ ] }, "method_id": { - "const": "F-001" + "const": "T1.state_weight_mass" }, "owner_ids": { "const": [ @@ -13545,10 +17239,10 @@ "minLength": 0, "pattern": "^[^\\u0000]*$" }, - "state_frequency": { + "state_mass": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -13561,7 +17255,7 @@ }, "required": [ "state_id", - "state_frequency" + "state_mass" ], "additionalProperties": false }, @@ -13591,42 +17285,107 @@ } ] }, - "weighted_state_masses": { + "distribution_basis": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "array", - "items": { - "type": "object", - "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "state_mass": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - } + "type": "object", + "properties": { + "input_basis": { + "type": "string", + "enum": [ + "empirical_assignments", + "explicit_counts_divided_by_included_records", + "weighted_record_mass", + "explicit_probability_vector" + ] }, - "required": [ - "state_id", - "state_mass" - ], - "additionalProperties": false + "analyzed_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "frequency_denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "supplied_probability_total": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "probability_residual": { + "anyOf": [ + { + "type": "number", + "minimum": -1.7976931348623157e+308, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "numerical_policy": { + "$ref": "#/$defs/numerical_policy" + } }, - "minItems": 0 + "required": [ + "input_basis", + "analyzed_record_count", + "frequency_denominator", + "denominator_basis", + "supplied_probability_total", + "probability_residual", + "numerical_policy" + ], + "additionalProperties": false }, { "type": "null" @@ -13634,7 +17393,7 @@ ] }, "unit": { - "const": "user_declared_weight_mass" + "const": "basis" }, "evidence_class": { "const": "derived_metric" @@ -13648,7 +17407,7 @@ ] }, "method_id": { - "const": "T1.state_weight_mass" + "const": "T1.validated_distribution_basis" }, "owner_ids": { "const": [ @@ -13893,36 +17652,101 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "object", - "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "state_mass": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - } + "type": "object", + "properties": { + "input_basis": { + "type": "string", + "enum": [ + "empirical_assignments", + "explicit_counts_divided_by_included_records", + "weighted_record_mass", + "explicit_probability_vector" + ] + }, + "analyzed_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "frequency_denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "supplied_probability_total": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "probability_residual": { + "anyOf": [ + { + "type": "number", + "minimum": -1.7976931348623157e+308, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] }, - "required": [ - "state_id", - "state_mass" - ], - "additionalProperties": false + "numerical_policy": { + "$ref": "#/$defs/numerical_policy" + } }, - "minItems": 0 + "required": [ + "input_basis", + "analyzed_record_count", + "frequency_denominator", + "denominator_basis", + "supplied_probability_total", + "probability_residual", + "numerical_policy" + ], + "additionalProperties": false } } } @@ -13947,507 +17771,649 @@ } } ] - }, - "distribution_basis": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "object", - "properties": { - "input_basis": { - "type": "string", - "enum": [ - "empirical_assignments", - "explicit_counts_divided_by_included_records", - "weighted_record_mass", - "explicit_probability_vector" - ] - }, - "analyzed_record_count": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - "frequency_denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "denominator_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "supplied_probability_total": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "probability_residual": { - "anyOf": [ - { - "type": "number", - "minimum": -1.7976931348623157e+308, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "numerical_policy": { - "$ref": "#/$defs/numerical_policy" - } - }, - "required": [ - "input_basis", - "analyzed_record_count", - "frequency_denominator", - "denominator_basis", - "supplied_probability_total", - "probability_residual", - "numerical_policy" - ], - "additionalProperties": false - }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "basis" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "T1.validated_distribution_basis" - }, - "owner_ids": { - "const": [ - "T1" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] + } + }, + "required": [], + "additionalProperties": false + } + }, + "gini_simpson_diversity_delta": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": -1, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "dimensionless" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "F-018" + }, + "owner_ids": { + "const": [ + "T1" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, - "coverage_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" - } - ] + "reason_codes": { + "minItems": 1 }, - "denominator": { - "anyOf": [ + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": -1, + "maximum": 1, + "oneOf": [ { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] - }, - "denominator_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" - } + } + } + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" ] - }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + }, + "tail": { + "type": "object", + "properties": { + "tail_support_size": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } + { + "type": "null" + } + ] + }, + "unit": { + "const": "states" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T2.declared_tail_rule" + }, + "owner_ids": { + "const": [ + "T2" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 - } - } + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "else": { - "properties": { - "value": { - "type": "object", - "properties": { - "input_basis": { - "type": "string", - "enum": [ - "empirical_assignments", - "explicit_counts_divided_by_included_records", - "weighted_record_mass", - "explicit_probability_vector" - ] - }, - "analyzed_record_count": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - "frequency_denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "denominator_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "supplied_probability_total": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "probability_residual": { - "anyOf": [ - { - "type": "number", - "minimum": -1.7976931348623157e+308, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "numerical_policy": { - "$ref": "#/$defs/numerical_policy" - } - }, - "required": [ - "input_basis", - "analyzed_record_count", - "frequency_denominator", - "denominator_basis", - "supplied_probability_total", - "probability_residual", - "numerical_policy" - ], - "additionalProperties": false - } - } + { + "exclusiveMaximum": 0 } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 } - ] + } + }, + "else": { + "properties": { + "value": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + } } }, - "required": [], - "additionalProperties": false - } + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] }, - "gini_simpson_diversity_delta": { + "tail_record_share": { "type": "object", "properties": { "value": { "anyOf": [ { "type": "number", - "minimum": -1, + "minimum": 0, "maximum": 1, "oneOf": [ { @@ -14464,7 +18430,7 @@ ] }, "unit": { - "const": "dimensionless" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -14478,11 +18444,11 @@ ] }, "method_id": { - "const": "F-018" + "const": "T2.tail_record_share" }, "owner_ids": { "const": [ - "T1" + "T2" ] }, "theory_map_ids": { @@ -14724,7 +18690,7 @@ "properties": { "value": { "type": "number", - "minimum": -1, + "minimum": 0, "maximum": 1, "oneOf": [ { @@ -14758,23 +18724,59 @@ } } ] - } - }, - "required": [], - "additionalProperties": false - }, - "tail": { - "type": "object", - "properties": { - "tail_support_size": { + }, + "rarity_ranking": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "state_frequency": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "rarity_rank": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "in_tail": { + "type": "boolean" + } + }, + "required": [ + "state_id", + "state_count", + "state_frequency", + "rarity_rank", + "in_tail" + ], + "additionalProperties": false + }, + "minItems": 0 }, { "type": "null" @@ -14782,7 +18784,7 @@ ] }, "unit": { - "const": "states" + "const": "ordinal_rank" }, "evidence_class": { "const": "derived_metric" @@ -14796,7 +18798,7 @@ ] }, "method_id": { - "const": "T2.declared_tail_rule" + "const": "T2.frequency_count_unicode_order" }, "owner_ids": { "const": [ @@ -15041,9 +19043,52 @@ "else": { "properties": { "value": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "state_frequency": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "rarity_rank": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "in_tail": { + "type": "boolean" + } + }, + "required": [ + "state_id", + "state_count", + "state_frequency", + "rarity_rank", + "in_tail" + ], + "additionalProperties": false + }, + "minItems": 0 } } } @@ -15069,23 +19114,20 @@ } ] }, - "tail_record_share": { + "tail_states": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true }, { "type": "null" @@ -15093,7 +19135,7 @@ ] }, "unit": { - "const": "ratio" + "const": "set_of_states" }, "evidence_class": { "const": "derived_metric" @@ -15107,7 +19149,7 @@ ] }, "method_id": { - "const": "T2.tail_record_share" + "const": "T2.declared_tail_rule" }, "owner_ids": { "const": [ @@ -15351,18 +19393,15 @@ }, "else": { "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "value": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true } } } @@ -15388,58 +19427,107 @@ } ] }, - "rarity_ranking": { + "tail_rule": { + "type": "string", + "enum": [ + "singleton_count", + "count_at_or_below", + "frequency_at_or_below", + "state_list" + ] + }, + "selection": { + "$ref": "#/$defs/tail_selection" + } + }, + "required": [], + "additionalProperties": false + }, + "provenance": { + "type": "object", + "properties": { + "source_type_shares": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "array", - "items": { - "type": "object", - "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "state_count": { - "type": "integer", - "minimum": 1, - "maximum": 1.7976931348623157e+308 - }, - "state_frequency": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "rarity_rank": { - "type": "integer", - "minimum": 1, - "maximum": 1.7976931348623157e+308 - }, - "in_tail": { - "type": "boolean" - } + "type": "object", + "properties": { + "human": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "required": [ - "state_id", - "state_count", - "state_frequency", - "rarity_rank", - "in_tail" - ], - "additionalProperties": false + "synthetic": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "mixed": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "sensor": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "unknown": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - "minItems": 0 + "required": [ + "human", + "synthetic", + "mixed", + "sensor", + "unknown" + ], + "additionalProperties": false }, { "type": "null" @@ -15447,7 +19535,7 @@ ] }, "unit": { - "const": "ordinal_rank" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -15461,11 +19549,11 @@ ] }, "method_id": { - "const": "T2.frequency_count_unicode_order" + "const": "F-007" }, "owner_ids": { "const": [ - "T2" + "PR-005" ] }, "theory_map_ids": { @@ -15706,91 +19794,170 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "object", - "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "state_count": { - "type": "integer", - "minimum": 1, - "maximum": 1.7976931348623157e+308 - }, - "state_frequency": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "rarity_rank": { - "type": "integer", - "minimum": 1, - "maximum": 1.7976931348623157e+308 - }, - "in_tail": { - "type": "boolean" - } + "type": "object", + "properties": { + "human": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "required": [ - "state_id", - "state_count", - "state_frequency", - "rarity_rank", - "in_tail" - ], - "additionalProperties": false + "synthetic": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "mixed": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "sensor": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "unknown": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - "minItems": 0 - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" + "required": [ + "human", + "synthetic", + "mixed", + "sensor", + "unknown" + ], + "additionalProperties": false } } } } ] }, - "tail_states": { + "weighted_source_type_shares": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "type": "object", + "properties": { + "human": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "synthetic": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "mixed": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "sensor": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "unknown": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - "minItems": 0, - "uniqueItems": true + "required": [ + "human", + "synthetic", + "mixed", + "sensor", + "unknown" + ], + "additionalProperties": false }, { "type": "null" @@ -15798,7 +19965,7 @@ ] }, "unit": { - "const": "set_of_states" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -15812,11 +19979,11 @@ ] }, "method_id": { - "const": "T2.declared_tail_rule" + "const": "F-007" }, "owner_ids": { "const": [ - "T2" + "PR-005" ] }, "theory_map_ids": { @@ -16057,59 +20224,89 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "type": "object", + "properties": { + "human": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "synthetic": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "mixed": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "sensor": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "unknown": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - "minItems": 0, - "uniqueItems": true - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" + "required": [ + "human", + "synthetic", + "mixed", + "sensor", + "unknown" + ], + "additionalProperties": false } } } } ] }, - "tail_rule": { - "type": "string", - "enum": [ - "singleton_count", - "count_at_or_below", - "frequency_at_or_below", - "state_list" - ] - }, - "selection": { - "$ref": "#/$defs/tail_selection" - } - }, - "required": [], - "additionalProperties": false - }, - "provenance": { - "type": "object", - "properties": { - "source_type_shares": { + "weighted_source_type_masses": { "type": "object", "properties": { "value": { @@ -16120,7 +20317,7 @@ "human": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16133,7 +20330,7 @@ "synthetic": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16146,7 +20343,7 @@ "mixed": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16159,7 +20356,7 @@ "sensor": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16172,7 +20369,7 @@ "unknown": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16198,7 +20395,7 @@ ] }, "unit": { - "const": "ratio" + "const": "user_declared_weight_mass" }, "evidence_class": { "const": "derived_metric" @@ -16212,7 +20409,7 @@ ] }, "method_id": { - "const": "F-007" + "const": "PR-005.source_weight_mass" }, "owner_ids": { "const": [ @@ -16462,7 +20659,7 @@ "human": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16475,7 +20672,7 @@ "synthetic": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16488,7 +20685,7 @@ "mixed": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16501,7 +20698,7 @@ "sensor": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16514,7 +20711,7 @@ "unknown": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16539,88 +20736,23 @@ } ] }, - "weighted_source_type_shares": { + "total_weight": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "object", - "properties": { - "human": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "synthetic": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "mixed": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "sensor": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "unknown": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + { + "exclusiveMaximum": 0 } - }, - "required": [ - "human", - "synthetic", - "mixed", - "sensor", - "unknown" - ], - "additionalProperties": false + ] }, { "type": "null" @@ -16628,7 +20760,7 @@ ] }, "unit": { - "const": "ratio" + "const": "user_declared_weight_mass" }, "evidence_class": { "const": "derived_metric" @@ -16642,7 +20774,7 @@ ] }, "method_id": { - "const": "F-007" + "const": "PR-005.total_weight" }, "owner_ids": { "const": [ @@ -16876,181 +21008,51 @@ "value": { "type": "null" }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "object", - "properties": { - "human": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "synthetic": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "mixed": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "sensor": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "unknown": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - } - }, - "required": [ - "human", - "synthetic", - "mixed", - "sensor", - "unknown" - ], - "additionalProperties": false - } - } - } - } - ] - }, - "weighted_source_type_masses": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "object", - "properties": { - "human": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "synthetic": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "mixed": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "sensor": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } + } + ] + }, + "missing_provenance_weight": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "unknown": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + { + "exclusiveMaximum": 0 } - }, - "required": [ - "human", - "synthetic", - "mixed", - "sensor", - "unknown" - ], - "additionalProperties": false + ] }, { "type": "null" @@ -17072,7 +21074,7 @@ ] }, "method_id": { - "const": "PR-005.source_weight_mass" + "const": "PR-005.missing_provenance_weight" }, "owner_ids": { "const": [ @@ -17317,89 +21319,24 @@ "else": { "properties": { "value": { - "type": "object", - "properties": { - "human": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "synthetic": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "mixed": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "sensor": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "unknown": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + { + "exclusiveMaximum": 0 } - }, - "required": [ - "human", - "synthetic", - "mixed", - "sensor", - "unknown" - ], - "additionalProperties": false + ] } } } } ] }, - "total_weight": { + "missing_provenance_share": { "type": "object", "properties": { "value": { @@ -17407,7 +21344,7 @@ { "type": "number", "minimum": 0, - "maximum": 1.7976931348623157e+308, + "maximum": 1, "oneOf": [ { "minimum": 0 @@ -17423,7 +21360,7 @@ ] }, "unit": { - "const": "user_declared_weight_mass" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -17437,11 +21374,11 @@ ] }, "method_id": { - "const": "PR-005.total_weight" + "const": "PR-004.one_minus_row_coverage" }, "owner_ids": { "const": [ - "PR-005" + "PR-004" ] }, "theory_map_ids": { @@ -17684,7 +21621,7 @@ "value": { "type": "number", "minimum": 0, - "maximum": 1.7976931348623157e+308, + "maximum": 1, "oneOf": [ { "minimum": 0 @@ -17699,7 +21636,7 @@ } ] }, - "missing_provenance_weight": { + "weighted_missing_provenance_share": { "type": "object", "properties": { "value": { @@ -17707,7 +21644,7 @@ { "type": "number", "minimum": 0, - "maximum": 1.7976931348623157e+308, + "maximum": 1, "oneOf": [ { "minimum": 0 @@ -17723,7 +21660,7 @@ ] }, "unit": { - "const": "user_declared_weight_mass" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -17737,7 +21674,7 @@ ] }, "method_id": { - "const": "PR-005.missing_provenance_weight" + "const": "PR-005.missing_provenance_weight_share" }, "owner_ids": { "const": [ @@ -17984,7 +21921,7 @@ "value": { "type": "number", "minimum": 0, - "maximum": 1.7976931348623157e+308, + "maximum": 1, "oneOf": [ { "minimum": 0 @@ -17993,623 +21930,623 @@ "exclusiveMaximum": 0 } ] - } - } - } - } - ] - }, - "missing_provenance_share": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + }, + "closure_exposure": { + "type": "object", + "properties": { + "direct": { + "type": "object", + "properties": { + "lower_bound": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "F-009" + }, + "owner_ids": { + "const": [ + "T3" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ { - "minimum": 0 + "$ref": "#/$defs/representation" }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "ratio" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "PR-004.one_minus_row_coverage" - }, - "owner_ids": { - "const": [ - "PR-004" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + "coverage_reason": { + "anyOf": [ { - "minimum": 0 + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, { - "exclusiveMaximum": 0 + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ - { + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ { - "type": "null" + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } } ] }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ + "upper_bound": { + "type": "object", + "properties": { + "value": { + "anyOf": [ { - "minimum": 0 + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "denominator_reason": { - "anyOf": [ - { + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "enum": [ + "available", + "partial", + "unavailable" + ] }, - { - "type": "null" - } - ] - }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - } - }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } - }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 + "method_id": { + "const": "F-010" + }, + "owner_ids": { + "const": [ + "T3" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" ] - } - } - } - } - ] - }, - "weighted_missing_provenance_share": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ { - "minimum": 0 + "$ref": "#/$defs/representation" }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "ratio" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "PR-005.missing_provenance_weight_share" - }, - "owner_ids": { - "const": [ - "PR-005" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + "coverage": { + "anyOf": [ { - "minimum": 0 + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] }, - { - "type": "null" - } - ] - }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ { - "minimum": 0 + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "denominator_reason": { - "anyOf": [ - { + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - { - "type": "null" - } - ] - }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - } - }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" } }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 + "then": { + "properties": { + "value": { + "type": "null" }, - { - "exclusiveMaximum": 0 + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 } - ] + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } } } - } - } - ] - } - }, - "required": [], - "additionalProperties": false - }, - "closure_exposure": { - "type": "object", - "properties": { - "direct": { - "type": "object", - "properties": { - "lower_bound": { + ] + }, + "interval_width": { "type": "object", "properties": { "value": { @@ -18647,7 +22584,7 @@ ] }, "method_id": { - "const": "F-009" + "const": "T3.upper_minus_lower" }, "owner_ids": { "const": [ @@ -18909,7 +22846,20 @@ } ] }, - "upper_bound": { + "classification_basis": { + "const": "toolkit_operationalization" + }, + "confidence_disclosure": { + "const": "provenance_confidence_is_separate_and_does_not_discount_grounding" + } + }, + "required": [], + "additionalProperties": false + }, + "lineage": { + "type": "object", + "properties": { + "lower_bound": { "type": "object", "properties": { "value": { @@ -18947,7 +22897,7 @@ ] }, "method_id": { - "const": "F-010" + "const": "T3.lineage_closure" }, "owner_ids": { "const": [ @@ -19209,7 +23159,7 @@ } ] }, - "interval_width": { + "upper_bound": { "type": "object", "properties": { "value": { @@ -19247,7 +23197,7 @@ ] }, "method_id": { - "const": "T3.upper_minus_lower" + "const": "T3.lineage_closure" }, "owner_ids": { "const": [ @@ -19509,20 +23459,7 @@ } ] }, - "classification_basis": { - "const": "toolkit_operationalization" - }, - "confidence_disclosure": { - "const": "provenance_confidence_is_separate_and_does_not_discount_grounding" - } - }, - "required": [], - "additionalProperties": false - }, - "lineage": { - "type": "object", - "properties": { - "lower_bound": { + "interval_width": { "type": "object", "properties": { "value": { @@ -19552,7 +23489,12 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "T3.lineage_closure" @@ -19626,813 +23568,1682 @@ "representation": { "anyOf": [ { - "$ref": "#/$defs/representation" + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [], + "additionalProperties": false + }, + "lineage": { + "type": "object", + "properties": { + "resolved_parent_edge_coverage": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] + { + "type": "null" + } + ] + }, + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "PR-008.resolved_edges_over_declared" + }, + "owner_ids": { + "const": [ + "PR-008" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" }, - "coverage_reason": { - "anyOf": [ + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "denominator_reason": { - "anyOf": [ + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "no_declared_parents": { + "type": "boolean" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "reason_codes": { + "minItems": 1 }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "required_evidence": { + "minItems": 1 + } } }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } - }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + { + "exclusiveMaximum": 0 } - } + ] } } + } + } + ] + }, + "resolved_lineage_coverage": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T4.resolved_records_over_scope" + }, + "owner_ids": { + "const": [ + "T4" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } ] }, - "upper_bound": { - "type": "object", - "properties": { - "value": { - "anyOf": [ + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "unit": { - "const": "ratio" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "const": "unavailable" - }, - "method_id": { - "const": "T3.lineage_closure" + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "owner_ids": { - "const": [ - "T3" + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } ] }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" - } - ] - }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "denominator_reason": { - "anyOf": [ + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } + } + ] + }, + "external_ancestry_coverage": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true + { + "type": "null" + } + ] + }, + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T4.external_roots_over_scope" + }, + "owner_ids": { + "const": [ + "T4" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" }, - "weighting": { - "$ref": "#/$defs/weighting" + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "input_basis": { + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "redaction": { - "$ref": "#/$defs/redaction" + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "method": { + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } + } + ] + }, + "distinct_external_root_count": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" } + ] + }, + "unit": { + "const": "roots" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T4.root_set_union" + }, + "owner_ids": { + "const": [ + "T4" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } - }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - } - } - } + "$ref": "#/$defs/representation" + }, + { + "type": "null" } ] }, - "interval_width": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "ratio" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "const": "unavailable" - }, - "method_id": { - "const": "T3.lineage_closure" - }, - "owner_ids": { - "const": [ - "T3" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "denominator_reason": { - "anyOf": [ + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + { + "type": "null" } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + } + } + } + ] + }, + "top_shared_ancestors": { + "type": "object", + "properties": { + "value": { + "anyOf": [ { - "if": { - "properties": { - "status": { - "const": "unavailable" - } + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "incidence_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "incidence_share": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "incidence_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "fractional_mass": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "normalized_weight": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "weight_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "record_key", + "incidence_count", + "incidence_share", + "incidence_denominator", + "fractional_mass", + "normalized_weight", + "weight_denominator" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true } }, - "then": { - "properties": { - "value": { - "type": "null" + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } }, - "reason_codes": { - "minItems": 1 + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } }, - "required_evidence": { - "minItems": 1 + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "incidence_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "incidence_share": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "incidence_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "fractional_mass": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "normalized_weight": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "weight_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "record_key", + "incidence_count", + "incidence_share", + "incidence_denominator", + "fractional_mass", + "normalized_weight", + "weight_denominator" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 }, - { - "exclusiveMaximum": 0 + "omission_reasons": { + "maxItems": 0 } - ] + } } - } - } - } - ] - } - }, - "required": [], - "additionalProperties": false - } - }, - "required": [], - "additionalProperties": false - }, - "lineage": { - "type": "object", - "properties": { - "resolved_parent_edge_coverage": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 }, { - "exclusiveMaximum": 0 + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } } ] }, @@ -20442,7 +25253,7 @@ ] }, "unit": { - "const": "ratio" + "const": "records" }, "evidence_class": { "const": "derived_metric" @@ -20456,11 +25267,11 @@ ] }, "method_id": { - "const": "PR-008.resolved_edges_over_declared" + "const": "T4.incidence_ranking" }, "owner_ids": { "const": [ - "PR-008" + "T4" ] }, "theory_map_ids": { @@ -20701,15 +25512,287 @@ "else": { "properties": { "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "incidence_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "incidence_share": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "incidence_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "fractional_mass": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "normalized_weight": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "weight_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "record_key", + "incidence_count", + "incidence_share", + "incidence_denominator", + "fractional_mass", + "normalized_weight", + "weight_denominator" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ { - "minimum": 0 + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "incidence_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "incidence_share": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "incidence_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "fractional_mass": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "normalized_weight": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "weight_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "record_key", + "incidence_count", + "incidence_share", + "incidence_denominator", + "fractional_mass", + "normalized_weight", + "weight_denominator" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } }, { - "exclusiveMaximum": 0 + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } } ] } @@ -20718,23 +25801,15 @@ } ] }, - "resolved_lineage_coverage": { + "grounded_record_count": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 }, { "type": "null" @@ -20742,16 +25817,21 @@ ] }, "unit": { - "const": "ratio" + "const": "records" }, "evidence_class": { "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { - "const": "T4.resolved_records_over_scope" + "const": "T4.grounded_record_count" }, "owner_ids": { "const": [ @@ -20996,40 +26076,24 @@ "else": { "properties": { "value": { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 } } } } ] }, - "external_ancestry_coverage": { + "closed_record_count": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 }, { "type": "null" @@ -21037,16 +26101,21 @@ ] }, "unit": { - "const": "ratio" + "const": "records" }, "evidence_class": { "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { - "const": "T4.external_roots_over_scope" + "const": "T4.closed_record_count" }, "owner_ids": { "const": [ @@ -21291,24 +26360,16 @@ "else": { "properties": { "value": { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 } } } } ] }, - "distinct_external_root_count": { + "unresolved_record_count": { "type": "object", "properties": { "value": { @@ -21324,16 +26385,21 @@ ] }, "unit": { - "const": "roots" + "const": "records" }, "evidence_class": { "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { - "const": "T4.root_set_union" + "const": "T4.unresolved_record_count" }, "owner_ids": { "const": [ @@ -21587,32 +26653,15 @@ } ] }, - "top_shared_ancestors": { + "records_with_resolved_external_ancestry": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "array", - "items": { - "type": "object", - "properties": { - "record_key": { - "$ref": "#/$defs/record_key" - }, - "incidence_count": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - } - }, - "required": [ - "record_key", - "incidence_count" - ], - "additionalProperties": false - }, - "minItems": 0 + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 }, { "type": "null" @@ -21626,10 +26675,15 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { - "const": "T4.incidence_ranking" + "const": "T4.records_with_resolved_external_ancestry" }, "owner_ids": { "const": [ @@ -21874,26 +26928,9 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "object", - "properties": { - "record_key": { - "$ref": "#/$defs/record_key" - }, - "incidence_count": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - } - }, - "required": [ - "record_key", - "incidence_count" - ], - "additionalProperties": false - }, - "minItems": 0 + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 } } } @@ -21930,7 +26967,12 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "F-012" @@ -22225,7 +27267,12 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "F-013" @@ -22512,7 +27559,12 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "PR-009.maximum_resolved_parent_depth" @@ -23342,7 +28394,12 @@ "const": "proxy_signal" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "T4.shared_ancestry_dependence" @@ -23514,7 +28571,7 @@ "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 1, + "minItems": 0, "uniqueItems": true }, "required_evidence": { @@ -23524,14 +28581,19 @@ "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 1, + "minItems": 0, "uniqueItems": true }, "signal": { "const": "shared_ancestry_dependence" }, "level": { - "const": "indeterminate" + "type": "string", + "enum": [ + "present", + "not_present", + "indeterminate" + ] }, "basis_fields": { "type": "array", diff --git a/src/recursive_integrity_toolkit/lineage/ancestry.py b/src/recursive_integrity_toolkit/lineage/ancestry.py index 9f51e0f..ddd48f1 100644 --- a/src/recursive_integrity_toolkit/lineage/ancestry.py +++ b/src/recursive_integrity_toolkit/lineage/ancestry.py @@ -1,6 +1,6 @@ """Resolve external roots, coverage, concentration and shared-root evidence. -Owner IDs: T4; Phase 5 Step 6. +Owner IDs: T4; Phase 5 Step 7. Validated explicit grounding supplies anchors. Iterative dependency scheduling unions complete root sets; unknown branches never supply exact roots. Logical @@ -8,9 +8,8 @@ The allocation is topological and makes no causal or scientific quality claim. Current phase status: - Phase 5 Step 6 adds an explicitly requested descriptive shared-root proxy. - Reports remain a later step. No file or network I/O, generation calculation - or implicit invocation. + Phase 5 Step 7 binds explicit report handoffs to relevant input evidence. + No file or network I/O, generation calculation or implicit invocation. """ from __future__ import annotations @@ -24,6 +23,7 @@ from ..io.validation import join_provenance from ..models import BundleValidationResult, RecordKey, ValidationMessage, ValidationSeverity from ..result import ExecutionStatus, ReportStatus +from ..utils.hashing import canonical_json_bytes, sha256_canonical from .cycles import CycleAnalysis, DepthAssessment, analyze_cycles, _local_depth_reasons from .graph import ( LineageGraph, LineageLimits, LineageResourceUsage, LineageScope, @@ -64,6 +64,63 @@ def _execution(messages: tuple[ValidationMessage, ...], complete: int, unresolve return ExecutionStatus.COMPLETED, () +def _lineage_input_signature(validation: BundleValidationResult) -> str: + """Bind report handoffs to relevant declarations without retaining raw input. + + This internal digest detects stale same-key evidence, not authenticity. + Content, URI, path, notes and extras are excluded. No graph or metric runs. + """ + if type(validation) is not BundleValidationResult or validation.parent_validation is None: + raise _invalid("lineage input binding requires retained validation evidence") + names = ("dataset_version", "record_id", "parent_ids", "external_grounding", + "source_type", "provenance_confidence", "transformation", "batch_id", "timestamp") + + def identity(key): + return None if key is None else (key.dataset_version, key.record_id) + + def diagnostics(messages): + return sorted([(message.code, message.severity.value, identity(message.record_key), + message.field, None if message.file_role is None else message.file_role.value) + for message in messages], key=canonical_json_bytes) + + def rows(items): + return tuple((identity(row.record_key), + getattr(row, "kind", None), + None if row.location.file_role is None else row.location.file_role.value, + tuple((name, name in row.values, row.values.get(name)) for name in names), + tuple((name, row.field_states.get(name)) for name in names) + if hasattr(row, "field_states") else ()) + for row in sorted(items, key=lambda row: row.record_key)) + + def chronology(order): + return (order.loaded_versions, order.order, order.order_source, dict(order.source_orders), + dict(order.declarations), order.invocation_order, diagnostics(order.messages)) + + batch = validation.parent_validation + parents = [] + for item in batch.assessments: + parent = item.result + refs = None if parent is None else tuple(sorted([( + tuple(sorted(ref.source_references)), ref.canonical_reference, + identity(ref.parent_key), ref.resolution_status.value, ref.temporal_status) + for ref in parent.references], key=canonical_json_bytes)) + parents.append((identity(item.child_key), item.provenance_available, + item.declared_reference_count, item.resolved_reference_count, + item.invalid_self_reference_count, + None if parent is None else parent.declaration_state, refs, + None if parent is None else parent.graph_validation_deferred, + None if parent is None else diagnostics(parent.messages), + diagnostics(item.messages))) + try: + return sha256_canonical(( + rows(validation.records), None if validation.provenance is None else rows(validation.provenance), + chronology(validation.version_order), chronology(batch.version_order), + tuple(parents), batch.promoted_warning_codes, diagnostics(batch.messages), + diagnostics(validation.validation_messages))) + except (ValueError, TypeError): + raise _invalid("lineage input binding requires canonical declarations") from None + + @dataclass(frozen=True, slots=True) class RecordAncestry: """Exact complete roots, including known-empty, or explicit uncertainty.""" @@ -184,8 +241,12 @@ class LineageAnalysisResult: effective_external_root_count: float | None resource_usage: LineageResourceUsage messages: tuple[ValidationMessage, ...] + input_signature: str = field(repr=False) def __post_init__(self) -> None: + if (type(self.input_signature) is not str or len(self.input_signature) != 64 + or any(character not in "0123456789abcdef" for character in self.input_signature)): + raise _invalid("lineage result requires its internal declaration binding") if type(self.scope) is not LineageScope or type(self.cycles) is not CycleAnalysis: raise _invalid("ancestry analysis requires typed scope and cycle observations") scope = replace(self.scope) @@ -538,6 +599,7 @@ def analyze_lineage( graph = build_lineage_graph(validation, target_dataset_version=target_dataset_version, limits=limits) cycles = analyze_cycles(graph) metadata, metadata_messages = _metadata(validation, graph) + input_signature = _lineage_input_signature(validation) messages = _messages(cycles.messages + metadata_messages) target_evidence = tuple(graph.parent_evidence_by_record[key] for key in graph.scope.target_record_keys) count_known = all(item.declared_reference_count is not None for item in target_evidence) @@ -561,7 +623,7 @@ def analyze_lineage( None, None, None, None, None, None, (error.reason_code,), **reference_fields, root_contributions=None, distinct_external_root_count=None, ancestry_concentration_hhi=None, effective_external_root_count=None, - resource_usage=error.resource_usage, messages=messages) + resource_usage=error.resource_usage, messages=messages, input_signature=input_signature) records = tuple(RecordAncestry( key, "unresolved" if roots[key] is None else "grounded" if roots[key] else "closed", roots[key], reasons[key], cycles.depths_by_record[key].lineage_depth, @@ -579,4 +641,4 @@ def analyze_lineage( () if total else ("EMPTY_TARGET_SCOPE",), **reference_fields, root_contributions=contributions, distinct_external_root_count=distinct, ancestry_concentration_hhi=hhi, effective_external_root_count=effective, - resource_usage=budget.usage(), messages=messages) + resource_usage=budget.usage(), messages=messages, input_signature=input_signature) diff --git a/src/recursive_integrity_toolkit/reports/assembly.py b/src/recursive_integrity_toolkit/reports/assembly.py index 6b3621f..ae84ed2 100644 --- a/src/recursive_integrity_toolkit/reports/assembly.py +++ b/src/recursive_integrity_toolkit/reports/assembly.py @@ -2,16 +2,16 @@ Owner IDs: PR-012, PR-014, PR-015, PR-016, PR-018; each public field retains its registered owner. Theory Map IDs: inherited through the frozen public field registry. -Inputs: already validated BundleValidationResult and explicitly supplied Phase 3 results. +Inputs: validated BundleValidationResult and explicitly supplied metric and lineage results. Outputs: immutable CanonicalReport and explicitly selected SafeReportView. Assumptions: supplied typed results are evidence handoffs, not authenticity certificates. Limits: no ingestion, classification, metric execution, graph traversal, simulation, filesystem access, rendering or CLI orchestration. -Current phase status: Phase 4 Step 4 additive privacy views and explicit run metadata. +Current phase status: Phase 5 Step 7 canonical lineage evidence and privacy views. """ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, fields from math import isfinite, isclose from types import MappingProxyType from pathlib import PosixPath, WindowsPath @@ -25,7 +25,10 @@ FileRole, FileFormat, VersionOrderResult, ProvenanceJoinResult, ProvenanceMatch, ProvenanceAssessment, ExplicitPairContext, RowMappingEvidence, RowLocation, GenerationValidationResult, RecordStateAssignment, TailSelectionOptions, ) -from ..metrics.bounds import DirectClosureExposureBounds +from ..metrics.bounds import DirectClosureExposureBounds, LineageClosureExposureBounds +from ..lineage.ancestry import LineageAnalysisResult, SharedAncestryDependence, RootContribution, RecordAncestry, _lineage_input_signature +from ..lineage.cycles import CycleAnalysis +from ..lineage.graph import LineageScope, LineageResourceUsage, LineageLimits from ..metrics.diversity import DistributionMetrics, StateDistributionResult, SupportComparison, StateFrequency from ..metrics.duplicates import ExactDuplicateResult, ExactDuplicateGroup from ..metrics.provenance import ProvenanceCompositionResult, DeclaredComposition, WeightedSourceComposition, DirectGroundingBasis, DirectGroundingAssignment @@ -1066,7 +1069,7 @@ def _inputs(payload, bundle): _diagnostics(payload, bundle.generation.messages) -def _capabilities(payload, bundle, operations, failures): +def _capabilities(payload, bundle, operations, failures, lineage=None): matrix = {} for key in CapabilityKey: original = bundle.observability.capabilities[key] @@ -1081,12 +1084,14 @@ def _capabilities(payload, bundle, operations, failures): reasons.append("R_ANALYSIS_NOT_REQUESTED") if key is CapabilityKey.DATASET_LONGITUDINAL and completed: execution, reasons = "partial", ["R_LONGITUDINAL_FAMILIES_DEFERRED"] - if key in (CapabilityKey.LINEAGE, CapabilityKey.MODEL_LONGITUDINAL): - execution, reasons = "deferred", ["R_LINEAGE_EXECUTION_DEFERRED" if key is CapabilityKey.LINEAGE else "R_MODEL_ANALYSIS_DEFERRED"] + if key is CapabilityKey.MODEL_LONGITUDINAL: + execution, reasons = "deferred", ["R_MODEL_ANALYSIS_DEFERRED"] + if key is CapabilityKey.LINEAGE and lineage is not None: + execution, reasons = lineage.execution_status.value, list(lineage.execution_reason_codes) family_codes = [error["code"] for error in payload["errors"] if key.value in error["effect_on_capabilities"]] - if family_codes and key not in (CapabilityKey.LINEAGE, CapabilityKey.MODEL_LONGITUDINAL): + if family_codes and key is not CapabilityKey.MODEL_LONGITUDINAL and (key is not CapabilityKey.LINEAGE or key.value in failures): execution = "partial" if completed else "failed" - reasons = list(dict.fromkeys(reasons + family_codes)) + reasons = list(dict.fromkeys(([] if key is CapabilityKey.LINEAGE else reasons) + family_codes)) if key is CapabilityKey.INGESTION and any("ingestion" in error["effect_on_capabilities"] for error in payload["errors"]): execution = "partial" if bundle.records else "failed" reasons = ["R_INPUT_VALIDATION_ERROR"] @@ -1114,7 +1119,7 @@ def _capabilities(payload, bundle, operations, failures): payload["capabilities"] = matrix payload["observability"] = {"maximum_level": bundle.observability.maximum_level, "level_label": LEVEL_LABELS[bundle.observability.maximum_level], "basis": list(bundle.observability.basis), - "limitations": list(dict.fromkeys(bundle.observability.limitations + ("Input eligibility is distinct from executed analysis; Phase 5 lineage remains deferred.",))), + "limitations": list(dict.fromkeys(bundle.observability.limitations + ("Input eligibility is distinct from explicitly supplied executed analysis.",))), "partial_evidence": list(dict.fromkeys(reason for capability in matrix.values() for reason in capability["reason_codes"])), "capabilities": matrix} @@ -1144,11 +1149,198 @@ def _lineage_observations(payload, bundle): "all_resolved_edges_follow_order": True}, scope, limitations=("This retained earlier-version ordering certificate is not general graph-cycle traversal.",)) target["cycle_status"] = _envelope("observed_facts.lineage.cycle_status", None, scope, - status="unavailable", reasons=("R_GRAPH_EXECUTION_DEFERRED",), required=("Phase_5_graph_analysis",), - limitations=("No general graph-cycle traversal executes in Phase 4.",)) + status="unavailable", reasons=("R_ANALYSIS_NOT_REQUESTED",), required=("explicit_lineage_result",), + limitations=("No executed graph-cycle result was supplied to this report.",)) payload["observed_facts"]["lineage"] = target +def _lineage_scope(value, bundle): + """Bind population identities without rerunning the graph or mathematical owners.""" + _typed(value, LineageScope, "lineage scope") + LineageScope(**{field.name: getattr(value, field.name) for field in fields(LineageScope)}) + loaded = tuple(sorted(row.record_key for row in bundle.records)) + target = tuple(key for key in loaded if key.dataset_version == value.target_dataset_version) + primary = {row.record_key.dataset_version for row in bundle.records + if row.location.file_role is FileRole.RECORDS_PRIMARY} + _require(not primary or primary == {value.target_dataset_version}, + "lineage target differs from the validated primary version") + _require(primary or not any(row.record_key.dataset_version == value.target_dataset_version and + row.location.file_role is FileRole.RECORDS_COMPARE for row in bundle.records), + "loaded comparison context cannot become a lineage target") + _require(value.target_dataset_version is not None or not bundle.records or + any(row.location.file_role is not None for row in bundle.records), + "unassigned loaded records require an explicit lineage target version") + _require(value.target_record_keys == target and value.loaded_record_count == len(loaded) + and value.loaded_dataset_versions == tuple(sorted({key.dataset_version for key in loaded})), + "lineage scope differs from the validated target or loaded context") + _require(value.target_dataset_version is None or value.target_dataset_version in + set(value.loaded_dataset_versions) | set(bundle.version_order.order), + "lineage target version was not declared by the validated bundle") + return {"dataset_versions": [] if value.target_dataset_version is None else [value.target_dataset_version], + "record_count": value.target_record_count, "excluded_record_count": 0, + "denominator_basis": "all_valid_records_in_selected_dataset_scope", "scope_id": "lineage_target", + "included_record_keys": [_key(key) for key in value.target_record_keys], "excluded_record_keys": []} + + +def _lineage_detail(items, total=None): + """Limit presentation only; values and rank were computed by the owner.""" + total = len(items) if total is None else total + shown = list(items[:100]) + omitted = total - len(shown) + return {"items": shown, "total_count": total, "returned_count": len(shown), + "omitted_count": omitted, "limit": 100, + "detail_status": "truncated" if omitted else "complete", + "omission_reasons": ["diagnostic_limit"] if omitted else []} + + +def _lineage_usage(value): + _typed(value, LineageResourceUsage, "lineage resource usage") + LineageResourceUsage(**{field.name: getattr(value, field.name) for field in fields(LineageResourceUsage)}) + result = {field.name: getattr(value, field.name) for field in fields(LineageResourceUsage)} + result["limits"] = {field.name: getattr(value.limits, field.name) for field in fields(LineageLimits)} + return result + + +def _lineage_result(payload, value, bundle, bounds, proxy): + """Serialize supplied owner results; no graph traversal or formula evaluation.""" + _typed(value, LineageAnalysisResult, "lineage analysis") + _require(value.input_signature == _lineage_input_signature(bundle), + "lineage result was computed from different retained input evidence") + scope = _lineage_scope(value.scope, bundle) + _typed(value.cycles, CycleAnalysis, "lineage cycle result") + _require(value.cycles.scope == value.scope and + set(value.cycles.depths_by_record) == {row.record_key for row in bundle.records}, + "lineage cycle observations differ from the validated loaded identities") + _require(value.records is None or type(value.records) is tuple and + all(type(record) is RecordAncestry for record in value.records) and + tuple(record.record_key for record in value.records) == value.scope.target_record_keys, + "lineage records differ from the selected target population") + _require(value.root_contributions is None or type(value.root_contributions) is tuple and + all(type(item) is RootContribution for item in value.root_contributions), + "root contributions require immutable typed rows") + total = value.scope.target_record_count + coverage = value.resolved_lineage_coverage + aborted = value.records is None + root_reasons = ("LINEAGE_RESOURCE_LIMIT_EXCEEDED",) if aborted else ("UNRESOLVED_ANCESTRY",) if value.unresolved_record_count else () + assumptions = ("theory_guided_operationalization", "Only explicit supplied parent and grounding evidence defines ancestry.") + limitations = ("Root allocation describes supplied topology and does not establish causal contribution or source independence.",) + graph_scope = _bundle_scope(bundle) + observed = {} + graph_value = {field.name: getattr(value.scope, field.name) for field in fields(LineageScope) + if field.name != "target_record_keys"} + graph_value["loaded_dataset_versions"] = list(graph_value["loaded_dataset_versions"]) + observed["graph_scope"] = _envelope("observed_facts.lineage.graph_scope", graph_value, + graph_scope, denominator=value.scope.loaded_record_count, + limitations=("Context supplies ancestry evidence without entering target denominators.",)) + cycles = value.cycles + components = [{"component_index": item.component_index, "member_count": item.member_count, + "target_member_count": item.target_member_count, + "witness_record_keys": None if item.witness_record_keys is None else [_key(key) for key in item.witness_record_keys], + "witness_edge_count": item.witness_edge_count, "witness_reason": item.witness_reason} + for item in cycles.component_details] + cycle_value = {name: getattr(cycles, name) for name in ( + "detected", "cycle_count", "counting_method", "cycle_member_count", "target_cycle_member_count", + "affected_record_count", "target_affected_record_count")} + cycle_value.update({"components": _lineage_detail(components, cycles.cycle_count), + "cycle_member_record_keys": _lineage_detail([_key(key) for key in cycles.cycle_member_record_keys[:100]], cycles.cycle_member_count), + "affected_record_keys": _lineage_detail([_key(key) for key in cycles.affected_record_keys[:100]], cycles.affected_record_count)}) + cycle_limits = ("Engineering graph-validity observation over accepted edges and explicit self references.", + "An acyclic accepted subgraph does not certify rejected parent declarations.") + cycle_reasons = tuple(sorted({message.code for message in cycles.messages + if message.severity in (ValidationSeverity.ERROR, ValidationSeverity.FATAL)})) + for name, result in (("cycle_analysis", cycle_value), ("cycle_status", cycles.cycle_status)): + observed[name] = _envelope("observed_facts.lineage." + name, result, graph_scope, + status="partial" if cycle_reasons else "available", reasons=cycle_reasons, + assumptions=(cycles.cycle_status_scope,), limitations=cycle_limits) + observed["depth_summary"] = _envelope("observed_facts.lineage.depth_summary", { + "depth_resolved_record_count": value.depth_resolved_record_count, + "maximum_resolved_target_depth": value.maximum_resolved_target_depth, + "target_record_count": total}, scope, denominator=total, + limitations=("Resolved structural depth does not establish resolved external ancestry.",)) + unresolved = [] + for record in value.records or (): + if record.classification == "unresolved": + unresolved.append({"record_key": _key(record.record_key), "reason_codes": list(record.reason_codes)}) + if len(unresolved) == 100: + break + observed["unresolved_record_details"] = _envelope("observed_facts.lineage.unresolved_record_details", + None if aborted else _lineage_detail(unresolved, value.unresolved_record_count), scope, denominator=total, coverage=coverage, + status="unavailable" if aborted else "available", reasons=root_reasons if aborted else (), + required=("completed_target_partition",) if aborted else ()) + observed["resource_usage"] = _envelope("observed_facts.lineage.resource_usage", _lineage_usage(value.resource_usage), + graph_scope, limitations=("Defined admission and root-work units do not measure peak memory or scientific thresholds.",)) + for name, count in (("declared_parent_edge_count", value.declared_parent_reference_count), + ("resolved_parent_edge_count", value.resolved_parent_reference_count), + ("unresolved_parent_edge_count", value.unresolved_parent_reference_count)): + observed[name] = _envelope("observed_facts.lineage." + name, count, scope, + denominator=value.declared_parent_reference_count, coverage=value.resolved_parent_edge_coverage, + status="unavailable" if count is None else "available", reasons=value.reference_coverage_reason_codes, + required=("countable_original_parent_references",) if count is None else (), + limitations=("Counts preserve original target reference multiplicity; graph adjacency deduplicates canonical edges.",)) + payload["observed_facts"]["lineage"] = observed + metrics = {} + for name in ("grounded_record_count", "closed_record_count", "unresolved_record_count", "records_with_resolved_external_ancestry"): + metrics[name] = _envelope("derived_metrics.lineage." + name, getattr(value, name), scope, + denominator=total, coverage=coverage, status="unavailable" if aborted else "available", + reasons=root_reasons if aborted else (), required=("completed_target_partition",) if aborted else (), + assumptions=assumptions, limitations=limitations) + for name in ("resolved_lineage_coverage", "external_ancestry_coverage"): + number = getattr(value, name) + metrics[name] = _envelope("derived_metrics.lineage." + name, number, scope, + denominator=total, coverage=coverage, status="unavailable" if number is None else "available", + reasons=value.ancestry_coverage_reason_codes, required=("nonempty_completed_target_partition",) if number is None else (), + assumptions=assumptions, limitations=limitations) + metrics["resolved_parent_edge_coverage"] = _envelope("derived_metrics.lineage.resolved_parent_edge_coverage", + value.resolved_parent_edge_coverage, scope, denominator=value.declared_parent_reference_count, + coverage=value.resolved_parent_edge_coverage, status="unavailable" if value.resolved_parent_edge_coverage is None else "available", + reasons=value.reference_coverage_reason_codes, + required=("countable_original_parent_references",) if value.resolved_parent_edge_coverage is None else (), + limitations=("Identity resolution coverage remains distinct from accepted graph edges and complete ancestry.",)) + metrics["resolved_parent_edge_coverage"]["no_declared_parents"] = value.no_declared_parents + roots = None if aborted else [{field.name: _key(item.record_key) if field.name == "record_key" else getattr(item, field.name) + for field in fields(RootContribution)} for item in value.root_contributions[:100]] + for name, result in (("distinct_external_root_count", value.distinct_external_root_count), + ("top_shared_ancestors", None if aborted else _lineage_detail(roots, len(value.root_contributions)))): + metrics[name] = _envelope("derived_metrics.lineage." + name, result, scope, denominator=total, + coverage=value.external_ancestry_coverage, status=value.root_metrics_status.value, + reasons=root_reasons, required=("completed_root_resolution",) if aborted else (), + assumptions=assumptions, limitations=limitations) + for name in ("ancestry_concentration_hhi", "effective_external_root_count"): + metrics[name] = _envelope("derived_metrics.lineage." + name, getattr(value, name), scope, + denominator=value.grounded_record_count, coverage=value.external_ancestry_coverage, + status=value.concentration_status.value, reasons=value.concentration_reason_codes or root_reasons, + required=("nonempty_resolved_external_root_population",) if getattr(value, name) is None else (), + assumptions=assumptions, limitations=limitations + ("Concentration uses the complete grounded subset and is computed before detail limits.",)) + metrics["lineage_depth"] = _envelope("derived_metrics.lineage.lineage_depth", value.lineage_depth, scope, + denominator=total, status="unavailable" if value.lineage_depth is None else "available", + reasons=cycles.depth_reason_codes, required=("complete_target_structural_depth",) if value.lineage_depth is None else (), + limitations=("Whole-target depth is unavailable when any required structural path is unresolved.",)) + payload["derived_metrics"]["lineage"] = metrics + if bounds is not None: + _typed(bounds, LineageClosureExposureBounds, "lineage bounds") + _require(bounds.source == value, "lineage bounds source differs from the supplied lineage analysis") + target = {} + for name in ("lower_bound", "upper_bound", "interval_width"): + target[name] = _envelope("derived_metrics.closure_exposure.lineage." + name, getattr(bounds, name), scope, + denominator=bounds.denominator, coverage=coverage, status=bounds.status.value, + reasons=bounds.reason_codes, required=("nonempty_completed_target_partition",) if bounds.reason_codes else (), + assumptions=(bounds.operationalization_label,), limitations=bounds.limitations) + payload["derived_metrics"].setdefault("closure_exposure", {})["lineage"] = target + if proxy is not None: + _typed(proxy, SharedAncestryDependence, "shared ancestry proxy") + _require(proxy.source == value, "shared ancestry source differs from the supplied lineage analysis") + signal = _base("proxy_signals.shared_ancestry_dependence", scope, denominator=total, coverage=coverage, + status=proxy.status.value, reasons=proxy.reason_codes, + required=("complete_ancestry_or_shared_root_witness",) if proxy.status.value == "unavailable" else (), + assumptions=(proxy.operationalization_label,), limitations=proxy.limitations) + signal.update({"signal": "shared_ancestry_dependence", "level": proxy.level, + "basis_fields": ["derived_metrics.lineage.top_shared_ancestors", "derived_metrics.lineage.resolved_lineage_coverage", + "derived_metrics.lineage.external_ancestry_coverage"], + "trigger_rule": "Present when a complete external root supports at least two targets; absence requires complete target ancestry."}) + payload["proxy_signals"]["shared_ancestry_dependence"] = signal + _diagnostics(payload, value.messages, "lineage") + + def _bundle_check(bundle): _typed(bundle, BundleValidationResult, "validated bundle") _typed(bundle.version_order, VersionOrderResult, "retained version order") @@ -1223,6 +1415,9 @@ def assemble_report(bundle: BundleValidationResult, *, run: dict, tail: TailSelectionResult | None = None, comparison: SupportComparison | None = None, closure: DirectClosureExposureBounds | None = None, + lineage: LineageAnalysisResult | None = None, + lineage_bounds: LineageClosureExposureBounds | None = None, + shared_ancestry: SharedAncestryDependence | None = None, expected_diversity: ExpectedDiversityResult | None = None, resampling: ResamplingSimulation | None = None, extinction: tuple[ExtinctionProbabilityResult, ...] = (), @@ -1243,6 +1438,8 @@ def assemble_report(bundle: BundleValidationResult, *, run: dict, _require(set(bundle.observability.capabilities) == set(CapabilityKey), "bundle capability matrix is incomplete") _require(type(distributions) is tuple and type(extinction) is tuple and type(family_errors) is tuple, "explicit result collections require immutable tuples") _require(expected_diversity is None or resampling is None, "one closed-resampling report slot cannot hold two different method results") + _require(lineage is not None or lineage_bounds is None and shared_ancestry is None, + "lineage derivatives require their explicit source lineage result") payload = {key: {} if index < 8 else [] for index, key in enumerate(SECTION_ORDER)} payload["run"] = dict(run) _inputs(payload, bundle) @@ -1254,6 +1451,8 @@ def assemble_report(bundle: BundleValidationResult, *, run: dict, FamilyFailure(failure.capability, failure.messages) failures.setdefault(failure.capability.value, []).extend(failure.messages) _diagnostics(payload, failure.messages, failure.capability.value) + _require(lineage is None or "lineage" not in failures, + "a supplied lineage result cannot also have a separate lineage family failure") empirical_representations = [] for supplied in distributions: if type(supplied) is StateDistributionResult: @@ -1305,8 +1504,20 @@ def assemble_report(bundle: BundleValidationResult, *, run: dict, payload["inputs"]["representation"] = _representation(empirical_representations[0]) elif empirical_representations: payload["inputs"]["limitations"].append("Supplied calculation families use multiple explicit representations; inspect each result envelope.") - _lineage_observations(payload, bundle) - _capabilities(payload, bundle, operations, failures) + if lineage is None: + _lineage_observations(payload, bundle) + if "lineage" in failures: + payload["observed_facts"]["lineage"]["cycle_status"]["reason_codes"] = ["R_LINEAGE_EXECUTION_FAILED"] + else: + _lineage_result(payload, lineage, bundle, lineage_bounds, shared_ancestry) + operations["lineage"].append("supplied_lineage_graph_and_depth") + if lineage.records is not None: + operations["lineage"].append("supplied_lineage_roots_and_concentration") + if lineage_bounds is not None: + operations["lineage"].append("supplied_lineage_closure_interval") + if shared_ancestry is not None: + operations["lineage"].append("supplied_shared_ancestry_proxy") + _capabilities(payload, bundle, operations, failures, lineage) if payload["errors"]: payload["run"]["run_status"] = "failed" if any(error["severity"] == "fatal" for error in payload["errors"]) else "partial" if bundle.records or distributions or payload["simulations"] else "failed" for error in payload["errors"]: @@ -1428,7 +1639,6 @@ def _disclosures(payload): "This signal reports unresolved supplied provenance evidence and does not estimate factual truth or source independence.", "A not_present signal is limited to the cited fields and does not certify universal integrity."], ) - # No shared_ancestry_dependence proxy: P4-D01 prohibits Phase 4 ancestry inference. _disclosure_unavailable( payload, "model_performance_decline", "PR-014", "model_longitudinal", "The supplied audit evidence cannot establish model-performance decline.", @@ -1439,8 +1649,8 @@ def _disclosures(payload): _disclosure_unavailable( payload, "causal_ancestor_effect", "T4", "lineage", "A causal effect of an ancestor is unavailable.", - ["R_CAUSAL_EVIDENCE_MISSING", "R_LINEAGE_EXECUTION_DEFERRED"], - ["Phase 4 does not compute ancestry; topology alone would not identify a causal contribution."], + ["R_CAUSAL_EVIDENCE_MISSING"], + ["Supplied parent topology and fractional root allocation do not identify a causal contribution."], ["An identified ancestor, measured outcomes and a controlled or otherwise justified causal design."], "Parent declarations and graph topology do not establish causal effects.", ) @@ -1476,20 +1686,31 @@ def _disclosures(payload): ) lineage = payload["capabilities"].get("lineage") if lineage is not None: - reasons = ["R_LINEAGE_EXECUTION_DEFERRED"] - if lineage["status"] != "available": - reasons.extend(reason for reason in lineage["reason_codes"] if reason not in reasons) - for name, owner, statement in ( - ("lineage_analysis", "PR-014", "General lineage graph analysis is deferred to Phase 5."), - ("lineage_closure_exposure", "T3", "Lineage closure exposure is deferred to Phase 5."), - ("external_ancestry", "T4", "External ancestry results are deferred to Phase 5."), - ): - _disclosure_unavailable( - payload, name, owner, "lineage", statement, list(reasons), - ["Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes."], - ["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."], - "Implementation is deferred even when the existing input classifier marks lineage available.", - ) + if lineage["execution_status"] in ("not_requested", "failed"): + _disclosure_unavailable(payload, "lineage_analysis", "PR-014", "lineage", + "A completed whole-target lineage analysis is unavailable in this report.", + list(lineage["execution_reason_codes"]), + ["No complete successful lineage execution was supplied; independently completed observations remain visible."], + ["Supply an explicit lineage result for the validated target and its loaded context."], + "Input eligibility alone does not establish executed ancestry analysis.") + lineage_metrics = derived.get("lineage", {}) + coverage = lineage_metrics.get("resolved_lineage_coverage", {}) + if coverage.get("value") is None or coverage["value"] < 1: + _disclosure_unavailable(payload, "external_ancestry", "T4", "lineage", + "Complete external ancestry for every target is unavailable.", + list(coverage.get("reason_codes") or lineage["execution_reason_codes"] or ["UNRESOLVED_ANCESTRY"]), + ["Missing, invalid or unknown ancestry remains unresolved; any complete subset retains its reported coverage."], + ["Complete the required parent references, chronology and external-grounding evidence."], + "Partial ancestry cannot be promoted to whole-target completeness.") + lineage_interval = derived.get("closure_exposure", {}).get("lineage", {}) + lower = lineage_interval.get("lower_bound") + if lower is None or lower["status"] == "unavailable": + _disclosure_unavailable(payload, "lineage_closure_exposure", "T3", "lineage", + "A lineage closure interval is unavailable in this report.", + list(lower["reason_codes"] if lower is not None else ["R_ANALYSIS_NOT_REQUESTED"]), + ["No available explicitly supplied lineage interval covers a nonempty completed target partition."], + ["Supply lineage closure bounds from the same explicit lineage result."], + "Unavailable intervals are never replaced by a guessed numeric value.") if payload["simulations"]: _disclosure_unavailable( payload, "empirical_intervention_effect", "T5", "intervention_simulation", @@ -1530,7 +1751,7 @@ def _disclosures(payload): "The supplied capability assessment identifies missing chronology.") if lineage is not None and any(reason in lineage["reason_codes"] for reason in ("R_PARENT_UNRESOLVED", "R_PARENT_AMBIGUOUS", "R_PARENT_DECLARATION_MISSING", "R_PARENT_INVALID")): _disclosure_recommend(payload, 3, "composite_parent_references", "records with unresolved or invalid parent declarations", - ["more complete immediate-parent validation; future Phase 5 analysis remains deferred"], + ["more complete parent validation and explicitly requested ancestry coverage"], "The existing classifier identifies incomplete or invalid parent evidence.") _disclosure_recommend(payload, 4, "versioned_model_outcomes", "models and comparable evaluation conditions", ["evidence needed by a future model-longitudinal implementation"], @@ -1541,6 +1762,76 @@ def _disclosures(payload): # capability owners. Unknown caller prose never becomes trusted by a prefix, # regular expression, or similarity to one of these reviewed literal strings. _PRIVACY_SAFE_TEXT = frozenset(( + 'R_LINEAGE_EXECUTION_FAILED', + 'Input eligibility is distinct from explicitly supplied executed analysis.', + 'No executed graph-cycle result was supplied to this report.', + 'explicit_lineage_result', + 'A completed whole-target lineage analysis is unavailable in this report.', + 'No complete successful lineage execution was supplied; independently completed observations remain visible.', + 'Supply an explicit lineage result for the validated target and its loaded context.', + 'Input eligibility alone does not establish executed ancestry analysis.', + 'Complete external ancestry for every target is unavailable.', + 'Missing, invalid or unknown ancestry remains unresolved; any complete subset retains its reported coverage.', + 'Complete the required parent references, chronology and external-grounding evidence.', + 'Partial ancestry cannot be promoted to whole-target completeness.', + 'A lineage closure interval is unavailable in this report.', + 'No available explicitly supplied lineage interval covers a nonempty completed target partition.', + 'Supply lineage closure bounds from the same explicit lineage result.', + 'Unavailable intervals are never replaced by a guessed numeric value.', + 'more complete parent validation and explicitly requested ancestry coverage', + 'Supplied parent topology and fractional root allocation do not identify a causal contribution.', + 'supplied_lineage_graph_and_depth', + 'supplied_lineage_roots_and_concentration', + 'supplied_lineage_closure_interval', + 'supplied_shared_ancestry_proxy', + 'Only explicit supplied parent and grounding evidence defines ancestry.', + 'Root allocation describes supplied topology and does not establish causal contribution or source independence.', + 'Context supplies ancestry evidence without entering target denominators.', + 'Engineering graph-validity observation over accepted edges and explicit self references.', + 'An acyclic accepted subgraph does not certify rejected parent declarations.', + 'Resolved structural depth does not establish resolved external ancestry.', + 'Defined admission and root-work units do not measure peak memory or scientific thresholds.', + 'Counts preserve original target reference multiplicity; graph adjacency deduplicates canonical edges.', + 'Identity resolution coverage remains distinct from accepted graph edges and complete ancestry.', + 'Concentration uses the complete grounded subset and is computed before detail limits.', + 'Whole-target depth is unavailable when any required structural path is unresolved.', + 'Present when a complete external root supports at least two targets; absence requires complete target ancestry.', + 'completed_target_partition', + 'countable_original_parent_references', + 'nonempty_completed_target_partition', + 'completed_root_resolution', + 'nonempty_resolved_external_root_population', + 'complete_target_structural_depth', + 'complete_ancestry_or_shared_root_witness', + 'accepted_edges_and_explicit_self_references', + 'theory_guided_operationalization', + 'INCOMPLETE_TARGET_DEPTH', + 'MISSING_PROVENANCE', + 'MISSING_REQUIRED_PROVENANCE', + 'UNKNOWN_GROUNDING', + 'PARENT_DECLARATION_UNAVAILABLE', + 'INVALID_PARENT_REFERENCE', + 'UNRESOLVED_PARENT_REFERENCE', + 'VERSION_ORDER_UNAVAILABLE', + 'CYCLE_AFFECTED', + 'INCOMPLETE_PARENT_ANCESTRY', + 'INCOMPLETE_PARENT_DEPTH', + 'GROUNDED_PARENT_RULE_UNSUPPORTED', + 'INVALID_LINEAGE_INPUT', + 'UNRESOLVED_ANCESTRY', + 'LINEAGE_RESOURCE_LIMIT_EXCEEDED', + 'E_LINEAGE_RESOURCE_LIMIT_EXCEEDED', + 'EMPTY_TARGET_SCOPE', + 'PARENT_REFERENCE_COUNT_UNAVAILABLE', + 'NO_RESOLVED_EXTERNAL_ROOTS', + 'root_contributions.incidence_count', + 'resolved_lineage_coverage', + 'external_ancestry_coverage', + 'Descriptive shared-root topology under supplied metadata; no calibrated risk level.', + 'Does not establish causal contribution, semantic error, independent information, biological relatedness, universal integrity or collapse.', + 'Lineage exposure is relative to the audited loop and supplied metadata.', + 'Unresolved ancestry remains in the conservative interval; availability does not certify input validity.', + 'No midpoint, calibrated risk threshold or causal claim.', 'earlier_positive_mass_support_in_harmonized_representation', 'explicit_version_order', 'identifier_secret_file_path', @@ -2657,6 +2948,20 @@ def _privacy_value(value, contract, schema, *, mode, record_id_mode, protection, if "const" in contract or "enum" in contract or value is None or type(value) in (int, float, bool): return value if type(value) is dict: + # Lineage detail collections contain identities in several required row + # shapes. Omit the collection as a whole, retaining presentation counts, + # rather than making individual rows invalid by deleting identity keys. + if record_id_mode == "omit" and set(value) == { + "items", "total_count", "returned_count", "omitted_count", + "limit", "detail_status", "omission_reasons"}: + reasons = list(value["omission_reasons"]) + if "redacted_identity_details" not in reasons: + reasons.append("redacted_identity_details") + omissions.append(("field", ".".join(path), "")) + return {"items": None, "total_count": value["total_count"], + "returned_count": 0, "omitted_count": value["total_count"], + "limit": value["limit"], "detail_status": "omitted", + "omission_reasons": reasons} if set(value) == {"dataset_version", "record_id"}: return {"dataset_version": _privacy_alias(protection, "dataset_version", value["dataset_version"]) if mode == "redacted" else value["dataset_version"], "record_id": _privacy_alias(protection, "record_id", [value["dataset_version"], value["record_id"]]) if record_id_mode == "hash" else value["record_id"]} @@ -2700,7 +3005,8 @@ def _privacy_value(value, contract, schema, *, mode, record_id_mode, protection, if type(value) is list: return [_privacy_value(item, contract.get("items", {}), schema, mode=mode, record_id_mode=record_id_mode, protection=protection, path=path, omissions=omissions) for item in value] - version_fields = ("dataset_version", "dataset_versions", "version_order", "earlier_version", "later_version") + version_fields = ("dataset_version", "dataset_versions", "version_order", "earlier_version", "later_version", + "target_dataset_version", "loaded_dataset_versions") state_fields = ("state_id", "state_ids", "missing_state_id", "state_order", "source_state", "source_states", "target_state", "original_earlier_support", "original_later_support", "harmonized_earlier_support", "harmonized_later_support", "support") identity_fields = ("run_id", "scope_id", "group_id", "representation_name", "representation_source", "representation_version", @@ -2839,7 +3145,7 @@ def build_run_metadata(*, options, run_id: str, operation="python_api", started_ if type(run_id) is not str or not run_id or "\x00" in run_id: raise ValueError("run identifier must be nonempty literal text") command = None if operation == "python_api" else "rit " + operation + (" --strict" if options.strict_mode else "") + (" --redacted" if options.privacy_mode == "redacted" else "") - result = {"run_id": run_id, "toolkit_version": __version__, "report_schema_version": "1.0", + result = {"run_id": run_id, "toolkit_version": __version__, "report_schema_version": "1.1", "started_at": started_at, "completed_at": completed_at, "duration_seconds": duration_seconds, "python_version": python_version, "platform": platform, "command": command, "config_hash": phase4_config_hash(options), "random_seed": random_seed, diff --git a/src/recursive_integrity_toolkit/reports/markdown_report.py b/src/recursive_integrity_toolkit/reports/markdown_report.py index 1b3efad..88e089e 100644 --- a/src/recursive_integrity_toolkit/reports/markdown_report.py +++ b/src/recursive_integrity_toolkit/reports/markdown_report.py @@ -102,6 +102,10 @@ def _null_reason(key: str | int, owner: dict) -> str: return "Withheld by the selected privacy view." if key == "record_keys" and owner.get("redaction"): return "Identity details omitted by the declared redaction policy." + if key == "items" and owner.get("omission_reasons"): + return "Detail omitted: " + _code(owner["omission_reasons"]) + if key in {"witness_record_keys", "witness_edge_count"} and owner.get("witness_reason"): + return "Cycle witness omitted: " + _code(owner["witness_reason"]) if key == "representation": return "No applicable representation supplied; consult status and required evidence." if key in {"record_count", "excluded_record_count"}: diff --git a/src/recursive_integrity_toolkit/result.py b/src/recursive_integrity_toolkit/result.py index 9d6cfad..6c32f26 100644 --- a/src/recursive_integrity_toolkit/result.py +++ b/src/recursive_integrity_toolkit/result.py @@ -14,7 +14,7 @@ No ingestion, classifier, metric, simulation, adapter, renderer or CLI runs. A valid contract does not certify the truth of supplied evidence. Current phase status: - Phase 4 Step 4 adds a safe-view wrapper; the Step 2 schema remains frozen. + Phase 5 Step 7 adds bounded lineage evidence under schema 1.1. """ from __future__ import annotations @@ -144,6 +144,31 @@ def _ref(name: str) -> dict: return {"$ref": "#/$defs/" + name} +def _lineage_details(item: dict) -> dict: + """Bound identity-bearing diagnostics independently of analytical values.""" + items = _array(item) + items["maxItems"] = 100 + result = _object({ + "items": _nullable(items), "total_count": _integer(), + "returned_count": {**_integer(), "maximum": 100}, "omitted_count": _integer(), + "limit": {"const": 100}, "detail_status": _enum("complete", "truncated", "omitted"), + "omission_reasons": _array(_enum("diagnostic_limit", "redacted_identity_details"), unique=True), + }) + result["allOf"] = [ + {"if": {"properties": {"detail_status": {"const": "omitted"}}}, + "then": {"properties": {"items": {"type": "null"}, "returned_count": {"const": 0}, + "omission_reasons": {"minItems": 1}}}, + "else": {"properties": {"items": items}}}, + {"if": {"properties": {"detail_status": {"const": "complete"}}}, + "then": {"properties": {"omitted_count": {"const": 0}, "omission_reasons": {"maxItems": 0}}}}, + {"if": {"properties": {"detail_status": {"const": "truncated"}}}, + "then": {"properties": {"omitted_count": {"minimum": 1}, + "returned_count": {"const": 100}, + "omission_reasons": {"const": ["diagnostic_limit"]}}}}, + ] + return result + + def _base_envelope(evidence: str, unit: str, owner: str, method: str) -> dict: return { "unit": {"const": unit}, "evidence_class": {"const": evidence}, @@ -239,7 +264,7 @@ def _build_contract() -> dict: }, ("status", "reason_codes", "coverage", "coverage_reason", "requirements_met", "requirements_missing", "notes", "execution_status", "execution_scope", "execution_reason_codes")) run = _object({ - "run_id": _text(), "toolkit_version": _text(), "report_schema_version": {"const": "1.0"}, + "run_id": _text(), "toolkit_version": _text(), "report_schema_version": {"const": "1.1"}, "started_at": _nullable(_text()), "completed_at": _nullable(_text()), "duration_seconds": _nullable(_number(minimum=0)), "python_version": _nullable(_text()), "platform": _nullable(_text()), "command": _nullable(_text()), @@ -325,7 +350,49 @@ def _build_contract() -> dict: observed_lineage = {name: _observed("observed_facts.lineage." + name, _integer(), "PR-008", "edges", "PR-008." + name, 3, "integer") for name in ("declared_parent_edge_count", "resolved_parent_edge_count", "unresolved_parent_edge_count")} observed_lineage["cycle_status"] = _observed("observed_facts.lineage.cycle_status", _enum("acyclic", "cyclic"), - "T6", "status", "T6.graph_cycle_check", 3, "enum", boundary="deferred_phase5") + "T6", "status", "T6.graph_cycle_check", 3, "enum") + graph_scope = _object({ + "target_dataset_version": _nullable(_text()), "target_record_count": _integer(), + "loaded_record_count": _integer(), "context_record_count": _integer(), + "loaded_dataset_versions": _strings(), + }) + witness = _array(_ref("record_key"), minimum=2) + witness["maxItems"] = 65 + component = _object({ + "component_index": _integer(minimum=1), "member_count": _integer(minimum=1), + "target_member_count": _integer(), "witness_record_keys": _nullable(witness), + "witness_edge_count": _nullable({**_integer(minimum=1), "maximum": 64}), + "witness_reason": _nullable({"const": "diagnostic_limit"}), + }) + cycle_analysis = _object({ + "detected": {"type": "boolean"}, "cycle_count": _integer(), + "counting_method": {"const": "cyclic_strongly_connected_components"}, + "cycle_member_count": _integer(), "target_cycle_member_count": _integer(), + "affected_record_count": _integer(), "target_affected_record_count": _integer(), + "components": _lineage_details(component), + "cycle_member_record_keys": _lineage_details(_ref("record_key")), + "affected_record_keys": _lineage_details(_ref("record_key")), + }) + depth_summary = _object({"depth_resolved_record_count": _integer(), + "maximum_resolved_target_depth": _nullable(_integer()), "target_record_count": _integer()}) + resource_usage = _object({ + "admitted_node_count": _integer(), "admitted_edge_count": _integer(), + "stored_root_membership_count": _integer(), "root_union_visit_count": _integer(), + "limits": _object({name: _integer(minimum=1) for name in ( + "max_nodes", "max_edges", "max_root_memberships", "max_root_union_visits")}), + "exhausted_limit": _nullable(_enum("max_nodes", "max_edges", "max_root_memberships", "max_root_union_visits")), + "attempted_value": _nullable(_integer(minimum=1)), + }) + for name, value, owner, unit, method, value_type in ( + ("graph_scope", graph_scope, "T4", "scope", "T4.graph_scope", "lineage_scope"), + ("cycle_analysis", cycle_analysis, "T6", "cycles", "T6.cyclic_strongly_connected_components", "cycle_analysis"), + ("depth_summary", depth_summary, "PR-009", "edges", "PR-009.resolved_depth_summary", "depth_summary"), + ("unresolved_record_details", _lineage_details(_object({"record_key": _ref("record_key"), + "reason_codes": _strings(minimum=1)})), "T4", "records", "T4.unresolved_record_details", "unresolved_record_details"), + ("resource_usage", resource_usage, "PR-015", "work_units", "PR-015.lineage_resource_usage", "lineage_resource_usage"), + ): + observed_lineage[name] = _observed("observed_facts.lineage." + name, value, + owner, unit, method, 3, value_type) observed_lineage["ordering_certificate"] = _observed("observed_facts.lineage.ordering_certificate", _object({ "method": {"const": "declared_earlier_version_order"}, "version_order": _strings(), "all_resolved_edges_follow_order": {"type": "boolean"}, @@ -416,20 +483,30 @@ def _build_contract() -> dict: ("lower_bound", "F-009"), ("upper_bound", "F-010"), ("interval_width", "T3.upper_minus_lower"))} direct["classification_basis"] = {"const": "toolkit_operationalization"} direct["confidence_disclosure"] = {"const": "provenance_confidence_is_separate_and_does_not_discount_grounding"} - lineage_bounds = {name: _metric("derived_metrics.closure_exposure.lineage." + name, _number(minimum=0, maximum=1), "T3", "ratio", "T3.lineage_closure", 3, "ratio", boundary="deferred_phase5") for name in ("lower_bound", "upper_bound", "interval_width")} + lineage_bounds = {name: _metric("derived_metrics.closure_exposure.lineage." + name, _number(minimum=0, maximum=1), "T3", "ratio", "T3.lineage_closure", 3, "ratio") for name in ("lower_bound", "upper_bound", "interval_width")} + root_contribution = _object({ + "record_key": _ref("record_key"), "incidence_count": _integer(minimum=1), + "incidence_share": _number(minimum=0, maximum=1), "incidence_denominator": _integer(minimum=1), + "fractional_mass": _number(minimum=0), "normalized_weight": _number(minimum=0, maximum=1), + "weight_denominator": _integer(minimum=1), + }) lineage = {} for name, owner, unit, method, value, value_type in ( ("resolved_parent_edge_coverage", "PR-008", "ratio", "PR-008.resolved_edges_over_declared", _number(minimum=0, maximum=1), "ratio"), ("resolved_lineage_coverage", "T4", "ratio", "T4.resolved_records_over_scope", _number(minimum=0, maximum=1), "ratio"), ("external_ancestry_coverage", "T4", "ratio", "T4.external_roots_over_scope", _number(minimum=0, maximum=1), "ratio"), ("distinct_external_root_count", "T4", "roots", "T4.root_set_union", _integer(), "integer"), - ("top_shared_ancestors", "T4", "records", "T4.incidence_ranking", _array(_object({"record_key": _ref("record_key"), "incidence_count": _integer()})), "ancestor_incidence[]"), + ("top_shared_ancestors", "T4", "records", "T4.incidence_ranking", _lineage_details(root_contribution), "root_contribution_details"), + ("grounded_record_count", "T4", "records", "T4.grounded_record_count", _integer(), "integer"), + ("closed_record_count", "T4", "records", "T4.closed_record_count", _integer(), "integer"), + ("unresolved_record_count", "T4", "records", "T4.unresolved_record_count", _integer(), "integer"), + ("records_with_resolved_external_ancestry", "T4", "records", "T4.records_with_resolved_external_ancestry", _integer(), "integer"), ("ancestry_concentration_hhi", "T4", "ratio", "F-012", _number(minimum=0, maximum=1), "ratio"), ("effective_external_root_count", "T4", "roots", "F-013", _number(minimum=0), "number"), ("lineage_depth", "PR-009", "edges", "PR-009.maximum_resolved_parent_depth", _integer(), "integer"), ): - lineage[name] = _metric("derived_metrics.lineage." + name, value, owner, unit, method, 3, value_type, - boundary="current" if name == "resolved_parent_edge_coverage" else "deferred_phase5") + lineage[name] = _metric("derived_metrics.lineage." + name, value, owner, unit, method, 3, value_type) + lineage["resolved_parent_edge_coverage"]["properties"]["no_declared_parents"] = {"type": "boolean"} derived = _object({"support": _object(support, ()), "diversity": diversity, "tail": _object(tail, ()), "provenance": _object(provenance_metrics, ()), "closure_exposure": _object({"direct": _object(direct, ()), "lineage": _object(lineage_bounds, ())}, ()), @@ -441,19 +518,13 @@ def _build_contract() -> dict: props.update({"signal": {"const": name}, "level": _enum("present", "not_present", "indeterminate"), "basis_fields": _strings(minimum=1), "trigger_rule": _text()}) props["limitations"] = _strings(minimum=1) - if name == "shared_ancestry_dependence": - props["status"] = {"const": "unavailable"} - props["level"] = {"const": "indeterminate"} - props["reason_codes"] = _strings(minimum=1) - props["required_evidence"] = _strings(minimum=1) proxies[name] = _object(props) proxies[name]["allOf"] = [{ "if": {"properties": {"status": {"const": "unavailable"}}}, "then": {"properties": {"level": {"const": "indeterminate"}, "reason_codes": {"minItems": 1}, "required_evidence": {"minItems": 1}}}, }] - _field_entries.append(FieldDefinition("proxy_signals." + name, owner, "proxy_signal", "signal", owner + "." + name, level, "proxy", False, - "deferred_phase5" if name == "shared_ancestry_dependence" else "current")) + _field_entries.append(FieldDefinition("proxy_signals." + name, owner, "proxy_signal", "signal", owner + "." + name, level, "proxy")) simulations = {} for name, owner, level in (("closed_resampling", "T1", 5), ("tail_extinction", "T2", 1)): props = _base_envelope("simulation", "scenario", owner, owner + "." + name) @@ -571,8 +642,8 @@ def _build_contract() -> dict: "warnings": _array(warning), "errors": _array(error)}) return {"$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://recursive-integrity-toolkit.example/schemas/report.schema.json", - "title": "Recursive Integrity Toolkit canonical report 1.0", - "description": "Phase 4 Step 2 public contract. Product metadata and five analytical evidence classes remain separate.", + "title": "Recursive Integrity Toolkit canonical report 1.1", + "description": "Phase 5 public contract with bounded lineage evidence. Product metadata and five analytical evidence classes remain separate.", **schema, "$defs": defs} @@ -724,6 +795,8 @@ def _check(value, schema: dict, path: str) -> None: if type(value) is list: if len(value) < schema.get("minItems", 0): _fail(path, "too few array items") + if "maxItems" in schema and len(value) > schema["maxItems"]: + _fail(path, "too many array items") if schema.get("uniqueItems"): identities = {_equality_key(item) for item in value} if len(identities) != len(value): @@ -784,6 +857,244 @@ def _check_nullable_reason(value: dict, name: str, reason_name: str, path: str) _fail(path, "null requires its reason; a present value cannot carry a null reason") +def _lineage_detail_checks(detail: dict) -> None: + path = "$.lineage.details" + for name in ("total_count", "returned_count", "omitted_count", "limit"): + if type(detail[name]) is not int: + _fail(path, "detail counts require exact integers") + total, returned, omitted = (detail[name] for name in ("total_count", "returned_count", "omitted_count")) + if total != returned + omitted: + _fail(path, "detail counts do not partition the total") + status, items, reasons = detail["detail_status"], detail["items"], detail["omission_reasons"] + if status == "omitted": + expected = {"redacted_identity_details"} + if total > 100: + expected.add("diagnostic_limit") + if items is not None or returned or omitted != total or set(reasons) != expected: + _fail(path, "privacy omission must retain the complete aggregate and applicable cap reason") + elif items is None or returned != len(items) or returned != min(total, 100): + _fail(path, "visible detail must contain the bounded prefix of the supplied total") + elif (status == "complete") != (omitted == 0): + _fail(path, "detail status disagrees with omitted count") + if items: + identities = [row.get("record_key", row) for row in items] + if len({_equality_key(key) for key in identities}) != len(identities): + _fail(path, "detail identities must be unique") + + +def _lineage_semantic_checks(payload: dict) -> None: + """Check supplied lineage evidence consistency without traversing a graph.""" + observed = payload["observed_facts"].get("lineage", {}) + metrics = payload["derived_metrics"].get("lineage", {}) + bounds = payload["derived_metrics"].get("closure_exposure", {}).get("lineage", {}) + proxy = payload["proxy_signals"].get("shared_ancestry_dependence") + capability = payload["capabilities"].get("lineage", {}) + execution = capability.get("execution_status", "not_requested") + new_observed = ("graph_scope", "cycle_analysis", "depth_summary", "unresolved_record_details", "resource_usage", "cycle_status") + new_metrics = tuple(name for name in metrics if name != "resolved_parent_edge_coverage") + + def value(group, name): + return group.get(name, {}).get("value") + + active = (any(value(observed, name) is not None for name in new_observed) + or any(value(metrics, name) is not None for name in new_metrics) + or any(entry["value"] is not None for entry in bounds.values()) + or (proxy is not None and proxy["status"] != "unavailable")) + if execution in ("not_requested", "deferred"): + if active or "no_declared_parents" in metrics.get("resolved_parent_edge_coverage", {}): + _fail("$.capabilities.lineage", "unexecuted lineage cannot contain analytical results") + return + scope = value(observed, "graph_scope") + if scope is None: + if execution == "failed" and not active: + return + _fail("$.observed_facts.lineage.graph_scope", "executed lineage requires its explicit target and loaded scope") + target, loaded, context = (scope[name] for name in ("target_record_count", "loaded_record_count", "context_record_count")) + if any(type(number) is not int for number in (target, loaded, context)) or loaded != target + context: + _fail("$.observed_facts.lineage.graph_scope", "loaded scope must partition into target and context") + version = scope["target_dataset_version"] + versions = scope["loaded_dataset_versions"] + if (target and (version is None or version not in versions)) or (loaded and not versions): + _fail("$.observed_facts.lineage.graph_scope", "scope counts require their declared versions") + target_versions = [] if version is None else [version] + graph_fields = {"graph_scope", "cycle_analysis", "cycle_status", "resource_usage"} + for name in graph_fields & observed.keys(): + entry_scope = observed[name]["scope"] + if entry_scope["record_count"] != loaded or set(entry_scope["dataset_versions"]) != set(versions): + _fail("$.lineage.scope", "graph diagnostics must retain the complete loaded population") + entries = [entry for name, entry in observed.items() if name not in graph_fields] + list(metrics.values()) + list(bounds.values()) + if proxy is not None: + entries.append(proxy) + for entry in entries: + if entry["scope"]["record_count"] != target or entry["scope"]["dataset_versions"] != target_versions: + _fail("$.lineage.scope", "lineage evidence must retain the selected target population") + usage = value(observed, "resource_usage") + if usage is None: + _fail("$.observed_facts.lineage.resource_usage", "executed lineage requires work accounting") + counter_names = { + "max_nodes": "admitted_node_count", "max_edges": "admitted_edge_count", + "max_root_memberships": "stored_root_membership_count", "max_root_union_visits": "root_union_visit_count", + } + for limit, counter in counter_names.items(): + if type(usage[counter]) is not int or type(usage["limits"][limit]) is not int or usage[counter] > usage["limits"][limit]: + _fail("$.lineage.resource_usage", "consumed work must remain within its exact integer limit") + exhausted = usage["exhausted_limit"] + if exhausted is None: + if usage["attempted_value"] is not None: + _fail("$.lineage.resource_usage", "unexhausted work cannot contain a rejected attempt") + elif (execution != "failed" or type(usage["attempted_value"]) is not int + or usage["attempted_value"] <= usage["limits"][exhausted]): + _fail("$.lineage.resource_usage", "resource exhaustion requires failed execution and an over-limit attempt") + cycles, depth = value(observed, "cycle_analysis"), value(observed, "depth_summary") + if execution in ("completed", "partial") and (cycles is None or depth is None): + _fail("$.capabilities.lineage", "completed traversal requires cycle and depth observations") + if cycles is None: + if value(observed, "cycle_status") is not None: + _fail("$.lineage.cycle_status", "cycle status requires a completed cycle analysis") + else: + if usage["admitted_node_count"] != loaded: + _fail("$.lineage.resource_usage", "completed graph evidence requires every loaded node") + if any(type(cycles[name]) is not int for name in ( + "cycle_count", "cycle_member_count", "target_cycle_member_count", + "affected_record_count", "target_affected_record_count")): + _fail("$.lineage.cycle_analysis", "cycle counts require exact integers") + total_members, target_members = cycles["cycle_member_count"], cycles["target_cycle_member_count"] + affected, target_affected = cycles["affected_record_count"], cycles["target_affected_record_count"] + if (cycles["detected"] != (cycles["cycle_count"] > 0) + or not cycles["cycle_count"] <= total_members <= affected <= loaded + or not target_members <= total_members or not target_members <= target_affected <= target + or target_affected > affected or (not cycles["detected"] and affected)): + _fail("$.lineage.cycle_analysis", "cycle counts disagree with their loaded and target scopes") + if execution == "completed" and cycles["detected"]: + _fail("$.capabilities.lineage", "cyclic input cannot claim completed valid lineage") + if value(observed, "cycle_status") != ("cyclic" if cycles["detected"] else "acyclic"): + _fail("$.lineage.cycle_status", "cycle status disagrees with cycle diagnostics") + for name, expected in (("components", cycles["cycle_count"]), ("cycle_member_record_keys", total_members), ("affected_record_keys", affected)): + if cycles[name]["total_count"] != expected: + _fail("$.lineage.cycle_analysis", "detail total disagrees with its aggregate") + rows = cycles["components"]["items"] + if rows is not None: + for index, row in enumerate(rows, 1): + if (any(type(row[name]) is not int for name in ("component_index", "member_count", "target_member_count")) + or row["component_index"] != index or row["target_member_count"] > row["member_count"]): + _fail("$.lineage.cycle_analysis", "component order or member counts are inconsistent") + witness = row["witness_record_keys"] + if witness is None: + if row["witness_edge_count"] is not None or row["witness_reason"] != "diagnostic_limit": + _fail("$.lineage.cycle_analysis", "omitted witness requires a diagnostic-limit reason") + elif (type(row["witness_edge_count"]) is not int or witness[0] != witness[-1] + or row["witness_edge_count"] != len(witness) - 1 + or row["witness_reason"] is not None or len(witness) - 1 > row["member_count"]): + _fail("$.lineage.cycle_analysis", "witness must be closed with a consistent bounded edge count") + if cycles["components"]["detail_status"] == "complete" and ( + sum(row["member_count"] for row in rows) != total_members + or sum(row["target_member_count"] for row in rows) != target_members): + _fail("$.lineage.cycle_analysis", "component rows disagree with aggregate membership") + if depth is not None: + resolved_depth = depth["depth_resolved_record_count"] + maximum = depth["maximum_resolved_target_depth"] + if (type(resolved_depth) is not int or type(depth["target_record_count"]) is not int + or (maximum is not None and type(maximum) is not int) + or depth["target_record_count"] != target or resolved_depth > target + or (maximum is None) != (resolved_depth == 0)): + _fail("$.lineage.depth_summary", "depth subset must retain its actual target coverage") + expected_depth = maximum if target and resolved_depth == target else None + if value(metrics, "lineage_depth") != expected_depth: + _fail("$.derived_metrics.lineage.lineage_depth", "whole-target depth cannot claim a subset maximum") + count_names = ("grounded_record_count", "closed_record_count", "unresolved_record_count", "records_with_resolved_external_ancestry") + counts = [value(metrics, name) for name in count_names] + if any(count is None for count in counts): + if any(count is not None for count in counts) or execution in ("completed", "partial"): + _fail("$.derived_metrics.lineage", "an exact target partition requires all G/C/U counts") + if any(value(metrics, name) is not None for name in new_metrics if name != "lineage_depth") or any(entry["value"] is not None for entry in bounds.values()): + _fail("$.derived_metrics.lineage", "incomplete root traversal cannot supply dependent values") + else: + grounded, closed, unresolved, resolved = counts + if any(type(count) is not int for count in counts) or grounded + closed + unresolved != target or resolved != grounded + closed: + _fail("$.derived_metrics.lineage", "G/C/U counts must exactly partition the target") + if any(metrics[name]["denominator"] != target for name in count_names): + _fail("$.derived_metrics.lineage", "partition counts must disclose the complete target denominator") + if exhausted is not None or (execution == "completed" and unresolved): + _fail("$.capabilities.lineage", "execution status disagrees with root completeness") + for name, numerator in (("resolved_lineage_coverage", resolved), ("external_ancestry_coverage", grounded)): + entry = metrics.get(name) + if entry is None or entry["denominator"] != target: + _fail("$.derived_metrics.lineage", "ancestry coverage must disclose the whole target denominator") + ratio = entry["value"] + if (target == 0 and ratio is not None) or (target and (ratio is None or not math.isclose(ratio * target, numerator, rel_tol=1e-12, abs_tol=1e-12))): + _fail("$.derived_metrics.lineage", "ancestry coverage disagrees with the supplied partition") + details = value(observed, "unresolved_record_details") + if details is None or details["total_count"] != unresolved: + _fail("$.lineage.unresolved_record_details", "unresolved detail total disagrees with the target partition") + root_details = value(metrics, "top_shared_ancestors") + distinct = value(metrics, "distinct_external_root_count") + if type(distinct) is not int or root_details is None or root_details["total_count"] != distinct: + _fail("$.lineage.top_shared_ancestors", "root detail total disagrees with distinct roots") + if grounded == 0 and (distinct != 0 or any(value(metrics, name) is not None for name in ("ancestry_concentration_hhi", "effective_external_root_count"))): + _fail("$.derived_metrics.lineage", "empty grounded population has no concentration value") + if grounded and (not distinct or any(value(metrics, name) is None for name in ("ancestry_concentration_hhi", "effective_external_root_count"))): + _fail("$.derived_metrics.lineage", "grounded population requires its root distribution and concentration") + if grounded and (value(metrics, "ancestry_concentration_hhi") <= 0 + or value(metrics, "effective_external_root_count") <= 0 + or not math.isclose(value(metrics, "ancestry_concentration_hhi") + * value(metrics, "effective_external_root_count"), + 1.0, rel_tol=1e-12, abs_tol=1e-12)): + _fail("$.derived_metrics.lineage", "effective roots and positive concentration must be reciprocal") + for name in ("distinct_external_root_count", "top_shared_ancestors", "ancestry_concentration_hhi", "effective_external_root_count"): + entry = metrics[name] + concentration = name in ("ancestry_concentration_hhi", "effective_external_root_count") + expected_status = "unavailable" if concentration and not grounded else "partial" if unresolved else "available" + if (entry["status"] != expected_status or entry["denominator"] != (grounded if concentration else target) + or entry["coverage"] != value(metrics, "external_ancestry_coverage")): + _fail("$.derived_metrics.lineage", "root metrics must disclose their grounded subset and unresolved coverage") + rows = root_details["items"] + if rows is not None: + for row in rows: + if (any(type(row[name]) is not int for name in ("incidence_count", "incidence_denominator", "weight_denominator")) + or row["incidence_denominator"] != target or row["weight_denominator"] != grounded + or row["incidence_count"] > grounded or row["fractional_mass"] <= 0 + or row["fractional_mass"] > row["incidence_count"] + or row["incidence_share"] <= 0 or row["normalized_weight"] <= 0 + or not math.isclose(row["incidence_share"] * target, row["incidence_count"], rel_tol=1e-12, abs_tol=1e-12) + or not math.isclose(row["normalized_weight"] * grounded, row["fractional_mass"], rel_tol=1e-12, abs_tol=1e-12)): + _fail("$.lineage.top_shared_ancestors", "root contribution contradicts its explicit count basis") + if any(left["incidence_count"] < right["incidence_count"] for left, right in zip(rows, rows[1:])): + _fail("$.lineage.top_shared_ancestors", "root contributions must retain decreasing incidence rank") + for name, numerator in (("lower_bound", closed), ("upper_bound", closed + unresolved), ("interval_width", unresolved)): + if name not in bounds: + continue + entry = bounds[name] + invalid_ratio = (entry["value"] is not None) if target == 0 else ( + entry["value"] is None or not math.isclose( + entry["value"] * target, numerator, rel_tol=1e-12, abs_tol=1e-12)) + if entry["denominator"] != target or invalid_ratio: + _fail("$.derived_metrics.closure_exposure.lineage", "lineage interval contradicts its target partition") + if proxy is not None: + required_basis = {"derived_metrics.lineage.top_shared_ancestors", "derived_metrics.lineage.resolved_lineage_coverage", "derived_metrics.lineage.external_ancestry_coverage"} + if not required_basis.issubset(proxy["basis_fields"]): + _fail("$.proxy_signals.shared_ancestry_dependence", "shared-root signal must cite incidence and coverage evidence") + if proxy["denominator"] != target or proxy["coverage"] != value(metrics, "resolved_lineage_coverage"): + _fail("$.proxy_signals.shared_ancestry_dependence", "shared-root signal must disclose its target coverage") + if proxy["level"] == "not_present" and (not target or unresolved or proxy["status"] != "available"): + _fail("$.proxy_signals.shared_ancestry_dependence", "absence requires complete nonempty target evidence") + if proxy["level"] == "not_present" and rows and rows[0]["incidence_count"] >= 2: + _fail("$.proxy_signals.shared_ancestry_dependence", "absence contradicts a supplied shared-root witness") + if proxy["level"] == "present" and (grounded < 2 or proxy["status"] != ("partial" if unresolved else "available") or (rows is not None and (not rows or rows[0]["incidence_count"] < 2))): + _fail("$.proxy_signals.shared_ancestry_dependence", "presence requires a shared-root witness with honest coverage") + reference_counts = [value(observed, name) for name in ("declared_parent_edge_count", "resolved_parent_edge_count", "unresolved_parent_edge_count")] + reference = metrics.get("resolved_parent_edge_coverage") + if reference is None or "no_declared_parents" not in reference: + _fail("$.derived_metrics.lineage.resolved_parent_edge_coverage", "executed reference coverage requires its explicit zero-reference state") + if any(count is None for count in reference_counts): + if any(count is not None for count in reference_counts) or reference["value"] is not None or reference["no_declared_parents"]: + _fail("$.lineage.reference_coverage", "unknown reference cardinality cannot become a zero declaration") + else: + declared, resolved, unresolved = reference_counts + ratio = reference["value"] + if any(type(count) is not int for count in reference_counts) or declared != resolved + unresolved or reference["no_declared_parents"] != (declared == 0) or reference["denominator"] != declared or ratio is None or (declared == 0 and ratio != 1) or (declared and not math.isclose(ratio * declared, resolved, rel_tol=1e-12, abs_tol=1e-12)): + _fail("$.lineage.reference_coverage", "reference coverage contradicts its declared count basis") + + def _semantic_checks(payload: dict) -> None: run = payload["run"] null_fields = {key for key in RUN_NULLABLE_FIELDS if run[key] is None} @@ -805,8 +1116,7 @@ def _semantic_checks(payload: dict) -> None: _fail("$.capabilities", "noncompleted execution needs explicit reasons") if capability["execution_status"] in ("completed", "partial") and not capability["execution_scope"]: _fail("$.capabilities", "executed work must name its operation scope") - if key == "lineage" and capability["execution_status"] in ("completed", "partial"): - _fail("$.capabilities.lineage", "Phase 5 lineage execution is deferred") + _lineage_semantic_checks(payload) tail = payload["derived_metrics"].get("tail", {}) if "tail_rule" in tail and "selection" in tail and tail["tail_rule"] != tail["selection"]["rule"]: _fail("$.derived_metrics.tail", "tail rule declarations disagree") @@ -824,6 +1134,8 @@ def _semantic_checks(payload: dict) -> None: _fail("$.table", "typed table identity must be unique") if type(item) is not dict: continue + if "detail_status" in contract.get("properties", {}): + _lineage_detail_checks(item) if contract is _REPORT_SCHEMA["$defs"]["coverage"]: if item["numerator"] > item["denominator"]: _fail("$.coverage", "coverage numerator exceeds denominator") @@ -912,7 +1224,7 @@ def _semantic_checks(payload: dict) -> None: lower, upper = interval["lower_bound"]["value"], interval["upper_bound"]["value"] if lower > upper: _fail("$.derived_metrics.closure_exposure", "interval lower bound exceeds upper bound") - if family == "direct" and not math.isclose(interval["interval_width"]["value"], upper - lower, rel_tol=1e-12, abs_tol=1e-12): + if not math.isclose(interval["interval_width"]["value"], upper - lower, rel_tol=1e-12, abs_tol=1e-12): _fail("$.derived_metrics.closure_exposure", "interval width disagrees with endpoints") for name, scenario in payload["simulations"].items(): parameters = scenario["parameters"] diff --git a/src/recursive_integrity_toolkit/utils/logging.py b/src/recursive_integrity_toolkit/utils/logging.py index 4b6ad48..a865561 100644 --- a/src/recursive_integrity_toolkit/utils/logging.py +++ b/src/recursive_integrity_toolkit/utils/logging.py @@ -50,6 +50,7 @@ "E_PARENT_AMBIGUOUS": ("A parent reference is ambiguous.", "Supply an explicit dataset-version and record-ID pair."), "E_PARENT_FUTURE_VERSION": ("A parent reference conflicts with declared version order.", "Check the parent reference against the declared version order."), "E_LINEAGE_CYCLE": ("Declared parent references contain a validated cycle.", "Correct the parent declarations before requesting dependent work."), + "E_LINEAGE_RESOURCE_LIMIT_EXCEEDED": ("Lineage analysis reached a configured resource limit.", "Review the reported work counters and configured limit before reducing the input scope or explicitly adjusting that limit."), "E_VERSION_ORDER_CONFLICT": ("Version-order declarations conflict.", "Supply one consistent explicit version order."), "E_REPRESENTATION_INCOMPATIBLE": ("The supplied representations are incompatible.", "Supply compatible explicit representation declarations."), "E_MAPPING_SOURCE_FIELD_MISSING": ("A declared mapping source field is missing.", "Correct the explicit mapping or supply its source field."), diff --git a/tests/golden/phase4_hero_redacted.json b/tests/golden/phase4_hero_redacted.json index 5ca0a7d..b705537 100644 --- a/tests/golden/phase4_hero_redacted.json +++ b/tests/golden/phase4_hero_redacted.json @@ -33,7 +33,7 @@ "python_version": null, "random_seed": null, "redacted_mode": true, - "report_schema_version": "1.0", + "report_schema_version": "1.1", "resolved_options": { "comparison_requested": true, "privacy_mode": "redacted", @@ -47,7 +47,7 @@ "run_status": "complete", "started_at": "2000-01-01T00:00:00+00:00", "strict_mode": false, - "toolkit_version": "0.1.0.dev2" + "toolkit_version": "0.1.0.dev3" }, "inputs": { "artifacts": [ @@ -401,10 +401,10 @@ }, "coverage_reason": null, "execution_reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "execution_scope": [], - "execution_status": "deferred", + "execution_status": "not_requested", "notes": [ "Input eligibility only; downstream analytical implementations remain deferred.", "Reference-entry resolution coverage is not ancestry or external-root coverage.", @@ -486,7 +486,7 @@ "limitations": [ "R_MODEL_EVIDENCE_MISSING", "R_SCENARIO_NOT_CONFIGURED", - "Input eligibility is distinct from executed analysis; Phase 5 lineage remains deferred." + "Input eligibility is distinct from explicitly supplied executed analysis." ], "maximum_level": 4, "partial_evidence": [ @@ -607,10 +607,10 @@ }, "coverage_reason": null, "execution_reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "execution_scope": [], - "execution_status": "deferred", + "execution_status": "not_requested", "notes": [ "Input eligibility only; downstream analytical implementations remain deferred.", "Reference-entry resolution coverage is not ancestry or external-root coverage.", @@ -698,18 +698,18 @@ "denominator_reason": "denominator_not_applicable_or_unavailable", "evidence_class": "observed_fact", "limitations": [ - "No general graph-cycle traversal executes in Phase 4." + "No executed graph-cycle result was supplied to this report." ], "method_id": "T6.graph_cycle_check", "owner_ids": [ "T6" ], "reason_codes": [ - "R_GRAPH_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "representation": null, "required_evidence": [ - "Phase_5_graph_analysis" + "explicit_lineage_result" ], "scope": { "dataset_versions": [ @@ -5650,7 +5650,7 @@ { "assumptions": [], "blocking_evidence": [ - "Phase 4 does not compute ancestry; topology alone would not identify a causal contribution." + "Supplied parent topology and fractional root allocation do not identify a causal contribution." ], "conclusion": "causal_ancestor_effect", "coverage": null, @@ -5666,8 +5666,7 @@ "T4" ], "reason_codes": [ - "R_CAUSAL_EVIDENCE_MISSING", - "R_LINEAGE_EXECUTION_DEFERRED" + "R_CAUSAL_EVIDENCE_MISSING" ], "related_capability": "lineage", "representation": null, @@ -6222,7 +6221,7 @@ { "assumptions": [], "blocking_evidence": [ - "Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes." + "No complete successful lineage execution was supplied; independently completed observations remain visible." ], "conclusion": "lineage_analysis", "coverage": null, @@ -6238,15 +6237,15 @@ "PR-014" ], "reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "related_capability": "lineage", "representation": null, "required_evidence": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Supply an explicit lineage result for the validated target and its loaded context." ], "required_next_metadata": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Supply an explicit lineage result for the validated target and its loaded context." ], "scope": { "dataset_versions": [ @@ -6325,19 +6324,19 @@ "record_count": 16, "scope_id": "hmac-sha256:fe3080b69d042a9eca2d392f8429412803f06dd3eb9e71fd16178d05e5971ac3" }, - "statement": "General lineage graph analysis is deferred to Phase 5.", + "statement": "A completed whole-target lineage analysis is unavailable in this report.", "status": "unavailable", "theory_map_ids": [], - "theory_or_product_limit": "Implementation is deferred even when the existing input classifier marks lineage available.", + "theory_or_product_limit": "Input eligibility alone does not establish executed ancestry analysis.", "trace_ids": [], "unit": "conclusion" }, { "assumptions": [], "blocking_evidence": [ - "Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes." + "Missing, invalid or unknown ancestry remains unresolved; any complete subset retains its reported coverage." ], - "conclusion": "lineage_closure_exposure", + "conclusion": "external_ancestry", "coverage": null, "coverage_reason": "This conclusion has no measured evidence coverage.", "denominator": null, @@ -6346,20 +6345,20 @@ "limitations": [ "Unavailable means the supplied evidence does not support this conclusion; it does not establish that the conclusion is false." ], - "method_id": "T3.unavailable_conclusion", + "method_id": "T4.unavailable_conclusion", "owner_ids": [ - "T3" + "T4" ], "reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "related_capability": "lineage", "representation": null, "required_evidence": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Complete the required parent references, chronology and external-grounding evidence." ], "required_next_metadata": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Complete the required parent references, chronology and external-grounding evidence." ], "scope": { "dataset_versions": [ @@ -6438,21 +6437,21 @@ "record_count": 16, "scope_id": "hmac-sha256:fe3080b69d042a9eca2d392f8429412803f06dd3eb9e71fd16178d05e5971ac3" }, - "statement": "Lineage closure exposure is deferred to Phase 5.", + "statement": "Complete external ancestry for every target is unavailable.", "status": "unavailable", "theory_map_ids": [], - "theory_or_product_limit": "Implementation is deferred even when the existing input classifier marks lineage available.", + "theory_or_product_limit": "Partial ancestry cannot be promoted to whole-target completeness.", "trace_ids": [ - "T3" + "T4" ], "unit": "conclusion" }, { "assumptions": [], "blocking_evidence": [ - "Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes." + "No available explicitly supplied lineage interval covers a nonempty completed target partition." ], - "conclusion": "external_ancestry", + "conclusion": "lineage_closure_exposure", "coverage": null, "coverage_reason": "This conclusion has no measured evidence coverage.", "denominator": null, @@ -6461,20 +6460,20 @@ "limitations": [ "Unavailable means the supplied evidence does not support this conclusion; it does not establish that the conclusion is false." ], - "method_id": "T4.unavailable_conclusion", + "method_id": "T3.unavailable_conclusion", "owner_ids": [ - "T4" + "T3" ], "reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "related_capability": "lineage", "representation": null, "required_evidence": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Supply lineage closure bounds from the same explicit lineage result." ], "required_next_metadata": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Supply lineage closure bounds from the same explicit lineage result." ], "scope": { "dataset_versions": [ @@ -6553,12 +6552,12 @@ "record_count": 16, "scope_id": "hmac-sha256:fe3080b69d042a9eca2d392f8429412803f06dd3eb9e71fd16178d05e5971ac3" }, - "statement": "External ancestry results are deferred to Phase 5.", + "statement": "A lineage closure interval is unavailable in this report.", "status": "unavailable", "theory_map_ids": [], - "theory_or_product_limit": "Implementation is deferred even when the existing input classifier marks lineage available.", + "theory_or_product_limit": "Unavailable intervals are never replaced by a guessed numeric value.", "trace_ids": [ - "T4" + "T3" ], "unit": "conclusion" } diff --git a/tests/golden/phase4_hero_redacted.md b/tests/golden/phase4_hero_redacted.md index ff78d13..7f5222c 100644 --- a/tests/golden/phase4_hero_redacted.md +++ b/tests/golden/phase4_hero_redacted.md @@ -4,7 +4,7 @@ - Run ID: `"phase4-step9-golden"`. - Maximum observability level: 4; `"longitudinal_dataset_observability"`. -- Capability statuses: Ingestion: `"available"` (execution `"completed"`); Content diagnostics: `"available"` (execution `"completed"`); Provenance: `"available"` (execution `"completed"`); Lineage: `"available"` (execution `"deferred"`); Dataset longitudinal: `"available"` (execution `"partial"`); Model longitudinal: `"unavailable"` (execution `"deferred"`); Intervention simulation: `"unavailable"` (execution `"not_requested"`). +- Capability statuses: Ingestion: `"available"` (execution `"completed"`); Content diagnostics: `"available"` (execution `"completed"`); Provenance: `"available"` (execution `"completed"`); Lineage: `"available"` (execution `"not_requested"`); Dataset longitudinal: `"available"` (execution `"partial"`); Model longitudinal: `"unavailable"` (execution `"deferred"`); Intervention simulation: `"unavailable"` (execution `"not_requested"`). - Record scope: `"hmac-sha256:fe3080b69d042a9eca2d392f8429412803f06dd3eb9e71fd16178d05e5971ac3"`; dataset versions `["hmac-sha256:4e02dbb7e57fd003c1b4a2d16e058bca5c848e94f5b8730261ad64c1175dfef4", "hmac-sha256:70696b56c6ef6dbd4145752d88c4da6f71498a8f35a521e3946d89acc2a56a5d"]`; records 16; excluded records 0. - Representation: `"hmac-sha256:220b787af129ef50eab4ee9f3c90a00a7bce0a525c6e49bfc40fe9bb8af1d0b8"`; source `"hmac-sha256:1faa75c58dd3f727bec66d81de5cab7c2e64afbebcd03c32653ba5432a5c4369"`; version `"hmac-sha256:b6d6724490a6778f4b6e234f43fff3602f2707d6b9ab74acb0cc8e83c96d3da8"`. Complete mapping metadata appears in Input inventory. - Warning entries: 0; error entries: 0. @@ -37,7 +37,7 @@ Quoted code literals represent supplied data, including identifiers and labels. | `["python_version"]` | Unavailable (null): `"private_or_nonstandard_metadata_omitted"` | | `["random_seed"]` | Unavailable (null): `"not_recorded"` | | `["redacted_mode"]` | true | -| `["report_schema_version"]` | `"1.0"` | +| `["report_schema_version"]` | `"1.1"` | | `["resolved_options"]["comparison_requested"]` | true | | `["resolved_options"]["privacy_mode"]` | `"redacted"` | | `["resolved_options"]["record_id_mode"]` | `"hash"` | @@ -49,7 +49,7 @@ Quoted code literals represent supplied data, including identifiers and labels. | `["run_status"]` | `"complete"` | | `["started_at"]` | `"2000-01-01T00:00:00+00:00"` | | `["strict_mode"]` | false | -| `["toolkit_version"]` | `"0.1.0.dev2"` | +| `["toolkit_version"]` | `"0.1.0.dev3"` | **`["identifier_protection"]["limitations"]`** @@ -193,7 +193,7 @@ Quoted code literals represent supplied data, including identifiers and labels. - `"R_MODEL_EVIDENCE_MISSING"` - `"R_SCENARIO_NOT_CONFIGURED"` -- `"Input eligibility is distinct from executed analysis; Phase 5 lineage remains deferred."` +- `"Input eligibility is distinct from explicitly supplied executed analysis."` The equal observability.capabilities compatibility mirror is represented once in the Capability matrix section. @@ -204,7 +204,7 @@ The equal observability.capabilities compatibility mirror is represented once in | Ingestion | `"available"` | 1.0 | `[]` | `"completed"` | `[]` | | Content diagnostics | `"available"` | 1.0 | `[]` | `"completed"` | `[]` | | Provenance | `"available"` | 1.0 | `[]` | `"completed"` | `[]` | -| Lineage | `"available"` | 1.0 | `[]` | `"deferred"` | `["R_LINEAGE_EXECUTION_DEFERRED"]` | +| Lineage | `"available"` | 1.0 | `[]` | `"not_requested"` | `["R_ANALYSIS_NOT_REQUESTED"]` | | Dataset longitudinal | `"available"` | 1.0 | `[]` | `"partial"` | `["R_LONGITUDINAL_FAMILIES_DEFERRED"]` | | Model longitudinal | `"unavailable"` | Unavailable (null): `"coverage_not_supplied_or_empty_denominator"` | `["R_MODEL_EVIDENCE_MISSING"]` | `"deferred"` | `["R_MODEL_ANALYSIS_DEFERRED"]` | | Intervention simulation | `"unavailable"` | Unavailable (null): `"coverage_not_supplied_or_empty_denominator"` | `["R_SCENARIO_NOT_CONFIGURED"]` | `"not_requested"` | `["R_ANALYSIS_NOT_REQUESTED"]` | @@ -323,7 +323,7 @@ Coverage details and unresolved counts, when supplied, follow below. Related dia Status: `"unavailable"`; evidence class: `"observed_fact"`; unit: `"status"`; method: `"T6.graph_cycle_check"`. -Value: Unavailable (null): reason codes `["R_GRAPH_EXECUTION_DEFERRED"]`. +Value: Unavailable (null): reason codes `["R_ANALYSIS_NOT_REQUESTED"]`. Denominator: Unavailable (null): `"denominator_not_applicable_or_unavailable"`; coverage (ratio): Unavailable (null): `"coverage_not_supplied_for_this_result"`. @@ -337,9 +337,9 @@ Denominator: Unavailable (null): `"denominator_not_applicable_or_unavailable"`; | `["evidence_class"]` | `"observed_fact"` | | `["method_id"]` | `"T6.graph_cycle_check"` | | `["owner_ids"]` | `["T6"]` | -| `["reason_codes"]` | `["R_GRAPH_EXECUTION_DEFERRED"]` | +| `["reason_codes"]` | `["R_ANALYSIS_NOT_REQUESTED"]` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | -| `["required_evidence"]` | `["Phase_5_graph_analysis"]` | +| `["required_evidence"]` | `["explicit_lineage_result"]` | | `["scope"]["dataset_versions"]` | `["hmac-sha256:4e02dbb7e57fd003c1b4a2d16e058bca5c848e94f5b8730261ad64c1175dfef4", "hmac-sha256:70696b56c6ef6dbd4145752d88c4da6f71498a8f35a521e3946d89acc2a56a5d"]` | | `["scope"]["denominator_basis"]` | `"all_validated_bundle_records"` | | `["scope"]["excluded_record_count"]` | 0 | @@ -350,11 +350,11 @@ Denominator: Unavailable (null): `"denominator_not_applicable_or_unavailable"`; | `["theory_map_ids"]` | `[]` | | `["trace_ids"]` | `["T6"]` | | `["unit"]` | `"status"` | -| `["value"]` | Unavailable (null): reason codes `["R_GRAPH_EXECUTION_DEFERRED"]` | +| `["value"]` | Unavailable (null): reason codes `["R_ANALYSIS_NOT_REQUESTED"]` | **`["limitations"]`** -- `"No general graph-cycle traversal executes in Phase 4."` +- `"No executed graph-cycle result was supplied to this report."` **`["scope"]["included_record_keys"]`** @@ -3763,7 +3763,7 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; | Field | Value | |---|---| | `["assumptions"]` | `[]` | -| `["blocking_evidence"]` | `["Phase 4 does not compute ancestry; topology alone would not identify a causal contribution."]` | +| `["blocking_evidence"]` | `["Supplied parent topology and fractional root allocation do not identify a causal contribution."]` | | `["conclusion"]` | `"causal_ancestor_effect"` | | `["coverage"]` | Unavailable (null): `"This conclusion has no measured evidence coverage."` | | `["coverage_reason"]` | `"This conclusion has no measured evidence coverage."` | @@ -3772,7 +3772,7 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; | `["evidence_class"]` | `"unavailable_conclusion"` | | `["method_id"]` | `"T4.unavailable_conclusion"` | | `["owner_ids"]` | `["T4"]` | -| `["reason_codes"]` | `["R_CAUSAL_EVIDENCE_MISSING", "R_LINEAGE_EXECUTION_DEFERRED"]` | +| `["reason_codes"]` | `["R_CAUSAL_EVIDENCE_MISSING"]` | | `["related_capability"]` | `"lineage"` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | | `["required_evidence"]` | `["An identified ancestor, measured outcomes and a controlled or otherwise justified causal design."]` | @@ -4078,7 +4078,7 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; | Field | Value | |---|---| | `["assumptions"]` | `[]` | -| `["blocking_evidence"]` | `["Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes."]` | +| `["blocking_evidence"]` | `["No complete successful lineage execution was supplied; independently completed observations remain visible."]` | | `["conclusion"]` | `"lineage_analysis"` | | `["coverage"]` | Unavailable (null): `"This conclusion has no measured evidence coverage."` | | `["coverage_reason"]` | `"This conclusion has no measured evidence coverage."` | @@ -4087,21 +4087,21 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; | `["evidence_class"]` | `"unavailable_conclusion"` | | `["method_id"]` | `"PR-014.unavailable_conclusion"` | | `["owner_ids"]` | `["PR-014"]` | -| `["reason_codes"]` | `["R_LINEAGE_EXECUTION_DEFERRED"]` | +| `["reason_codes"]` | `["R_ANALYSIS_NOT_REQUESTED"]` | | `["related_capability"]` | `"lineage"` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | -| `["required_evidence"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | -| `["required_next_metadata"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | +| `["required_evidence"]` | `["Supply an explicit lineage result for the validated target and its loaded context."]` | +| `["required_next_metadata"]` | `["Supply an explicit lineage result for the validated target and its loaded context."]` | | `["scope"]["dataset_versions"]` | `["hmac-sha256:4e02dbb7e57fd003c1b4a2d16e058bca5c848e94f5b8730261ad64c1175dfef4", "hmac-sha256:70696b56c6ef6dbd4145752d88c4da6f71498a8f35a521e3946d89acc2a56a5d"]` | | `["scope"]["denominator_basis"]` | `"all_validated_bundle_records"` | | `["scope"]["excluded_record_count"]` | 0 | | `["scope"]["excluded_record_keys"]` | `[]` | | `["scope"]["record_count"]` | 16 | | `["scope"]["scope_id"]` | `"hmac-sha256:fe3080b69d042a9eca2d392f8429412803f06dd3eb9e71fd16178d05e5971ac3"` | -| `["statement"]` | `"General lineage graph analysis is deferred to Phase 5."` | +| `["statement"]` | `"A completed whole-target lineage analysis is unavailable in this report."` | | `["status"]` | `"unavailable"` | | `["theory_map_ids"]` | `[]` | -| `["theory_or_product_limit"]` | `"Implementation is deferred even when the existing input classifier marks lineage available."` | +| `["theory_or_product_limit"]` | `"Input eligibility alone does not establish executed ancestry analysis."` | | `["trace_ids"]` | `[]` | | `["unit"]` | `"conclusion"` | @@ -4134,38 +4134,38 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; `["unavailable_conclusions"][7]` -Status: `"unavailable"`; evidence class: `"unavailable_conclusion"`; unit: `"conclusion"`; method: `"T3.unavailable_conclusion"`. +Status: `"unavailable"`; evidence class: `"unavailable_conclusion"`; unit: `"conclusion"`; method: `"T4.unavailable_conclusion"`. Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; coverage (ratio): Unavailable (null): `"This conclusion has no measured evidence coverage."`. | Field | Value | |---|---| | `["assumptions"]` | `[]` | -| `["blocking_evidence"]` | `["Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes."]` | -| `["conclusion"]` | `"lineage_closure_exposure"` | +| `["blocking_evidence"]` | `["Missing, invalid or unknown ancestry remains unresolved; any complete subset retains its reported coverage."]` | +| `["conclusion"]` | `"external_ancestry"` | | `["coverage"]` | Unavailable (null): `"This conclusion has no measured evidence coverage."` | | `["coverage_reason"]` | `"This conclusion has no measured evidence coverage."` | | `["denominator"]` | Unavailable (null): `"This conclusion has no scalar denominator."` | | `["denominator_reason"]` | `"This conclusion has no scalar denominator."` | | `["evidence_class"]` | `"unavailable_conclusion"` | -| `["method_id"]` | `"T3.unavailable_conclusion"` | -| `["owner_ids"]` | `["T3"]` | -| `["reason_codes"]` | `["R_LINEAGE_EXECUTION_DEFERRED"]` | +| `["method_id"]` | `"T4.unavailable_conclusion"` | +| `["owner_ids"]` | `["T4"]` | +| `["reason_codes"]` | `["R_ANALYSIS_NOT_REQUESTED"]` | | `["related_capability"]` | `"lineage"` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | -| `["required_evidence"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | -| `["required_next_metadata"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | +| `["required_evidence"]` | `["Complete the required parent references, chronology and external-grounding evidence."]` | +| `["required_next_metadata"]` | `["Complete the required parent references, chronology and external-grounding evidence."]` | | `["scope"]["dataset_versions"]` | `["hmac-sha256:4e02dbb7e57fd003c1b4a2d16e058bca5c848e94f5b8730261ad64c1175dfef4", "hmac-sha256:70696b56c6ef6dbd4145752d88c4da6f71498a8f35a521e3946d89acc2a56a5d"]` | | `["scope"]["denominator_basis"]` | `"all_validated_bundle_records"` | | `["scope"]["excluded_record_count"]` | 0 | | `["scope"]["excluded_record_keys"]` | `[]` | | `["scope"]["record_count"]` | 16 | | `["scope"]["scope_id"]` | `"hmac-sha256:fe3080b69d042a9eca2d392f8429412803f06dd3eb9e71fd16178d05e5971ac3"` | -| `["statement"]` | `"Lineage closure exposure is deferred to Phase 5."` | +| `["statement"]` | `"Complete external ancestry for every target is unavailable."` | | `["status"]` | `"unavailable"` | | `["theory_map_ids"]` | `[]` | -| `["theory_or_product_limit"]` | `"Implementation is deferred even when the existing input classifier marks lineage available."` | -| `["trace_ids"]` | `["T3"]` | +| `["theory_or_product_limit"]` | `"Partial ancestry cannot be promoted to whole-target completeness."` | +| `["trace_ids"]` | `["T4"]` | | `["unit"]` | `"conclusion"` | **`["limitations"]`** @@ -4197,38 +4197,38 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; `["unavailable_conclusions"][8]` -Status: `"unavailable"`; evidence class: `"unavailable_conclusion"`; unit: `"conclusion"`; method: `"T4.unavailable_conclusion"`. +Status: `"unavailable"`; evidence class: `"unavailable_conclusion"`; unit: `"conclusion"`; method: `"T3.unavailable_conclusion"`. Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; coverage (ratio): Unavailable (null): `"This conclusion has no measured evidence coverage."`. | Field | Value | |---|---| | `["assumptions"]` | `[]` | -| `["blocking_evidence"]` | `["Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes."]` | -| `["conclusion"]` | `"external_ancestry"` | +| `["blocking_evidence"]` | `["No available explicitly supplied lineage interval covers a nonempty completed target partition."]` | +| `["conclusion"]` | `"lineage_closure_exposure"` | | `["coverage"]` | Unavailable (null): `"This conclusion has no measured evidence coverage."` | | `["coverage_reason"]` | `"This conclusion has no measured evidence coverage."` | | `["denominator"]` | Unavailable (null): `"This conclusion has no scalar denominator."` | | `["denominator_reason"]` | `"This conclusion has no scalar denominator."` | | `["evidence_class"]` | `"unavailable_conclusion"` | -| `["method_id"]` | `"T4.unavailable_conclusion"` | -| `["owner_ids"]` | `["T4"]` | -| `["reason_codes"]` | `["R_LINEAGE_EXECUTION_DEFERRED"]` | +| `["method_id"]` | `"T3.unavailable_conclusion"` | +| `["owner_ids"]` | `["T3"]` | +| `["reason_codes"]` | `["R_ANALYSIS_NOT_REQUESTED"]` | | `["related_capability"]` | `"lineage"` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | -| `["required_evidence"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | -| `["required_next_metadata"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | +| `["required_evidence"]` | `["Supply lineage closure bounds from the same explicit lineage result."]` | +| `["required_next_metadata"]` | `["Supply lineage closure bounds from the same explicit lineage result."]` | | `["scope"]["dataset_versions"]` | `["hmac-sha256:4e02dbb7e57fd003c1b4a2d16e058bca5c848e94f5b8730261ad64c1175dfef4", "hmac-sha256:70696b56c6ef6dbd4145752d88c4da6f71498a8f35a521e3946d89acc2a56a5d"]` | | `["scope"]["denominator_basis"]` | `"all_validated_bundle_records"` | | `["scope"]["excluded_record_count"]` | 0 | | `["scope"]["excluded_record_keys"]` | `[]` | | `["scope"]["record_count"]` | 16 | | `["scope"]["scope_id"]` | `"hmac-sha256:fe3080b69d042a9eca2d392f8429412803f06dd3eb9e71fd16178d05e5971ac3"` | -| `["statement"]` | `"External ancestry results are deferred to Phase 5."` | +| `["statement"]` | `"A lineage closure interval is unavailable in this report."` | | `["status"]` | `"unavailable"` | | `["theory_map_ids"]` | `[]` | -| `["theory_or_product_limit"]` | `"Implementation is deferred even when the existing input classifier marks lineage available."` | -| `["trace_ids"]` | `["T4"]` | +| `["theory_or_product_limit"]` | `"Unavailable intervals are never replaced by a guessed numeric value."` | +| `["trace_ids"]` | `["T3"]` | | `["unit"]` | `"conclusion"` | **`["limitations"]`** @@ -4281,7 +4281,7 @@ No entries supplied. Empty list: `[]`. **Report footer** -Toolkit version: `"0.1.0.dev2"`; report schema version: `"1.0"`. +Toolkit version: `"0.1.0.dev3"`; report schema version: `"1.1"`. Evidence-class legend: diff --git a/tests/golden/phase4_hero_report.json b/tests/golden/phase4_hero_report.json index e0af097..09cc0e5 100644 --- a/tests/golden/phase4_hero_report.json +++ b/tests/golden/phase4_hero_report.json @@ -34,7 +34,7 @@ "python_version": null, "random_seed": null, "redacted_mode": false, - "report_schema_version": "1.0", + "report_schema_version": "1.1", "resolved_options": { "comparison_requested": true, "privacy_mode": "standard", @@ -48,7 +48,7 @@ "run_status": "complete", "started_at": "2000-01-01T00:00:00+00:00", "strict_mode": false, - "toolkit_version": "0.1.0.dev2" + "toolkit_version": "0.1.0.dev3" }, "inputs": { "artifacts": [ @@ -402,10 +402,10 @@ }, "coverage_reason": null, "execution_reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "execution_scope": [], - "execution_status": "deferred", + "execution_status": "not_requested", "notes": [ "Input eligibility only; downstream analytical implementations remain deferred.", "Reference-entry resolution coverage is not ancestry or external-root coverage.", @@ -487,7 +487,7 @@ "limitations": [ "R_MODEL_EVIDENCE_MISSING", "R_SCENARIO_NOT_CONFIGURED", - "Input eligibility is distinct from executed analysis; Phase 5 lineage remains deferred." + "Input eligibility is distinct from explicitly supplied executed analysis." ], "maximum_level": 4, "partial_evidence": [ @@ -608,10 +608,10 @@ }, "coverage_reason": null, "execution_reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "execution_scope": [], - "execution_status": "deferred", + "execution_status": "not_requested", "notes": [ "Input eligibility only; downstream analytical implementations remain deferred.", "Reference-entry resolution coverage is not ancestry or external-root coverage.", @@ -699,18 +699,18 @@ "denominator_reason": "denominator_not_applicable_or_unavailable", "evidence_class": "observed_fact", "limitations": [ - "No general graph-cycle traversal executes in Phase 4." + "No executed graph-cycle result was supplied to this report." ], "method_id": "T6.graph_cycle_check", "owner_ids": [ "T6" ], "reason_codes": [ - "R_GRAPH_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "representation": null, "required_evidence": [ - "Phase_5_graph_analysis" + "explicit_lineage_result" ], "scope": { "dataset_versions": [ @@ -5651,7 +5651,7 @@ { "assumptions": [], "blocking_evidence": [ - "Phase 4 does not compute ancestry; topology alone would not identify a causal contribution." + "Supplied parent topology and fractional root allocation do not identify a causal contribution." ], "conclusion": "causal_ancestor_effect", "coverage": null, @@ -5667,8 +5667,7 @@ "T4" ], "reason_codes": [ - "R_CAUSAL_EVIDENCE_MISSING", - "R_LINEAGE_EXECUTION_DEFERRED" + "R_CAUSAL_EVIDENCE_MISSING" ], "related_capability": "lineage", "representation": null, @@ -6223,7 +6222,7 @@ { "assumptions": [], "blocking_evidence": [ - "Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes." + "No complete successful lineage execution was supplied; independently completed observations remain visible." ], "conclusion": "lineage_analysis", "coverage": null, @@ -6239,15 +6238,15 @@ "PR-014" ], "reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "related_capability": "lineage", "representation": null, "required_evidence": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Supply an explicit lineage result for the validated target and its loaded context." ], "required_next_metadata": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Supply an explicit lineage result for the validated target and its loaded context." ], "scope": { "dataset_versions": [ @@ -6326,19 +6325,19 @@ "record_count": 16, "scope_id": "validated_bundle" }, - "statement": "General lineage graph analysis is deferred to Phase 5.", + "statement": "A completed whole-target lineage analysis is unavailable in this report.", "status": "unavailable", "theory_map_ids": [], - "theory_or_product_limit": "Implementation is deferred even when the existing input classifier marks lineage available.", + "theory_or_product_limit": "Input eligibility alone does not establish executed ancestry analysis.", "trace_ids": [], "unit": "conclusion" }, { "assumptions": [], "blocking_evidence": [ - "Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes." + "Missing, invalid or unknown ancestry remains unresolved; any complete subset retains its reported coverage." ], - "conclusion": "lineage_closure_exposure", + "conclusion": "external_ancestry", "coverage": null, "coverage_reason": "This conclusion has no measured evidence coverage.", "denominator": null, @@ -6347,20 +6346,20 @@ "limitations": [ "Unavailable means the supplied evidence does not support this conclusion; it does not establish that the conclusion is false." ], - "method_id": "T3.unavailable_conclusion", + "method_id": "T4.unavailable_conclusion", "owner_ids": [ - "T3" + "T4" ], "reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "related_capability": "lineage", "representation": null, "required_evidence": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Complete the required parent references, chronology and external-grounding evidence." ], "required_next_metadata": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Complete the required parent references, chronology and external-grounding evidence." ], "scope": { "dataset_versions": [ @@ -6439,21 +6438,21 @@ "record_count": 16, "scope_id": "validated_bundle" }, - "statement": "Lineage closure exposure is deferred to Phase 5.", + "statement": "Complete external ancestry for every target is unavailable.", "status": "unavailable", "theory_map_ids": [], - "theory_or_product_limit": "Implementation is deferred even when the existing input classifier marks lineage available.", + "theory_or_product_limit": "Partial ancestry cannot be promoted to whole-target completeness.", "trace_ids": [ - "T3" + "T4" ], "unit": "conclusion" }, { "assumptions": [], "blocking_evidence": [ - "Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes." + "No available explicitly supplied lineage interval covers a nonempty completed target partition." ], - "conclusion": "external_ancestry", + "conclusion": "lineage_closure_exposure", "coverage": null, "coverage_reason": "This conclusion has no measured evidence coverage.", "denominator": null, @@ -6462,20 +6461,20 @@ "limitations": [ "Unavailable means the supplied evidence does not support this conclusion; it does not establish that the conclusion is false." ], - "method_id": "T4.unavailable_conclusion", + "method_id": "T3.unavailable_conclusion", "owner_ids": [ - "T4" + "T3" ], "reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "related_capability": "lineage", "representation": null, "required_evidence": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Supply lineage closure bounds from the same explicit lineage result." ], "required_next_metadata": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Supply lineage closure bounds from the same explicit lineage result." ], "scope": { "dataset_versions": [ @@ -6554,12 +6553,12 @@ "record_count": 16, "scope_id": "validated_bundle" }, - "statement": "External ancestry results are deferred to Phase 5.", + "statement": "A lineage closure interval is unavailable in this report.", "status": "unavailable", "theory_map_ids": [], - "theory_or_product_limit": "Implementation is deferred even when the existing input classifier marks lineage available.", + "theory_or_product_limit": "Unavailable intervals are never replaced by a guessed numeric value.", "trace_ids": [ - "T4" + "T3" ], "unit": "conclusion" } diff --git a/tests/golden/phase4_hero_report.md b/tests/golden/phase4_hero_report.md index cf26dfc..f4d1cc3 100644 --- a/tests/golden/phase4_hero_report.md +++ b/tests/golden/phase4_hero_report.md @@ -4,7 +4,7 @@ - Run ID: `"phase4-step9-golden"`. - Maximum observability level: 4; `"longitudinal_dataset_observability"`. -- Capability statuses: Ingestion: `"available"` (execution `"completed"`); Content diagnostics: `"available"` (execution `"completed"`); Provenance: `"available"` (execution `"completed"`); Lineage: `"available"` (execution `"deferred"`); Dataset longitudinal: `"available"` (execution `"partial"`); Model longitudinal: `"unavailable"` (execution `"deferred"`); Intervention simulation: `"unavailable"` (execution `"not_requested"`). +- Capability statuses: Ingestion: `"available"` (execution `"completed"`); Content diagnostics: `"available"` (execution `"completed"`); Provenance: `"available"` (execution `"completed"`); Lineage: `"available"` (execution `"not_requested"`); Dataset longitudinal: `"available"` (execution `"partial"`); Model longitudinal: `"unavailable"` (execution `"deferred"`); Intervention simulation: `"unavailable"` (execution `"not_requested"`). - Record scope: `"validated_bundle"`; dataset versions `["v1", "v2"]`; records 16; excluded records 0. - Representation: `"topic"`; source `"topic_field"`; version `"hero-topic-v1"`. Complete mapping metadata appears in Input inventory. - Warning entries: 0; error entries: 0. @@ -37,7 +37,7 @@ Quoted code literals represent supplied data, including identifiers and labels. | `["python_version"]` | Unavailable (null): `"private_or_nonstandard_metadata_omitted"` | | `["random_seed"]` | Unavailable (null): `"not_recorded"` | | `["redacted_mode"]` | false | -| `["report_schema_version"]` | `"1.0"` | +| `["report_schema_version"]` | `"1.1"` | | `["resolved_options"]["comparison_requested"]` | true | | `["resolved_options"]["privacy_mode"]` | `"standard"` | | `["resolved_options"]["record_id_mode"]` | `"preserve"` | @@ -49,7 +49,7 @@ Quoted code literals represent supplied data, including identifiers and labels. | `["run_status"]` | `"complete"` | | `["started_at"]` | `"2000-01-01T00:00:00+00:00"` | | `["strict_mode"]` | false | -| `["toolkit_version"]` | `"0.1.0.dev2"` | +| `["toolkit_version"]` | `"0.1.0.dev3"` | **`["identifier_protection"]["limitations"]`** @@ -194,7 +194,7 @@ Quoted code literals represent supplied data, including identifiers and labels. - `"R_MODEL_EVIDENCE_MISSING"` - `"R_SCENARIO_NOT_CONFIGURED"` -- `"Input eligibility is distinct from executed analysis; Phase 5 lineage remains deferred."` +- `"Input eligibility is distinct from explicitly supplied executed analysis."` The equal observability.capabilities compatibility mirror is represented once in the Capability matrix section. @@ -205,7 +205,7 @@ The equal observability.capabilities compatibility mirror is represented once in | Ingestion | `"available"` | 1.0 | `[]` | `"completed"` | `[]` | | Content diagnostics | `"available"` | 1.0 | `[]` | `"completed"` | `[]` | | Provenance | `"available"` | 1.0 | `[]` | `"completed"` | `[]` | -| Lineage | `"available"` | 1.0 | `[]` | `"deferred"` | `["R_LINEAGE_EXECUTION_DEFERRED"]` | +| Lineage | `"available"` | 1.0 | `[]` | `"not_requested"` | `["R_ANALYSIS_NOT_REQUESTED"]` | | Dataset longitudinal | `"available"` | 1.0 | `[]` | `"partial"` | `["R_LONGITUDINAL_FAMILIES_DEFERRED"]` | | Model longitudinal | `"unavailable"` | Unavailable (null): `"coverage_not_supplied_or_empty_denominator"` | `["R_MODEL_EVIDENCE_MISSING"]` | `"deferred"` | `["R_MODEL_ANALYSIS_DEFERRED"]` | | Intervention simulation | `"unavailable"` | Unavailable (null): `"coverage_not_supplied_or_empty_denominator"` | `["R_SCENARIO_NOT_CONFIGURED"]` | `"not_requested"` | `["R_ANALYSIS_NOT_REQUESTED"]` | @@ -324,7 +324,7 @@ Coverage details and unresolved counts, when supplied, follow below. Related dia Status: `"unavailable"`; evidence class: `"observed_fact"`; unit: `"status"`; method: `"T6.graph_cycle_check"`. -Value: Unavailable (null): reason codes `["R_GRAPH_EXECUTION_DEFERRED"]`. +Value: Unavailable (null): reason codes `["R_ANALYSIS_NOT_REQUESTED"]`. Denominator: Unavailable (null): `"denominator_not_applicable_or_unavailable"`; coverage (ratio): Unavailable (null): `"coverage_not_supplied_for_this_result"`. @@ -338,9 +338,9 @@ Denominator: Unavailable (null): `"denominator_not_applicable_or_unavailable"`; | `["evidence_class"]` | `"observed_fact"` | | `["method_id"]` | `"T6.graph_cycle_check"` | | `["owner_ids"]` | `["T6"]` | -| `["reason_codes"]` | `["R_GRAPH_EXECUTION_DEFERRED"]` | +| `["reason_codes"]` | `["R_ANALYSIS_NOT_REQUESTED"]` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | -| `["required_evidence"]` | `["Phase_5_graph_analysis"]` | +| `["required_evidence"]` | `["explicit_lineage_result"]` | | `["scope"]["dataset_versions"]` | `["v1", "v2"]` | | `["scope"]["denominator_basis"]` | `"all_validated_bundle_records"` | | `["scope"]["excluded_record_count"]` | 0 | @@ -351,11 +351,11 @@ Denominator: Unavailable (null): `"denominator_not_applicable_or_unavailable"`; | `["theory_map_ids"]` | `[]` | | `["trace_ids"]` | `["T6"]` | | `["unit"]` | `"status"` | -| `["value"]` | Unavailable (null): reason codes `["R_GRAPH_EXECUTION_DEFERRED"]` | +| `["value"]` | Unavailable (null): reason codes `["R_ANALYSIS_NOT_REQUESTED"]` | **`["limitations"]`** -- `"No general graph-cycle traversal executes in Phase 4."` +- `"No executed graph-cycle result was supplied to this report."` **`["scope"]["included_record_keys"]`** @@ -3764,7 +3764,7 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; | Field | Value | |---|---| | `["assumptions"]` | `[]` | -| `["blocking_evidence"]` | `["Phase 4 does not compute ancestry; topology alone would not identify a causal contribution."]` | +| `["blocking_evidence"]` | `["Supplied parent topology and fractional root allocation do not identify a causal contribution."]` | | `["conclusion"]` | `"causal_ancestor_effect"` | | `["coverage"]` | Unavailable (null): `"This conclusion has no measured evidence coverage."` | | `["coverage_reason"]` | `"This conclusion has no measured evidence coverage."` | @@ -3773,7 +3773,7 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; | `["evidence_class"]` | `"unavailable_conclusion"` | | `["method_id"]` | `"T4.unavailable_conclusion"` | | `["owner_ids"]` | `["T4"]` | -| `["reason_codes"]` | `["R_CAUSAL_EVIDENCE_MISSING", "R_LINEAGE_EXECUTION_DEFERRED"]` | +| `["reason_codes"]` | `["R_CAUSAL_EVIDENCE_MISSING"]` | | `["related_capability"]` | `"lineage"` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | | `["required_evidence"]` | `["An identified ancestor, measured outcomes and a controlled or otherwise justified causal design."]` | @@ -4079,7 +4079,7 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; | Field | Value | |---|---| | `["assumptions"]` | `[]` | -| `["blocking_evidence"]` | `["Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes."]` | +| `["blocking_evidence"]` | `["No complete successful lineage execution was supplied; independently completed observations remain visible."]` | | `["conclusion"]` | `"lineage_analysis"` | | `["coverage"]` | Unavailable (null): `"This conclusion has no measured evidence coverage."` | | `["coverage_reason"]` | `"This conclusion has no measured evidence coverage."` | @@ -4088,21 +4088,21 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; | `["evidence_class"]` | `"unavailable_conclusion"` | | `["method_id"]` | `"PR-014.unavailable_conclusion"` | | `["owner_ids"]` | `["PR-014"]` | -| `["reason_codes"]` | `["R_LINEAGE_EXECUTION_DEFERRED"]` | +| `["reason_codes"]` | `["R_ANALYSIS_NOT_REQUESTED"]` | | `["related_capability"]` | `"lineage"` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | -| `["required_evidence"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | -| `["required_next_metadata"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | +| `["required_evidence"]` | `["Supply an explicit lineage result for the validated target and its loaded context."]` | +| `["required_next_metadata"]` | `["Supply an explicit lineage result for the validated target and its loaded context."]` | | `["scope"]["dataset_versions"]` | `["v1", "v2"]` | | `["scope"]["denominator_basis"]` | `"all_validated_bundle_records"` | | `["scope"]["excluded_record_count"]` | 0 | | `["scope"]["excluded_record_keys"]` | `[]` | | `["scope"]["record_count"]` | 16 | | `["scope"]["scope_id"]` | `"validated_bundle"` | -| `["statement"]` | `"General lineage graph analysis is deferred to Phase 5."` | +| `["statement"]` | `"A completed whole-target lineage analysis is unavailable in this report."` | | `["status"]` | `"unavailable"` | | `["theory_map_ids"]` | `[]` | -| `["theory_or_product_limit"]` | `"Implementation is deferred even when the existing input classifier marks lineage available."` | +| `["theory_or_product_limit"]` | `"Input eligibility alone does not establish executed ancestry analysis."` | | `["trace_ids"]` | `[]` | | `["unit"]` | `"conclusion"` | @@ -4135,38 +4135,38 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; `["unavailable_conclusions"][7]` -Status: `"unavailable"`; evidence class: `"unavailable_conclusion"`; unit: `"conclusion"`; method: `"T3.unavailable_conclusion"`. +Status: `"unavailable"`; evidence class: `"unavailable_conclusion"`; unit: `"conclusion"`; method: `"T4.unavailable_conclusion"`. Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; coverage (ratio): Unavailable (null): `"This conclusion has no measured evidence coverage."`. | Field | Value | |---|---| | `["assumptions"]` | `[]` | -| `["blocking_evidence"]` | `["Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes."]` | -| `["conclusion"]` | `"lineage_closure_exposure"` | +| `["blocking_evidence"]` | `["Missing, invalid or unknown ancestry remains unresolved; any complete subset retains its reported coverage."]` | +| `["conclusion"]` | `"external_ancestry"` | | `["coverage"]` | Unavailable (null): `"This conclusion has no measured evidence coverage."` | | `["coverage_reason"]` | `"This conclusion has no measured evidence coverage."` | | `["denominator"]` | Unavailable (null): `"This conclusion has no scalar denominator."` | | `["denominator_reason"]` | `"This conclusion has no scalar denominator."` | | `["evidence_class"]` | `"unavailable_conclusion"` | -| `["method_id"]` | `"T3.unavailable_conclusion"` | -| `["owner_ids"]` | `["T3"]` | -| `["reason_codes"]` | `["R_LINEAGE_EXECUTION_DEFERRED"]` | +| `["method_id"]` | `"T4.unavailable_conclusion"` | +| `["owner_ids"]` | `["T4"]` | +| `["reason_codes"]` | `["R_ANALYSIS_NOT_REQUESTED"]` | | `["related_capability"]` | `"lineage"` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | -| `["required_evidence"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | -| `["required_next_metadata"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | +| `["required_evidence"]` | `["Complete the required parent references, chronology and external-grounding evidence."]` | +| `["required_next_metadata"]` | `["Complete the required parent references, chronology and external-grounding evidence."]` | | `["scope"]["dataset_versions"]` | `["v1", "v2"]` | | `["scope"]["denominator_basis"]` | `"all_validated_bundle_records"` | | `["scope"]["excluded_record_count"]` | 0 | | `["scope"]["excluded_record_keys"]` | `[]` | | `["scope"]["record_count"]` | 16 | | `["scope"]["scope_id"]` | `"validated_bundle"` | -| `["statement"]` | `"Lineage closure exposure is deferred to Phase 5."` | +| `["statement"]` | `"Complete external ancestry for every target is unavailable."` | | `["status"]` | `"unavailable"` | | `["theory_map_ids"]` | `[]` | -| `["theory_or_product_limit"]` | `"Implementation is deferred even when the existing input classifier marks lineage available."` | -| `["trace_ids"]` | `["T3"]` | +| `["theory_or_product_limit"]` | `"Partial ancestry cannot be promoted to whole-target completeness."` | +| `["trace_ids"]` | `["T4"]` | | `["unit"]` | `"conclusion"` | **`["limitations"]`** @@ -4198,38 +4198,38 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; `["unavailable_conclusions"][8]` -Status: `"unavailable"`; evidence class: `"unavailable_conclusion"`; unit: `"conclusion"`; method: `"T4.unavailable_conclusion"`. +Status: `"unavailable"`; evidence class: `"unavailable_conclusion"`; unit: `"conclusion"`; method: `"T3.unavailable_conclusion"`. Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; coverage (ratio): Unavailable (null): `"This conclusion has no measured evidence coverage."`. | Field | Value | |---|---| | `["assumptions"]` | `[]` | -| `["blocking_evidence"]` | `["Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes."]` | -| `["conclusion"]` | `"external_ancestry"` | +| `["blocking_evidence"]` | `["No available explicitly supplied lineage interval covers a nonempty completed target partition."]` | +| `["conclusion"]` | `"lineage_closure_exposure"` | | `["coverage"]` | Unavailable (null): `"This conclusion has no measured evidence coverage."` | | `["coverage_reason"]` | `"This conclusion has no measured evidence coverage."` | | `["denominator"]` | Unavailable (null): `"This conclusion has no scalar denominator."` | | `["denominator_reason"]` | `"This conclusion has no scalar denominator."` | | `["evidence_class"]` | `"unavailable_conclusion"` | -| `["method_id"]` | `"T4.unavailable_conclusion"` | -| `["owner_ids"]` | `["T4"]` | -| `["reason_codes"]` | `["R_LINEAGE_EXECUTION_DEFERRED"]` | +| `["method_id"]` | `"T3.unavailable_conclusion"` | +| `["owner_ids"]` | `["T3"]` | +| `["reason_codes"]` | `["R_ANALYSIS_NOT_REQUESTED"]` | | `["related_capability"]` | `"lineage"` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | -| `["required_evidence"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | -| `["required_next_metadata"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | +| `["required_evidence"]` | `["Supply lineage closure bounds from the same explicit lineage result."]` | +| `["required_next_metadata"]` | `["Supply lineage closure bounds from the same explicit lineage result."]` | | `["scope"]["dataset_versions"]` | `["v1", "v2"]` | | `["scope"]["denominator_basis"]` | `"all_validated_bundle_records"` | | `["scope"]["excluded_record_count"]` | 0 | | `["scope"]["excluded_record_keys"]` | `[]` | | `["scope"]["record_count"]` | 16 | | `["scope"]["scope_id"]` | `"validated_bundle"` | -| `["statement"]` | `"External ancestry results are deferred to Phase 5."` | +| `["statement"]` | `"A lineage closure interval is unavailable in this report."` | | `["status"]` | `"unavailable"` | | `["theory_map_ids"]` | `[]` | -| `["theory_or_product_limit"]` | `"Implementation is deferred even when the existing input classifier marks lineage available."` | -| `["trace_ids"]` | `["T4"]` | +| `["theory_or_product_limit"]` | `"Unavailable intervals are never replaced by a guessed numeric value."` | +| `["trace_ids"]` | `["T3"]` | | `["unit"]` | `"conclusion"` | **`["limitations"]`** @@ -4282,7 +4282,7 @@ No entries supplied. Empty list: `[]`. **Report footer** -Toolkit version: `"0.1.0.dev2"`; report schema version: `"1.0"`. +Toolkit version: `"0.1.0.dev3"`; report schema version: `"1.1"`. Evidence-class legend: diff --git a/tests/golden/phase4_report_cases.md b/tests/golden/phase4_report_cases.md index 65888ac..c05b15a 100644 --- a/tests/golden/phase4_report_cases.md +++ b/tests/golden/phase4_report_cases.md @@ -61,7 +61,7 @@ observations may state 8 declared, 8 resolved and 0 unresolved references and th sufficient declared-earlier-version ordering certificate. They cannot claim graph traversal or a general cycle verdict. -Maximum input observability stays Level 4. Lineage execution remains deferred; +Maximum input observability stays Level 4. Lineage execution is not requested; dataset-longitudinal execution is partial with the sole explicit-pair support and diversity operation identified. The successful run remains complete. Model longitudinal is unavailable/deferred, default intervention simulation is @@ -161,7 +161,7 @@ Earlier minus later is battery,lizard,turtle; intersection contains five states. The selected v2 scope has eight matching valid provenance rows: four human/yes and four synthetic/no. Each coverage is 8/8; each nonzero source share is 4/8. Direct interval=(closed/N,(closed+unresolved)/N)=[4/8,4/8]. -All eight supplied v2 parent references resolve to explicitly earlier v1 records. This supports reference counts and a declared-order certificate only. Graph traversal and all ancestry metrics remain deferred. +All eight supplied v2 parent references resolve to explicitly earlier v1 records. This supports reference counts and a declared-order certificate only. This CLI invocation does not request graph traversal or ancestry metrics. ### hero_redacted @@ -281,9 +281,9 @@ Required retained errors: `E_CONFIG_INVALID`. Driver: `cli`. Expected exit: `0`. -Of two later parent references one resolves and one is absent: 2 declared,1 resolved,1 unresolved. Missing references retain a warning and cannot become external roots. One valid resolved path and one unresolved reference give partial input capability under the frozen lineage classification; execution remains deferred. +Of two later parent references one resolves and one is absent: 2 declared,1 resolved,1 unresolved. Missing references retain a warning and cannot become external roots. One valid resolved path and one unresolved reference give partial input capability under the frozen lineage classification; execution is not requested. -Independent ordered supports {A,B} and {A,C} both have size2, intersection1, retention1/2; lineage execution remains deferred. +Independent ordered supports {A,B} and {A,C} both have size2, intersection1, retention1/2; lineage execution is not requested. Required retained warnings: `W_PARENT_UNRESOLVED`. @@ -398,3 +398,11 @@ Acceptance requires every case, unchanged original math cases, literal JSON and Markdown review, normalization mutation rejection, privacy/format/reordering metamorphic checks and the governing regression gates. This oracle alone makes no claim that current implementation, installed artifacts or remote jobs have passed. + +## Phase 5 Step 7 current-schema consolidation + +Current canonical fixtures use report schema 1.1 and package 0.1.0.dev3. +Ordinary CLI calls report lineage execution as `not_requested`; explicit lineage +report assembly is covered by the current integration suite. Original numeric +expectations are retained. Prior schema 1.0 fixtures and the original authority +hashes remain auditable in Git history. No source-binding migration is added. diff --git a/tests/golden/phase4_report_expected.json b/tests/golden/phase4_report_expected.json index 53bfc7d..c104b89 100644 --- a/tests/golden/phase4_report_expected.json +++ b/tests/golden/phase4_report_expected.json @@ -156,8 +156,8 @@ }, "expected_exit": 0, "expected": { - "/run/report_schema_version": "1.0", - "/run/toolkit_version": "0.1.0.dev2", + "/run/report_schema_version": "1.1", + "/run/toolkit_version": "0.1.0.dev3", "/run/run_status": "complete", "/run/network_call_count": 0, "/run/network_count_scope": "toolkit_managed_outbound_operations", @@ -244,7 +244,7 @@ "/capabilities/ingestion/execution_status": "completed", "/capabilities/content_diagnostics/execution_status": "completed", "/capabilities/provenance/execution_status": "completed", - "/capabilities/lineage/execution_status": "deferred", + "/capabilities/lineage/execution_status": "not_requested", "/capabilities/dataset_longitudinal/execution_status": "partial", "/capabilities/dataset_longitudinal/execution_scope": [ "supplied_explicit_pair_support_and_diversity" @@ -468,7 +468,7 @@ "v2 topic counts cat=3,dog=2,bird=fish=refund=1 give support 5; sum squared frequencies=16/64=1/4 and D=3/4.", "Earlier minus later is battery,lizard,turtle; intersection contains five states. Delta=5-8=-3, retention=5/8, diversity delta=3/4-7/8=-1/8.", "The selected v2 scope has eight matching valid provenance rows: four human/yes and four synthetic/no. Each coverage is 8/8; each nonzero source share is 4/8. Direct interval=(closed/N,(closed+unresolved)/N)=[4/8,4/8].", - "All eight supplied v2 parent references resolve to explicitly earlier v1 records. This supports reference counts and a declared-order certificate only. Graph traversal and all ancestry metrics remain deferred." + "All eight supplied v2 parent references resolve to explicitly earlier v1 records. This supports reference counts and a declared-order certificate only. This CLI invocation does not request graph traversal or ancestry metrics." ] }, { @@ -485,8 +485,8 @@ }, "expected_exit": 0, "expected": { - "/run/report_schema_version": "1.0", - "/run/toolkit_version": "0.1.0.dev2", + "/run/report_schema_version": "1.1", + "/run/toolkit_version": "0.1.0.dev3", "/run/run_status": "complete", "/run/network_call_count": 0, "/run/network_count_scope": "toolkit_managed_outbound_operations", @@ -545,7 +545,7 @@ "/capabilities/ingestion/execution_status": "completed", "/capabilities/content_diagnostics/execution_status": "completed", "/capabilities/provenance/execution_status": "completed", - "/capabilities/lineage/execution_status": "deferred", + "/capabilities/lineage/execution_status": "not_requested", "/capabilities/dataset_longitudinal/execution_status": "partial", "/capabilities/dataset_longitudinal/execution_scope": [ "supplied_explicit_pair_support_and_diversity" @@ -650,8 +650,8 @@ }, "expected_exit": 0, "expected": { - "/run/report_schema_version": "1.0", - "/run/toolkit_version": "0.1.0.dev2", + "/run/report_schema_version": "1.1", + "/run/toolkit_version": "0.1.0.dev3", "/run/run_status": "complete", "/run/network_call_count": 0, "/run/network_count_scope": "toolkit_managed_outbound_operations", @@ -738,7 +738,7 @@ "/capabilities/ingestion/execution_status": "completed", "/capabilities/content_diagnostics/execution_status": "completed", "/capabilities/provenance/execution_status": "completed", - "/capabilities/lineage/execution_status": "deferred", + "/capabilities/lineage/execution_status": "not_requested", "/capabilities/dataset_longitudinal/execution_status": "partial", "/capabilities/dataset_longitudinal/execution_scope": [ "supplied_explicit_pair_support_and_diversity" @@ -2412,7 +2412,7 @@ "/run/run_status": "complete", "/observability/maximum_level": 4, "/capabilities/lineage/status": "partial", - "/capabilities/lineage/execution_status": "deferred", + "/capabilities/lineage/execution_status": "not_requested", "/observed_facts/lineage/declared_parent_edge_count/value": 2, "/observed_facts/lineage/resolved_parent_edge_count/value": 1, "/observed_facts/lineage/unresolved_parent_edge_count/value": 1, @@ -2442,8 +2442,8 @@ "json_value_evidence_status_parity" ], "numeric_rationale": [ - "Of two later parent references one resolves and one is absent: 2 declared,1 resolved,1 unresolved. Missing references retain a warning and cannot become external roots. One valid resolved path and one unresolved reference give partial input capability under the frozen lineage classification; execution remains deferred.", - "Independent ordered supports {A,B} and {A,C} both have size2, intersection1, retention1/2; lineage execution remains deferred." + "Of two later parent references one resolves and one is absent: 2 declared,1 resolved,1 unresolved. Missing references retain a warning and cannot become external roots. One valid resolved path and one unresolved reference give partial input capability under the frozen lineage classification; execution is not requested.", + "Independent ordered supports {A,B} and {A,C} both have size2, intersection1, retention1/2; lineage execution is not requested." ] }, { @@ -2574,7 +2574,7 @@ "/derived_metrics/support/by_version/later/support_size/scope/record_count": 2, "/run/run_status": "partial", "/observability/maximum_level": 4, - "/capabilities/lineage/execution_status": "deferred" + "/capabilities/lineage/execution_status": "not_requested" }, "required_error_codes": [ "E_LINEAGE_CYCLE" @@ -3550,7 +3550,7 @@ "/capabilities/ingestion/execution_status": "completed", "/capabilities/content_diagnostics/execution_status": "completed", "/capabilities/provenance/execution_status": "completed", - "/capabilities/lineage/execution_status": "deferred", + "/capabilities/lineage/execution_status": "not_requested", "/capabilities/dataset_longitudinal/execution_status": "partial", "/capabilities/dataset_longitudinal/execution_scope": [ "supplied_explicit_pair_support_and_diversity" diff --git a/tests/golden/test_phase4_reports.py b/tests/golden/test_phase4_reports.py index 0fc3ad0..bd67fb8 100644 --- a/tests/golden/test_phase4_reports.py +++ b/tests/golden/test_phase4_reports.py @@ -90,22 +90,13 @@ def test_phase4_step9_literal_json_and_markdown_goldens(redacted, phase4_step9_h stem = "phase4_hero_redacted" if redacted else "phase4_hero_report" json_path, markdown_path = GOLDEN / (stem + ".json"), GOLDEN / (stem + ".md") before = {path: path.read_bytes() for path in (json_path, markdown_path)} - # Step 11 synchronizes only the expected package-version metadata. Frozen - # fixture bytes and the actual report's closed normalization stay unchanged. + # Current schema fixtures retain the independently authored numeric oracle. + # Prior schema fixtures remain recoverable from Git history. expected_json = json.loads(before[json_path]) - assert expected_json["run"]["toolkit_version"] == "0.1.0.dev2" + assert expected_json["run"]["toolkit_version"] == __version__ assert actual["report"]["run"]["toolkit_version"] == __version__ assert actual["normalized"]["run"]["toolkit_version"] == __version__ - expected_json["run"]["toolkit_version"] = __version__ expected_markdown = before[markdown_path].decode("utf-8") - for old, new in ( - ('| `["toolkit_version"]` | `"0.1.0.dev2"` |', - '| `["toolkit_version"]` | `"' + __version__ + '"` |'), - ('Toolkit version: `"0.1.0.dev2"`; report schema version:', - 'Toolkit version: `"' + __version__ + '"`; report schema version:'), - ): - assert expected_markdown.count(old) == 1 - expected_markdown = expected_markdown.replace(old, new, 1) NORMALIZER["assert_golden_pair"](actual["normalized"], actual["normalized_markdown"], expected_json, expected_markdown) assert {path: path.read_bytes() for path in before} == before @@ -498,9 +489,6 @@ def phase4_step9_assert_oracle(case, actual, schema): assert markdown.startswith("# Recursive Integrity Audit Report\n") assert [line for line in markdown.splitlines() if line.startswith("## ")] == ["## " + value for value in headings] for pointer, expected in case.get("expected", {}).items(): - if pointer == "/run/toolkit_version": - assert expected == "0.1.0.dev2" - expected = __version__ got = phase4_step9_pointer(payload, pointer) assert got == expected, (case["case_id"], pointer, got, expected) if isinstance(expected, bool): diff --git a/tests/integration/test_current_verification.py b/tests/integration/test_current_verification.py index 1f42f4e..3859961 100644 --- a/tests/integration/test_current_verification.py +++ b/tests/integration/test_current_verification.py @@ -25,10 +25,14 @@ def protected_copy(repo_root, tmp_path): return tmp_path -@pytest.mark.parametrize("mutation", ["formula", "new_module", "deleted_module"]) +@pytest.mark.parametrize("mutation", ["formula", "new_module", "deleted_module", + "nested_metadata", "similar_metadata"]) def test_current_scope_rejects_unauthorized_product_changes(current_tools, protected_copy, mutation): - """Lineage closure and proxy do not authorize unrelated product changes.""" + """Lineage report integration does not authorize unrelated product changes.""" verify = current_tools["verify_source_scope"] + generated = protected_copy / "src/recursive_integrity_toolkit.egg-info" + generated.mkdir(exist_ok=True) + (generated / "PKG-INFO").write_text("Generated wheel metadata\n", encoding="utf-8") verify(protected_copy) target = protected_copy / "src/recursive_integrity_toolkit/metrics/diversity.py" if mutation == "formula": @@ -37,8 +41,15 @@ def test_current_scope_rejects_unauthorized_product_changes(current_tools, prote target.write_text(source.replace("1.0 -", "1.0 +", 1), encoding="utf-8") elif mutation == "new_module": target.with_name("unauthorized_metric.py").write_text("def score(): return 1\n", encoding="utf-8") - else: + elif mutation == "deleted_module": target.unlink() + else: + relative = ("src/recursive_integrity_toolkit/recursive_integrity_toolkit.egg-info" + if mutation == "nested_metadata" else + "src/recursive_integrity_toolkit_other.egg-info") + extra = protected_copy / relative + extra.mkdir() + (extra / "unauthorized.py").write_text("def score(): return 1\n", encoding="utf-8") with pytest.raises(ValueError): verify(protected_copy) diff --git a/tests/integration/test_hero_end_to_end.py b/tests/integration/test_hero_end_to_end.py index 25e1ec2..e20d2c0 100644 --- a/tests/integration/test_hero_end_to_end.py +++ b/tests/integration/test_hero_end_to_end.py @@ -284,7 +284,7 @@ def phase4_step8_hero_values(report): direct = report["derived_metrics"]["closure_exposure"]["direct"] assert [direct[n]["value"] for n in ("lower_bound", "upper_bound", "interval_width")] == [0.5, 0.5, 0] assert report["observability"]["maximum_level"] == 4 and report["capabilities"] == report["observability"]["capabilities"] - assert report["capabilities"]["lineage"]["execution_status"] == "deferred" + assert report["capabilities"]["lineage"]["execution_status"] == "not_requested" assert report["capabilities"]["dataset_longitudinal"]["execution_status"] == "partial" assert report["capabilities"]["model_longitudinal"]["status"] == "unavailable" assert report["capabilities"]["intervention_simulation"]["status"] == "unavailable" diff --git a/tests/integration/test_partial_lineage_report.py b/tests/integration/test_partial_lineage_report.py index 1a63873..c13aae2 100644 --- a/tests/integration/test_partial_lineage_report.py +++ b/tests/integration/test_partial_lineage_report.py @@ -96,7 +96,7 @@ def test_phase2_step9_content_limits_and_invalid_utf8(tmp_path): def phase4_step3_run(): return { "run_id": "step3-independent-case", "toolkit_version": "0.1.0.dev2", - "report_schema_version": "1.0", "started_at": None, "completed_at": None, + "report_schema_version": "1.1", "started_at": None, "completed_at": None, "duration_seconds": None, "python_version": None, "platform": None, "command": None, "config_hash": None, "random_seed": None, "strict_mode": False, "redacted_mode": False, "network_call_count": 0, @@ -130,14 +130,14 @@ def phase4_step3_schema(report, repo_root): return payload -def test_phase4_step3_level_four_keeps_lineage_input_available_and_execution_deferred(tmp_path, repo_root): +def test_phase4_step3_level_four_keeps_lineage_input_available_and_execution_not_requested(tmp_path, repo_root): from recursive_integrity_toolkit.reports.assembly import assemble_report bundle = validate_bundle(_bundle(tmp_path, ["v1::p"]), configuration=_config()) report = phase4_step3_schema(assemble_report(bundle, run=phase4_step3_run()), repo_root) assert report["observability"]["maximum_level"] == 4 lineage = report["capabilities"]["lineage"] - assert lineage["status"] == "available" and lineage["execution_status"] == "deferred" + assert lineage["status"] == "available" and lineage["execution_status"] == "not_requested" assert lineage["execution_reason_codes"] assert report["capabilities"]["dataset_longitudinal"]["status"] == "available" assert report["capabilities"]["dataset_longitudinal"]["execution_status"] == "not_requested" @@ -171,7 +171,7 @@ def test_phase4_step3_unresolved_parent_warning_survives_without_observability_d report = phase4_step3_schema(assemble_report(bundle, run=phase4_step3_run()), repo_root) assert report["observability"]["maximum_level"] == 4 assert report["capabilities"]["lineage"]["status"] == "unavailable" - assert report["capabilities"]["lineage"]["execution_status"] == "deferred" + assert report["capabilities"]["lineage"]["execution_status"] == "not_requested" assert any(item["code"] == WarningCode.PARENT_UNRESOLVED.value for item in report["warnings"]) assert not report["errors"] and report["run"]["run_status"] == "complete" @@ -204,7 +204,7 @@ def phase4_step3_forbid(*args, **kwargs): monkeypatch.setattr(assembly, name, phase4_step3_forbid) report = phase4_step3_schema(assembly.assemble_report(bundle, run=phase4_step3_run()), repo_root) assert report["observability"]["maximum_level"] == 4 - assert report["capabilities"]["lineage"]["execution_status"] == "deferred" + assert report["capabilities"]["lineage"]["execution_status"] == "not_requested" @pytest.mark.parametrize("redacted", [False, True]) @@ -240,7 +240,7 @@ def test_phase4_step9_parent_failure_survives_pair_cli_and_both_formats( assert all(path.read_bytes() == raw for path, raw in before.items()) assert report["observability"]["maximum_level"] == 4 lineage = report["capabilities"]["lineage"] - assert lineage["status"] == "unavailable" and lineage["execution_status"] == "deferred" + assert lineage["status"] == "unavailable" and lineage["execution_status"] == "not_requested" assert lineage == report["observability"]["capabilities"]["lineage"] support = report["derived_metrics"]["support"] assert support["support_delta"]["value"] == 0 diff --git a/tests/integration/test_partial_provenance_report.py b/tests/integration/test_partial_provenance_report.py index 5c2e8fe..f784865 100644 --- a/tests/integration/test_partial_provenance_report.py +++ b/tests/integration/test_partial_provenance_report.py @@ -391,7 +391,7 @@ def test_PR004_pipeline_incomplete_assessment_preserves_failure_evidence(repo_ro def phase4_step3_run(): return { "run_id": "step3-independent-case", "toolkit_version": "0.1.0.dev2", - "report_schema_version": "1.0", "started_at": None, "completed_at": None, + "report_schema_version": "1.1", "started_at": None, "completed_at": None, "duration_seconds": None, "python_version": None, "platform": None, "command": None, "config_hash": None, "random_seed": None, "strict_mode": False, "redacted_mode": False, "network_call_count": 0, diff --git a/tests/integration/test_phase4_cli.py b/tests/integration/test_phase4_cli.py index 83332d6..fdd9484 100644 --- a/tests/integration/test_phase4_cli.py +++ b/tests/integration/test_phase4_cli.py @@ -51,7 +51,7 @@ def test_phase4_step7_explicit_single_version_metrics_match_hand_counts(tmp_path assert report["run"]["resolved_options"]["weighted"] is False assert "weighted_support_size" not in report["derived_metrics"]["support"]["by_version"]["v1"] assert report["simulations"] == {} and "tail" not in report["derived_metrics"] - assert report["capabilities"]["lineage"]["execution_status"] == "deferred" + assert report["capabilities"]["lineage"]["execution_status"] == "not_requested" assert report["capabilities"] == report["observability"]["capabilities"] assert all(path.read_bytes() == data for path, data in before.items()) assert {artifact["role"] for artifact in report["inputs"]["artifacts"]} == {"config", "records_primary", "provenance_manifest"} @@ -391,7 +391,7 @@ def test_phase4_step7_existing_lineage_error_retains_metrics_and_precedence(tmp_ assert "E_PARENT_FORMAT" in {item["code"] for item in report["errors"]} assert report["observed_facts"]["record_counts"]["v1"]["value"] == 4 assert "closure_exposure" in report["derived_metrics"] - assert report["capabilities"]["lineage"]["execution_status"] == "deferred" + assert report["capabilities"]["lineage"]["execution_status"] == "not_requested" if not unsupported: assert report["derived_metrics"]["support"]["by_version"]["v1"]["support_size"]["value"] == 3 diff --git a/tests/integration/test_phase5_lineage_privacy.py b/tests/integration/test_phase5_lineage_privacy.py new file mode 100644 index 0000000..7ce1c47 --- /dev/null +++ b/tests/integration/test_phase5_lineage_privacy.py @@ -0,0 +1,301 @@ +"""Lineage identities cross the existing privacy boundary without data loss. + +Small real inputs exercise roots, cycle witnesses, unresolved rows and public +scope labels. One 202-record case reaches the existing 100-row presentation cap. +""" + +from copy import deepcopy +import json + +import pytest +from jsonschema import Draft202012Validator + +from recursive_integrity_toolkit.io.validation import validate_bundle +from recursive_integrity_toolkit.lineage.ancestry import analyze_lineage, shared_ancestry_dependence +from recursive_integrity_toolkit.metrics.bounds import lineage_closure_exposure +from recursive_integrity_toolkit.models import AuditBundle, FileRole, InputSource +from recursive_integrity_toolkit.reports.assembly import assemble_report, privacy_view +from recursive_integrity_toolkit.reports.json_report import render_json +from recursive_integrity_toolkit.reports.markdown_report import render_markdown +from recursive_integrity_toolkit.result import CanonicalReport +from recursive_integrity_toolkit.utils.hashing import IdentifierProtection + + +_VERSION_OLD = "VERSION_SENTINEL_old" +_VERSION_TARGET = "VERSION_SENTINEL_target" +_ROOT_ID = "RECORD_SENTINEL_root|`" +_SECRET = b"lineage-privacy-test-secret-only!!" +_RAW_PAYLOAD = ("CONTENT_SENTINEL", "NOTES_SENTINEL", "URI_SENTINEL") + + +def _run(): + nullable = ("started_at", "completed_at", "duration_seconds", "python_version", + "platform", "command", "config_hash", "random_seed") + return { + "run_id": "RUN_SENTINEL", "toolkit_version": "0.1.0.dev3", + "report_schema_version": "1.1", **dict.fromkeys(nullable), + "strict_mode": False, "redacted_mode": False, "network_call_count": 0, + "deterministic": True, "privacy_mode": "standard", "run_status": "complete", + "null_reasons": {key: "No execution metadata supplied." for key in nullable}, + } + + +def _report(tmp_path, *, capped=False): + tmp_path.mkdir(parents=True, exist_ok=True) + # A matching record_id in different versions is two distinct identities. + if capped: + declarations = { + **{(_VERSION_TARGET, f"RECORD_SENTINEL_root_{i:03}"): ([], "yes") for i in range(101)}, + **{(_VERSION_TARGET, f"RECORD_SENTINEL_cycle_{i:03}"): + ([f"{_VERSION_TARGET}::RECORD_SENTINEL_cycle_{i:03}"], "no") for i in range(101)}, + } + else: + declarations = { + (_VERSION_OLD, _ROOT_ID): ([], "yes"), + (_VERSION_OLD, "RECORD_SENTINEL_cycle_a"): + ([f"{_VERSION_OLD}::RECORD_SENTINEL_cycle_b"], "no"), + (_VERSION_OLD, "RECORD_SENTINEL_cycle_b"): + ([f"{_VERSION_OLD}::RECORD_SENTINEL_cycle_a"], "no"), + (_VERSION_TARGET, _ROOT_ID): ([], "yes"), + (_VERSION_TARGET, "RECORD_SENTINEL_child"): + ([f"{_VERSION_OLD}::{_ROOT_ID}"], "no"), + (_VERSION_TARGET, "RECORD_SENTINEL_child_two"): + ([f"{_VERSION_OLD}::{_ROOT_ID}"], "no"), + (_VERSION_TARGET, "RECORD_SENTINEL_closed"): ([], "no"), + (_VERSION_TARGET, "RECORD_SENTINEL_unknown"): ([], "unknown"), + (_VERSION_TARGET, "RECORD_SENTINEL_affected"): + ([f"{_VERSION_OLD}::RECORD_SENTINEL_cycle_a"], "no"), + } + records, provenance = [], [] + for (version, record_id), (parents, grounding) in declarations.items(): + identity = {"dataset_version": version, "record_id": record_id} + records.append({**identity, "content": "CONTENT_SENTINEL", + "notes": "NOTES_SENTINEL"}) + provenance.append({**identity, "source_type": "human" if grounding == "yes" else "synthetic", + "provenance_confidence": "confirmed", "external_grounding": grounding, + "parent_ids": parents, "transformation": "generate", + "source_uri": "https://invalid.example/URI_SENTINEL", + "notes": "NOTES_SENTINEL"}) + sources = [] + for name, role, rows in ( + ("primary", FileRole.RECORDS_PRIMARY, [row for row in records if row["dataset_version"] == _VERSION_TARGET]), + ("context", FileRole.RECORDS_COMPARE, [row for row in records if row["dataset_version"] != _VERSION_TARGET]), + ("provenance", FileRole.PROVENANCE_MANIFEST, provenance), + ): + if rows: + path = tmp_path / ("PATH_SENTINEL_" + name + ".jsonl") + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + sources.append(InputSource(role, path)) + versions = [_VERSION_TARGET] if capped else [_VERSION_OLD, _VERSION_TARGET] + bundle = validate_bundle(AuditBundle(tuple(sources)), configuration={"version_order": versions}) + lineage = analyze_lineage(bundle, target_dataset_version=_VERSION_TARGET) + return assemble_report(bundle, run=_run(), lineage=lineage, + lineage_bounds=lineage_closure_exposure(lineage), + shared_ancestry=shared_ancestry_dependence(lineage)) + + +def _view(report, mode="redacted", record_id_mode="hash", *, secret=_SECRET): + return privacy_view(report, mode=mode, record_id_mode=record_id_mode, + protection=IdentifierProtection.create(secret=secret)) + + +def _validate(view, repo_root): + value = view.to_dict() + schema = json.loads((repo_root / "schemas/report.schema.json").read_text(encoding="utf-8")) + Draft202012Validator(schema).validate(value) + CanonicalReport.from_dict(value) + assert json.loads(render_json(view)) == value + return value + + +def _details(value): + observed = value["observed_facts"]["lineage"] + cycle = observed["cycle_analysis"]["value"] + return { + **{name: cycle[name] for name in ("components", "cycle_member_record_keys", "affected_record_keys")}, + "unresolved_record_details": observed["unresolved_record_details"]["value"], + "top_shared_ancestors": value["derived_metrics"]["lineage"]["top_shared_ancestors"]["value"], + } + + +def _identities(value, path=()): + if isinstance(value, dict): + if set(value) == {"dataset_version", "record_id"}: + yield path, value + else: + for name, item in value.items(): + yield from _identities(item, (*path, name)) + elif isinstance(value, list): + for index, item in enumerate(value): + yield from _identities(item, (*path, index)) + + +def _lineage_identity_evidence(value): + return { + "observations": value["observed_facts"]["lineage"], + "metrics": value["derived_metrics"]["lineage"], + "bounds": value["derived_metrics"]["closure_exposure"]["lineage"], + "proxy": value["proxy_signals"]["shared_ancestry_dependence"], + "errors": value["errors"], "warnings": value["warnings"], + } + + +def _quantitative_evidence(value): + """Exclude presentation identities while comparing exact analytical values.""" + observed = value["observed_facts"]["lineage"] + cycle = observed["cycle_analysis"]["value"] + metrics = value["derived_metrics"]["lineage"] + graph_scope = observed["graph_scope"]["value"] + return { + "run_status": value["run"]["run_status"], + "capability": value["capabilities"]["lineage"], + "cycle_counts": {name: item for name, item in cycle.items() if not isinstance(item, dict)}, + "graph_counts": {name: item for name, item in graph_scope.items() if type(item) is int}, + "depth": observed["depth_summary"]["value"], + "resources": observed["resource_usage"]["value"], + "metrics": {name: {field: item for field, item in envelope.items() + if field in {"value", "status", "coverage", "denominator", "evidence_class", "unit"}} + for name, envelope in metrics.items() if name != "top_shared_ancestors"}, + "closure": {name: {field: item for field, item in envelope.items() + if field in {"value", "status", "coverage", "denominator", "evidence_class", "unit"}} + for name, envelope in value["derived_metrics"]["closure_exposure"]["lineage"].items()}, + "proxy": {name: item for name, item in value["proxy_signals"]["shared_ancestry_dependence"].items() + if name in {"status", "level", "coverage", "denominator", "evidence_class"}}, + "diagnostics": [(name, row["severity"], row.get("effect_on_run"), row["effect_on_capabilities"]) + for name in ("errors", "warnings") for row in value[name]], + } + + +@pytest.mark.parametrize("mode,record_id_mode", [ + ("standard", "preserve"), ("redacted", "hash"), + ("redacted", "omit"), ("redacted", "preserve"), +]) +def test_lineage_privacy_modes_preserve_analytical_values_and_safe_rendering( + tmp_path, repo_root, mode, record_id_mode): + report = _report(tmp_path) + original = deepcopy(report.to_dict()) + view = _view(report, mode, record_id_mode) + actual = _validate(view, repo_root) + assert _quantitative_evidence(actual) == _quantitative_evidence(original) + assert report.to_dict() == original + for rendered in (render_json(view), render_markdown(view)): + assert all(sentinel not in rendered for sentinel in _RAW_PAYLOAD) + assert ("VERSION_SENTINEL" in rendered) is (mode == "standard") + assert ("PATH_SENTINEL" in rendered) is (mode == "standard") + assert ("RECORD_SENTINEL" in rendered) is (record_id_mode == "preserve") + markdown = render_markdown(view) + assert "