diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb5db74..849fe9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,15 +1,31 @@ -name: Phase 4 Step 11 CI +name: Current verification on: pull_request: + types: [opened, synchronize, reopened, labeled] paths-ignore: - README.md - CHANGELOG.md - - "docs/**" + - docs/architecture.md + - docs/cli.md + - docs/data_schema.md + - docs/privacy.md + - docs/release_process.md + - docs/report_schema.md + - docs/theory_traceability.md - PHASE_4_COMPLETION.md - PHASE_4_VALIDATION_REPORT.md - PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md + - PHASE_5_STEP_*.md + - PHASE_5_COMPLETION.md workflow_dispatch: + inputs: + gate: + description: Focused development checks or complete candidate verification + required: true + default: focused + type: choice + options: [focused, candidate] permissions: contents: read @@ -19,10 +35,37 @@ defaults: shell: bash jobs: + focused: + if: (github.event_name != 'workflow_dispatch' || inputs.gate != 'candidate') && github.event.action != 'labeled' + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + PYTHONDONTWRITEBYTECODE: "1" + RIT_TEST_PARQUET: "0" + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install ".[test]" + - name: Check current lineage, reports and affected boundaries + run: | + set -euo pipefail + python scripts/check_spec_consistency.py + python scripts/check_traceability.py + python scripts/release_check.py + python -m pytest -p no:cacheprovider -q tests/unit/test_PR009_generation.py tests/unit/test_PR008_parent_resolution.py tests/unit/test_PR007_version_order.py tests/unit/test_parent_batch.py tests/unit/test_PR001_identity.py tests/unit/test_PR002_loaders.py tests/unit/test_PR003_schema_mapping.py tests/unit/test_PR004_coverage.py tests/unit/test_PR010_observability.py tests/unit/test_lineage_context_inputs.py tests/unit/test_lineage_graph.py tests/unit/test_T6_cycles.py tests/unit/test_T4_ancestry.py tests/unit/test_lineage_root_resources.py tests/unit/test_lineage_concentration.py tests/unit/test_lineage_bounds_proxy.py tests/unit/test_lineage_fixture_inputs.py tests/unit/test_phase3_contracts.py tests/unit/test_PR016_determinism.py + python -m pytest -p no:cacheprovider -q tests/integration/test_phase3_metric_pipeline.py tests/integration/test_phase5_lineage_cli.py tests/integration/test_phase5_lineage_reports.py tests/integration/test_phase5_lineage_privacy.py tests/integration/test_phase4_cli.py tests/integration/test_hero_end_to_end.py tests/integration/test_partial_lineage_report.py tests/golden/test_phase4_reports.py tests/unit/test_PR013_report_schema.py tests/unit/test_PR015_redaction.py tests/integration/test_no_network.py tests/integration/test_current_verification.py tests/integration/test_ci_workflows.py tests/integration/test_no_algorithms.py tests/integration/test_repository_structure.py + core: + if: (github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate') || (github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == 'verification:candidate') name: Core (${{ matrix.dependencies }}) on ${{ matrix.os }} with Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} - timeout-minutes: 105 + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -66,32 +109,32 @@ jobs: fi - name: Verify actual optional dependency absence run: python -c "import importlib.util; assert importlib.util.find_spec('pyarrow') is None" - - name: Run the complete core suite + - name: Run complete canonical regression run: | set -euo pipefail mkdir -p "$RUNNER_TEMP/rit-core" - python -m pytest -p no:cacheprovider -q --junitxml="$RUNNER_TEMP/rit-core/core.xml" --maxfail=1 | tee "$RUNNER_TEMP/rit-core/phase4_test_results.log" - python scripts/release_check.py --phase 4 --step 11 --junit "$RUNNER_TEMP/rit-core/core.xml" --minimum-tests 3768 - - name: Check current authorized scope and canonical suite + python -m pytest -p no:cacheprovider -q --ignore=tests/performance --junitxml="$RUNNER_TEMP/rit-core/core.xml" --maxfail=1 | tee "$RUNNER_TEMP/rit-core/tests.log" + python scripts/release_check.py --junit "$RUNNER_TEMP/rit-core/core.xml" + - name: Check current source, specifications and traceability run: | set -euo pipefail - python scripts/check_spec_consistency.py --phase 4 --step 11 - python scripts/check_traceability.py --phase 4 --step 11 - python scripts/release_check.py --phase 4 --step 11 --diff - python scripts/release_check.py --phase 4 --step 11 --baseline-evidence "$RUNNER_TEMP/rit-core" + python scripts/check_spec_consistency.py + python scripts/check_traceability.py + python scripts/release_check.py - name: Retain core evidence without masking failure if: always() uses: actions/upload-artifact@v4 with: - name: phase4-current-core-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.dependencies }} + name: current-core-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.dependencies }} path: ${{ runner.temp }}/rit-core/ if-no-files-found: error retention-days: 30 parquet: + if: (github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate') || (github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == 'verification:candidate') name: Real optional Parquet on Ubuntu with Python 3.12 runs-on: ubuntu-latest - timeout-minutes: 75 + timeout-minutes: 30 env: PYTHONDONTWRITEBYTECODE: "1" RIT_TEST_PARQUET: "1" @@ -108,7 +151,7 @@ jobs: python-version: "3.12" - name: Install approved real Parquet extra run: python -m pip install ".[test,parquet]" - - name: Run the complete extra-enabled suite + - name: Run complete canonical regression with Parquet run: | set -euo pipefail mkdir -p "$RUNNER_TEMP/rit-parquet" @@ -117,19 +160,60 @@ jobs: python -c "import pyarrow; print('PyArrow', pyarrow.__version__)" git rev-parse HEAD > "$RUNNER_TEMP/rit-parquet/source-commit.txt" free --bytes | tee "$RUNNER_TEMP/rit-parquet/host-memory-bytes.txt" - python -m pytest -p no:cacheprovider -q --junitxml="$RUNNER_TEMP/rit-parquet/parquet.xml" --maxfail=1 | tee "$RUNNER_TEMP/rit-parquet/phase4_parquet_results.log" - python scripts/release_check.py --phase 4 --step 11 --junit "$RUNNER_TEMP/rit-parquet/parquet.xml" --require-parquet --minimum-tests 3771 - python scripts/release_check.py --phase 4 --step 11 --baseline-evidence "$RUNNER_TEMP/rit-parquet" + python -m pytest -p no:cacheprovider -q --ignore=tests/performance --junitxml="$RUNNER_TEMP/rit-parquet/parquet.xml" --maxfail=1 | tee "$RUNNER_TEMP/rit-parquet/tests.log" + python scripts/release_check.py --junit "$RUNNER_TEMP/rit-parquet/parquet.xml" --require-parquet - name: Retain Parquet evidence without masking failure if: always() uses: actions/upload-artifact@v4 with: - name: phase4-current-real-parquet + name: current-real-parquet path: ${{ runner.temp }}/rit-parquet/ if-no-files-found: error retention-days: 30 + performance: + if: (github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate') || (github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == 'verification:candidate') + runs-on: ubuntu-latest + timeout-minutes: 75 + env: + PYTHONDONTWRITEBYTECODE: "1" + RIT_TEST_PARQUET: "0" + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install ".[test]" + - name: Measure reference profile including required 100k workload + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/rit-performance" + python -m pip freeze > "$RUNNER_TEMP/rit-performance/dependencies.txt" + python -c "import platform; print(platform.platform(), platform.python_version())" + free --bytes + python -m pytest -p no:cacheprovider -q tests/performance --junitxml="$RUNNER_TEMP/rit-performance/performance.xml" | tee "$RUNNER_TEMP/rit-performance/tests.log" + python scripts/release_check.py --junit "$RUNNER_TEMP/rit-performance/performance.xml" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: current-reference-performance + path: ${{ runner.temp }}/rit-performance/ + if-no-files-found: error + retention-days: 30 + + hero: + if: (github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate') || (github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == 'verification:candidate') + uses: ./.github/workflows/golden.yml + + security: + if: (github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate') || (github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == 'verification:candidate') + uses: ./.github/workflows/security.yml + delivery: name: Build verified candidate - needs: [core, parquet] + needs: [core, parquet, performance, hero, security] uses: ./.github/workflows/release.yml diff --git a/.github/workflows/golden.yml b/.github/workflows/golden.yml index 5fea5d7..1d4fc71 100644 --- a/.github/workflows/golden.yml +++ b/.github/workflows/golden.yml @@ -1,14 +1,7 @@ -name: Phase 4 Step 11 Hero and Mathematical Contract +name: Hero and mathematical contract on: - pull_request: - paths-ignore: - - README.md - - CHANGELOG.md - - "docs/**" - - PHASE_4_COMPLETION.md - - PHASE_4_VALIDATION_REPORT.md - - PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md + workflow_call: workflow_dispatch: permissions: @@ -41,16 +34,15 @@ jobs: run: | set -euo pipefail mkdir -p "$RUNNER_TEMP/rit-math" - python scripts/check_spec_consistency.py --phase 4 --step 11 - python -m pytest -p no:cacheprovider -q tests/integration/test_hero_structure.py tests/integration/test_hero_end_to_end.py tests/integration/test_phase3_metric_pipeline.py tests/golden/test_phase3_math.py tests/golden/test_phase4_reports.py --junitxml="$RUNNER_TEMP/rit-math/math.xml" | tee "$RUNNER_TEMP/rit-math/phase4_math_validation.log" - python scripts/release_check.py --phase 4 --step 11 --golden-junit "$RUNNER_TEMP/rit-math/math.xml" - python scripts/release_check.py --phase 4 --step 11 --diff + python scripts/check_spec_consistency.py + python -m pytest -p no:cacheprovider -q tests/integration/test_hero_structure.py tests/integration/test_hero_end_to_end.py tests/integration/test_phase3_metric_pipeline.py tests/golden/test_phase3_math.py tests/golden/test_phase4_reports.py tests/unit/test_T4_ancestry.py tests/unit/test_lineage_concentration.py tests/unit/test_lineage_bounds_proxy.py tests/integration/test_phase5_lineage_reports.py --junitxml="$RUNNER_TEMP/rit-math/math.xml" | tee "$RUNNER_TEMP/rit-math/tests.log" + python scripts/release_check.py --junit "$RUNNER_TEMP/rit-math/math.xml" + python scripts/release_check.py - name: Retain actual Hero and mathematical evidence if: always() uses: actions/upload-artifact@v4 with: - name: phase4-current-hero-mathematical + name: current-hero-mathematical path: ${{ runner.temp }}/rit-math/ if-no-files-found: error retention-days: 30 -# Frozen mathematical expectations are unchanged; Step 11 Hero reports use independent assertions. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1a9558f..55d243f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Phase 4 Step 11 Build and Candidate Evidence +name: Package and candidate verification on: workflow_call: @@ -33,19 +33,19 @@ jobs: - name: Reuse complete core results from this candidate run uses: actions/download-artifact@v4 with: - name: phase4-current-core-ubuntu-latest-3.12-current - path: ${{ runner.temp }}/rit-phase4-current/core-evidence/ + name: current-core-ubuntu-latest-3.12-current + path: ${{ runner.temp }}/rit-candidate/core-evidence/ - name: Reuse complete real-Parquet results from this candidate run uses: actions/download-artifact@v4 with: - name: phase4-current-real-parquet - path: ${{ runner.temp }}/rit-phase4-current/parquet-evidence/ + name: current-real-parquet + path: ${{ runner.temp }}/rit-candidate/parquet-evidence/ - name: Install approved build and test tools run: python -m pip install ".[release,test,parquet]" "setuptools>=82" - name: Check authority, declarations and current-step scope run: | set -euo pipefail - evidence="$RUNNER_TEMP/rit-phase4-current" + evidence="$RUNNER_TEMP/rit-candidate" test "$(cat "$evidence/core-evidence/source-commit.txt")" = "$(git rev-parse HEAD)" test "$(cat "$evidence/parquet-evidence/source-commit.txt")" = "$(git rev-parse HEAD)" cp "$evidence/core-evidence/core.xml" "$evidence/core.xml" @@ -53,17 +53,17 @@ jobs: python -m pip freeze > "$evidence/build-dependencies.txt" python -m pip check python -c "import pyarrow; print('PyArrow', pyarrow.__version__)" - python scripts/check_spec_consistency.py --phase 4 --step 11 - python scripts/check_traceability.py --phase 4 --step 11 - python scripts/release_check.py --phase 4 --step 11 --diff + python scripts/check_spec_consistency.py + python scripts/check_traceability.py + python scripts/release_check.py rit version - - name: Build and repeat the dev3 artifacts from the same committed source + - name: Build and repeat the development artifacts from the same committed source run: | set -euo pipefail - evidence="$RUNNER_TEMP/rit-phase4-current" + evidence="$RUNNER_TEMP/rit-candidate" export SOURCE_DATE_EPOCH="$(git show -s --format=%ct HEAD)" printf '%s\n' "$SOURCE_DATE_EPOCH" > "$evidence/source-date-epoch.txt" - python -m build --no-isolation --outdir "$evidence/dist" | tee "$evidence/phase4_build_results.log" + python -m build --no-isolation --outdir "$evidence/dist" | tee "$evidence/build.log" python -m twine check --strict "$evidence/dist/"* mkdir -p "$evidence/rebuild-source" git archive HEAD | tar -x -C "$evidence/rebuild-source" @@ -79,7 +79,7 @@ jobs: from pathlib import Path import tarfile - evidence = Path(os.environ["RUNNER_TEMP"]) / "rit-phase4-current" + evidence = Path(os.environ["RUNNER_TEMP"]) / "rit-candidate" first = {p.name: p for p in (evidence / "dist").iterdir() if p.is_file()} second = {p.name: p for p in (evidence / "rebuild-dist").iterdir() if p.is_file()} assert first.keys() == second.keys() and len(first) == 2 @@ -109,17 +109,17 @@ jobs: (evidence / "reproducibility.json").write_text(json.dumps(result, indent=2) + "\n") print(json.dumps(result, indent=2)) PY - - name: Verify installed packages and produce the byte-verified Step 11 candidate + - name: Verify installed packages and produce the byte-verified current candidate run: | set -euo pipefail - python scripts/release_check.py --phase 4 --step 11 --candidate "$RUNNER_TEMP/rit-phase4-current" | tee "$RUNNER_TEMP/rit-phase4-current/installed-candidate.log" + python scripts/release_check.py --candidate "$RUNNER_TEMP/rit-candidate" | tee "$RUNNER_TEMP/rit-candidate/installed-candidate.log" - name: Retain candidate or failure evidence without masking failure if: always() uses: actions/upload-artifact@v4 with: - name: recursive-integrity-toolkit-phase4-current-candidate + name: recursive-integrity-toolkit-current-candidate path: | - ${{ runner.temp }}/rit-phase4-current/ - !${{ runner.temp }}/rit-phase4-current/rebuild-source/ + ${{ runner.temp }}/rit-candidate/ + !${{ runner.temp }}/rit-candidate/rebuild-source/ if-no-files-found: error retention-days: 90 diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index e193294..200ac39 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -1,14 +1,7 @@ -name: Phase 4 Step 11 Security Boundary +name: Security boundaries on: - pull_request: - paths-ignore: - - README.md - - CHANGELOG.md - - "docs/**" - - PHASE_4_COMPLETION.md - - PHASE_4_VALIDATION_REPORT.md - - PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md + workflow_call: workflow_dispatch: permissions: @@ -52,6 +45,7 @@ jobs: tests/unit/test_PR017_content_refs.py \ tests/integration/test_hero_end_to_end.py tests/golden/test_phase4_reports.py \ tests/integration/test_partial_lineage_report.py \ + tests/integration/test_phase5_lineage_privacy.py tests/unit/test_lineage_root_resources.py \ tests/unit/test_T1_representation.py \ tests/unit/test_PR006_duplicates.py \ tests/unit/test_T1_support.py \ @@ -73,17 +67,15 @@ jobs: tests/integration/test_partial_provenance_report.py \ tests/unit/test_phase4_output_safety.py \ tests/integration/test_phase4_cli.py \ - tests/unit/test_phase4_contracts.py \ - tests/integration/test_phase4_gates.py \ - --junitxml="$RUNNER_TEMP/rit-security/security.xml" | tee "$RUNNER_TEMP/rit-security/phase4_current_security_results.log" - python scripts/release_check.py --phase 4 --step 11 --junit "$RUNNER_TEMP/rit-security/security.xml" - python scripts/check_traceability.py --phase 4 --step 11 - python scripts/release_check.py --phase 4 --step 11 + --junitxml="$RUNNER_TEMP/rit-security/security.xml" | tee "$RUNNER_TEMP/rit-security/tests.log" + python scripts/release_check.py --junit "$RUNNER_TEMP/rit-security/security.xml" + python scripts/check_traceability.py + python scripts/release_check.py - name: Retain actual security test evidence if: always() uses: actions/upload-artifact@v4 with: - name: phase4-current-security-boundary + name: current-security-boundary path: ${{ runner.temp }}/rit-security/ if-no-files-found: error retention-days: 30 diff --git a/CHANGELOG.md b/CHANGELOG.md index e2ae3c6..0776d06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## 0.1.0.dev4: Phase 5 lineage candidate + +Added explicit canonical lineage graphs, iterative cycles and structural depth, +strict external ancestry, root incidence/fractional mass, ancestry concentration, +lineage closure intervals and descriptive shared-root evidence. Schema 1.1 and +standard/redacted JSON/Markdown retain partial, unavailable and unknown states. +`audit --lineage` and `example --lineage` support dedicated local ancestor context +without changing primary metric denominators. Input validation remains analytical- +execution-free. Finite node, edge, root-membership and work limits are explicit. + +Replaced measured quadratic generation scans with immediate dependency +propagation while preserving declarations and scientific results. The actual +100k reverse-chain input/ancestry API baseline is 123.36 seconds and 1.71 GiB +peak RSS; full audit serialization is excluded. Seven bounded mutations were +detected by current behavioral tests. Step 10 verifies supported environments, +packaging, installed ordinary/lineage Hero and reproducibility through the +existing candidate workflow. Actual acceptance is in `PHASE_5_COMPLETION.md`. + +Historical source-migration machinery was consolidated into current behavioral +controls and recoverable Git evidence. Phase 6A/6B, CLI simulation, HTML, main +merge, tag and publication remain outside this development milestone. + ## 0.1.0.dev3: Phase 4 report and CLI milestone Added a validated twelve-section JSON report contract, Markdown rendering, explicit evidence assembly, standard/redacted privacy views and safe local report publication. The installed CLI supports input-only `validate`, single-version `audit`, one explicitly ordered compatible pair and a packaged local `example`; help/version remain import-safe. Reports preserve five evidence classes, unknown/unavailable states, scope/denominator metadata, partial errors and deferred capabilities. diff --git a/PHASE_5_COMPLETION.md b/PHASE_5_COMPLETION.md new file mode 100644 index 0000000..958508c --- /dev/null +++ b/PHASE_5_COMPLETION.md @@ -0,0 +1,187 @@ +# Phase 5 Completion and Candidate Verification + +Status: **PHASE 5 COMPLETE; ALL 13 REQUIRED CANDIDATE JOBS PASSED**. + +The Theory Owner instructed `Phase 5 Step 10` on 2026-09-26 UTC. Development +version is `0.1.0.dev4`; report schema remains `1.1`. + +| Reference | Value | +|---|---| +| Starting Step 9 commit | `15d50c3d992265e716b0b27e69c43107d947d21d` | +| Tested code candidate | `53802fb0e99ea867a27bb82210cc9fa32d8e02d1` | +| Tested candidate tree | `5f235a4e008f65d0cb76d0af2e5b7d929271f018` | +| Branch / draft PR | `phase5-lineage`; [PR #2](https://github.com/DavidWallstructurallaw/recursive-integrity-toolkit/pull/2), based on the accepted unmerged Phase 4 branch | +| Candidate workflow | [Run 36270399128](https://github.com/DavidWallstructurallaw/recursive-integrity-toolkit/actions/runs/36270399128) | + +## Candidate scope + +Steps 1-9 implement retained parent evidence, canonical graphs, cycles and depth, +strict external ancestry, G/C/U partition, root concentration, lineage closure +bounds, descriptive shared-root evidence, schema 1.1 and explicit CLI/context +integration. Step 10 updates version metadata, public documentation and the +existing candidate verification path. All analytical source, frozen scientific +definitions, canonical Hero inputs and schema resources retain Step 9 bytes. + +Two existing Parquet context tests now use the explicit dependency profile, +matching the original three real-PyArrow cases. Candidate selection adds a PR +label event to the existing workflow because the default branch still contains +the early workflow without manual dispatch. No additional workflow, source +migration framework or evidence registry is introduced. + +## Acceptance gates + +The following completed jobs checked out the exact candidate above. JUnit +artifacts and dependency profiles were retrieved from this run. Every executed +test in these jobs passed, with zero failures, errors or skips. + +| Role | Python / NumPy / pandas | Passed | +|---|---|---:| +| Ubuntu current | 3.11.16 / 2.4.6 / 3.0.6 | 2,960 | +| Ubuntu current | 3.12.14 / 2.5.3 / 3.0.6 | 2,960 | +| Ubuntu minimum | 3.11.16 / 2.0.0 / 2.2.2 | 2,960 | +| Ubuntu minimum | 3.12.14 / 2.0.0 / 2.2.2 | 2,960 | +| Windows current | 3.11.9 / 2.4.6 / 3.0.6 | 2,960 | +| Windows current | 3.12.10 / 2.5.3 / 3.0.6 | 2,960 | +| Windows minimum | 3.11.9 / 2.0.0 / 2.2.2 | 2,960 | +| Windows minimum | 3.12.10 / 2.0.0 / 2.2.2 | 2,960 | +| Ubuntu real Parquet | 3.12.14 / 2.5.3 / 3.0.6; PyArrow 25.0.1 | 2,965 | +| Hero and mathematical contracts | Ubuntu / Python 3.12 | 279 | +| Security and local boundaries | Ubuntu / Python 3.12 | 1,992 | +| Designated reference performance | Ubuntu / Python 3.12 | 13 | +| Dependent package and delivery checks | Ubuntu / Python 3.12 | PASS | + +The eight core jobs verify actual PyArrow absence. Real Parquet includes all +five optional cases, including both lineage context paths. Current dependencies +are the resolved compatible versions for each Python, rather than one shared +NumPy pin. These supported profiles do not certify every Python/OS/version +combination permitted by broad package declarations. + +Local preflight ran all 2,965 canonical cases with real PyArrow in 157.46 seconds +and 136 affected cases in an actual no-PyArrow environment. Source/specification/ +traceability checks and affected document links passed. Golden edits in Step 10 +were verified to change package-version strings only. + +Candidate run 36270399128 completed successfully on its first attempt. Its +unselected focused job was intentionally skipped; no required test was skipped. +The PR-open focused run 36270393683 also passed. There were no failed candidate +jobs or hidden retries. Completed logs and all candidate artifacts were +retrieved successfully. + +## Performance observations and review + +The designated hosted profile was Ubuntu 24.04.5, Linux 6.17.0-1022-azure, +glibc 2.39, AMD EPYC 7763 with four visible CPUs, 16,766,410,752 bytes host memory +and 3,221,221,376 bytes configured swap. Python was 3.12.14, NumPy 2.5.3, +pandas 3.0.6, pytest 9.1.1 and PyArrow absent. The 13-case performance suite +passed in 954.63 seconds. Full-scale tests ran once on this designated profile. + +| Workload and scope | Outer wall seconds | Recorded whole-process peak RSS bytes | Output bytes | +|---|---:|---:|---| +| Hero with lineage, attempt 1 | 0.683194 | 42,614,784 | JSON 240,782; Markdown 216,519 | +| Hero with lineage, attempt 2 | 0.676312 | 42,614,784 | JSON 240,782; Markdown 216,519 | +| Hero with lineage, attempt 3 | 0.678967 | 42,614,784 | JSON 240,783; Markdown 216,520 | +| Actual 100k reverse-chain input and ancestry API | 66.225214 | 1,848,307,712 | API summary 937; no audit report | +| 1,000-target chain, complete lineage CLI | 8.440556 | 1,785,204,736* | JSON 6,201,924; Markdown 1,531,924 | +| 64 context roots and 64 targets, complete lineage CLI | 1.653459 | 1,785,204,736* | JSON 610,566; Markdown 284,286 | +| Retained 100k metadata audit, complete ordinary CLI | 780.729803 | 7,681,183,744 | JSON 442,740,533; Markdown 165,119,562 | + +All three untraced lineage Hero attempts meet the under-five-second target on +this profile. Ordinary Hero attempts were 0.660752, 0.654029 and 0.655411 seconds. +Its separate allocation-traced attempt was 3.560648 seconds with 14,652,609 bytes +peak traced Python allocations; it is not an untraced throughput measurement. + +The 100k chain has 100,000 target nodes, 99,999 edges, no context nodes, +100,000 root memberships, 99,999 union visits and depth 99,999. G/C/U is +100,000/0/0, with one root, HHI 1 and effective roots 1. Validation took +26.480915 seconds and ancestry 39.209371 seconds. This scope includes loading, +generation validation and ancestry, and excludes ordinary metrics and full +JSON/Markdown audit serialization. The bounded fan-in case has 4,096 edges, +4,160 memberships, 4,096 visits and depth 1. The metadata audit checks all +100,000 records, independent arithmetic and 66,000 retained warnings. + +*Memory review:* both bounded hosted CLI cases already recorded a +1,785,204,736-byte high-water mark before CLI invocation, unchanged afterward. +That pre-invocation floor cannot establish a new 1.66 GiB CLI memory requirement. +A narrowly selected two-case recheck on the unchanged candidate runtime passed +in 13.59 seconds. On the Step 9 reference container, the 1,000-target CLI took +11.099089 seconds with a 29,306,880-byte pre-CLI mark and a 131,702,784-byte peak; +fan-in took 1.964126 seconds with a 48,889,856-byte mark both before and after. +The fan-in observation remains bounded by its pre-CLI floor and is not an +incremental allocation estimate. This recheck used Python 3.12.14, NumPy 2.3.5, +pandas 2.2.3 and installed but unused PyArrow 25.0.1 on the Xeon 8370C container +documented in Step 9. It did not rerun the 100k workloads or candidate matrix. + +The comparable local chain result remains close to Step 9's 11.152764 seconds +and 131,334,144-byte peak, below the approved regression-review thresholds. +Hosted/container timing differences are separate observations, not a claimed +product speedup. Raw pre-invocation RSS, every measured attempt and all scopes +remain in their respective JUnit records. No timing guard or resource threshold +was relaxed, and no product change was needed for the review. + +## Packaging, installation and reproducibility + +Delivery job 108485664963 passed on the exact candidate after all prerequisite +jobs. It reused this run's complete core/Parquet JUnit with source-commit checks, +then verified complete current test identities, strict wheel/sdist metadata, +40 shipped Python modules, seven canonical resources and clean installed +behavior outside the checkout. Installed checks covered import/input-only +boundaries, mathematical APIs, report/privacy/output behavior and ordinary plus +explicit-lineage Hero in standard/redacted modes with network blocked. Both +wheel and sdist paths passed the existing installed checks. + +Two builds used the same committed source, recorded build environment and +`SOURCE_DATE_EPOCH=1790455223`. Wheel bytes were identical. Every sdist file +payload matched; compressed sdist container bytes differed because timestamps +may differ. This establishes the documented same-environment reproducibility +scope. It does not claim identical bytes across arbitrary toolchains. + +| Delivered artifact | Bytes | SHA-256 | +|---|---:|---| +| `recursive_integrity_toolkit-0.1.0.dev4-py3-none-any.whl` | 284,740 | `e7d763ea05fc30cb2fdfea298cf1ef7f1ecbe8c0a273907d35676afdb41fbb9a` | +| `recursive_integrity_toolkit-0.1.0.dev4.tar.gz` | 256,685 | `5b0c6d88f4488150ccded959bdedeabdafa93f97ad4af513704b7d5e69822655` | +| `recursive-integrity-toolkit-source.zip` | 1,307,831 | `0df49b1a3f3dd837be2082bc98836a71377148715d72ca61a93bca96fce20551` | + +The source ZIP was checked against all 273 tracked candidate files and their +exact bytes. [Delivery artifact 10915637271](https://github.com/DavidWallstructurallaw/recursive-integrity-toolkit/actions/runs/36270399128/artifacts/10915637271) +contains the packages, candidate source ZIP, dependency records, complete reused +JUnit, build/install logs and reproducibility results. [Performance artifact +10916225143](https://github.com/DavidWallstructurallaw/recursive-integrity-toolkit/actions/runs/36270399128/artifacts/10916225143) +contains the raw scale observations. These hosted ZIPs were retrieved and their +downloaded bytes matched the reported artifact digests. GitHub retention is +90 days for delivery and 30 days for other job evidence; this completion record +and prior phase records remain recoverable from Git. + +## Administrative handoff + +The final administrative successor changes only this completion report. The +other 272 tracked entries retain the tested candidate's exact mode, type and +blob identity. Existing source/specification/traceability and affected document +checks pass. Candidate evidence is reused under the approved governance policy; +no second full matrix is claimed. The archived code candidate contains the +earlier pending form of this report; the final report records its completed +results separately. A skipped administrative CI run does not imply merge-ready +branch-protection status. + +## Scientific and operational limits + +Root identity and structural depth describe declared topology. They do not +certify independently generated evidence, truth, quality or causal dependence. +Missing, unknown and unresolved evidence remains explicit. The primary version +owns the target denominators; context records supply ancestors only. The +shared-root proxy is descriptive and does not provide a calibrated risk score. + +The frozen Hero has eight resolved v2 targets, five roots, HHI 1/4, effective +roots 4 and lineage bounds [0,0], while direct bounds remain [1/2,1/2]. The +independent partial oracle and adversarial cases preserve incomplete evidence. + +[Step 9](PHASE_5_STEP_9.md) records seven detected meaningful mutations and the +actual 100k reverse-chain input/ancestry run: 123.36 seconds and 1.71 GiB peak RSS. +That measurement excludes ordinary metrics and full audit serialization. The +1,000-record complete lineage CLI took 11.15 seconds. These scopes differ from +the historical Phase 4 metadata-only 100k audit. Timeouts and finite graph/root +limits are engineering resource guards, not scientific thresholds or performance +SLAs. Inputs are intended for trusted stable local filesystems as documented. + +Phase 6A longitudinal orchestration and Phase 6B reopening/experiments remain +deferred. This milestone does not merge a PR, tag or publish a package, or claim +final v0.1 completion. diff --git a/PHASE_5_DECISIONS.md b/PHASE_5_DECISIONS.md new file mode 100644 index 0000000..47b5afd --- /dev/null +++ b/PHASE_5_DECISIONS.md @@ -0,0 +1,313 @@ +# Phase 5 Decisions and Current Authorization + +Status: **PLAN APPROVED; STEPS 1-9 COMPLETE; STEP 10 AUTHORIZED**. + +On 2026-09-23 UTC the Theory Owner instructed: +`批准，Phase 5 step 1开始`. +This accepts the delivered Phase 5 plan and its eleven decisions without stated +exceptions, and authorizes Step 1. Execution remains one authorized step at a +time. Step 2, a main merge, a tag and software publication are not authorized by +this instruction. + +The accepted starting commit is +`1db3b1a460c233242ff37fbe45b8fac74d6101ef`, on `phase4-reports-cli`. +The Phase 5 implementation branch is `phase5-lineage`. The package remains +`0.1.0.dev3` and the executable report schema remains `1.0` during Step 1. +The original standalone plan records its draft submission; the repository copy +and this document record the subsequent actual approval. No new machine-readable +approval registry or historical source-binding chain is introduced. + +## Approved contract decisions + +| Decision | Status | Implementation boundary | +|---|---|---| +| P5-D01 | APPROVED | One primary target population; explicitly loaded context supplies paths and roots without entering target denominators. Empty targets have unavailable ratios and no invented version. | +| P5-D02 | APPROVED | Reuse canonical reference resolution, chronology, aliases and declaration states. Retain per-record failures and successful reference evidence in a bounded batch refactor. Original declarations determine reference coverage; unique edges determine graph behavior. | +| P5-D03 | APPROVED | Iterative cycle analysis; cycle count means cyclic strongly connected components. Distinguish cycle members and affected descendants. Depth measures fully resolved structural parent paths. | +| P5-D04 | APPROVED | Explicitly parentless grounded anchors; grounded single-parent carryover inherits roots. Unsupported grounded-with-parent forms remain unresolved. New external inputs use separately loaded explicit anchors. | +| P5-D05 | APPROVED | Complete root sets only for exact ancestry metrics; partition targets into grounded, closed and unresolved. Unknown required-path grounding and unresolved branches propagate uncertainty. | +| P5-D06 | APPROVED | Incidence shares use all targets N; fractional root weights use the complete nonempty-root population G. HHI and effective roots retain F-012/F-013 and their topological meaning. | +| P5-D07 | APPROVED | Lineage closure is `[C/N, (C+U)/N]`; direct closure is unchanged. Shared-root presence is descriptive and discloses partial coverage. | +| P5-D08 | APPROVED | Schema 1.1 in Step 7; typed immutable results, twelve existing report sections and equal capability mirrors. Protect every new identity; omitted detail remains schema-valid. | +| P5-D09 | APPROVED | Explicit `--lineage` on audit/example and repeatable local lineage context inputs. Primary/context versions stay disjoint; validation remains input-only. | +| P5-D10 | APPROVED | Finite graph, edge, root-membership and root-union work limits; bounded diagnostics; honest unavailable outcomes after exhaustion. | +| P5-D11 | APPROVED | Consolidate current verification around behavior and bounded candidate gates. Recover obsolete implementation checks from Git/history without requiring them in every execution. | + +The operational field names, shapes, defaults and edge cases are fixed in +[docs/lineage_contract.md](docs/lineage_contract.md). That document specifies +future implementation contracts; Step 1 adds no graph execution or schema 1.1 +runtime support. Implementation steps retain their dependencies in the approved +plan. + +## Step 1 clarifications within the approved decisions + +1. **Carryover aliases:** exactly one parent means one canonical resolved parent + after the existing resolver groups aliases/repeated declarations. Extra + unresolved declarations still prevent complete inheritance. Multiplicity is + retained in reference coverage, never ancestry mass. +2. **Depth versus grounding:** depth depends on complete structural parent paths. + An explicitly parentless unknown-grounding record has depth zero while its + external-root set remains unresolved. A whole-target maximum is unavailable if + any target's required structural path is incomplete. A resolved-subset maximum + is a separately labeled diagnostic. +3. **Proxy wire vocabulary:** the conceptual absent outcome uses the existing + `not_present` value. Unavailable uses `status: unavailable` with + `level: indeterminate`; no synonymous enum or risk scale is added. +4. **Diagnostic bounds:** each lineage detail table/key collection returns at most + 100 entries. A cycle witness contains at most 64 edges. Longer witnesses are + omitted with an explicit size reason; a nonclosed prefix is never presented as + a cycle witness. Exact aggregate counts survive detail omission. +5. **Resource interface:** the approved defaults are 200,000 loaded nodes, + 1,000,000 unique edges, 1,000,000 stored root memberships and 10,000,000 root-union + candidate visits. Limits reject booleans, nulls, nonintegers and nonpositive + values. Existing input limits retain their separate meaning. +6. **Empty scope:** N = G = C = U = 0 is a valid empty analytical scope. Coverage, + closure and concentration ratios are unavailable. Exact zero declared + references retain the executed-lineage convention of coverage 1.0 with + `no_declared_parents=true`; this does not establish external grounding or a + negative shared-root finding. + +These clarify implementation details requested by Step 1. They add no scientific +formula, inferred parent, external-input evidence schema or automatic longitudinal +behavior. A genuine conflict with the approved contracts must be surfaced before +dependent implementation. + +## Current verification boundary + +The user-approved Verification Governance and Complexity Control instruction +governs consolidation. Preserve direct mathematical, input, provenance, privacy, +CLI and packaging guarantees. Identify the behavior of each retired historical +check in the Step 1 change explanation and retain its equivalent current check. +Git history and archived phase records preserve prior bodies and receipts. + +Current consistency/dispatch checks continue to reject unauthorized product +changes where that boundary applies. Frozen Phase 0 specifications, the six +canonical Hero files, runtime modules, executable schemas and package version +remain unchanged in Step 1. Current canonical regression remains executable; +release verification retains compatibility, clean installs, reproducibility and +artifact integrity at suitable candidate gates. + +Do not add a new Tier D mechanism without identifying its exact failure, why +Tier A/B/C cannot detect it, why Git/archive cannot suffice, maintenance cost and +an exit/consolidation path. This step proposes no such mechanism. Historical +Phase 4 approval records retain their historical scope and are not rewritten as +new Phase 5 authority. + +The Step 1 consolidation retains these current protections while retiring the +corresponding historical execution machinery: + +| Historical mechanism consolidated | Protected behavior retained in current checks | +|---|---| +| Snapshot fixtures, staged path/migration assertions and historical-only Phase 4 gate suites | Current positive checks and negative tests reject unauthorized Step 1 product changes and frozen-specification tampering. | +| Exact owner-wrapper/docstring/placeholder assertions around implemented modules | Direct mathematical, representation, provenance, bounds, reference, generation, report, privacy and CLI tests exercise current supported behavior. The PR-003 operation inventory and T5 deferred public boundary remain explicit. | +| Historical package/schema/Hero source bindings | Current checks reject packaged schema or Hero corruption and protect frozen Hero inputs/expectations. | +| Repeated historical workflow/phase dispatch assertions | One current dispatch path and current workflow behavior checks retain focused ordinary gates and candidate compatibility/delivery roles. | + +Input-only validation and absence of analytical/network/IO side effects retain +their direct integration/security cases. Historical source forms remain +recoverable from the accepted starting commit and archived Phase 4 evidence. +This table explains consolidation; it is not a new executable registry. + +## Acceptance and handoff + +Step 1 requires reviewed typed contracts, independent fixture expectations and +affected consistency/dispatch/behavioral boundary checks. It does not require a +release matrix or demonstrate graph correctness. Actual changes, check results, +failures and unresolved limitations belong in the Step 1 completion report. +This approval record makes no claim that those checks have already passed. + +Step 1 completed before the following separate execution instruction. + +## Step 2 execution instruction + +On 2026-09-23 UTC the Theory Owner instructed `Phase 5 Step 2开始`. +This authorizes the bounded reference-retention refactor and immutable validated +parent graph in the approved plan. Work starts from Step 1 commit +`5e946d1a42f87f89d4f97e9da4fd76cb05517caa` on `phase5-lineage`. + +The current source boundary opens four existing modules: `models.py`, `errors.py`, +`io/validation.py` and `lineage/graph.py`. Direct reference/graph tests, affected +generation/input neighbors, documentation and the current focused gate are in +scope. This updates the single current control without adding a registry or +historical migration. Package version and executable report schema remain +`0.1.0.dev3` and `1.0`. No cycle/depth/root algorithm, lineage metric, new CLI role, +full environment matrix or performance measurement is authorized by this step. + +Stop after Step 2; subsequent execution needs its own instruction. Existing +authorization to synchronize this branch is retained. Merge, tag and publication +remain separate actions. + +## Step 3 execution instruction + +On 2026-09-23 UTC the Theory Owner instructed `Phase 5 Step 3 继续`. +This authorizes iterative strongly connected component detection, affected +descendants, bounded cycle witnesses, topological ordering and structural depth. +Work starts from Step 2 commit +`e40543ff5ed3d2793a41bf58d7e4831bd2a9bfe9` on `phase5-lineage`. + +Runtime implementation is confined to the existing `lineage/cycles.py` owner. +The existing T6 test module becomes a direct behavior suite; input/graph and +generation neighbors remain the affected gate. The single current source check +advances to the completed Step 2 tree and opens only this implementation module. +No additional historical dispatcher or source-binding registry is introduced. +Root propagation, ancestry metrics, schema/report integration and CLI lineage +dispatch remain subsequent steps. Stop after Step 3; merge, tag and publication +remain outside this instruction. + +## Step 4 execution instruction + +On 2026-09-23 UTC the Theory Owner instructed `Phase 5 Step 4 继续`. +This authorizes external-root anchors, carryover inheritance, complete root +unions, G/C/U classification and coverage, including the approved root resource +guards. Work starts from Step 3 commit +`11192a3c353292f11d85eb3725c8edc73a71967b` on `phase5-lineage`. + +Runtime changes are confined to `lineage/ancestry.py`. Direct T4 and root-resource +tests exercise the new behavior; graph/cycle, provenance and input-only neighbors +retain their relevant checks. The existing source boundary advances its single +comparison commit to Step 3 and opens only the ancestry module. No new registry +or historical migration is added. Incidence, concentration, bounds, shared-root +proxy, report/schema integration and CLI lineage dispatch remain later steps. + +The conflict rule follows frozen `DEFINITIONS_AND_UNITS.md` section 5.8 and +`DATA_AND_PROVENANCE_SPEC.md` section 9.12: unresolved grounding/ancestry conflicts +block root classification. A `batch_id` or `timestamp` difference between record +and provenance namespaces alone does not establish such a conflict. Original +metadata is preserved; no new grounding inference is made from labels or URIs. + +Stop after Step 4; merge, tag and publication remain outside this instruction. + +## Step 5 execution instruction + +On 2026-09-23 UTC the Theory Owner instructed `Phase 5 Step 5 继续`. +This authorizes typed root contributions, incidence with the full target +denominator, fractional root mass with the grounded target denominator, +ancestry HHI and effective external-root count. Work starts from Step 4 commit +`2450100d08355d8cb1e0be931345f44572861e4a` on `phase5-lineage`. + +Runtime changes remain confined to `lineage/ancestry.py`. Direct rational +oracles, deterministic invariance checks and affected ancestry/resource +neighbors verify the new calculations. The current source check advances its +single comparison commit to Step 4 and continues to open only the ancestry +module. No additional governance framework or historical migration is added. +Package version and executable report schema remain `0.1.0.dev3` and `1.0`. + +Stop after Step 5. Bounds, shared-root proxy, report/schema integration and CLI +lineage dispatch remain later steps. Existing branch synchronization approval +continues; merge, tag and publication remain separate actions. + +## Step 6 execution instruction + +On 2026-09-23 UTC the Theory Owner instructed `ok 继续Phase 5 Step 6`. +This authorizes lineage closure bounds and the descriptive shared-ancestry +dependence proxy under P5-D07. Work starts from Step 5 commit +`d81966561502b94701712a3d4b4b7fd50783cba9` on `phase5-lineage`. + +Runtime changes are confined to `metrics/bounds.py` and `lineage/ancestry.py`. +Lineage bounds use the complete target G/C/U partition and preserve direct +bounds. The proxy uses exact root incidence with explicit coverage and no +calibrated risk interpretation. Its absence requires complete nonempty target +evidence; incomplete evidence without a witness remains unavailable. + +The current source check advances its single comparison commit to Step 5 and +opens these two existing owners. Direct oracle, uncertainty, input-boundary and +direct-bound regression checks cover the change; no additional governance +architecture is introduced. Package version and executable report schema stay +`0.1.0.dev3` and `1.0`. + +Stop after Step 6. Report/schema and CLI integration remain Steps 7-8. Existing +branch synchronization approval continues; merge, tag and publication remain +separate actions. + +## Step 7 execution instruction + +On 2026-09-23 UTC the Theory Owner instructed `Phase 5 Step 7 开始`. +This authorizes schema 1.1, explicit lineage report assembly and identity-safe +JSON/Markdown output under P5-D08. Work starts from Step 6 commit +`e2666515d98cfe41483b5c43345c642bab02ced0` on `phase5-lineage`. + +Runtime changes are confined to the existing result/schema, report assembly, +Markdown renderer, safe diagnostics and ancestry owners. The ancestry change binds supplied +results to relevant validated declarations so that same-key stale results cannot +enter a new report; it performs no graph or metric recalculation. Both report +schema resources advance together. No new product module is introduced. + +Current report fixtures are consolidated to schema 1.1 with their independently +authored scientific values preserved. Schema 1.0 fixtures remain recoverable in +Git history. Direct semantic, malformed-wire, redaction and report compatibility +checks cover this public boundary. The single current source control advances +to Step 6 and opens only the seven changed product paths. No new migration registry +or meta-verification architecture is added. + +Package version stays `0.1.0.dev3`. Step 8 CLI flags, context-role ingestion and +dispatch remain separate work. Stop after Step 7. Existing branch synchronization +approval continues; merge, tag and publication remain separate actions. + +## Step 8 execution instruction + +On 2026-09-26 UTC the Theory Owner instructed `Phase 5 Step 8 开始`. +This authorizes the explicit CLI and installed Hero integration under P5-D09, +including the finite configured resource limits already specified under P5-D10. +Work starts from Step 7 commit +`267504e8fee4784f6d041e445a9e690c982e61a3` on `phase5-lineage`. + +`audit` and `example` gain explicit lineage opt-in. Repeatable local context +inputs supply ancestors without entering the primary population's denominators +or automatically requesting comparison. Context versions must remain disjoint +from primary and comparison versions. Configuration resolution remains inert; +`validate` may load context for input/reference validation and never executes +lineage or analytical families. Existing empty-file rejection remains unchanged. + +The current source check advances its single comparison commit to Step 7 and +opens the existing configuration, input-role, CLI and report-schema owners needed +for this integration. Direct CLI/config/input tests, affected metric/report +neighbors and installed Hero checks protect the public boundary. CSV/JSONL and +real Parquet context inputs receive relevant loader checks. No new historical +migration mechanism or meta-verification registry is authorized or required. + +Package version stays `0.1.0.dev3`; report schema stays `1.1`. Step 9 scale work +and Step 10 candidate verification remain subsequent steps. Stop after Step 8. +Existing branch synchronization approval continues; merge, tag and publication +remain separate actions. + +## Step 9 execution instruction + +On 2026-09-26 UTC the Theory Owner instructed `Phase 5 Step 9 开始`. +Work starts from Step 8 commit +`f1c5309deda3ae4107c78f025cb2a7bb85ba2881` on `phase5-lineage`. +This authorizes bounded adversarial and mutation checks, actual 100k sparse +lineage execution on the designated reference profile, and full CLI/Hero +measurements under P5-D10 and the approved performance policy. + +The initial reverse-chain probe demonstrates quadratic generation scans and +identity lookup. The plan's explicit narrow allowance applies: replace those +operations with behavior-preserving immediate-dependency propagation and a key +set, with generation and input/lineage neighbor regression. No scientific +definition or unrelated kernel is opened. The single current source boundary +advances to Step 8; no new governance mechanism is introduced. + +Package version remains `0.1.0.dev3`; report schema remains `1.1`. Step 10 owns +the complete candidate environment matrix, release checks and dev4 handoff. +Stop after Step 9. Existing branch synchronization approval continues; merge, +tag and publication remain separate actions. + +## Step 10 execution instruction + +On 2026-09-26 UTC the Theory Owner instructed `Phase 5 Step 10`. +Work starts from Step 9 commit +`15d50c3d992265e716b0b27e69c43107d947d21d` on `phase5-lineage`. +This authorizes the planned dev4 version update, public documentation and +traceability, complete canonical regression, supported candidate matrix, +packaging, clean installs, reproducibility and concise completion handoff. + +The single source boundary advances to Step 9 and opens only `pyproject.toml` +and the package root version declaration. Frozen definitions, schemas, Hero +inputs and analytical code remain protected. Existing workflow triggers may be +adapted for explicit candidate selection; existing optional-Parquet collection +must distinguish the actual installed and absent dependency profiles without +skips. These are bounded repairs to the existing delivery path. + +`PHASE_5_COMPLETION.md` records actual candidate status and limits. Administrative +report successors reuse applicable verified candidate evidence without another +matrix. Stop at the Phase 5 boundary. Existing branch synchronization approval +continues; Phase 6A/6B, merge, tag and publication remain separate actions. diff --git a/PHASE_5_PLAN.md b/PHASE_5_PLAN.md new file mode 100644 index 0000000..8c9faf0 --- /dev/null +++ b/PHASE_5_PLAN.md @@ -0,0 +1,278 @@ +# PHASE_5_PLAN + +## Document control + +| Field | Value | +|---|---| +| Project | Recursive Integrity Toolkit | +| Phase | Phase 5: lineage graph and ancestry | +| Document version / date | 1.0 / 2026-09-23 | +| Status | APPROVED; Step 1 authorized | +| Theory Owner | Xiangyu Guo | +| Accepted starting commit | `1db3b1a460c233242ff37fbe45b8fac74d6101ef` | +| Accepted source tree | `51d8854d9869b8a0aeb498230e02b03797ed8634` | +| Baseline branch / package | `phase4-reports-cli` / `0.1.0.dev3` | +| Proposed implementation branch | `phase5-lineage`, from the accepted commit after Step 1 authorization | +| Proposed completed-phase version | `0.1.0.dev4`; this is not a final v0.1 release | +| Authorization recorded | Plan approved and Step 1 started by the Theory Owner on 2026-09-23 | +| Merge, tag, publication authorization | None | + +The Theory Owner approved this plan and authorized Step 1 on 2026-09-23: “批准，Phase 5 step 1开始”. All P5-D decisions below are approved without stated exceptions. Execution continues one authorized step at a time; this approval does not automatically execute all ten steps. See `PHASE_5_DECISIONS.md` for the Step 1 contract record. + +## 1. Starting point and governing constraints + +The inspected checkout is clean at the accepted commit: 246 tracked files and 40 Python package modules. All sixteen frozen Phase 0 specifications match the existing approved baseline. PR #1 remains open and draft, with the accepted commit as its head. Starting the proposed Phase 5 branch does not require merging that PR. + +Phase 4 delivers input validation, existing metrics, JSON/Markdown reports, privacy modes, a local audit CLI, one explicitly requested comparison pair and an installed Hero example. General graph traversal and ancestry remain deferred. The lineage modules and their designated tests/fixtures provide the intended implementation locations. + +Historical Phase 4 evidence includes twelve successful candidate jobs: eight supported core profiles at 3,965 tests each, a real-Parquet profile at 3,968, Hero at 203 and security at 3,277, plus delivery checks. The verified code candidate was `ceb0cf702284dbdbe50b14722026ec26fbcc21ba`; the accepted successor changed three administrative reports. These are preservation evidence, not Phase 5 results. No product tests were rerun to write this plan. + +Apply `PROJECT_INSTRUCTIONS.md` section 4.4 and the approved UD decisions. The primary Phase 5 owners are **T4, T6, PR-008 full graph behavior, PR-009 lineage depth, and T3 lineage bounds**. P3-D01 reserves these functions for Phase 5 despite older overlapping roadmap wording. The plan introduces no new interpretation of the theory papers or new scientific formula. + +The user's Verification Governance and Complexity Control instruction governs this phase: + +- Strengthen direct product/scientific correctness and real security boundaries. +- Keep compatibility, packaging and reproducibility gates at suitable candidate milestones. +- Consolidate historical implementation checks when equivalent current behavioral coverage exists and Git/archive evidence remains recoverable. +- Add no new approval registry, source-body migration chain, execution-receipt framework or evidence-of-evidence layer by default. + +## 2. Objective and exclusions + +Deliver an explicitly requested, local lineage analysis over validated records: parent graph, cycle detection, ancestry resolution, external-root incidence and concentration, lineage coverage, depth and lineage-aware closure bounds. Integrate the results into typed reports and the CLI while preserving existing metric scopes and privacy guarantees. + +Phase 6A retains automatic multi-version orchestration, trajectories, lineage-change comparisons and additional longitudinal formulas. Phase 6B retains experiment/reopening workflows. Also excluded are network retrieval, inferred parents, causal root weights, semantic ancestry, HTML implementation, model-performance conclusions, universal risk thresholds, new runtime graph dependencies and unrestricted graph/path exports. + +## 3. Approved contract decisions + +### P5-D01. Separate the target population from the supporting graph + +Use canonical `RecordKey(dataset_version, record_id)` identities. The analyzed target contains N selected records from one primary version. The loaded graph may also contain explicitly supplied ancestor records. Only target records enter target counts, provenance shares, closure denominators, ancestry incidence shares and concentration allocation. Context records can supply roots and paths without becoming additional observations. + +Reports identify the target version/count and loaded graph counts separately. Existing representation eligibility exclusions remain specific to their metrics; they must not silently shrink the provenance or lineage denominator. An empty target produces explicit unavailable ratios and no invented version identity. + +### P5-D02. Reuse validated references and preserve declaration semantics + +Consume existing parent-resolution results, aliases, version order and declaration states. Do not implement a competing interpretation of parent IDs. Preserve the distinction between an absent/null declaration and explicit `parent_ids=[]`. + +The current bundle validator loses its aggregate generation/parent result after the first malformed, ambiguous, future-parent or self-parent exception. Step 2 therefore permits a narrow batch resolver refactor that retains per-record successes, failures and explicit self-reference evidence, reusing one identity lookup index. Preserve public validation/error and generation semantics. Invalid references never become accepted ancestry edges; self-reference evidence remains available for cycle diagnostics. This retention is necessary to report unaffected records honestly and must not require rebuilding an O(V) index for each record. + +Resolve cross-version edges only under the existing explicit chronology. Same-version edges are allowed when acyclic. Duplicate canonical identities fail validation. Missing parents remain unresolved and warning-level by default, with existing strict-mode escalation; ambiguity and future-parent violations remain errors. + +Reference coverage counts original declared references, including aliases or repeated references, using the existing resolver's denominator. Graph adjacency and root unions deduplicate canonical edges/roots. Thus repeated declarations cannot inflate ancestry mass. With no declared references, edge coverage is 1.0 and `no_declared_parents=true`; this alone proves neither grounding nor ancestry completeness. + +This zero-reference convention follows the frozen definition and deliberately replaces the current report's null/unavailable convention when lineage executes under schema 1.1. With lineage disabled, retain the existing immediate-reference observation and null reason for its zero denominator, clearly labeled as validation evidence; do not synthesize executed graph coverage. Test both modes explicitly. + +### P5-D03. Define cycles, affected records and depth precisely + +Use iterative graph algorithms. Report `cycle_count` as the number of **cyclic strongly connected components**, explicitly labeled with that method: a component has multiple nodes or a self-loop. This resolves the previously unspecified counting convention without enumerating every simple cycle. + +Distinguish cycle members from affected descendants along parent-to-child edges. Return a bounded deterministic witness per reported component, total counts and explicit detail-omission metadata. A detected cycle is a lineage validation error. Affected target records have unresolved ancestry; independent families and unaffected target ancestry may remain reportable with partial status and precise scope. Never label the entire loaded graph acyclic when a disconnected cycle exists, and never return a successful CLI audit status for a lineage error. + +Depth is the maximum number of edges to loaded parentless roots on fully resolved paths. An incomplete or invalid required path produces null with a reason. Known partial depth may be diagnostic only when labeled as a lower bound. Generation and lineage depth remain distinct: a carryover can have generation zero and positive depth. + +### P5-D04. Adopt a conservative, schema-preserving external-root rule + +An explicitly parentless record with validated `external_grounding=yes` anchors an external root. A parentless record with grounding `no` has a known-empty root set; grounding `unknown`, missing required provenance or an undeclared parent boundary makes root resolution incomplete. + +A grounded `transformation=carryover` record with exactly one resolved parent inherits that parent's external roots. It does not create a new root merely because its own grounding is `yes`. A carryover with additional unresolved declarations is incomplete. + +Represent a new independent external input as a separately loaded, explicitly parentless, grounded record and link it as an ordinary parent. No evidence URI, source label, review flag or transformation string alone certifies independence in Phase 5. This pattern supports truthfully ungrounded child transformations and unambiguous single-parent grounded carryovers. + +A grounded record with parents outside the single-parent carryover rule remains unresolved in Phase 5, even when a separate grounded anchor is present. A different rule would require an explicit contract decision. Multi-parent carryover is not silently collapsed. Never change a child's actual grounding label merely to fit the supported representation. + +Strict ancestors exclude the record itself. Separately, an external root supports itself: its support-root set is `{its own key}`. This convention must be named explicitly wherever root support is exposed. + +### P5-D05. Require complete root sets for exact ancestry metrics + +For each target record r, compute its unique external-root set A_r only when every required path is resolved under P5-D02 through P5-D04. Unknown grounding anywhere on a required path propagates uncertainty. One known root plus an unresolved branch is insufficient for an exact root distribution. + +Partition targets into grounded G, closed C and unresolved U: + +- G: complete, nonempty external-root set. +- C: complete, known-empty external-root set. +- U: incomplete or invalid ancestry, including missing/ambiguous links, cycles, unknown grounding, conflicting declarations and undeclared boundaries. + +N = |G| + |C| + |U|. `records_with_resolved_external_ancestry` includes G and C. Resolved lineage coverage is (|G|+|C|)/N; external ancestry coverage is |G|/N. Optional observed roots on incomplete paths are diagnostics only and cannot enter exact root metrics. + +### P5-D06. Preserve the approved incidence and fractional-mass formulas + +For each unique root a among G: + +```text +I_a = count of r in G for which a belongs to A_r +incidence_share_a = I_a / N +M_a = sum over r in G containing a of 1 / |A_r| +w_a = M_a / |G| +ancestry_HHI = sum_a w_a^2 +effective_external_root_count = 1 / ancestry_HHI +``` + +Each grounded target contributes total mass one, including a target that is itself an external root. Rank roots by decreasing incidence, breaking ties with canonical record key. Distinct roots are the union over G. All reported shares state their denominators. If G is empty, HHI and effective-root count are unavailable/undefined with reasons; they are not zero or infinity. + +These are topological allocation conventions. They do not estimate causal contribution, semantic error, independent information, biological relatedness or universal integrity. Multiple paths to the same root must not multiply that root's contribution. + +### P5-D07. Add lineage bounds without changing direct bounds + +For nonempty targets, lineage closure bounds are: + +```text +lower = |C| / N +upper = (|C| + |U|) / N +width = |U| / N +``` + +Keep direct closure calculations and their classifications unchanged. Report coverage and unresolved reasons alongside lineage bounds. A shared-root dependence signal is a descriptive proxy: `present` when a completely resolved root supports at least two targets; `absent` only when all target ancestry is resolved and no such root exists; otherwise `unavailable`. A witnessed `present` on partial coverage discloses that coverage. This is not a calibrated risk level. + +T4 retains its theory-guided operationalization label. T6 is an engineering graph-validity rule, not a theorem establishing the theory's scientific claims. + +### P5-D08. Use one report schema revision and protect new identifiers + +Use `report_schema_version: "1.1"` for all Phase 5 reports. Retain the twelve top-level sections, equal capability mirrors, finite values, explicit unavailable reasons and unchanged non-lineage numerical semantics. The root schema and packaged schema remain identical. + +Extend typed immutable results for lineage scope, resolution counts, roots/incidence, concentration, bounds, depth and cycles. Values are computed before rendering. JSON and Markdown share the same canonical result; renderers own no graph or mathematical logic. + +Lineage execution defaults to `not_requested`; explicit execution yields `completed`, `partial` or `failed` according to actual coverage/errors. Input observability remains distinct from execution. More loaded versions do not mean a comparison ran. + +Apply existing privacy modes to every new key, path and witness. Redacted output supports an explicit aggregate-only or omitted-detail representation that still validates against the schema. Removing required row fields while leaving invalid rows is unacceptable. Preserve no-network, safe-output and deterministic identifier behavior. + +Strict schema-1.0 consumers must explicitly adopt 1.1. This plan does not promise backward acceptance by those readers or add a report migration framework. + +### P5-D09. Make lineage opt-in and context inputs explicit + +Add `--lineage` to `audit` and `example`. Add repeatable `--lineage-records PATH`, backed by a dedicated `FileRole.LINEAGE_CONTEXT` and existing local loaders. For `audit`, context inputs require lineage opt-in. Record/provenance row schemas remain unchanged; config input-role and report inventory enums expand coherently. + +`--records` selects exactly one primary version. `--compare` retains its existing CLI limit of one explicit earlier comparison input and all current compatibility declarations. With lineage requested, that comparison input may also supply ancestors; supplying context does not automatically request comparison. + +Context versions must be disjoint from primary and comparison versions. This keeps the existing whole-version provenance join unambiguous. Same-version lineage is supported when the needed same-version records are in the primary input. Splitting one version across target and context roles is outside this phase. Duplicate keys across repeated context inputs are rejected. + +All ordinary metric calls explicitly select the primary population instead of pooling loaded versions. Preserve existing Python/config multiplicity for other supported input roles; a CLI restriction must not accidentally narrow those APIs. + +`validate` may ingest context inputs for existing input/reference validation, but rejects the lineage execution flag and never runs graph metrics. `validate_bundle(...)` remains input-only. `rit example --lineage --out ...` executes the frozen Hero with installed resources outside the checkout. Plain `rit example` retains ordinary calculations with lineage `not_requested` under schema 1.1. + +### P5-D10. Bound graph work and disclose resource failures + +Use standard-library iterative adjacency/SCC/topological algorithms. Graph traversal should be O(V+E); root-set propagation can grow with V times the number of roots. Sparse edges alone do not guarantee bounded root-set memory. + +Approved initial configurable limits for explicitly requested lineage are 200,000 loaded nodes, 1,000,000 unique edges, 1,000,000 stored record/root memberships and 10,000,000 root-union candidate visits. A visit means processing one candidate root identity into a child's union. These finite engineering guards bound algorithmic work; they do not guarantee a particular peak RSS or establish scientific thresholds. Check cumulative node admission across primary, comparison and context batches before graph materialization, edge admission before adjacency insertion, and root memberships/work before union insertion. Retain the separate inherited input byte/row/parent-list limits: graph guards do not bound allocations already incurred while loading a table. Caller overrides must be explicit positive finite integers. + +Guard details and counters must be deterministic. On exhaustion, provide a resource-limit reason and no exact lineage values based on truncated work. Preserve independently completed non-lineage families. Bound report diagnostics and witness output separately, disclosing omitted counts; do not export all paths or a full transitive closure. Step 1 fixes the diagnostic row cap using existing report conventions; Step 9 measures whether the proposed defaults need a documented adjustment. No silent limit increase is permitted to make a gate pass. + +### P5-D11. Consolidate current verification instead of extending its history + +Maintain three practical bodies: current canonical regression, candidate/release verification, and historical evidence recoverable from Git and archived phase records. Use existing test locations and tooling; this classification does not require new registries. + +At this phase boundary, replace obsolete placeholder, exact historical test-body and source-wrapper assertions with equivalent or stronger current behavioral tests. Identify the protected behavior in the existing decision/PR explanation, retain its direct test and keep the old evidence recoverable. For example, unauthorized product mutation must still be rejected where that boundary applies; the exact old Python method body need not remain executable forever. + +Consolidate the existing phase dispatch in spec/traceability/release checks and workflows into one current path. Do not clone the Phase 4 framework into a new permanent chain. Preserve frozen source specifications and historical evidence; do not rewrite them to manufacture approval. + +Any proposed new Tier D control needs all five justifications: exact failure; why A/B/C cannot detect it; why Git/archive is insufficient; maintenance cost; consolidation/exit path. Otherwise omit it. No new mechanism of that kind is proposed here. + +## 4. Ten implementation steps + +Each step ends with its actual changes, relevant checks and remaining limitations. Dependent work starts only after its preceding contracts are settled. Failed checks are repaired before dependent execution; no step is complete merely because its files exist. + +| Step | Deliverable and dependencies | Acceptance and smallest sufficient gate | +|---|---|---| +| **1. Contracts and current verification boundary** | Record actual approval/any exceptions in a concise Phase 5 decision document; finalize typed field names, bounded diagnostic detail and resource interfaces; specify independent fixture expectations. Consolidate obsolete current-phase dispatch/assertions under P5-D11. | Every behavior retired from historical execution has current equivalent coverage and recoverable evidence. Check affected consistency/dispatch and behavioral boundary tests only. No graph completion claims. | +| **2. Validated parent graph** | Implement graph types/construction in `lineage/graph.py`, consuming the resolver with the bounded retention refactor in P5-D02. Establish target/context scope and immutable adjacency. Depends on 1. | Identity, duplicate declarations, aliases, same-version edges, chronology, missing/ambiguous parents, retained partial results and role scope cases pass. Run parent-resolution and relevant generation neighbors. | +| **3. Cycles, topology and depth** | Implement iterative SCC detection, affected descendants, bounded witnesses, topological ordering and depth. Depends on 2. | Self-loop, two-node/long/disconnected cycles, descendant propagation and a chain beyond Python recursion depth behave as specified. Errors never claim completed whole-graph lineage. | +| **4. Root resolution and coverage** | Implement external anchors, carryover inheritance, unique root unions and G/C/U classification in `lineage/ancestry.py`. Depends on 2-3. | Parentless yes/no/unknown, absent versus empty declarations, multi-parent unions, incomplete branches and conservative grounded-parent cases match independent fixtures. | +| **5. Incidence and concentration** | Add typed root incidence, denominator metadata, fractional mass, HHI and effective roots. Depends on 4. | Hero and multi-root arithmetic match exact/rational oracles; renaming, permutation and duplicate-path invariance hold. Empty G has explicit unavailable values. | +| **6. Lineage closure and descriptive proxy** | Integrate lineage bounds and shared-root proxy, leaving direct bounds unchanged. Depends on 4-5. | G/C/U partition and interval identities hold; missing and unknown propagation are tested independently; direct-bound regressions remain unchanged. | +| **7. Canonical reports and privacy** | Integrate typed results, schema 1.1 and its packaged mirror, assembly, JSON/Markdown and redaction. Depends on 3-6. | Non-lineage values/scopes remain stable; complete/partial/failed/not-requested results validate; new keys and witnesses obey privacy; no renderer-owned calculation or invalid redacted row. | +| **8. Explicit CLI and installed Hero** | Wire `--lineage`, context role/flags and explicit primary selection; document usable commands. Depends on 7. | Audit opt-in, context-only ancestry, existing comparison behavior, input-only validate, meaningful exit codes and installed Hero pass. Exercise core inputs and real Parquet where the new role crosses that loader boundary. | +| **9. Bounded adversarial, mutation and scale work** | Resolve remaining graph/resource defects, measure lineage workloads, run a finite meaningful mutation set. Depends on 2-8. | Resource failure is honest; no recursion dependence; 100k sparse graph completes on the designated reference profile; end-to-end lineage is measured; meaningful seeded mutations are killed or remaining gaps explained and corrected. | +| **10. Candidate verification and handoff** | Update public docs/traceability and package to dev4; complete canonical regression, supported candidate matrix and delivery checks; record concise completion and limitations. Depends on 1-9. | Frozen Hero expectations hold, required owners implemented, supported installs and reproducibility verified, no Phase 6 execution or publication implied. Administrative report successors reuse unchanged candidate evidence. | + +The existing generation validator uses repeated propagation scans. If the deep-chain/100k workload demonstrates a bottleneck there, Steps 3 or 9 may replace that propagation with a behavior-preserving iterative dependency algorithm. Preserve existing generation semantics and run its direct/dependency-neighbor tests. This narrow allowance does not authorize unrelated input or metric redesign. + +## 5. Independent acceptance oracles + +### 5.1 Frozen Hero + +Keep the six canonical Hero input/expectation files unchanged. Construct current report goldens from their approved values rather than editing the oracle to fit output. + +| Quantity | Required value | +|---|---| +| Primary target / loaded records | v2: 8 / v1 plus v2: 16 | +| v1/v2 support | 8 / 5 | +| v1/v2 diversity / retention | 0.875 / 0.75 / 0.625 | +| v2 human/synthetic shares | 0.5 / 0.5 | +| Direct / lineage closure interval | [0.5, 0.5] / [0, 0] | +| Declared / resolved target references | 8 / 8 | +| Cycle detected / unresolved references | false / 0 | +| Distinct supporting external roots | 5 | +| Top root / incidence / share | `v1::v1_01` / 3 / 3/8 | +| Root masses / normalized shares | [3, 2, 1, 1, 1] / 8 | +| Ancestry HHI / effective roots | 0.25 / 4.0 | +| Input observability / lineage execution | Level 4 / completed when requested | +| Shared-ancestry proxy | present, with topological limitations | + +Required unavailable conclusions remain model-performance decline, causal ancestor effect, universal integrity and universal collapse prediction. No simulation runs by default. + +### 5.2 Proposed multi-root and uncertainty oracle + +Use four ungrounded target records in a later version. Context anchors a and b are explicitly parentless and grounded; context u is explicitly parentless with unknown grounding. Target t1 has parent a; t2 has parents a,b; t3 has explicit empty parents; t4 has parents a,u,and a missing key. + +| Quantity | Independent expected value | +|---|---| +| Complete root sets | t1: {a}; t2: {a,b}; t3: {}; t4: unresolved | +| N / G / C / U | 4 / 2 / 1 / 1 | +| Declared/resolved target references | 6 / 5; edge coverage 5/6 | +| Resolved lineage / external ancestry coverage | 3/4 / 1/2 | +| Incidence a,b / shares using N | 2,1 / 1/2,1/4 | +| Fractional masses a,b / normalized weights | 3/2,1/2 / 3/4,1/4 | +| Distinct roots / HHI / effective roots | 2 / 5/8 / 8/5 | +| Lineage lower / upper / width | 1/4 / 1/2 / 1/4 | + +Create separate variants removing the missing branch while retaining unknown grounding, and resolving unknown grounding while retaining the missing branch. Either defect independently keeps t4 unresolved. Additional compact fixtures cover carryover, repeated paths, tied ranks, all-closed, all-unresolved, empty targets, cycle components and context/target denominator isolation. + +## 6. Verification effort and performance policy + +| Change/risk | Gate | +|---|---| +| Documentation or non-authoritative metadata | Formatting and affected consistency only | +| Graph/root/math implementation | Affected unit/oracle tests plus resolver, generation or bounds neighbors | +| Input role, public schema, API/CLI or privacy boundary | Relevant integration, compatibility and boundary regression | +| Stable PR candidate | Complete canonical regression and supported OS/Python/dependency matrix | +| Release/delivery candidate | Candidate checks plus build, wheel/sdist, clean install, canonical examples, reproducibility and required security checks | +| Administrative successor with unchanged executable/authoritative bytes | Direct document consistency and evidence reuse; no automatic matrix rerun | + +Keep Ubuntu/Windows, Python 3.11/3.12, current/minimum dependency coverage and a real optional-Parquet profile. Adapt existing workflow triggers or dispatch so ordinary implementation updates use the affected gate and a stable candidate receives the full matrix. Reuse existing jobs instead of creating parallel governance workflows. + +Run the expensive 100,000-record sparse lineage benchmark on one designated reference profile; use bounded structural cases across the compatibility matrix. Preserve the required actual 100k execution without multiplying it across every environment. Candidate failures require rerunning affected gates; unchanged successful environments may be reused only when their tested candidate remains applicable and that relationship is clear. + +Measure elapsed time, peak RSS, graph size, target/context counts, total root memberships and output size. Test a deep sparse chain and a small high-fan-in graph, because root-set growth can dominate graph traversal. Include at least one full CLI lineage measurement through report serialization. Kernel timing alone cannot establish end-to-end performance. + +The existing Hero target is under five seconds on its stated reference environment, including input loading, validation, observability, required metrics and JSON/Markdown. Measure it afresh with lineage enabled. The sparse-lineage 100k requirement has no approved hard wall-time/RSS SLA. Establish a reproducible baseline and a measured operational CI timeout; an execution timeout is not a product performance target. + +Phase 4's metadata-only 100k audit took approximately 1,362.47 seconds and 7,640,743,936 bytes peak RSS. It did not execute general ancestry. Use that result to understand inherited report overhead, not to claim Phase 5 performance. A >20% time or >50% memory regression on a comparable workload requires review; changed workloads must be labeled. Reuse the existing scope pooling and avoid duplicating per-record graph evidence across report sections. + +Bound mutation work to meaningful deterministic defects: wrong N versus G denominator, set union replaced by duplicate counting, carryover minted as a root, unknown treated as closed, omitted cycle propagation, reversed bounds and incorrect HHI inversion. Record the attempted mutation and whether a current test detects it in the existing validation report. No permanent mutation registry or test-count target is required. + +## 7. Change scope and completion boundary + +Expected implementation locations are `lineage/`, narrowly shared models/config/errors, input-role adapters, bounded parent-result retention and necessary generation propagation, lineage branches of bounds, result/report assembly, JSON/Markdown, CLI, config/report schemas and their packaged mirrors. Relevant fixtures, tests, public docs, existing verification tooling and version declarations may change. These are ownership boundaries, not a new hash-based file authorization system. + +Do not change unrelated mathematical kernels, normalization semantics, direct provenance definitions or theory-source documents. Preserve frozen Phase 0 files. Approved Phase 5 clarifications are additive decisions with source references. An actual conflict or newly required independent-input evidence schema is surfaced before dependent implementation, with the affected step identified. + +Phase 5 is complete when its required owners, Hero and advanced oracles, privacy/resource boundaries, installed CLI, candidate compatibility and delivery checks pass, and limitations accurately describe partial evidence and topological meaning. Completion records identify the tested candidate and actual results without requiring historical implementation forms to execute forever. + +The handoff leaves Phase 6A and Phase 6B explicitly deferred. It does not merge a PR, tag a release, publish a package or claim final v0.1 completion. Future verification growth should primarily follow new product capabilities. + +## 8. Source basis for review + +Repository-relative references below identify the existing authority; they are not additional executable approval artifacts. + +- `PROJECT_INSTRUCTIONS.md`, Phase 5 and lineage/ancestry rules; section 4.4 authority order. +- `V0.1_PRODUCT_SPEC.md`, phase sequence and lineage requirements. +- `PHASE_3_DECISIONS.md`, P3-D01 scope reservation. +- `DEFINITIONS_AND_UNITS.md`, strict ancestry, roots, incidence, mass/concentration, coverage and bounds. +- `DATA_AND_PROVENANCE_SPEC.md`, sections 11.7-11.10 and version-order rules. +- `THEORY_TO_CODE_TRACEABILITY.md`, Phase 5 owners T4/T6/PR-008/PR-009/T3. +- `REPOSITORY_ARCHITECTURE.md`, lineage ownership and phase responsibilities. +- `OBSERVABILITY_AND_REPORTING.md`, typed evidence, explicit unavailable states and interpretive boundaries. +- `VALIDATION_PLAN.md`, sections 61.1, 62.3 and 81; `SUCCESS_CRITERIA.md`. +- `examples/hero/EXPECTED_OUTPUTS.md`, frozen full-product Hero expectations. +- `PHASE_4_COMPLETION.md`, `PHASE_4_VALIDATION_REPORT.md`, `PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md` and accepted candidate evidence. +- User instruction: Verification Governance and Complexity Control, supplied during Phase 4. diff --git a/PHASE_5_STEP_1.md b/PHASE_5_STEP_1.md new file mode 100644 index 0000000..1d300a2 --- /dev/null +++ b/PHASE_5_STEP_1.md @@ -0,0 +1,116 @@ +# Phase 5 Step 1 Completion + +Date: 2026-09-23 UTC. Status: **STEP 1 COMPLETE; PHASE 5 INCOMPLETE**. + +The Theory Owner approved the Phase 5 plan and instructed Step 1 to start. +Work begins from `1db3b1a460c233242ff37fbe45b8fac74d6101ef` on the new +`phase5-lineage` branch. This record covers contracts, fixtures and verification +consolidation only. It grants no subsequent-step execution, merge, tag or +publication. + +## Delivered + +- Approved `PHASE_5_PLAN.md` and P5-D01 through P5-D11 in + `PHASE_5_DECISIONS.md`. +- `docs/lineage_contract.md`: immutable type/field interfaces, explicit scope, + root/uncertainty/depth rules, status semantics, bounded privacy-safe detail and + resource interfaces for later implementation. +- Twelve independent acceptance cases across the three existing lineage fixture + directories, plus separate frozen-Hero lineage expectations. Exact oracle + arithmetic is recorded independently of future algorithms. +- One current path in the three existing verification scripts. Historical + source-body migrations, snapshot fixtures and staged authorization assertions + no longer run by default. Earlier forms remain recoverable from Git. +- Focused ordinary CI and explicitly selected candidate CI. The supported matrix, + real Parquet, Hero, security, installed packages and reproducibility remain + required at candidate gates; the expensive performance directory runs once on + a designated reference profile. + +The executable product is unchanged: forty Python modules, five root schemas, +seven packaged resource copies, six canonical Hero files and package metadata +match the accepted Phase 4 product. Package version remains `0.1.0.dev3`; report +schema remains `1.0`. General lineage algorithms are still deferred. + +## Contract details resolved + +One canonical parent remains one parent even when its declaration has multiple +aliases. Reference coverage retains those declarations, while adjacency/root mass +deduplicates them. Unknown grounding can coexist with known structural depth. +Other grounded-with-parent forms remain unresolved outside the approved +single-parent carryover rule. Empty analytical scopes do not imply current CLI +acceptance of an empty records file. + +Detail collections are capped at 100 rows; complete cycle witnesses at 64 edges. +Long witnesses are explicitly omitted rather than presented as nonclosed prefixes. +The four initial lineage budgets are 200,000 nodes, 1,000,000 edges, 1,000,000 +logical root memberships and 10,000,000 root-union visits. These are future graph +work guards, separate from ingestion limits and measured RSS. + +Current normalization preserves an absent parent field in +`CanonicalRow.field_states` while its normalized value is null. Step 2 must use +that retained evidence to preserve absent/null declaration semantics. + +## Verification performed + +| Check | Actual result | +|---|---| +| Three current source/specification/traceability commands | PASS: 59 protected product files unchanged, 16 frozen specifications, 7 exact resource copies and 40 owned modules | +| Exact focused CI test selection | 212 passed in 3.62 seconds | +| Affected current-control and neighboring tests | 263 passed in 3.01 seconds; overlaps the focused selection and is not added to it | +| Workflow scheduling/JUnit boundary tests after final path-filter fix | 18 passed | +| New fixture input validation | 12 passed; these also belong to the focused selection | +| Final canonical collection, excluding designated performance directory | 2,629 collected without errors; this is collection, not a full passing run | +| Independent fixture arithmetic | Hero and rational multi-root partitions, coverage, bounds, HHI and effective-root values checked | +| Four workflow files | YAML parsed; candidate dependencies and read-only permissions checked | +| Retained same-named test/helper definitions | 746 compared structurally; executable bodies unchanged, excluding explanatory docstrings | +| Diff formatting and independent review | PASS; no remaining Step 1 blocker | + +Checks used Python 3.12, pytest 9.1.1, NumPy 2.5.3 and pandas 3.0.6. Missing test +dependencies in the resumed local environment were installed before execution. +No full regression, cross-platform matrix, real-Parquet profile, release build, +installed-artifact check or 100k performance run was performed for Step 1. +Hosted execution of the changed workflows is not claimed by this record. + +The local focused command was: + +```bash +python -m pytest -p no:cacheprovider -q \ + tests/integration/test_current_verification.py \ + tests/integration/test_ci_workflows.py \ + tests/unit/test_lineage_fixture_inputs.py \ + tests/unit/test_phase3_contracts.py \ + tests/integration/test_phase3_metric_pipeline.py \ + tests/unit/test_PR016_determinism.py \ + tests/integration/test_no_algorithms.py \ + tests/integration/test_repository_structure.py +``` + +## Guarantees retained through consolidation + +Direct current tests continue to protect mathematical formulas, representation +semantics, provenance and direct bounds, input-only validation, unavailable +states, report schemas, privacy, no-network behavior, safe paths and CLI output. +Current negative checks reject unauthorized runtime mutation, self-authorized +specification edits, corrupted/extra packaged resources, damaged or incomplete +wheel/sdist payloads, duplicate archive entries and unsafe extraction members. +Existing installed behavioral checks remain in candidate verification. + +The three verification scripts shrink from 17,316 to 1,444 lines by removing +historical dispatch and repeated source-preservation machinery. This count +describes the consolidation and is not an acceptance target. The behavior map is +in `PHASE_5_DECISIONS.md`; historical bodies and evidence remain recoverable at +the accepted Phase 4 commit and its earlier history. No new baseline registry, +historical source-binding migration or evidence-of-evidence mechanism was added. + +Independent review found that a blanket documentation path exclusion would hide +changes to the new authoritative lineage contract. The workflow now excludes only +named nonauthoritative documents; a lineage-contract change receives focused +checks. The final workflow tests cover that distinction. + +## Handoff + +Step 2 will construct the validated parent graph and retain per-record reference +successes/failures using one identity lookup. Its algorithms and runtime changes +are not implemented here. Phase 6A and Phase 6B remain deferred. + +Continue only when Phase 5 Step 2 is instructed. diff --git a/PHASE_5_STEP_2.md b/PHASE_5_STEP_2.md new file mode 100644 index 0000000..182efad --- /dev/null +++ b/PHASE_5_STEP_2.md @@ -0,0 +1,90 @@ +# Phase 5 Step 2 Completion + +Date: 2026-09-23 UTC. Status: **STEP 2 COMPLETE; PHASE 5 INCOMPLETE**. + +The Theory Owner instructed `Phase 5 Step 2开始`. Work starts from +`5e946d1a42f87f89d4f97e9da4fd76cb05517caa` on `phase5-lineage`. +This step implements retained immediate-parent validation and immutable graph +construction. It does not authorize subsequent steps, a merge, tag or release. + +## Delivered behavior + +`resolve_parent_batch` shares single-reference interpretation with the existing +fail-fast resolver and builds one identity lookup per batch. Its immutable +per-record results preserve accepted and unresolved siblings when another +reference is ambiguous, malformed, a future parent or a direct self-reference. +The existing standalone resolver and generation API retain their behavior. +`validate_bundle` retains the batch without invoking analytical algorithms. + +Original declaration counts include duplicate aliases. Identity-resolved counts +remain known when chronology is unavailable; cross-version graph edges require +chronology. Adjacency deduplicates canonical parent identities. Missing +provenance, absent declarations, null declarations and explicit empty lists stay +distinct. Malformed containers with unknown cardinality never become zero +declared parents. Existing ingestion still rejects malformed serialized input at +its established boundary; the retained batch also handles typed per-reference +failures without losing successful siblings. + +`build_lineage_graph` supplies explicit target/context scope, canonical sorted +nodes, immutable parent and child adjacency, sanitized per-record evidence and +separate self-reference evidence. It revalidates retained input evidence and +rejects stale or forged handoffs. Same-version edges remain available for later +cycle analysis. Graph construction performs no file or network access. + +`LineageLimits` enforces node and unique-edge admission. Exceeding a limit raises +a typed resource error and returns no partial exact graph. Root membership and +union-work limits are validated parameters reserved for Step 4. These limits do +not claim to bound already-loaded input memory or process RSS. + +## Verification + +| Check | Actual result | +|---|---| +| Final exact focused CI selection | 438 passed in 3.98 seconds | +| Provenance, observability, capability, evidence and CLI neighbors | 501 passed in 12.18 seconds | +| Direct parent-batch and graph cases within the focused selection | 15 batch cases and 40 graph cases passed | +| Current specification, traceability and source-scope checks | PASS | +| Diff formatting and independent implementation review | PASS; no remaining concrete blocker | + +The earlier 398-test affected run is superseded by the final focused selection +and is not added to it. Independent review found two direct graph-construction +gaps: target nodes could be omitted with adjusted counts, and resolved reference +multiplicity could be falsified. Both are fixed and protected by direct +regressions. An initial graph static check found a missing implementation-status +annotation; it was restored and the complete static checks passed. The initial +graph test run had two empty-scope fixture-construction errors, corrected to use +the approved typed analytical input before the final passing run. + +The final focused command matches `.github/workflows/ci.yml`, with local JUnit +output. The neighbor selection was: + +```bash +python -m pytest -p no:cacheprovider -q \ + tests/unit/test_PR004_coverage.py \ + tests/unit/test_PR010_observability.py \ + tests/unit/test_PR011_capabilities.py \ + tests/unit/test_PR012_evidence_classes.py \ + tests/integration/test_phase4_cli.py +``` + +The three current static checks protect 55 unchanged product files, permit the +four approved implementation modules, retain the exact product inventory, and +verify 16 frozen specifications, 7 packaged resource copies and 40 owned modules. +The existing source-scope control and focused CI selection are updated in place; +no historical migration, approval registry or evidence framework is added. + +Local environment: Python 3.12.14, pytest 9.1.1, NumPy 2.5.3 and pandas 3.0.6. +This step does not run the full regression, OS/Python/dependency matrix, optional +Parquet profile, release builds, clean installs or 100k performance workloads. +Hosted CI success is not claimed. + +## Handoff + +Package version remains `0.1.0.dev3`; report schema remains `1.0`. Graph execution +is a direct opt-in API. The audit/validate CLI, report assembly and ordinary +metric pipeline do not dispatch lineage analysis. General cycles, depth, roots, +ancestry metrics, schema 1.1 and lineage CLI/context inputs remain later steps. + +Step 3 may consume immediate adjacency and retained self-reference evidence to +implement iterative cycle analysis and structural depth after its separate +execution instruction. diff --git a/PHASE_5_STEP_3.md b/PHASE_5_STEP_3.md new file mode 100644 index 0000000..e6d84f9 --- /dev/null +++ b/PHASE_5_STEP_3.md @@ -0,0 +1,88 @@ +# Phase 5 Step 3 Completion + +Date: 2026-09-23 UTC. Status: **STEP 3 COMPLETE; PHASE 5 INCOMPLETE**. + +The Theory Owner instructed `Phase 5 Step 3 继续`. Work starts from +`e40543ff5ed3d2793a41bf58d7e4831bd2a9bfe9` on `phase5-lineage`. +This step covers cycle detection, topology and structural depth. It does not +authorize later steps, a merge, tag or release. + +## Implementation + +`lineage.cycles.analyze_cycles` consumes and revalidates the immutable Step 2 +graph. Iterative strongly connected component detection counts cyclic components +once, including explicit self-parent evidence retained outside accepted adjacency. +Affected records include members and descendants along parent-to-child edges. +Disconnected loaded cycles remain errors even when target paths are unaffected. + +Canonical traversal makes component ordering, witnesses, unaffected topology and +depth deterministic. Component detail is limited to 100 rows. A witness is a +closed path of actual edges, with explicit self-reference evidence permitted for +a self-loop. Witnesses longer than 64 edges are omitted with a reason. Internal +membership and affected sets remain complete for the next computation stage. + +Structural depth is zero for an explicitly parentless record and one plus the +maximum parent depth only when all required paths are valid and complete. +Unknown grounding alone leaves structural depth available. Missing provenance, +absent/null declarations, invalid or unresolved parents, unavailable chronology +and cycle-affected paths produce explicit unavailable reasons. Whole-target depth +is unavailable if any target is unresolved or the target is empty; the resolved +subset maximum is separate. Generation semantics are unchanged. + +Results use frozen data and detached read-only mappings. Diagnostics retain only +safe codes, severity and canonical identities. Analysis performs no file/network +access and is not dispatched by current validation, CLI audit or comparison. + +## Verification + +| Check | Actual result | +|---|---| +| Final focused regression command in `ci.yml` | 481 passed in 4.48 seconds | +| Separate focused CLI validation/comparison isolation command | 2 passed in 0.59 seconds | +| Direct T6 cases included in the 481-test run | 43 passed | +| Current specification, traceability and source-scope checks | PASS | +| Workflow YAML parsing and diff formatting | PASS | +| Independent implementation and oracle review | PASS; no remaining blocker | + +The focused gate covers T6 behavior, parent graph/retention, reference resolution, +generation, input-only metric boundaries and current source/workflow checks. +The 43 T6 cases include the frozen cycle and depth fixtures, an independent +reachability oracle over 64 three-node directed graphs, a 1,500-node chain, +64/65-edge witness boundaries and 101 cyclic components. These bounded cases +establish direct behavior; they are not substitutes for the Step 9 measurements. +No raw test-count target was used. + +Independent review found typed graph handoffs could retain a resolved reference +with no parent key, hide a self-reference outside retained self evidence, or +mislabel same-version chronology. Step 3 rejects these inconsistent inputs before +producing analytical results. Regression cases cover each failure. Fatal input +diagnostics also remain errors. Missing unrelated provenance metadata does not +erase complete structural depth. + +The first separate CLI isolation run had one pass and one failure. The existing +test imported dependent modules while their helpers were already patched, leaving +a temporary blocking function cached in another module. Importing all relevant +modules before applying patches fixes the test-order dependency. The final two +cases pass and explicitly block graph/cycle/ancestry dispatch. No product CLI +code changed, and no failed run is counted as success. + +Commands match the two focused invocations in `.github/workflows/ci.yml`; the +local regression also produced JUnit output. The local environment used Python +3.12.14 and pytest 9.1.1. No full matrix or release-level verification was run. + +The existing source-scope check advances its single comparison commit to the +completed Step 2 and permits only `lineage/cycles.py`. The remaining 58 product +files and exact file inventory stay protected. Sixteen frozen specifications and +seven packaged resource copies remain unchanged. No new registry, historical +source migration or evidence framework is introduced. + +## Handoff + +Package version remains `0.1.0.dev3`; report schema remains `1.0`. This direct API +does not compute external roots, ancestry metrics or lineage bounds, and does +not integrate lineage into reports or CLI dispatch. The full environment matrix, +release builds, optional Parquet profile and actual 100k workloads remain their +designated candidate/performance gates. Hosted CI success is not claimed. + +Step 4 can consume cycle membership, affected records and unaffected topology to +propagate complete external-root sets after its separate execution instruction. diff --git a/PHASE_5_STEP_4.md b/PHASE_5_STEP_4.md new file mode 100644 index 0000000..4676156 --- /dev/null +++ b/PHASE_5_STEP_4.md @@ -0,0 +1,106 @@ +# Phase 5 Step 4 Completion + +Date: 2026-09-23 UTC. Status: **COMPLETE; PHASE INCOMPLETE**. + +The Theory Owner instructed `Phase 5 Step 4 继续`. Work starts from +`11192a3c353292f11d85eb3725c8edc73a71967b` on `phase5-lineage`. +This step implements external-root resolution and coverage. It does not +authorize incidence/concentration, bounds/proxy, report or CLI integration, +subsequent steps, a merge, tag or release. + +## Delivered behavior + +`lineage.ancestry.analyze_lineage` consumes the retained validation handoff, +reuses the existing graph and cycle owners, and revalidates provenance metadata. +An explicitly parentless grounded-yes record supports itself. A grounded-no +parentless record has a complete empty root set. Missing or unknown evidence is +represented by a null root set with reasons, preserving the distinction between +unknown ancestry and known closure. + +A grounded-yes single-parent carryover inherits its parent's complete root set, +including an empty set. Repeated aliases still mean one canonical parent. +Other grounded-with-parent forms remain unresolved under the approved rule. +Grounding-no descendants union unique roots only when every required branch is +complete. Cycles, invalid/unresolved parents, missing required provenance, +unknown grounding and undeclared boundaries propagate unresolved ancestry. +Source labels, evidence URIs and record content never mint roots. + +Only targets enter G/C/U classification and coverage denominators. Context nodes +support propagation without entering N. Original reference multiplicity remains +separate from deduplicated adjacency and root sets. Known identity resolution +with unavailable chronology can retain reference coverage while ancestry remains +unavailable. Malformed declarations with unknown cardinality do not become zero +declared references. Structural depth remains independent of root uncertainty. + +The conflict interpretation follows frozen definitions: blocking grounding or +ancestry contradictions are distinct from `batch_id` or `timestamp` differences +between metadata namespaces. Those metadata differences alone do not erase valid +roots, and their original values are not overwritten. + +## Resource boundaries + +Propagation processes loaded nodes in deterministic dependency order, choosing +canonical ready nodes and canonical parent/root order. Logical memberships are +charged before each new stored `(record, root)` association. Every candidate +parent-root visit is charged, including duplicates already seen from another +parent. An anchor self-membership costs one membership and zero union visits. +Unknown branches do not contribute partial observed-root sets as exact ancestry. + +Root-stage exhaustion yields a failed result with no records, G/C/U partition or +root coverage. Previously completed graph/cycle/depth observations and immediate +reference coverage remain available. Existing graph node/edge admission failures +continue to raise their typed error before an analytical result exists. Limits +describe defined work units, not process RSS or already-loaded input memory. + +The current implementation processes loaded context as well as targets; resource +usage includes context associations actually stored. Returned ancestry records +contain targets only. Result containers are immutable and exclude raw content, +locations and provenance URI values. + +## Verification + +Local verification used Python 3.12.14 with the core dependency profile. + +| Check | Result | +|---|---| +| Combined Step 4 direct and affected regression gate | 541 passed in 5.17 seconds | +| Additional provenance coverage/source-share neighbors | 203 passed in 0.51 seconds | +| Separate validation/pair CLI isolation checks | 2 passed in 0.48 seconds | +| Specification consistency, traceability and source scope | PASS | +| Workflow YAML parsing and whitespace check | PASS | +| Independent implementation and test review | PASS; no outstanding blockers | + +The combined gate includes 48 ancestry cases and 12 resource cases. Direct +checks cover all twelve frozen root/classification fixtures and Hero exact root +sets, separate unknown/missing branches, conservative carryover rules, original +reference multiplicity, context exclusion from target denominators, strict and +fatal outcomes, immutable results and input-only/no-I/O guarantees. Resource +checks cover exact membership and candidate-visit boundaries, duplicate roots +versus duplicate aliases, canonical ordering, failure retention and a 1,200-node +chain without recursion. Existing cycle, graph, parent, generation, metric and +current boundary checks remain in the same focused selection. + +The initial direct-test draft had two fixture-construction failures: an incorrect +provenance-match field name and invalid empty content rejected before lineage. +Both were corrected in the tests without changing frozen inputs. Independent +review also found a missing fatal-status branch and opportunities to construct +inconsistent result statuses or boolean coverage ratios. These were fixed with +direct regression cases. Completed cycle diagnostics remain preserved, and +complete root identities must belong to the loaded scope. + +The existing source check advances its single comparison commit to completed +Step 3 and opens only `lineage/ancestry.py`. The other 58 product files, exact +file inventory, 16 frozen specifications and seven packaged resource copies +remain protected. No new registry, historical migration or evidence framework +is added. + +## Handoff + +Package version remains `0.1.0.dev3`; report schema remains `1.0`. Current report +assembly and ordinary CLI flows continue their existing behavior. Step 5 may +add root incidence and fractional concentration from complete target root sets +after its separate execution instruction. + +No full regression matrix, release build, clean install, optional Parquet profile +or 100k performance measurement is claimed here. Those remain their designated +candidate/performance gates. Hosted CI success is not claimed. diff --git a/PHASE_5_STEP_5.md b/PHASE_5_STEP_5.md new file mode 100644 index 0000000..5375728 --- /dev/null +++ b/PHASE_5_STEP_5.md @@ -0,0 +1,100 @@ +# Phase 5 Step 5 Completion + +Date: 2026-09-23 UTC. Status: **COMPLETE; PHASE INCOMPLETE**. + +The Theory Owner instructed `Phase 5 Step 5 继续`. Work starts from +`2450100d08355d8cb1e0be931345f44572861e4a` on `phase5-lineage`. +This step implements root incidence, fractional mass, ancestry concentration +and effective external-root count in the existing ancestry owner. + +## Delivered behavior + +`analyze_lineage` extends its immutable result with typed `RootContribution` +rows and the complete distinct-root count. Each root's incidence counts complete +nonempty-root targets containing that root. Incidence shares divide by the full +target population N. Each grounded target contributes mass one, divided equally +among its unique roots. Root weights divide accumulated mass by G, the number +of targets with complete nonempty roots. HHI sums squared weights; effective +root count is its reciprocal. + +Context records supply ancestry without entering either denominator. Repeated +paths and aliases do not multiply a target's root contribution. Unresolved +targets supply no exact mass. Their presence marks conditional root metrics +partial, accompanied by existing G/N coverage. Complete known-empty targets +remain distinct from unresolved targets. + +Contributions retain all supporting roots and are ordered by decreasing +incidence with canonical-key ties. Numerical reduction is deterministic. The +future report detail limit does not truncate the calculation. The values +describe a topological allocation; they do not measure causal influence, +semantic error, independent information or universal source diversity. + +A completed G=0 partition returns an empty contribution tuple and root count +zero. HHI and effective roots are null with `NO_RESOLVED_EXTERNAL_ROOTS`. +Resource-aborted propagation returns null contributions and aggregate values +with `LINEAGE_RESOURCE_LIMIT_EXCEEDED`; independently completed structural and +reference observations retain their Step 4 behavior. + +## Verification + +Local verification used Python 3.12.14 with the core dependency profile. + +| Check | Result | +|---|---| +| Combined Step 5 direct and affected regression gate | 565 passed in 6.59 seconds | +| Separate validation/pair CLI isolation checks | 2 passed in 0.48 seconds | +| Specification consistency, traceability and source scope | PASS | +| Four workflow YAML files and whitespace check | PASS | +| Independent implementation and test review | PASS; no outstanding blockers | + +The combined gate includes 24 new concentration cases alongside the existing +ancestry, resource, graph, cycle, parent and dependency-neighbor cases. It covers +all twelve frozen ancestry fixtures and the Hero. Hero HHI/effective roots remain +`1/4` and `4`; the frozen mixed multi-root fixture remains `5/8` and `8/5`. +An additional unequal-support oracle with N=6, G=4, C=1 and U=1 produces HHI +`35/96` and effective roots `96/35` using exact rational expectations. + +Direct tests also exercise permutation, identity renaming, duplicate paths and +aliases, incidence-first ranking, a full 105-root distribution, unavailable +concentration, resource-aborted null distributions, and immutable typed +handoffs. The completed all-unresolved partition retains its empty observed +root distribution even when input errors make overall execution fail. +Independent review ran forty bounded rational/relabeling cases; the maximum +observed relative HHI rounding difference was approximately `3.16e-16`. +Its initial ad hoc `3e-16` tolerance was adjusted to `1e-15`; no product change +was required. The permanent new suite passed on its first execution. + +### Bounded mutation check + +One passing unequal-support test was rerun against three disposable source +variants. Each variant successfully imported, calculated and constructed its +result, then failed an independent assertion with exit code 1. Denominator +mutations changed the corresponding declared metadata as well as arithmetic, +so local row consistency alone could not reject them. + +| Seeded defect | Detection | +|---|---| +| Incidence denominator N replaced with G | Reported denominator 4 rejected against independent N=6. | +| Weight denominator G replaced with N | Reported denominator 6 rejected against independent G=4. | +| Effective count uses HHI without its reciprocal | `0.3645833333333333` rejected against `96/35`. | + +All three meaningful mutations were killed. No permanent mutation runner, +registry or additional execution layer was added. Broader adversarial and scale +work remains Step 9. + +The existing source check advances its single comparison commit to completed +Step 4 and continues to open only `lineage/ancestry.py`. The other 58 product +files, product inventory, 16 frozen specifications and seven packaged resource +copies remain protected. No new governance registry or historical migration +is introduced. + +## Handoff + +Package version remains `0.1.0.dev3`; executable report schema remains `1.0`. +Step 6 lineage bounds and shared-root proxy need their separate execution +instruction. Report and CLI integration remain Steps 7-8. + +No full compatibility matrix, release build, clean install, optional Parquet +profile or 100k performance measurement is claimed here. Their designated +candidate/performance gates remain unchanged. No merge, tag, publication or +hosted CI success is claimed. diff --git a/PHASE_5_STEP_6.md b/PHASE_5_STEP_6.md new file mode 100644 index 0000000..363da28 --- /dev/null +++ b/PHASE_5_STEP_6.md @@ -0,0 +1,98 @@ +# Phase 5 Step 6 Completion + +Date: 2026-09-23 UTC. Status: **COMPLETE; PHASE INCOMPLETE**. + +The Theory Owner instructed `ok 继续Phase 5 Step 6`. Work starts from +`d81966561502b94701712a3d4b4b7fd50783cba9` on `phase5-lineage`. +This step implements lineage closure bounds and the descriptive shared-root +proxy using the completed ancestry result. + +## Delivered behavior + +`metrics.bounds.lineage_closure_exposure` returns the conservative interval +`[C/N, (C+U)/N]` with width `U/N`. Only selected targets enter the denominator. +The immutable result retains the source G/C/U partition, coverage, unresolved +reasons and input diagnostics. A complete all-unresolved partition yields +`[0,1]`; empty targets and resource-aborted propagation have unavailable values +with explicit reasons. Available bounds do not erase input errors or imply +complete ancestry. Existing direct-bound behavior is preserved. + +`lineage.ancestry.shared_ancestry_dependence` returns `present` only when an +exact root supports at least two targets. It discloses partial status when U>0. +`not_present` requires nonempty targets with fully resolved ancestry and no +shared root. Incomplete evidence without a witness, empty targets and resource +failure remain `unavailable` with `indeterminate` level. Duplicate paths and +aliases cannot manufacture target incidence. + +Both results retain one validated immutable ancestry source, with computed +properties instead of duplicate record/root tables or independently mutable +summary fields. The proxy exposes a real root contribution witness and coverage +references. Neither operation starts ingestion, graph analysis, report assembly +or file/network access. Direct-only bounds retain their lazy dependency boundary. +The outputs remain operational descriptions of lineage topology and conservative +exposure, with no risk calibration or causal/scientific certification. + +## Verification + +Local verification used Python 3.12.14 with the core dependency profile. + +| Check | Result | +|---|---| +| Combined Step 6 direct and affected regression gate | 664 passed in 5.50 seconds | +| Separate validation/pair CLI isolation checks | 2 passed in 0.47 seconds | +| Specification consistency, traceability and source scope | PASS | +| Four workflow YAML files and whitespace check | PASS | +| Independent implementation and test review | PASS; no outstanding blockers | + +The combined gate includes 25 new lineage-bound/proxy cases, all 74 unchanged +direct-bound cases, and existing ancestry, concentration, graph, cycle, resource, +metric-pipeline and input-boundary checks. All twelve frozen ancestry fixtures +retain their interval values. Hero lineage closure remains `[0,0]`, while +its direct closure remains `[1/2,1/2]` before and after explicit lineage calls. + +The direct tests distinguish exact incidence one from two, target incidence from +context/path/alias multiplicity, partial positive witnesses from incomplete +negative evidence, missing from unknown branches, complete all-unresolved +classification from resource-aborted classification, and empty target scope. +Masking grounded or closed evidence into unresolved evidence can only widen the +conservative interval. Invalid typed handoffs are rejected, frozen snapshots +retain input errors, and blocked loaders/graph functions confirm no repeated +graph execution or I/O. + +Independent review additionally checked nine bounded semantic cases with graph, +cycle and provenance dispatch blocked. An isolated process imported the bounds +module and executed both existing direct entry points while all lineage modules +remained unloaded. The review harness needed its source import path and empty +fixture role corrected; no product defect or corresponding source change was +required. The new permanent suite passed on its first execution. + +### Bounded mutation check + +Three passing selected tests were rerun against three disposable source +variants. Each variant imported and executed successfully, then failed actual +semantic assertions with exit code 1. + +| Seeded defect | Detection | +|---|---| +| Upper bound omits U | Expected `2/3`; observed `1/3`. | +| Shared-root threshold changes from `>=2` to `>2` | Exactly two target records lost their required `present` signal. | +| Absence drops the U=0 requirement | Incomplete evidence incorrectly returned `available` instead of `unavailable`. | + +All three mutations were killed. The unchanged variant passed all three selected +tests. No permanent mutation runner or registry was added. + +The current source check advances its single comparison commit to completed +Step 5 and opens only the existing ancestry and bounds modules. The other 57 +product files, exact product inventory, 16 frozen specifications and seven +packaged resource copies remain protected. No additional governance framework, +historical migration chain or permanent mutation mechanism is introduced. + +## Handoff + +Package version remains `0.1.0.dev3`; executable report schema remains `1.0`. +Step 7 report/schema integration requires its separate execution instruction. +Ordinary reports and CLI calls retain their current behavior. + +Full compatibility/release verification and 100k lineage performance measurement +remain their designated candidate/scale gates. No merge, tag, software +publication or hosted CI success is claimed here. diff --git a/PHASE_5_STEP_7.md b/PHASE_5_STEP_7.md new file mode 100644 index 0000000..6f9e163 --- /dev/null +++ b/PHASE_5_STEP_7.md @@ -0,0 +1,110 @@ +# Phase 5 Step 7 Completion + +Date: 2026-09-23 UTC. Status: **COMPLETE; PHASE INCOMPLETE**. + +The Theory Owner instructed `Phase 5 Step 7 开始`. Work starts from +`e2666515d98cfe41483b5c43345c642bab02ced0` on `phase5-lineage`. +This step integrates explicitly computed lineage evidence with canonical reports, +schema 1.1, privacy protection and the existing JSON/Markdown renderers. + +## Delivered behavior + +`assemble_report` accepts explicit `lineage`, `lineage_bounds` and +`shared_ancestry` results. Bounds and proxy must refer to the supplied ancestry +result. Assembly does not execute ingestion, graph traversal, root propagation +or metric kernels. A private input digest binds the result to relevant validated +declarations, roles, chronology and diagnostic classifications. Same record keys +with changed grounding or parent declarations cannot silently reuse old results. +Raw content, URIs, paths and notes are excluded from that digest, and the digest +is never serialized. This is a product handoff check, not a new evidence registry. + +All reports now declare schema `1.1`, retaining the twelve top-level sections, +equal capability mirrors and existing non-lineage values and scopes. Reports +expose loaded graph scope separately from the target population, cycle members +and affected descendants, depth, resource usage, G/C/U, coverage, roots, incidence, +fractional mass, concentration, lineage bounds and the descriptive shared-root +signal. Hero retains G=8, C=U=0, five roots, HHI=1/4, four effective roots and +lineage bounds `[0,0]`; its independent direct bounds remain `[1/2,1/2]`. + +Lineage defaults to `not_requested`. Completed, partial and failed execution +remain distinct from input eligibility and individual field availability. +Pregraph failure cannot claim an acyclic graph; a root-budget failure retains +already completed cycle, depth and reference observations without inventing root +results. Empty and unresolved populations retain explicit unavailable reasons. + +Five identity-bearing detail collections are bounded to 100 rows. Cycle witnesses +contain at most 64 edges and 65 keys. Full counts, root metrics and denominators +are calculated before display truncation. Existing scoped HMAC protection covers +new versions, roots, cycle witnesses and record identities. Omit mode removes +whole detail collections with explicit counts and reasons; it retains prior +diagnostic-limit reasons. JSON and Markdown share the validated safe view. +Resource-limit diagnostics now have a fixed readable message and remediation. + +## Verification + +Local checks used Python 3.12.14 and the core dependency profile without PyArrow. + +| Check | Result | +|---|---| +| New report semantics and integration cases | 29 passed | +| New privacy and identity-boundary cases | 14 passed | +| Combined affected regression selection | 1,583 passed; four setup-related failures resolved below | +| Installed Hero and source-boundary failure rechecks | 4 passed in 1.58 seconds | +| Current verification and CI checks after metadata fix | 41 passed | +| Wheel module/resource identity and schema mirror | PASS; all 47 current files match | +| Specification consistency and module boundaries | PASS | +| Four workflow YAML files and whitespace check | PASS | +| Independent implementation, schema and privacy review | PASS; no remaining blockers | + +The combined gate covered report contracts, both validators, malformed wire +values, privacy, CLI isolation, safe publication, golden reports, optional +dependency absence, no-network behavior, lineage algorithms and affected input +neighbors. It also ran the bounded Hero runtime checks. The 43 new cases include +same-key stale handoffs, no-recalculation guards, 100-row detail limits, 64/65-edge +witness boundaries, malicious identifiers and unknown diagnostic text. Independent +review found and corrected a post-HMAC reason-order validation error; all four +supported privacy modes now preserve source ordering without requiring alias +lexical ordering. Unsupported standard/hash and standard/omit remain rejected. + +The initial legacy check found a registered resource-limit code missing from safe +diagnostic templates. Its existing behavior test detected the omission; the +template and privacy whitelist were corrected. Initial golden candidate generation +also required the source import path. The installed golden first lacked a wheel, +then correctly rejected a wheel built before the final schema-validator edit. +Rebuilding and reinstalling the current wheel resolved that mismatch. + +Three source-boundary checks encountered setuptools metadata produced by the +local build. The existing control now excludes only the exact external package +metadata prefix `src/recursive_integrity_toolkit.egg-info/`, after its symlink +check. Product bytes and package inventory stay protected. Existing unauthorized +mutation tests run with that metadata present; similarly named and nested +directories remain rejected. The four setup failures passed targeted rechecks; +the subsequent boundary gate passed all 41 cases. Together, 1,589 distinct cases +in the affected selection and these two added boundary cases have passed. + +No full OS/Python/dependency matrix, 100k lineage measurement or release-candidate +execution was repeated. Those remain their designated candidate/scale gates. + +## Verification consolidation + +Current report goldens and independently authored expected status/version fields +advance to schema 1.1. Before promotion, both standard and redacted candidates +were compared with the previous fixtures: all non-lineage numerical metrics, +scopes, proxy signals, simulations, diagnostics and metadata recommendations were +unchanged. Current literal fixtures now carry package `0.1.0.dev3`, removing the +old package-version substitution adapter. Original numerical oracles and the six +canonical Hero files remain unchanged. Prior fixture forms and authority records +remain recoverable from Git history. + +The single source boundary advances to completed Step 6 and permits the seven +changed product files. The remaining 52 product files, exact package inventory, +16 frozen specifications and seven resource copies remain protected. No new +historical migration chain, approval registry or evidence-of-evidence mechanism +is introduced. + +## Handoff + +Package version stays `0.1.0.dev3`; report schema is now `1.1`. Strict schema-1.0 +readers must explicitly adopt 1.1. Step 8 remains the separately authorized CLI +lineage flag, context-input role and dispatch work. No merge, tag, release +publication or hosted CI success is claimed here. diff --git a/PHASE_5_STEP_8.md b/PHASE_5_STEP_8.md new file mode 100644 index 0000000..7d17424 --- /dev/null +++ b/PHASE_5_STEP_8.md @@ -0,0 +1,113 @@ +# Phase 5 Step 8 Completion + +Date: 2026-09-26 UTC. Status: **COMPLETE; PHASE INCOMPLETE**. + +The Theory Owner instructed `Phase 5 Step 8 开始`. Work starts from +`267504e8fee4784f6d041e445a9e690c982e61a3` on `phase5-lineage`. +This step connects explicit CLI lineage requests, local context inputs and +configured resource limits to the accepted analysis and report owners. + +## Delivered behavior + +`rit audit --lineage` and `rit example --lineage` execute graph, ancestry, +lineage closure and shared-root analysis. Ordinary invocations retain lineage +`not_requested`. The CLI implements orchestration only; the accepted kernels +continue to own all calculations. No simulation is enabled by these flags. + +Repeated `--lineage-records PATH` declarations and configured +`inputs.lineage_context` use the existing CSV, JSONL and optional Parquet loaders. +Context can provide ancestors without entering primary support, diversity, +duplicate, provenance, direct-bound or tail denominators. It does not implicitly +request comparison. An explicitly requested comparison can also supply ancestors +while retaining its existing comparison semantics. + +Context versions must be disjoint from primary and comparison versions. +Duplicate composite identities fail before joining provenance. Repeated context +files may contain distinct records from the same context version. Existing +Python/config multiplicity for other input roles is preserved. Config-relative +and CLI-relative context paths retain their separate bases, even when their +relative spellings match. + +`validate --lineage-records ...` performs input and immediate-reference validation +only. Both its unsupported `--lineage` flag and config `lineage: true` are +rejected. Configuration resolution and the Python `validate_bundle` API remain +inert with respect to analysis. Existing empty-file rejection remains unchanged; +the CLI never substitutes context for a missing primary population. + +The four configured lineage budgets use the approved positive integer defaults: +200,000 nodes, 1,000,000 edges, 1,000,000 logical root memberships and 10,000,000 +root-union candidate visits. They remain separate from loader limits. A small +optional field on the existing `FamilyFailure` retains admission resource usage +when graph construction fails. Reports preserve exact admitted counts, the limit +and the rejected next unit without claiming completed graph, cycle or root work. +Root-stage failures retain already completed graph observations. Independently +completed ordinary values survive either failure. Resource errors use exit 1, +configuration errors exit 2, and parent/cycle errors exit 3 under the existing +exit policy. Fatal admission errors retain failed run status; ordinary errors +with usable surviving evidence can retain partial run status. + +Context identities and paths use the existing report/privacy boundary in every +output sink. Whole-list omission remains schema-valid. Executed lineage capability +notes no longer describe its graph implementation as deferred. Plain Hero stderr +now explains that lineage was not requested and names the opt-in command. + +## Verification + +Local verification used Python 3.12.14, NumPy 2.3.5 and pandas 2.2.3. The context +loader checks actually ran with PyArrow 25.0.1. A separate core environment had +no PyArrow installed and ran the installed-wheel Hero checks. + +| Check | Actual result | +|---|---| +| Combined affected regression, 48 test modules | 2,314 passed in 153.09 seconds; no failures or skips | +| New CLI integration cases, included above | 28 passed | +| New inert configuration cases, included above | 21 passed | +| New context/input and admission-report cases, included above | 28 passed | +| Actual installed ordinary and lineage Hero, standard and redacted | PASS with network blocked and PyArrow absent | +| Wheel/source identity and report schema mirror | PASS; all 47 module/resource files match | +| Specification, traceability and current source checks | PASS | +| Four workflow YAML files and whitespace check | PASS | +| Independent implementation and boundary review | PASS; no remaining blockers | + +The 77 new cases cover actual CSV/JSONL/Parquet context ingestion, opt-in versus +input-only dispatch, target denominators, explicit comparison, mixed path bases, +role overlap and duplicates, all four resource budgets, chronology recovery, +privacy, output collisions and Hero arithmetic. Direct wire mutations reject +false admission stages, counters and attempted values. Existing report goldens +passed unchanged, preserving ordinary numerical values, scope and configuration +hash behavior. + +Installed Hero confirms N=8 with eight context records, G=8 and C=U=0, five roots, +HHI=0.25, four effective roots, lineage bounds `[0,0]`, direct bounds `[1/2,1/2]` +and a present descriptive shared-root signal. Both schemas and renderers preserve +the same meaning; scientific unavailable conclusions remain explicit. The six +packaged Hero inputs match their canonical bytes and remain unchanged. + +### Findings resolved during implementation + +Review identified mixed config/CLI context paths incorrectly sharing one base; +the adapter now retains declaration provenance when resolving their paths. +Review also identified admission counters being lost by the previous generic +family-failure handoff. The bounded payload extension fixes that concrete product +reporting gap without adding a new failure framework. + +During the shared resource-validator refactor, tests caught a missing local +variable assignment that rejected otherwise valid lineage reports. It was fixed +before the clean focused and combined runs. A development-time source-snapshot +check also overlapped active edits; the unchanged check passed after the product +bytes stabilized. Final wheel identity verification and the combined regression +used those stable bytes. No failed check remains waived. + +## Handoff + +Package version remains `0.1.0.dev3`; report schema remains `1.1`. The current +source boundary advances to Step 7 and opens only the twelve changed product +paths. The other 47 product files, exact product inventory, 16 frozen +specifications and seven packaged resource copies remain protected. Existing CI +adds the affected cases and removes redundant CLI reruns. No new governance +registry, historical source migration or evidence-of-evidence layer is added. + +Step 9 remains bounded adversarial/mutation work and actual lineage scale +measurement. Step 10 remains the complete candidate matrix, release checks and +dev4 handoff. No full OS/Python/dependency matrix, 100k lineage measurement, merge, +tag, software publication or hosted CI success is claimed by this step. diff --git a/PHASE_5_STEP_9.md b/PHASE_5_STEP_9.md new file mode 100644 index 0000000..d2ebaef --- /dev/null +++ b/PHASE_5_STEP_9.md @@ -0,0 +1,176 @@ +# Phase 5 Step 9 Completion + +Date: 2026-09-26 UTC. Status: **COMPLETE; PHASE INCOMPLETE**. + +The Theory Owner instructed `Phase 5 Step 9 开始`. Work starts from +`f1c5309deda3ae4107c78f025cb2a7bb85ba2881` on `phase5-lineage`. +This step completes bounded adversarial/mutation work and actual lineage scale +measurements under the approved plan. Step 10 candidate verification is separate. + +## Concrete defect and bounded repair + +The reverse-ID chain exposed two quadratic operations in +`validate_generation_declarations`: tuple membership for each provenance row, +and repeated scans when each child sorts before its parent. The initial probe +ran against the accepted Step 8 implementation, with preconstructed inputs. + +| Records | Before: untraced seconds | After: untraced seconds | Before: generation line events / identity comparisons | After: same work observations | +|---|---:|---:|---:|---:| +| 128 | 0.02719 | 0.00819 | 61,652 / 17,784 | 6,542 / 1,528 | +| 512 | 0.25355 | 0.03486 | 934,676 / 267,768 | 26,126 / 6,136 | +| 2,048 | 4.20171 | 0.11949 | Not instrumented | Not instrumented | + +Tracing and timing are separate runs. Four times the input formerly produced +more than fifteen times the counted work. The current bounded regression permits +six times the work, leaving sorting/implementation headroom while detecting both +observed quadratic defects without a flaky wall-clock assertion. + +The sole runtime change is in `io/validation.py`: use a key set and an iterative +queue that consumes each validated immediate dependency once. Canonical input +sorting remains; the propagation itself is O(V+E) in time and auxiliary storage. +Grounding yes still resets expected generation to zero independently of its own +parents. Unknown, missing, invalid and stalled dependencies remain unavailable; +declarations never seed expectations and this validator never claims a cycle or +calculates ancestry/depth. Diagnostic contents, ordering and severity are retained. + +A one-off differential check compared the entire result and messages with the +accepted implementation for 100 deterministic mixed dependency graphs in both +ordinary and selected-strict-warning modes: all 200 comparisons matched. The +old implementation was obtained from Git only for this experiment. No historical +source adapter or permanent differential framework was added. + +The bounded layered fan-in check verifies 37 nodes, 133 edges and 192 logical +memberships. The full operation needs 257 candidate visits. A limit of 256 rejects +exactly visit 257 and returns unavailable root metrics, while preserving completed +structural observations. At 257 it completes with 32 roots and HHI=1/32. Existing +cycle, missing/unknown evidence, privacy and all four resource-boundary cases +remain in the canonical suite. + +## Actual reference measurements + +Reference: local Ubuntu 24.04.3 LTS, Linux 6.18.44 x86-64/glibc 2.39, +Intel Xeon Platinum 8370C at 2.80 GHz, nine visible logical CPUs, eight-CPU quota, +8 GiB cgroup memory limit and no swap. Python 3.12.14, NumPy 2.3.5, pandas 2.2.3, +pytest 9.1.1; PyArrow 25.0.1 was installed in the measuring environment, though +these JSONL/CSV workloads did not require it. This is local reference evidence, +not a claim that hosted CI or every supported environment has executed Step 9. + +All six observations below use fresh interpreters, untraced wall time and actual +whole-process peak RSS. Inputs are built before timing; startup/imports and work +through output completion are included in outer wall time. There is no warm-up +exclusion or best-run selection. None was killed, timed out or retried. + +| Workload and measured path | Wall seconds | Peak RSS bytes | Output bytes | +|---|---:|---:|---| +| 100,000 reverse-chain records, complete input validation plus ancestry API | 123.36372 | 1,836,056,576 | 939, API summary only | +| 1,000 reverse-chain records, full lineage CLI | 11.15276 | 131,334,144 | JSON 6,202,005; Markdown 1,532,005 | +| 64 roots and 64 targets, full high-fan-in lineage CLI | 2.04153 | 60,375,040 | JSON 610,673; Markdown 284,393 | +| Canonical Hero with lineage, attempt 1 | 0.98528 | 29,347,840 | JSON 240,712; Markdown 216,449 | +| Canonical Hero with lineage, attempt 2 | 1.08168 | 29,347,840 | JSON 240,713; Markdown 216,450 | +| Canonical Hero with lineage, attempt 3 | 1.01130 | 29,347,840 | JSON 240,712; Markdown 216,449 | + +| Workload | Nodes / edges | Targets / context | Logical root memberships / union visits | Depth | +|---|---:|---:|---:|---:| +| 100k API chain | 100,000 / 99,999 | 100,000 / 0 | 100,000 / 99,999 | 99,999 | +| 1k CLI chain | 1,000 / 999 | 1,000 / 0 | 1,000 / 999 | 999 | +| High-fan-in CLI | 128 / 4,096 | 64 / 64 | 4,160 / 4,096 | 1 | +| Each Hero attempt | 16 / 8 | 8 / 8 | 16 / 8 | 1 | + +The 100k worker spent 45.56481 seconds in import/loading/validation and 76.83787 +seconds in ancestry analysis. Every expected generation and target root set was +checked. G=100,000, C=U=0, one distinct root, HHI=1 and effective roots=1; no +cycles or resource exhaustion occurred. Actual input sizes were 10,300,000 bytes +of primary JSONL, 22,388,875 bytes of provenance JSONL and 154 bytes of config. +The generator, commands and independent expectations are documented in +`tests/performance/README.md` and `test_sparse_lineage_100k.py`. Ordinary JUnit +observations retain exact input hashes, process status, stdout/stderr and sizes. + +The **100k API measurement excludes full audit report serialization and ordinary +metric families**. Its small summary is not a 100k audit report. The separate +1k chain and high-fan-in CLI measurements include complete JSON and Markdown. +The 1k CLI probe and inherited large-report cost motivated keeping the expensive +100k run scoped to the approved graph requirement within this container's memory +budget. A 100k full lineage CLI audit was neither attempted nor certified. + +Hero's three complete lineage-enabled runs all satisfy the reference +under-five-second target. Independent assertions retain G=8, C=U=0, five roots, HHI=1/4, +four effective roots, lineage bounds `[0,0]` and direct bounds `[1/2,1/2]`. +There is no universal laptop SLA. Phase 4's 1,362.47-second, 7.12-GiB metadata +audit is a different workload and provides no comparable lineage speedup claim. + +Operational guards are 600 seconds for the actual 100k API, 180 seconds for +either bounded CLI case and the existing 60 seconds per Hero attempt. The +600-second guard is approximately 4.9 times the observed API runtime. The +existing 75-minute reference CI job is unchanged. These are execution guards, +not new product performance promises. Same-workload regressions above 20% time +or 50% memory still require review. Root-set growth beyond sparse traversal +remains constrained by the existing membership and union-visit budgets. + +## Finite mutation experiment + +The unmodified selected suite passed all 23 cases. Each of seven isolated +source copies received one compilable mutation and was tested against existing +behavioral assertions. Every mutant exited pytest with a test failure, not an +import/collection error. Runs stopped at the first detected failure. Test names +below omit their common `test_` prefix. No production file was mutated by the +experiment, and no mutation runner or registry is shipped. + +| Seeded defect | Existing detecting case | Result | +|---|---|---| +| Use G instead of N for incidence share and its denominator | `unequal_supports_use_n_for_incidence_and_g_for_weights` | Killed | +| Charge a new root membership for every duplicate candidate in a union | `root_union_counts_duplicate_candidates_but_deduplicates_alias_edges[False]` | Killed | +| Mint a grounded carryover record as its own root | `carryover_inherits_single_canonical_parent_including_known_empty[yes-expected_roots0]` | Killed | +| Omit the unknown-grounding failure branch, allowing a closed empty set | `missing_provenance_unknown_and_undeclared_boundaries_never_become_empty_roots` | Killed | +| Empty the initial queue for cycle-descendant propagation | `frozen_cycle_oracles_and_unaffected_topology[two_node_cycle_and_descendants]` | Killed | +| Exchange lineage lower and upper formulas | `frozen_partition_bounds_and_shared_root_evidence[unknown_absent_and_null_boundaries]` | Killed | +| Return HHI instead of its reciprocal as effective root count | `unequal_supports_use_n_for_incidence_and_g_for_weights` | Killed | + +The union mutation specifically checks duplicate-counting in logical storage and +the resulting premature resource failure. The cycle mutation is caught by the +existing incomplete-topology rejection. These seven results describe the bounded +experiment; they do not claim an exhaustive mutation score or prove absence of +other defects. + +## Regression and attempt accounting + +The final affected selection covers 34 modules and 1,602 distinct cases: +generation/parent inputs, loaders and normalization neighbors, observability, +graphs/cycles/ancestry/bounds, CLI, reports/goldens/privacy, no-network behavior +and existing source/workflow controls. Real Parquet cases ran in the environment +with PyArrow. This was not the full canonical or cross-platform candidate suite. + +The first combined run yielded **1,601 passed and one failed in 160.60 seconds**. +The installed-without-PyArrow golden detected that the test interpreter still +contained the Step 8 wheel: its installed `io/validation.py` hash disagreed with +the current checkout. The source-identity check correctly rejected that stale +installation. No product assertion, golden or runtime code was changed to bypass +it. After building/installing the current wheel, **all 72 report-golden cases +passed in 19.01 seconds**, including the failed installed case. + +The current effective result is passing coverage of all 1,602 distinct affected +cases, with that targeted rerun explicitly accounted for. The 72 cases are a +subset, not additional coverage. Unchanged successful cases and the source-based +performance measurements were retained; no automatic full-matrix rerun followed +the installation refresh. All 47 wheel module/resource payloads match current +source. Ordinary and lineage installed Hero checks also use a real core-only +environment with PyArrow absent and network blocked. + +Specification consistency, traceability, current source/resource checks, changed +Python syntax, all four workflow YAML files and whitespace checks passed. + +## Handoff and limits + +The current source boundary advances to Step 8 and opens only the changed +validation owner. The other 58 product files, exact product inventory, 16 frozen +specifications and seven packaged resource copies remain protected. Existing +focused CI selects the affected input/lineage neighbors. The actual 100k test +enters only the already established reference performance job. No new Tier D +architecture, source-binding migration, approval registry or evidence-of-evidence +layer is added. Scope pooling and report semantics are unchanged. + +Package version remains `0.1.0.dev3`; report schema remains `1.1`. The existing +metadata 100k candidate gate remains required and was not rerun in Step 9. +Step 10 owns complete canonical regression, OS/Python/dependency candidate +checks, build/reproducibility/delivery verification and the dev4 handoff. +No Phase 6 implementation, hosted CI success, main merge, tag or publication is +claimed. Stop after Step 9. diff --git a/PHASE_6A_PLAN.md b/PHASE_6A_PLAN.md new file mode 100644 index 0000000..95d5c23 --- /dev/null +++ b/PHASE_6A_PLAN.md @@ -0,0 +1,280 @@ +# PHASE_6A_PLAN + +## Document control + +| Field | Value | +|---|---| +| Project | Recursive Integrity Toolkit | +| Phase | Phase 6A: longitudinal dataset comparison | +| Document version / date | 1.0 / 2026-09-26 | +| Status | PROPOSED FOR THEORY OWNER REVIEW; implementation not started | +| Theory Owner | Xiangyu Guo | +| Accepted starting commit | `f09521907558a193e6438d9acb316adb9e191195` | +| Accepted source tree | `da6b0bc3ab64b417ef5be86050f9f932149bc712` | +| Baseline branch / package / report schema | `phase5-lineage` / `0.1.0.dev4` / `1.1` | +| Proposed implementation branch | `phase6a-longitudinal`, from the accepted Phase 5 history after Step 1 authorization | +| Proposed completed-phase package / schema | `0.1.0.dev5` / `1.2` | +| Current authorization | Prepare and deliver this plan | +| Merge, tag, publication authorization | None | + +Approval of this plan would approve the recommended P6A-D decisions below, subject to any stated exceptions. Execution continues one authorized step at a time. Preparing this document does not record approval, start Step 1, or authorize Phase 6B. + +## 1. Starting point and authority + +Phase 5 is complete. Its accepted checkout contains 273 tracked files and 40 Python package modules. The verified code candidate was `53802fb0e99ea867a27bb82210cc9fa32d8e02d1`; `f095219` adds the final completion record. All 13 required candidate jobs passed, including eight OS/Python/dependency profiles, real Parquet, security, performance and delivery. These are historical baseline results, not Phase 6A test results. No product regression or environment matrix is needed merely to write this plan. + +The existing implementation provides: + +- validated explicit chronology and representation declarations; +- a Python `compare_support` API with directed literal mappings, original and harmonized distributions, support sets, retention and diversity deltas; +- an audit CLI for one primary version and one explicitly requested earlier comparison; +- single-target provenance, direct closure and opt-in ancestry calculations; +- canonical JSON/Markdown, report schema 1.1, privacy handling and installed Hero resources. + +The remaining work is multi-version orchestration and truthful change reporting. The current CLI computes only content distributions for its earlier side. Its provenance and lineage summaries belong to the primary version. The existing graph API deliberately rejects comparison/context versions as primary targets. Those boundaries require explicit extensions before any multi-version loop is valid. + +Apply the contract order in `PROJECT_INSTRUCTIONS.md` section 4.4, approved UD decisions and accepted Phase 3-5 clarifications. Primary Phase 6A owners are **T1 observed longitudinal comparison, PR-007 ordering and PR-011 dataset-longitudinal capability**. Existing T2/T3/T4/T6, representation, coverage and report owners remain responsible for their reused calculations. Formula F-018 supplies the later-minus-earlier delta convention. No new theory formula is proposed. + +Continue the user's Verification Governance and Complexity Control policy and P5-D11: direct semantic correctness stays strict; candidate compatibility and delivery checks run at appropriate milestones; obsolete historical implementation checks remain recoverable through Git. This phase adds no approval registry, receipt chain, source-body migration framework or evidence-of-evidence layer. + +## 2. Objective and bounded scope + +Produce an explicitly requested series of two or more ordered dataset snapshots, their per-version summaries, adjacent comparisons and optional first-version comparisons. Report observed support/diversity changes, observed label/topic disappearance, provenance changes and available lineage changes with each side's scope, denominator, compatibility and coverage. + +Required outputs from product specification section 16.3 are record-count delta, support delta, diversity delta, added/missing states, provenance-coverage delta and source-share delta. This plan also implements earlier-tail disappearance when requested, the existing direct-closure interval changes, and the specified lineage changes when lineage is requested and computable. + +Excluded: Phase 6B experiment/reopening orchestration, automatic simulations, model-performance trends, causal intervention claims, permanent population extinction claims, universal scores or thresholds, automatic semantic remapping, all-pairs matrices, temporal regression/forecasting, HTML/dashboard work, remote retrieval and new runtime analytical dependencies. The approved relative-change formula remains available for a future explicit addition; this phase uses absolute deltas only. Avoid introducing percentages or ratios whose extra interpretation is unnecessary for the required outputs. + +## 3. Recommended contract decisions + +### P6A-D01. Require explicit series selection and retained chronology + +Add an explicit longitudinal request to the Python/report workflow and `--longitudinal` to `audit` and `example`. `audit --records` continues to identify exactly one primary version, which must be the latest selected snapshot. In longitudinal mode, repeatable `--compare PATH` inputs identify the other selected snapshots. Each CLI records file identifies one nonempty version; repeated version selections and duplicate canonical keys are rejected. Without longitudinal mode, retain the existing single-version/one-pair interface and behavior. + +The series contains the selected primary and comparison versions only. Ancestor/context inputs and versions mentioned only in a chronology document do not become snapshots. At least two selected versions are required. File enumeration order, filename spelling, CLI path order and lexical version order cannot supply chronology for the new series. Use retained explicit config/order-file declarations or valid timestamps through the existing order resolver, including conflict and timestamp-tie rules. The existing explicit two-version invocation-order contract remains supported in legacy pair mode. + +Order all selected snapshots by that validated chronology and retain its source. Chronology must also cover loaded context when cross-version parent validation requires it. A missing or conflicting order blocks cross-version results while valid independent snapshot results remain reportable. Do not silently sort conflicting evidence into a usable sequence. + +An empty file cannot identify a version by itself. The CLI returns a specific input/selection error instead of deriving a version from its filename. Typed Python calls may represent an explicitly identified empty snapshot; all zero-denominator and empty-distribution rules still apply. A nonempty snapshot whose representation excludes every row remains selected, with unavailable representation metrics. + +### P6A-D02. Compare adjacent snapshots; make baseline comparison explicit + +For selected order `v1, ..., vk`, default to `k-1` adjacent pairs. An optional `baseline=first` declaration additionally requests the first snapshot against each later snapshot. Deduplicate the first adjacent pair, so at most `2k-3` distinct comparisons execute. Default baseline is `none`. A baseline is a selected observed snapshot, not a model of the underlying population. + +Keep per-version original summaries separate from pair results. Each pair records its two version IDs, pair kind(s), compatibility evidence, original/harmonized basis and status. No hidden all-pairs expansion, endpoint substitution, pooled distribution or average-of-deltas is permitted. If an intermediate pair is unavailable, keep that gap. Independently requested valid baseline pairs can still be calculated. + +No regression slope, overall collapse classification, or inferred causal trajectory is generated. Loss, gain and reappearance remain local observations at their declared positions. + +### P6A-D03. Preserve exact representation and population contracts + +Reuse `RepresentationDescriptor`, `ExplicitPairContext`, `StateMappingDeclaration` and `validate_representation_compatibility`; do not create a competing interpretation of compatibility. Every selected snapshot has an explicit representation and literal state-meaning declaration. A common declaration may apply to all selected snapshots only when the user actually declares it. + +Support exact compatible descriptors and explicitly supplied directed literal mappings through the existing pair kernel. Mappings must identify source/target descriptors and meanings, cover the required supplied states, preserve missing-state rules and expose many-to-one collisions. No automatic mapping composition, semantic inference or silent fallback is allowed. Configuration may serialize these existing typed declarations; the currently accepted bare mapping/compatibility fields must not be promoted into sufficient mapping authority. + +Pair-local harmonization does not establish one common multi-version basis. Display original snapshot values with their descriptors; label mapped comparisons with their own harmonized basis. A connected common-basis trajectory is available only when all included snapshots explicitly share that basis and meaning. Otherwise retain separate basis groups and pair-local results, without a global trend claim or hidden transitive mapping. + +An incompatible pair blocks its comparison results and records the reason; valid single-version observations remain. This phase does not introduce a separate provenance-only bypass around the approved Level 4 representation gate. A failure in an unrelated optional calculation need not invalidate an otherwise eligible pair. + +Keep count-backed and probability-only inputs distinct. Reuse existing weighting and denominator checks: compare like with like; do not replace unweighted values with weighted values. CLI series uses the existing unweighted record analysis. Preserve supported explicit weighted Python pairs without automatically expanding them into a new CLI feature. + +Representation eligibility exclusions affect representation metrics only. Record count, provenance and lineage use their independently declared complete version populations. Every snapshot and pair records both sides' population counts, eligible/excluded counts, denominator basis and relevant weighting. Context records enter none of these target denominators. + +### P6A-D04. Define observed distribution changes and tail disappearance + +All deltas use `later - earlier` in the original metric's unit and have evidence class `derived_metric`; their source snapshots retain their existing evidence classes. Counts use record or state units; share and coverage deltas use ratios in `[-1,1]`, not a percentage-change formula. Reuse `compare_support` for support delta, loss/addition counts, retained states, retention and diversity delta. Add record-count delta from the complete version populations. + +For eligible pair supports `S_e` and `S_l`: + +```text +missing states = S_e \ S_l +added states = S_l \ S_e +retained states = S_e intersection S_l +support retention = |retained states| / |S_e| +``` + +State disappearance has evidence class `derived_metric` and the required interpretation: **extinct from the observed later version under the declared representation**. It does not establish permanent absence from the production process. A state absent at v2 and present at v3 appears in the relevant missing and added sets; earlier outputs remain valid observations. + +Tail disappearance is optional and requires the existing explicit tail rule/threshold. Select the earlier tail on the pair's comparison basis using that side's counts/frequencies, then intersect it with the pair's missing-state set. Retain the rule, threshold, earlier sample size and representation. Do not select the later tail or silently import a threshold from another snapshot. For mapped pairs, disclose that tail membership uses the harmonized earlier distribution and preserve its original distribution separately. No one-step extinction probability or resampling scenario executes. + +Empty/all-excluded distributions retain the accepted Phase 3 behavior: unavailable pair distribution values and null state sets. Do not report all earlier states extinct when later evidence is absent. An eligible comparison with no missing states reports an empty set and zero loss. Do not collapse this distinction during rendering or redaction. Independently defined record-count deltas may still exist for explicitly identified empty snapshots. + +### P6A-D05. Calculate provenance and direct closure per version + +Reuse validated joins, `summarize_provenance` and the current direct-closure kernel for each selected version. Calculate each snapshot once. Expose separate changes in provenance-row coverage, required-field coverage and grounding-field coverage. Never label all three as one unspecified coverage measure. + +Source-share changes retain `human`, `synthetic`, `mixed`, `sensor`, explicit `unknown` and missing-provenance categories under the existing denominator convention. Missing rows remain distinct from declared unknown; estimates remain distinct from confirmed/log-derived assertions. `mixed` is not redistributed. Changes in metadata availability are visible alongside changes in declared composition. + +Report direct-closure lower-bound, upper-bound and interval-width changes as endpoint/width arithmetic, preserving both original intervals and their populations. A narrower interval can result from improved evidence. An endpoint delta does not prove an equivalent change in a hidden true closure rate. + +Null values propagate to the affected delta with side-specific reasons; valid zero remains zero. Malformed provenance follows existing family failure rules. Do not pool versions to repair a missing side or interpolate unavailable shares. All provenance quantities remain based on supplied declarations. + +### P6A-D06. Extend target selection safely for lineage changes + +Lineage remains separately opt-in. Without `--lineage`, longitudinal lineage rows show `not_requested`; loading several versions never triggers graph analysis by itself. With lineage requested, use the union of explicitly loaded selected snapshots and disjoint context records as a common graph evidence scope for this invocation. + +Introduce a validated series-target selection interface that admits only versions explicitly selected as snapshots. Keep the current ordinary graph API's primary-only default and its context exclusion. Do not implement the series by loosely relabeling context rows, editing validation objects or bypassing the existing graph/result identity checks. Bind each result to the actual selected version, full target key set and retained input declarations. Step 1 fixes the narrow typed interface before this change is implemented. + +Reuse parent parsing, identity resolution, chronology, cycles, depth, grounding/carryover rules and G/C/U classification. Validate and build shared graph structure once where the existing integrity checks permit it; cache only invocation-local, immutable, input-bound work. Derive each selected target summary from that shared evidence without recomputing every entire audit or creating a persistent cache. A disconnected cycle remains a graph error under the Phase 5 contract. + +Compare distinct supporting external-root count, HHI, effective root count, unresolved reference count, resolved-reference coverage, resolved-lineage/external-ancestry coverage and lineage-closure interval endpoints/width. Retain each version's N/G/C/U and reference denominators. Count target-supported roots, not every anchor loaded into the graph. Root identity, allocation method and graph rules remain those approved in Phase 5. + +Both finite, semantically compatible values are needed for a numerical delta. A partially resolved result may provide its valid grounded-subset metric, with a partial delta and both sides' coverage attached. Such arithmetic describes the observed subsets; it must not claim a full-population ancestry change or that a change in visibility proves a change in actual roots. G=0 concentration, failed resource stages, unresolved metrics and unrequested analysis produce unavailable/unrequested affected deltas, never artificial zeros. Shared-root proxy states remain categorical; do not subtract or rank them. + +All snapshots in a run use the disclosed common loaded evidence scope. This is a retrospective comparison of the supplied versions, not a reconstruction of what was known at each historical date. Parent links to later versions under the validated chronology remain invalid. No historical visibility inference is added. Evidence additions in a later invocation can legitimately revise an earlier snapshot's coverage; reports must make the different context/input basis identifiable. + +### P6A-D07. Extend the canonical report once and protect every new field + +Use report schema `1.2` for the completed phase, including ordinary reports. Retain the twelve top-level sections, capability mirrors and finite-number/null rules. Add a typed longitudinal family under the existing sections, with ordered snapshot summaries, explicit pair results, selection/order evidence, representation bases, per-family statuses, resource limits and bounded details. Step 1 records the exact public field inventory and units before runtime work. + +Preserve existing single-target and explicit-pair values and meanings. Multi-version results have their own named fields; do not silently fill an old singular-pair field with an arbitrary adjacent or endpoint pair. The primary version can continue to populate existing ordinary summary fields. Strict schema-1.1 consumers must opt into 1.2; no report migration framework is introduced. + +JSON and Markdown consume the same canonical typed calculations. Renderers own no comparison, mapping, graph or delta arithmetic. Input observability remains distinct from execution status. Series execution distinguishes `not_requested`, `completed`, `partial` and `failed`, with family/pair reasons. No complete-series claim is allowed while a requested pair/family is unresolved, incompatible or uncomputed. Optional families not requested do not make an otherwise completed requested analysis partial. Replace the current blanket `R_LONGITUDINAL_FAMILIES_DEFERRED` classification only for the families actually implemented and requested; preserve accurate legacy pair capability limits. Existing error exit behavior still applies to actual validation/resource errors. + +Apply privacy to version/state/root identifiers, mapping keys and values, pair references, context evidence, paths, diagnostics and omitted-detail metadata. Consistent pseudonyms must join the same version/state across this report; hash/omit modes must not leak identifiers through a new dictionary key or prose field. Arbitrary user state-meaning declarations also require safe handling. Reuse canonical identifier protection and aggregate-only representations, preserving exact aggregates and schema validity. Standard output remains local, and no-network/output-path protections continue unchanged. + +### P6A-D08. Bound orchestration and keep configuration explicit + +Proposed engineering guard: at most **100 selected versions** by default, with a validated positive finite integer override. Adjacent plus optional first-baseline selection therefore has at most 197 distinct pairs at the default limit. Reject an over-limit request before materializing per-version calculations. Context inputs remain subject to inherited input/graph limits and never consume snapshot slots. + +Keep Phase 5 default guards for the union of loaded records: 200,000 nodes, 1,000,000 unique edges, 1,000,000 stored record/root memberships and 10,000,000 root-union candidate visits, alongside existing input byte/row/parent-list limits. Shared lineage work must retain meaningful deterministic counters. Independent reprocessing must not reset a work counter to evade the guard. Additional aggregate summaries should not duplicate all per-record ancestry evidence for every version. + +Limit new state/root detail tables to the current 100-row report convention per table, with exact total/shown/omitted counts. Compute aggregate counts and metrics from complete eligible data before applying the display cap. Do not emit all paths, all-pairs intersections or a full ordinary report for every snapshot. Reuse immutable per-version scopes internally and avoid copying complete identity lists into every new pair or metric. Step 1 defines any explicit scope-reference serialization needed for the new series fields. Retain bounded diagnostics and disclose known pre-existing ordinary-report size costs. + +Resource failure preserves independently completed results with explicit scope and failure reasons. Do not calculate a series aggregate from a truncated selection, label uncomputed pairs as zero, or quietly raise limits to pass a benchmark. These guards are operational limits, not scientific thresholds or a memory/time SLA. + +Configuration extends existing input-role lists with a bounded longitudinal options object for enablement, baseline selection, per-version declarations and explicit pair mappings. Exact serialization names and conflict validation are frozen in Step 1. The common-representation CLI form uses `--state-semantics`; richer declarations use local declarative config. Reject competing CLI/config singleton declarations. Preserve established Python/config input multiplicity outside the new CLI mode. + +`validate` may load repeated comparison/context inputs for input validation, but never executes a series, graph, metric or simulation. Reject execution-only flags there. Package imports and `validate_bundle` remain inert; extend the existing direct isolation checks to the new orchestrator. `example --longitudinal` runs the unchanged two-version Hero; adding `--lineage` also requests both versions' lineage snapshots. A separate compact three-version example exercises actual multi-step change. Existing `example` and `example --lineage` behavior remains available. + +## 4. Nine implementation steps + +Each step records actual changes, relevant checks and remaining limits in a concise completion note. Preserve the existing phased authorization model. Exact identifiers and serialization choices may be settled within the approved meaning; a material semantic change requires a specific owner decision before dependent implementation. + +| Step | Deliverable and dependency | Smallest sufficient acceptance gate | +|---|---|---| +| **1. Contracts and independent examples** | Record actual approval/exceptions in `PHASE_6A_DECISIONS.md`; freeze typed snapshot/pair/series fields, selection API, config declarations, statuses, limits and independent expectations. No runtime series execution. | Reconcile required fields to existing owners/formulas; hand-check the three-version oracle and incompatible/empty/partial cases; affected document/contract consistency only. | +| **2. Ordered selection and compatibility** | Validate selected versions, retained order, adjacent/baseline pair selection, per-version declarations and explicit pair mappings. Depends on 1. | Lexically misleading names, unloaded/context versions, conflicting timestamps/order, missing/repeated versions, pair deduplication, mapping direction/collisions and incompatible-basis gaps. Reuse order/compatibility regression. | +| **3. Snapshot distributions and observed changes** | Reuse distribution and pair kernels; add record-count delta, ordered summaries, support/diversity changes and optional earlier-tail disappearance. Depends on 2. | Independent oracle, reappearance, zero versus null, all-excluded/empty evidence, denominator separation, renaming/permutation invariance and pair-local mapping behavior. No new stochastic execution. | +| **4. Provenance and direct-closure changes** | Compute per-version provenance and direct bounds, then named coverage/source-share/interval deltas. Depends on 2-3. | Unknown versus missing, changing coverage, malformed rows, source categories, zero/partial/unavailable propagation and unchanged single-target values. Relevant provenance/bounds neighbors only. | +| **5. Available lineage changes** | Add validated selected-version targeting and shared immutable graph work; integrate target-specific summaries and compatible deltas. Depends on 2-4. | Primary/context isolation, full target selection, input-signature mismatch rejection, root-support counts, N/G/C/U, partial concentration, same shared context, disconnected cycles and resource-stage failure. Relevant graph/root/report-binding tests. | +| **6. Report schema and privacy integration** | Add canonical series results, schema 1.2 and packaged mirror, JSON/Markdown, capability execution and protected identities. Depends on 3-5. | Two/three-version available/partial/failed/not-requested reports, exact values, bounded details, null states, stable privacy joins, no dictionary-key leakage and legacy pair values. Update report goldens from independent expectations. | +| **7. CLI, configuration and installed examples** | Wire explicit longitudinal mode, repeated comparisons, optional baseline/lineage, richer config and separate three-version example; preserve input-only validate. Depends on 6. | Actual CSV/JSONL plus real optional Parquet, order/config conflict messages, output safety, meaningful exits, standard/redacted installed examples outside checkout and no-network boundaries. | +| **8. Bounded adversarial and scale preparation** | Resolve remaining semantic/resource defects; run the finite mutation set and small/medium multi-version end-to-end measurements; establish the large reference workload. Depends on 2-7. | Mutations detected by direct tests, many-version/pair-limit behavior, capped details with exact aggregates, shared-work scaling, and complete CLI serialization timing/RSS. No new permanent verification framework. | +| **9. Candidate verification and handoff** | Update docs, traceability, changelog and package to dev5; run canonical regression, supported candidate matrix, designated full-scale measurements and delivery checks; record completion. Depends on 1-8. | Required outputs and owners implemented; Hero and three-version oracles hold; installed wheel/sdist, privacy and reproducibility pass; exact limitations and remaining release work documented. No merge, release or Phase 6B execution implied. | + +## 5. Independent acceptance oracles + +### 5.1 Preserve the canonical Hero + +Keep all six canonical Hero source/expectation files unchanged. The existing pair remains v1 to v2: record counts 8/8, support 8/5, diversity 7/8 and 3/4, support delta -3, diversity delta -1/8, retention 5/8, and missing states `battery`, `lizard`, `turtle`. The new record-count delta is 0. Existing v2 direct bounds [1/2,1/2], five supporting roots, HHI 1/4, effective roots 4 and lineage bounds [0,0] remain unchanged. + +Step 1 derives new per-v1 provenance/lineage and cross-version expectations directly from the frozen input rows and approved formulas, without editing that oracle or copying implementation output. A two-version series must agree with the existing pair kernel on all shared fields. Schema/package metadata changes must not alter scientific values. + +### 5.2 Three-version observed-change fixture + +Use one unchanged declared topic representation and these record-count distributions: + +| Snapshot | A | B | C | D | Records | Support | Gini-Simpson diversity | +|---|---:|---:|---:|---:|---:|---:|---:| +| v1 | 2 | 1 | 1 | 0 | 4 | 3 | 5/8 | +| v2 | 2 | 0 | 0 | 2 | 4 | 2 | 1/2 | +| v3 | 1 | 1 | 0 | 1 | 3 | 3 | 2/3 | + +| Quantity | v1 to v2 | v2 to v3 | v1 to v3, only when baseline requested | +|---|---|---|---| +| Record-count delta | 0 | -1 | -1 | +| Support delta | -1 | +1 | 0 | +| Diversity delta | -1/8 | +1/6 | +1/24 | +| Missing states | {B,C} | empty | {C} | +| Added states | {D} | {B} | {D} | +| Support retention | 1/3 | 1 | 2/3 | +| Earlier-tail disappearance under singleton-count rule | {B,C} | empty | {C} | + +B disappears and then reappears. The equal first/last support sizes must not erase that intermediate loss. Compare exact rational expectations with the existing declared numerical tolerance, independently of report generation. + +For a separate provenance scenario over the same snapshot sizes: + +| Declared provenance population | v1 | v2 | v3 | +|---|---|---|---| +| Human / synthetic / explicit unknown / missing rows | 4 / 0 / 0 / 0 | 2 / 1 / 0 / 1 | 1 / 1 / 1 / 0 | +| Provenance-row coverage | 1 | 3/4 | 1 | +| Human share | 1 | 1/2 | 1/3 | +| Synthetic share | 0 | 1/4 | 1/3 | +| Explicit unknown share | 0 | 0 | 1/3 | +| Missing-provenance share | 0 | 1/4 | 0 | +| Grounding yes / no / unresolved counts | 4 / 0 / 0 | 2 / 1 / 1 | 1 / 1 / 1 | +| Direct-closure interval | [0,0] | [1/4,1/2] | [1/3,2/3] | + +Thus row-coverage deltas are -1/4 and +1/4; human-share deltas -1/2 and -1/6; synthetic-share deltas +1/4 and +1/12. Lower-bound deltas are +1/4 and +1/12; upper-bound deltas +1/2 and +1/6. Explicit unknown is never repaired into missing or synthetic. Freeze the remaining field-coverage expectations after specifying every manifest field in Step 1. + +### 5.3 Separate lineage oracle and failure variants + +Use two explicit parentless grounded context anchors a and b. In a complete scenario, v1 has four targets with root sets `{a},{a},{b},{b}`; v2 has four targets all supported by `{a}`; v3 has two targets supported by `{a},{b}`. All targets truthfully declare grounding `no` and reference their anchor. + +| Quantity | v1 | v2 | v3 | +|---|---:|---:|---:| +| N / G / C / U | 4/4/0/0 | 4/4/0/0 | 2/2/0/0 | +| Supporting external roots | 2 | 1 | 2 | +| HHI | 1/2 | 1 | 1/2 | +| Effective root count | 2 | 1 | 2 | +| Resolved-lineage coverage | 1 | 1 | 1 | +| Lineage-closure interval | [0,0] | [0,0] | [0,0] | + +Adjacent root-count deltas are -1/+1; HHI deltas +1/2/-1/2; effective-root deltas -1/+1. Loaded anchor count is two throughout and must not replace target-supported root count. + +Then add a third v3 target with an unresolved parent. Its G/C/U becomes 2/0/1, coverage 2/3 and bounds [0,1/3]. HHI remains 1/2 over its grounded subset; its delta is explicitly partial. A separate G=0 variant makes HHI/effective roots and their deltas unavailable. Further bounded cases cover unknown grounding, known-empty root sets, a disconnected cycle, resource exhaustion, unchanged record IDs with altered parent declarations, and a context-only version that never becomes a snapshot. + +### 5.4 Required semantic distinctions + +Include incompatible middle-version representation with valid independent snapshots; different pair-local mapping bases; many-to-one collision disclosure; missing timestamps/order; tied timestamps; v2/v10 lexical traps; unloaded versions in declared order; empty versus all-excluded snapshots; literal zero-mass states; unequal record populations; earlier-tail versus later-tail membership; changed context visibility; pair failures without zero-filled gaps; renamed/shuffled input invariance; and redaction across repeated state/version identities. + +Use a finite mutation set for distinct failure modes: reversed delta sign, lexical chronology, context inclusion in N, missing provenance folded into unknown, later-tail selection, incompatible pair accepted, unavailable value replaced with zero, and all-loaded roots substituted for target-supported roots. Record whether current tests detect each defect and fix genuine gaps. Do not create a new mutation registry or a numerical test-count target. + +## 6. Verification, performance and delivery policy + +| Change or boundary | Verification | +|---|---| +| This plan and later administrative notes | Document/authority consistency and independent review; no product matrix | +| Selection, compatibility or arithmetic | Direct examples/properties and affected existing kernel neighbors | +| Target selection, input roles, report schema, CLI or privacy | Relevant integration and boundary regression | +| Stable candidate | Complete current regression; Ubuntu/Windows, Python 3.11/3.12, minimum/current compatible direct dependencies | +| Optional input support | Actual no-PyArrow core profiles and a real-PyArrow profile | +| Delivery candidate | Existing build, wheel/sdist, clean installs, installed examples, no-network/security and documented reproducibility checks | +| Administrative successor with unchanged applicable runtime/authority | Direct consistency plus clearly identified reusable candidate evidence | + +Reuse existing CI, release and traceability tooling. Update the current path rather than adding a permanent phase dispatch layer. Full matrices belong to the stable candidate; ordinary implementation steps run affected gates. A failed check requires the relevant correction and rerun. Additional testing must address a concrete remaining risk. + +Measure complete series CLI execution, including input validation, selected per-version work, optional lineage, report assembly and JSON/Markdown publication. Record selected/context records, version/pair counts, states, graph edges/root memberships/union visits, elapsed time, fresh-process peak RSS and output bytes. Report inherited high-water marks if a fresh process cannot be used. Kernel timing cannot certify complete CLI performance. + +Use a compact many-version workload to detect repeated global scans and unintended quadratic pairing. Establish a three-version **100,000-record total loaded-input** workload, counting context in that total, and execute it on one designated reference profile at the candidate boundary. It must exercise longitudinal CLI serialization and available lineage changes. This is 100k combined records, not 100k per snapshot. Keep bounded cases across the compatibility matrix. Step 8 performs smaller preflight measurements; it need not repeat the eventual full-scale run. + +Retain required existing candidate gates. Run each required expensive workload once on the reference profile, not once per version, pair or OS/dependency combination. Reuse unchanged historical evidence only where the existing governance permits it and the workload/runtime relationship is demonstrated; do not present Phase 5 results as new Phase 6A measurements. There is no approved hard wall-time/RSS SLA for the new 100k series workload. Establish an observed baseline and a practical CI timeout without silently loosening resource defaults. + +Measure the two-version Hero with longitudinal mode, both with and without lineage, against the existing under-five-second reference target. Disclose profile and every retained attempt. A greater-than-20% time or greater-than-50% memory regression on a comparable workload triggers review; changed workloads require separate labels. + +Phase 5's hosted results provide scale context: 100k ancestry API took 66.23 seconds and about 1.85 GB peak RSS, excluding full reports; the ordinary 100k metadata report took 780.73 seconds and about 7.68 GB peak RSS, producing about 443 MB JSON and 165 MB Markdown. Those observations make one-full-report-per-version an unsuitable default design. Share validated work and scope references, cap new detail, and report actual total output size. If measurement reveals a bottleneck, permit a narrow behavior-preserving fix in the affected assembly or orchestration path, with direct regression. Broader redesign requires a separate scoped decision. + +## 7. Ownership, completion and next boundary + +Expected implementation locations are existing config/models/result types, `io/validation.py` selection adapters, `representations/compatibility.py`, `metrics/diversity.py`, `metrics/provenance.py`, `metrics/bounds.py`, lineage target/summary interfaces, report assembly/rendering, CLI, schemas and packaged resources. A small `metrics/longitudinal.py` orchestration owner is appropriate if it keeps reusable calculation outside the CLI; it must delegate existing kernels and must not become a generic plugin framework. Tests, a compact example, docs, traceability and existing candidate tools may change as needed. + +Preserve frozen Phase 0 files, canonical Hero inputs/expectations and approved scientific meanings. Record clarifications additively. Do not modify theory source PDFs, unrelated mathematical kernels, source-type meaning, direct grounding rules, generation meaning or parent identity conventions. Any actual authority conflict is reported with the affected step and proposed resolution before dependent implementation. + +Phase 6A completes when all required version deltas, explicit selection/compatibility, observed disappearance, provenance changes, available lineage changes, truthful partial states, schema/privacy, installed commands, candidate compatibility and delivery checks pass. Completion reports identify the tested candidate, actual checks, performance scope and known limitations. They do not require historical source-code forms to remain executable forever. + +The handoff leaves two distinct choices: optional Phase 6B, if separately approved, and formal v0.1 release preparation/acceptance. Phase 6A completion itself does not merge draft PRs, approve a release, create a tag, publish a package or claim model-performance validation. An unmerged accepted Phase 5 branch can serve as the development base; prior PR merges are not a prerequisite for writing or implementing this plan. + +## 8. Source basis + +- `PROJECT_INSTRUCTIONS.md`: authority order; Phase 6A/6B boundaries; evidence, chronology, representation and unavailable-conclusion rules. +- `SPEC_AUDIT.md` and `UNRESOLVED_DECISIONS.md`: approved UD-007 ordering and UD-017 phase split, plus representation and report decisions. +- `V0.1_PRODUCT_SPEC.md` section 16: required version deltas, observed-extinction wording, available lineage change and model boundary. +- `DEFINITIONS_AND_UNITS.md` sections 6, 10.10 and 14: representation/support, earlier-tail extinction, delta and relative-change definitions. +- `DATA_AND_PROVENANCE_SPEC.md`: identity, ordering, joins, grounding and parent declarations. +- `THEORY_TO_CODE_TRACEABILITY.md`: T1 observed support contraction, existing provenance/ancestry owners, PR-007/PR-011 and Phase 6A requirements. +- `VALIDATION_PLAN.md` section 82 and `SUCCESS_CRITERIA.md`: Phase 6A acceptance and candidate obligations. +- `PHASE_3_DECISIONS.md`: explicit-pair compatibility, mapping, weighting and empty-distribution contracts. +- `PHASE_5_PLAN.md`, `PHASE_5_DECISIONS.md`, `PHASE_5_COMPLETION.md`: accepted target/context, root/coverage, report-binding, resource and verification boundaries. +- `examples/hero/EXPECTED_OUTPUTS.md` and its canonical inputs: immutable two-version scientific oracle. +- Current code: `metrics/diversity.py`, `metrics/provenance.py`, `representations/compatibility.py`, `lineage/graph.py`, `lineage/ancestry.py`, `observability/levels.py`, `reports/assembly.py`, `config.py` and `cli.py`. +- User instruction: Verification Governance and Complexity Control, applied through the accepted Phase 5 consolidation. diff --git a/README.md b/README.md index 504d70d..658c02a 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,15 @@ A local-first research toolkit for examining recursive closure exposure in synthetic-data and recursive-data pipelines under explicit representations and assumptions. -## Current milestone: Phase 4 reports and CLI +## Current milestone: Phase 5 lineage candidate -Development version `0.1.0.dev3` provides local input validation, JSON and Markdown audit reports, privacy controls, one explicitly ordered dataset comparison and a packaged Hero example. The mathematical core includes literal topic/label and exact record-form representations, exact duplicates, support/diversity, provenance composition, direct closure-exposure bounds, tail ranking and explicitly invoked closed-resampling scenarios. Actual acceptance and execution evidence are recorded in `PHASE_4_COMPLETION.md`, `PHASE_4_VALIDATION_REPORT.md` and `PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md`. +Development version `0.1.0.dev4` provides local input validation, JSON and Markdown audit reports, privacy controls, one explicitly ordered dataset comparison and a packaged Hero example. The mathematical core includes literal topic/label and exact record-form representations, exact duplicates, support/diversity, provenance composition, direct closure-exposure bounds, tail ranking and explicitly invoked closed-resampling scenarios. Actual acceptance and execution evidence are recorded in `PHASE_4_COMPLETION.md`, `PHASE_4_VALIDATION_REPORT.md` and `PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md`. -General lineage/ancestry remains Phase 5 work; longitudinal orchestration remains Phase 6A work; external reopening and experiment orchestration remain Phase 6B work. The CLI does not execute simulations. HTML output is deferred. No stable release, main merge or registry publication is implied by this development milestone. +Phase 5 adds validated graphs, cycles, depth, external ancestry, root concentration, lineage closure bounds and descriptive shared-root evidence through explicit Python calls or `audit --lineage` and `example --lineage`. Repeatable local context inputs supply ancestors while the primary version keeps its own metric scopes. JSON and Markdown use report schema 1.1. Bounded mutation checks and actual 100k lineage measurement are complete. Step 10 verifies the dev4 candidate; [Phase 5 completion](PHASE_5_COMPLETION.md) records its actual gate status and limitations. Longitudinal orchestration remains Phase 6A work; external reopening and experiment orchestration remain Phase 6B work. The CLI does not execute simulations. HTML output is deferred. No stable release, main merge or registry publication is implied by this development milestone. ## Install and run -Python 3.11 and 3.12 on Ubuntu and Windows form the verified environment matrix. From a checkout: +Python 3.11 and 3.12 on Ubuntu and Windows form the supported candidate matrix. From a checkout: ```bash python -m pip install . @@ -24,21 +24,23 @@ After installation, the following command works in any local working directory w ```bash rit example --out ./hero-workspace +rit example --lineage --out ./hero-lineage-workspace ``` -It copies six packaged Hero files into `hero-workspace/inputs/` and writes `hero-workspace/reports/report.json` and `report.md`. It downloads nothing. The packaged `EXPECTED_OUTPUTS.md` also contains future lineage targets; the actual Phase 4 report explicitly defers those calculations. +Each invocation copies six packaged Hero files into its workspace's `inputs/` and writes `reports/report.json` and `report.md`. It downloads nothing. The packaged `EXPECTED_OUTPUTS.md` contains the independently specified ordinary and lineage targets. -The report compares v1 with v2: support **8 to 5**, retention **5/8**, diversity **7/8 to 3/4**, and missing topic states `battery`, `lizard`, `turtle`. Later-version human and synthetic shares are each **1/2**, and direct closure exposure is **[1/2, 1/2]**. Input observability is Level 4, lineage execution is deferred and simulations are empty. +Both reports compare v1 with v2: support **8 to 5**, retention **5/8**, diversity **7/8 to 3/4**, and missing topic states `battery`, `lizard`, `turtle`. Later-version human and synthetic shares are each **1/2**, and direct closure exposure is **[1/2, 1/2]**. Input observability is Level 4 and simulations are empty. Plain `example` keeps lineage `not_requested`. With `--lineage`, all eight v2 targets have resolved external ancestry, with **5** roots, ancestry HHI **1/4**, effective roots **4**, lineage exposure **[0, 0]**, and shared-ancestry evidence `present`. Use the extracted files for these independent commands, each with a fresh output destination: ```bash rit validate --records ./hero-workspace/inputs/records_v2.csv --out ./validation-report rit audit --records ./hero-workspace/inputs/records_v2.csv --config ./hero-workspace/inputs/config.json --out ./topic-report +rit audit --records ./hero-workspace/inputs/records_v2.csv --lineage --lineage-records ./hero-workspace/inputs/records_v1.csv --provenance ./hero-workspace/inputs/provenance.csv --config ./hero-workspace/inputs/config.json --version-order ./hero-workspace/inputs/version_order.json --out ./lineage-report rit audit --records ./hero-workspace/inputs/records_v2.csv --compare ./hero-workspace/inputs/records_v1.csv --provenance ./hero-workspace/inputs/provenance.csv --config ./hero-workspace/inputs/config.json --version-order ./hero-workspace/inputs/version_order.json --state-semantics "Hero topic labels retain their literal meaning across v1 and v2." --redacted --out ./private-pair-report ``` -`validate` writes a validation report without calculating metrics. `audit` uses only an explicitly declared representation; missing representation or provenance produces specific unavailable results. In a pair, `--compare` supplies the earlier version and `--records` the later version, with explicit chronology and shared state meaning. Reports are unweighted in this CLI. Existing report targets are never overwritten. Always check the exit code, since failed runs may retain useful partial reports. +`validate` writes a validation report without calculating metrics; it accepts context files and rejects lineage execution. `audit` uses only an explicitly declared representation; missing representation or provenance produces specific unavailable results. `--lineage-records` may repeat and requires lineage opt-in for audit. Context versions must be disjoint from primary/comparison versions, and duplicate composite keys fail. Context alone does not request comparison. In a pair, `--compare` supplies the earlier version and `--records` the later version, with explicit chronology and shared state meaning. Reports are unweighted in this CLI. Existing report targets are never overwritten. Always check the exit code, since failed runs may retain useful partial reports. `recursive-integrity` and `python -m recursive_integrity_toolkit` provide the same commands. See [CLI options and exits](docs/cli.md), [data contracts](docs/data_schema.md), [report fields](docs/report_schema.md) and [privacy boundaries](docs/privacy.md). @@ -118,7 +120,9 @@ Identity is `(dataset_version, record_id)`. Missing, null, declared unknown, zer Standard reports exclude raw content, private notes, full embeddings and secrets. `--redacted` additionally protects paths and structural identifiers; record IDs default to keyed hashes, with explicit preserve/omit options. Aggregate values and errors remain visible. Input/configuration hashes are linkable, and pseudonyms provide no statistical anonymity. Exact content hashes describe record form and cannot establish semantic equivalence or authorship. Content references are read only through explicit Python `LOCAL_REF` requests; CLI audit/validate do not activate that reader. -The Step 10 reference-container measurement completed the 100,000-record synthetic metadata audit in **1,362.4708 seconds**, with **7.12 GiB peak process RSS**, approximately **443 MB JSON** and **165 MB Markdown**. This is a demanding, verbose report workload; plan memory, disk space and runtime before applying it to large datasets. Those observations are neither an SLA nor evidence for unimplemented lineage performance. Full-scale Python allocation tracing was omitted after a separately recorded bounded tracing-overhead measurement. See [measurement method](tests/performance/README.md) and the phase validation record for actual environment and scope. +The Phase 4 Step 10 reference-container measurement completed the 100,000-record synthetic metadata audit in **1,362.4708 seconds**, with **7.12 GiB peak process RSS**, approximately **443 MB JSON** and **165 MB Markdown**. This is a demanding, verbose report workload; plan memory, disk space and runtime before applying it to large datasets. Those observations are neither an SLA nor evidence for lineage performance. Full-scale Python allocation tracing was omitted after a separately recorded bounded tracing-overhead measurement. + +[Phase 5 Step 9](PHASE_5_STEP_9.md) measured an actual 100,000-record deep chain through loading, complete validation and ancestry API analysis in **123.36 seconds**, with **1.71 GiB peak RSS**. This excludes full audit reports and ordinary metrics, so it is not comparable to the Phase 4 workload. A separate 1,000-record lineage CLI audit through JSON/Markdown took **11.15 seconds**; lineage-enabled Hero runs took **0.99, 1.08 and 1.01 seconds**. See the [measurement method](tests/performance/README.md) for exact workloads, resource guards and limitations. ## Verification and development artifacts diff --git a/docs/architecture.md b/docs/architecture.md index a1abb38..0be2691 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,5 +1,40 @@ # Architecture +## Current Phase 5 architecture + +Development version `0.1.0.dev4` implements the approved [lineage contract](lineage_contract.md). +The report schema is 1.1. The existing forty package modules, local-only runtime, +dependency declarations and canonical Hero inputs are retained. + +| Current owner | Responsibility and boundary | +|---|---| +| `io/validation.py` | Input joins, retained parent declarations, chronology and generation dependency propagation; never executes lineage or metrics. | +| `lineage/graph.py` | Immutable canonical graph, target/context scope and pre-insertion node/edge guards. | +| `lineage/cycles.py` | Iterative cyclic components, bounded witnesses, affected descendants and structural depth independent of generation. | +| `lineage/ancestry.py` | External-root resolution, strict G/C/U partition, incidence/fractional mass, concentration, coverage and descriptive shared-root evidence. Root memberships and work have finite guards. | +| `metrics/bounds.py` | Explicit direct and lineage closure intervals with distinct denominators and evidence. | +| `models.py`, `reports/assembly.py` | Typed schema 1.1 results, supplied-result binding and canonical evidence assembly; no ancestry recomputation in rendering. | +| `reports/json_report.py`, `reports/markdown_report.py` | Deterministic bounded output, local aliases, aggregate-preserving redaction and explicit unavailable states. | +| `cli.py`, `config.py` | Explicit `audit --lineage` / `example --lineage`, repeatable local context inputs and validated finite limits. `validate` remains input-only. | + +Primary records define the target population. Comparison and dedicated context +records can supply ancestors without entering primary metric denominators. +Unresolved paths, missing provenance and cycles remain visible. Topological +roots and depth do not establish independent causal evidence or semantic truth. + +Current verification uses one path in the existing consistency, traceability and +release scripts. Exact historical method bodies, snapshot fixtures and migration +chains are recoverable from Git, while direct current product/security tests +remain active. [Release process](release_process.md) defines focused and candidate +gates. Step 10 changes only package version metadata in runtime/package files; +the single current source boundary protects all other accepted product bytes. + +## Historical Phase 2-4 implementation notes + +The following step statements describe their original stage, including then +unimplemented owners. The current ownership table above and the lineage contract +describe the supported Phase 5 behavior. Historical test bodies remain in Git. + Status: Phase 3 Step 11 development milestone. Actual acceptance is recorded in `PHASE_3_COMPLETION.md`. The input-layer sections below describe the preserved Phase 2 implementation. Subsequent step sections retain the boundary at their historical stage. The current boundary is summarized here and in the final Step 11 section. diff --git a/docs/cli.md b/docs/cli.md index f30b772..911727d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,16 +1,19 @@ # CLI -Development version `0.1.0.dev3` supports `audit`, `validate`, `example`, `version`, `--version` and `--help`. The `recursive-integrity` alias and `python -m recursive_integrity_toolkit` use the same entry point. Help and version do not load analytical dependencies or user inputs. +Development version `0.1.0.dev4` supports `audit`, `validate`, `example`, `version`, `--version` and `--help`. The `recursive-integrity` alias and `python -m recursive_integrity_toolkit` use the same entry point. Help and version do not load analytical dependencies or user inputs. ## Packaged local example ```bash rit example --out ./hero-workspace +rit example --lineage --out ./hero-lineage-workspace ``` Run after installation from a trusted local directory. The workspace must be new and its parent must already exist. Even an existing empty workspace is rejected. Six exact packaged Hero files are copied under `inputs/`; `report.json` and `report.md` appear under `reports/`. No data download occurs. Add `--redacted` to protect identifiers and paths in reports and console diagnostics. Extracted inputs remain the public canonical Hero files. -The copied `inputs/EXPECTED_OUTPUTS.md` includes full-product lineage targets. Phase 4 reports explicitly defer lineage calculation. Their current targets are support 8 and 5, delta -3, retention 5/8, diversity 7/8 and 3/4, delta -1/8, and missing states `battery`, `lizard`, `turtle`. Later human/synthetic shares are each 1/2 and direct exposure is [1/2, 1/2]. Input observability is Level 4 and default simulations are empty. +Both commands retain support 8 and 5, delta -3, retention 5/8, diversity 7/8 and 3/4, delta -1/8, and missing states `battery`, `lizard`, `turtle`. Later human/synthetic shares are each 1/2 and direct exposure is [1/2, 1/2]. Input observability is Level 4 and default simulations are empty. + +With `--lineage`, v2 is the eight-record target and v1 supplies eight ancestor records. All eight targets have resolved external ancestry, supported by five distinct roots. Ancestry HHI is 1/4, effective root count is 4, lineage exposure is [0, 0], and the shared-ancestry proxy is `present`. The report retains the distinction between declared topology and causal evidence. Plain `example` leaves lineage `not_requested`. Both modes use schema 1.1 and the unchanged packaged `inputs/EXPECTED_OUTPUTS.md` oracle. Successful stdout names both report files. In redacted mode the fixed names are relative to the example's `reports/` directory. An audit failure may leave extracted inputs and an error report. Extraction failures clean only files owned by that attempt; incomplete cleanup is disclosed on stderr. Inspect a failed destination and choose a fresh one for retry. @@ -57,10 +60,50 @@ Without a representation, audit retains input/provenance evidence and explains u | `--tail-rule frequency_at_or_below --tail-threshold P` | Audit only; finite number in [0,1] | | `--compare FILE` | One earlier-version records file; current records are the later side | | `--state-semantics TEXT` | Audit pair only; explicit shared literal meaning across both sides | +| `--lineage` | Audit/example only; explicitly execute the validated parent graph and ancestry families | +| `--lineage-records PATH` | Repeatable audit/validate context input; audit requires lineage opt-in | + +Repeated singleton flags and competing CLI/config singleton declarations fail even if values match. `--lineage-records` is repeatable. Config output allows only `directory`, `record_id_mode` and `id_salt_file`. The CLI performs unweighted calculations even when weights are present and applies no implicit representation, tail threshold or simulation. Debug output, simulation requests, state-list tail selection and arbitrary state maps are unsupported. + +An ordinary audit requires one primary dataset version. Multiple versions in a calculated side cause an input error without pooling or choosing one. Validate can inspect multiple versions through `--records`, `--compare` and repeated `--lineage-records`; it rejects lineage execution, state-semantics and tail requests. Its `derived_metrics`, `proxy_signals` and `simulations` stay empty. Empty/malformed input can produce an error-only report. Missing provenance never becomes supplied unknown or fabricated source evidence. + +## Explicit lineage and context + +The extracted Hero inputs can run ancestry without requesting a comparison: + +```bash +rit audit --records ./hero-workspace/inputs/records_v2.csv --lineage --lineage-records ./hero-workspace/inputs/records_v1.csv --provenance ./hero-workspace/inputs/provenance.csv --version-order ./hero-workspace/inputs/version_order.json --config ./hero-workspace/inputs/config.json --out ./lineage-report +rit validate --records ./hero-workspace/inputs/records_v2.csv --lineage-records ./hero-workspace/inputs/records_v1.csv --provenance ./hero-workspace/inputs/provenance.csv --version-order ./hero-workspace/inputs/version_order.json --out ./lineage-input-check +``` + +Repeat `--lineage-records` for additional local context files. CSV, JSONL and optional Parquet use the existing local table loaders. A context file may contain multiple context versions. Context versions must be disjoint from the primary and comparison versions; duplicate composite keys across context inputs fail. Keep same-version ancestors in the primary records file. Splitting one version between primary/comparison and context roles is unsupported. The provenance manifest may describe all loaded versions. + +Ancestry targets exactly the primary `--records` version. Context contributes ancestors and appears in input inventory, graph scope and diagnostics. It does not enlarge the primary support, diversity, provenance, direct-bound, duplicate or tail denominator. Adding `--lineage` to an explicit pair lets the existing comparison input also supply ancestors. Context alone never requests comparison or state compatibility, and parent chronology still requires explicit declarations where needed. + +JSON/TOML config can declare `lineage: true`, repeated `inputs.lineage_context`, and the four finite graph limits. For example, a JSON config relative to its own directory may include: + +```json +{ + "lineage": true, + "inputs": { + "lineage_context": ["ancestors_v1.jsonl", "ancestors_v0.csv"] + }, + "resource_limits": { + "max_lineage_nodes": 200000, + "max_lineage_edges": 1000000, + "max_lineage_root_memberships": 1000000, + "max_lineage_root_union_visits": 10000000 + } +} +``` + +These are the defaults. Each override must be a positive integer; null, booleans and fractional values fail. They bound admitted nodes, unique edges, logical record/root memberships and candidate root-union visits. Existing input byte/row/parent-list limits remain separate. The limits do not guarantee a peak memory budget. Exhaustion produces a lineage error and unavailable dependent values; independently completed ordinary metrics survive. A cycle anywhere in the loaded graph remains an error, even if primary ancestry is unaffected. + +CLI context declarations extend the configured context list. Declare the lineage opt-in once, through either config or the CLI; an explicit config `lineage` value together with `--lineage` is a conflicting singleton declaration. -Repeated flags and competing CLI/config singleton declarations fail even if values match. Config output allows only `directory`, `record_id_mode` and `id_salt_file`. The CLI performs unweighted calculations even when weights are present and applies no implicit representation, tail threshold or simulation. Debug output, simulation requests, state-list tail selection and arbitrary state maps are unsupported. +`validate` accepts context declarations with lineage disabled, performs existing input and immediate-reference validation, and never runs graph metrics. Both `--lineage` and config `lineage: true` are rejected for `validate`. Config parsing and the Python `validate_bundle` function also remain input-only. `--strict` promotes only configured warning codes, including a configured missing-parent warning; ordinary unresolved ancestry remains explicit in coverage and bounds. -An ordinary audit requires one dataset version. Multiple versions in a side cause an input error without pooling or choosing one. Validate can inspect multiple versions and two files through `--records`/`--compare`; it rejects state-semantics and tail requests. Its `derived_metrics`, `proxy_signals` and `simulations` stay empty. Empty/malformed input can produce an error-only report. Missing provenance never becomes supplied unknown or fabricated source evidence. +Use `--redacted` for protected context paths, dataset labels, roots and cycle witnesses. `--record-ids omit` removes identity-bearing lineage detail collections while retaining counts, values and omission reasons. Resource failures and validation errors retain severity in both report formats and stderr. ## Explicit earlier/later pair @@ -85,9 +128,9 @@ Warnings carry an `affected_scope` summary of versions, record/exclusion counts, | Exit | Meaning | |---|---| | 0 | Supported work completed without error-severity diagnostics | -| 1 | Input/validation or output I/O failure | +| 1 | Input/validation, lineage resource-limit or output I/O failure | | 2 | Invalid invocation/configuration or unsupported request | -| 3 | Existing immediate-parent/lineage-family validation error | +| 3 | Immediate-parent format/resolution or lineage-cycle error | | 4 | Internal or invariant failure | Precedence is 4, then 2, then 3, then 1. Warnings alone return 0 unless configured strict promotion applies. Deferred/optional unavailable analyses alone do not cause a partial run. A report may contain useful results despite a nonzero exit, so file presence is insufficient to establish success. @@ -102,4 +145,4 @@ The output directory's existing ancestors must be stable and trusted. Ordinary a The Python helper `utils.paths.publish_reports(safe_view, output_directory, input_paths=...)` consumes a validated `SafeReportView`; supply every input path, including declared missing inputs. A `complete` result means both exact reports were verified and private staging removed. `E_OUTPUT_PATH_INVALID` maps to exit 2; collision, existing/unsafe target, I/O or cleanup failure maps to 1; rendering/invariant failure maps to 4. -Handled partial failures clean only files still identified as belonging to the attempt. An `incomplete` result discloses remaining or uncertain cleanup. Pair publication is not a filesystem transaction: another reader or process crash may observe one report before the other. See [privacy and filesystem limits](privacy.md) for platform permissions and race/crash boundaries. HTML, general ancestry and simulation orchestration remain deferred. +Handled partial failures clean only files still identified as belonging to the attempt. An `incomplete` result discloses remaining or uncertain cleanup. Pair publication is not a filesystem transaction: another reader or process crash may observe one report before the other. See [privacy and filesystem limits](privacy.md) for platform permissions and race/crash boundaries. HTML, automatic longitudinal analysis and simulation orchestration remain deferred. diff --git a/docs/data_schema.md b/docs/data_schema.md index 6bbacaf..d15f544 100644 --- a/docs/data_schema.md +++ b/docs/data_schema.md @@ -77,9 +77,9 @@ Typed incomplete required fields remain ProvenanceAssessment plus errors, never `resolve_version_order` validates parsed explicit lists, integer ranks, timezone timestamps and optional tie-break/invocation evidence. Every source covers loaded versions and agrees on shared pairs; extra declared versions remain recorded. Rank gaps/negative starting points are allowed, duplicates/conflicts are errors. Equal instants need explicit ties. Empty explicit documents fail. Retained declarations, not cached flags or filename spelling, are authoritative. -`parse_parent_ids` validates native arrays or explicitly CSV-encoded JSON arrays. Null stays null, blank CSV means empty, limits apply before deduplication. `resolve_parent_references` resolves immediate targets only. Bare identifiers require one match; ambiguity fails. Missing targets remain unresolved. Aliases/duplicates collapse deterministically with warnings and retained original spellings. Direct self-parent uses the existing cycle input code without implementing general T6. Declared future targets fail when chronology proves them future, even if unloaded. Same-version graph validation remains deferred. +`parse_parent_ids` validates native arrays or explicitly CSV-encoded JSON arrays. Null stays null, blank CSV means empty, limits apply before deduplication. `resolve_parent_references` resolves immediate targets only. Bare identifiers require one match; ambiguity fails. Missing targets remain unresolved. Aliases/duplicates collapse deterministically with warnings and retained original spellings. Direct self-parent uses the existing cycle input code. Declared future targets fail when chronology proves them future, even if unloaded. Explicit Phase 5 graph analysis separately detects general cycles, including same-version cycles. -`validate_generation_declarations` independently establishes expected counts from validated grounding/dependencies. Grounding yes establishes zero including carryovers. Ungrounded children require all parent counts before one plus their maximum. Declared generation never seeds expectation. Missing/unknown/invalid evidence and stalled dependencies retain unavailable reasons; stalling is not called a cycle. Mismatches warn or explicitly promote. Bounded monotone scans use flat indexes and existing version order, without root traversal, depth or topological graph algorithms. Some same-version chains have quadratic worst case; no large-scale performance certification is claimed. +`validate_generation_declarations` independently establishes expected counts from validated grounding/dependencies. Grounding yes establishes zero including carryovers. Ungrounded children require all parent counts before one plus their maximum. Declared generation never seeds expectation. Missing/unknown/invalid evidence and stalled dependencies retain unavailable reasons; stalling is not called a cycle. Mismatches warn or explicitly promote. A dependency queue and canonical-key set avoid repeated whole-input scans. This remains immediate generation propagation; explicit lineage analysis separately computes structural depth and roots. See Phase 5 Step 9 for the measured reverse-chain workload and its limits. ## Content, classification and internal handoff @@ -87,7 +87,7 @@ Content is read only by explicit LOCAL_REF requests through PR-017 containment a The bundle returns inventory, records, optional provenance, join evidence, chronology, generation when available, capability classification, mapping evidence, resolved-content identities and diagnostics. It writes no report. Check has_errors independently of maximum level. Content/parent-family failures can coexist with independent valid metadata; fatal structural input errors raise. -Hero input validation qualifies as Level 4 without metric calculation. Model longitudinal remains unavailable. Scenario declarations may establish experimental eligibility without execution. The separately invoked calculation calls below provide approved fields. Phase 4 CLI audit orchestrates these calls and renders reports through the separate report contract. General lineage and ancestry remain deferred. Input/configuration schemas retain their accepted meanings; `report.schema.json` defines the Phase 4 public report. +Hero input validation qualifies as Level 4 without metric calculation. Model longitudinal remains unavailable. Scenario declarations may establish experimental eligibility without execution. The separately invoked calculation calls below provide approved fields. CLI audit orchestrates these calls and renders reports through the separate report contract. Phase 5 adds explicit lineage/ancestry dispatch and dedicated context inputs; see the lineage contract. `report.schema.json` defines the current schema 1.1 public report. ## Literal field representations diff --git a/docs/lineage_contract.md b/docs/lineage_contract.md new file mode 100644 index 0000000..843a77e --- /dev/null +++ b/docs/lineage_contract.md @@ -0,0 +1,557 @@ +# Phase 5 Lineage Contract + +Status: **implemented Phase 5 contract; dev4 candidate verification in Step 10**. + +Authority: [Phase 5 decisions](../PHASE_5_DECISIONS.md), approved Phase 5 plan, +`PROJECT_INSTRUCTIONS.md` section 4.4 and the approved UD decisions. Relevant +frozen owners are T4, T6, PR-008, PR-009 and T3 lineage bounds. The original +definitions/specifications remain unchanged. + +This document defines the implemented interfaces from Steps 2-8. Step 2 implements retained batch +parent evidence and `build_lineage_graph` with immutable scope, adjacency and +node/edge limits. Step 3 adds `analyze_cycles` for iterative cycle detection, +unaffected topology and structural depth. Step 4 adds `analyze_lineage` for +external-root resolution, G/C/U counts and coverage. Step 5 adds root incidence, +fractional mass and ancestry concentration. Step 6 adds explicit lineage closure +and shared-root proxy calls. Step 7 integrates explicitly supplied lineage +results into schema 1.1, canonical assembly and privacy-safe JSON/Markdown. +Step 8 integrates explicit CLI lineage invocation and local context input options. +Step 9 adds measured scale/resource evidence and behavior-preserving generation +dependency propagation. The package is `0.1.0.dev4`; the executable report schema +is `1.1`. See [completion](../PHASE_5_COMPLETION.md) for candidate gate results. + +## 1. Scope and reference semantics + +Canonical identity is the existing `RecordKey(dataset_version, record_id)`; +transport spelling is `version::id`. Compare keys using their two string fields +in that order. Records in the primary version define the target population N. +Loaded comparison/context records support the graph without changing N. + +`LineageScope` has these immutable fields: + +| Field | Python type | Meaning | +|---|---|---| +| `target_dataset_version` | `str \| None` | Explicit selected primary version; null only when an empty input has no declared version. | +| `target_record_keys` | `tuple[RecordKey, ...]` | Unique sorted target identities; no representation eligibility exclusions. | +| `target_record_count` | `int` | N, equal to the tuple length. | +| `loaded_record_count` | `int` | Unique loaded nodes across all admitted roles. | +| `context_record_count` | `int` | Loaded nodes outside the target. | +| `loaded_dataset_versions` | `tuple[str, ...]` | Lexical inventory, not inferred chronological order. | + +The target keys must be a subset of the loaded keys. Counts are nonnegative +integers and never booleans. Context versions are disjoint from primary and +comparison versions. Same-version edges are supported within a loaded primary +version; split target/context roles for one version are outside this phase. +Duplicate canonical keys across any roles fail validation. + +Keep `ParentValidationResult.declaration_state` values `absent`, `null`, `empty` +and `declared`. The batch resolver uses retained `CanonicalRow.field_states` to +preserve absent versus null fields. Only `empty` explicitly declares no parents. Missing provenance +is separately identified. Reuse `ParentReference`, its canonical identity, +`source_references`, resolution status and chronology. The Step 2 batch resolver +retains a per-record result or per-record failure, original declaration counts +where knowable, and explicit self-parent evidence. It constructs one lookup +index for the batch. It never treats an ambiguous/future/invalid reference as an +accepted ancestry edge. + +For each syntactically valid reference list, declared count is the original +list length, including repeated aliases. Absent/null declarations and missing +provenance have zero original reference entries; this does not establish an +explicitly parentless record. Resolved count sums the lengths of +`source_references` whose existing resolution status is `resolved`, including +known identities with unavailable chronology. Graph edge admission separately +requires known chronology. Unresolved reference count is +declared minus resolved and includes invalid references whose entries can be +counted. A malformed declaration with unknown reference cardinality makes the +whole-target reference totals/coverage unavailable; retain known-subset counts +only with an explicitly named subset. Never turn malformed input into zero +declared parents. + +Graph adjacency deduplicates canonical edges. Grounded carryover with two +spellings of the same resolved parent still has one parent. An additional +unresolved spelling that cannot resolve to that key prevents complete ancestry. +Cross-version edges require the existing declared chronology; missing chronology +cannot certify a resolved required path. Same-version edges require cycle +analysis. Default missing-parent warnings and existing strict-mode error +promotion remain unchanged. Ambiguity, future-parent and cycle errors remain +errors. + +## 2. Computation ownership and immutable targets + +The target high-level computation entry is: + +```python +analyze_lineage( + validation: BundleValidationResult, + *, + target_dataset_version: str | None, + limits: LineageLimits = LineageLimits(), +) -> LineageAnalysisResult +``` + +`validation` supplies retained record, provenance, chronology and parent evidence +after the Step 2 retention refactor. It must cover context as well as target +records. Selection is explicit, revalidated and independent of ordinary metric +eligibility. `validate_bundle` remains input-only and never calls this function. +CLI orchestration calls it only after an explicit lineage request. + +Steps 4-5 implement this direct API for root resolution, coverage and concentration. Its +`LineageAnalysisResult.records` contains target records in canonical order; +context roots are intermediate computation data and do not enter target +denominators. The existing `ExecutionStatus` vocabulary applies. Step 6 derives +bounds and the descriptive proxy through separate explicit calls over that +typed result; `analyze_lineage` does not invoke either automatically. + +Graph admission failures retain the existing `LineageResourceLimitError` before +a valid analytical result can be created. A root-membership or union-work +failure after graph/cycle analysis returns a failed result with root records, +G/C/U counts and root coverage unavailable. Completed cycle/depth observations, +original-reference coverage and exact consumed-work counters survive. No partial +root traversal is converted into an exact target partition. + +Step 2 supplies the lower-level input-only handoff in `models.py`: +`ParentRecordValidation` retains provenance presence, an optional +`ParentValidationResult`, declared/resolved/self-reference counts and diagnostics; +`ParentBatchValidationResult` covers every loaded key in canonical order. +`io.validation.resolve_parent_batch` shares reference interpretation with the +existing fail-fast resolver and creates one identity lookup per batch. +`BundleValidationResult.parent_validation` is an appended optional field for +source compatibility. Graph construction requires this retained evidence and +revalidates it against the canonical inputs, rejecting stale or forged handoffs. + +`lineage.graph.build_lineage_graph(validation, *, target_dataset_version, limits)` +constructs the immediate graph only. When explicit primary record roles exist, +the selected version must be that primary version. Context-only typed inputs can +represent an empty target with no declared version; an explicit empty version +must be present in the declared version order. Input loading and its limits stay +independent of graph limits. Root-membership and union-work limits are reserved +for the later ancestry implementation. + +Ownership stays in existing modules: + +| Module | Typed handoff and responsibilities | +|---|---| +| `lineage/graph.py` | `LineageGraph`: scope, sorted node keys, deduplicated immutable `parents_by_child` and `children_by_parent`, retained per-record reference/declaration evidence, invalid-self-reference evidence and validation messages. | +| `lineage/cycles.py` | `CycleAnalysis`: cyclic components, member keys, affected descendant keys, topology for unaffected nodes and structural depth assessments. Iterative algorithms only. | +| `lineage/ancestry.py` | `RecordAncestry`, `RootContribution`, `LineageAnalysisResult`; complete root resolution, G/C/U partition, root incidence/mass and concentration. `SharedAncestryDependence` describes shared-root evidence. | +| `metrics/bounds.py` | `LineageClosureExposureBounds` from the typed G/C/U counts, preserving the direct-bound branch. | +| `result.py`, `reports/assembly.py` | Validate/assemble supplied typed results, scope/status metadata and canonical schema 1.1 values. No graph traversal or metric recomputation. | +| `reports/json_report.py`, `reports/markdown_report.py` | Render the same privacy-safe canonical result. No computation ownership. | + +All result objects use frozen typed data. Sequences are tuples; sets of roots are +frozensets; mappings are detached read-only mappings whose contents are also +immutable. Raw row content, raw URIs and local paths are not copied into ancestry +results. Invalid handoffs are rejected instead of silently trusted. + +`RecordAncestry` contains `record_key: RecordKey`, +`classification: Literal["grounded", "closed", "unresolved"]`, +`external_root_keys: frozenset[RecordKey] | None`, +`reason_codes: tuple[str, ...]`, `lineage_depth: int | None`, and +`depth_reason_codes: tuple[str, ...]`. A null root set means unresolved; an empty +frozenset means complete known-empty ancestry. Unknown and empty never collapse +into the same value. Complete records have no root-resolution failure reasons. +Internal complete sets are bounded by the resource budget and are not exported +as a full transitive-closure table. + +`RootContribution` contains `record_key: RecordKey`, `incidence_count: int`, +`incidence_share: float`, `incidence_denominator: int`, `fractional_mass: float`, +`normalized_weight: float`, and `weight_denominator: int`. Runtime values must be +finite. Independent fixture expectations use exact fractions where possible; +deterministic numeric reduction uses canonical target/root order. + +Step 5 returns all `root_contributions` in decreasing incidence order, with +canonical root-key ties. Its aggregate fields are +`distinct_external_root_count`, `ancestry_concentration_hhi` and +`effective_external_root_count`. These computations precede the Step 7 detail +limit; the direct result does not truncate the root distribution. + +Read-only `root_metrics_status` and `concentration_status` properties use the +existing `ReportStatus` vocabulary. A completed partition has available root +observations when U=0 and partial observations when U>0. A G-only concentration +uses that status when G>0; existing G/N coverage and contribution denominators +disclose its conditional population. With G=0, the contribution tuple is empty +and the distinct-root count is zero, while concentration is unavailable with +`concentration_reason_codes=("NO_RESOLVED_EXTERNAL_ROOTS",)`. A root-resource +abort instead makes contributions and all three aggregates null, with unavailable +statuses and `LINEAGE_RESOURCE_LIMIT_EXCEEDED`. These field states remain +separate from the overall execution status and its input diagnostics. + +`LineageAnalysisResult` contains `scope: LineageScope`, +`execution_status: ExecutionStatus`, `execution_reason_codes: tuple[str, ...]`, +`records: tuple[RecordAncestry, ...] | None`, `cycles: CycleAnalysis | None`, +`root_contributions: tuple[RootContribution, ...] | None`, typed counts/coverage, +concentration and depth values described below, `resource_usage: LineageResourceUsage` +and existing typed validation messages. Failed resource-dependent values are +nullable with reasons; the result never fills unprocessed records with a guessed +closed classification. A failed intermediate field uses null and its reason, +while the result identifies which computation stages actually completed. + +## 3. Cycles and structural depth + +The direct Step 3 entry is `lineage.cycles.analyze_cycles(graph: LineageGraph)`. +It revalidates the typed graph and returns immutable `CycleAnalysis` data without +calling loaders, generation, root propagation, metrics or rendering. Full SCC +membership, affected keys, unaffected topology and per-record depth assessments +are internal computation inputs for subsequent ancestry work. They retain all +loaded nodes. `component_details` holds at most 100 canonical component rows; +omitted component counts remain explicit. The general report detail envelope +and identity protection in section 5 belong to Step 7. + +Cycle status explicitly covers accepted edges and retained self-reference +evidence. Structural input reasons and validation errors remain separate, so an +acyclic accepted subgraph does not certify rejected declarations. A detected +cycle adds `E_LINEAGE_CYCLE` even when its component is disconnected from targets. + +Cycle analysis uses accepted resolved edges plus separately retained explicit +self-parent evidence. The self-edge remains invalid ancestry evidence while +permitting a truthful self-cycle diagnostic. Rejected ambiguous/future/malformed +references remain disclosed and cannot certify whole-input graph validity. + +`cycle_count` is the number of strongly connected components having more than one +node or a self-loop. The method label is `cyclic_strongly_connected_components`. +Components are ordered by their smallest canonical member. Members are the nodes +inside those components; affected records are the union of members and their +descendants along parent-to-child edges. Report total and target counts for both +sets. A disconnected loaded cycle remains detected even if no target descends +from it. + +A cycle witness is a closed directed sequence whose adjacent keys are actual +parent-to-child edges. Use a deterministic canonical-neighbor traversal within +each component. For a self-loop it is `(key, key)`. Return a witness only if it +contains at most 64 edges; otherwise return null with +`witness_reason="diagnostic_limit"`. A bounded search that does not find a short +witness may omit it; absence of a witness does not erase an SCC-based finding. +Do not enumerate all simple cycles or store every ancestry path. + +Depth is the maximum number of structural edges from a record to loaded, +explicitly parentless records, provided every required structural path is valid +and complete. Such a parentless node has depth zero even when its grounding is +unknown. A missing/invalid edge, unavailable parent declaration/chronology or +cycle on a required path makes depth null. Unknown grounding alone prevents an +exact external-root set but does not invalidate an otherwise complete structural +depth. Generation remains its existing non-grounding step count; it is not depth. + +`lineage_depth` is the whole-target maximum and is null for an empty target or +when any target depth is unavailable. `maximum_resolved_target_depth` and +`depth_resolved_record_count` may separately describe the complete-depth subset; +the former is null if that subset is empty. A subset maximum must not occupy the +whole-target field. + +## 4. External roots, coverage and values + +An explicit parentless grounded-yes record supports itself with root set `{key}`. +Strict ancestry still excludes self. Explicit parentless grounding-no produces a +known-empty set. Grounding-unknown, missing required provenance, conflicting +declarations or an undeclared boundary produces an unresolved set. + +Conflicts here concern grounding or required ancestry, as defined in frozen +`DEFINITIONS_AND_UNITS.md` section 5.8. Record/provenance `batch_id` or `timestamp` +differences remain separate metadata observations under +`DATA_AND_PROVENANCE_SPEC.md` section 9.12; they do not alone invalidate roots. +Provenance-specific declarations retain their manifest authority. Labels, URI +spelling and metadata disagreements do not introduce inferred grounding values. + +A grounding-yes carryover with exactly one deduplicated resolved canonical parent +inherits the parent's set. Any unresolved additional declaration blocks it. +Other grounded-with-parent forms, including multi-parent grounded carryover, +remain unresolved under `GROUNDED_PARENT_RULE_UNSUPPORTED`. Evidence labels and +URIs do not mint independent roots. A new external input is a separately loaded, +explicitly parentless grounded anchor linked as an ordinary parent; actual child +grounding must never be relabeled to fit the supported rule. + +Grounding-no descendants union their parents' complete unique root sets. Unknown +grounding anywhere on a required path propagates unresolved ancestry. Every +required branch must resolve; observed roots on incomplete branches are excluded +from exact incidence and mass. Cycle-affected records remain unresolved. Topology +and roots are never inferred from record IDs, filenames or semantic content. + +For complete nonempty-root targets G, complete empty-root targets C and unresolved +targets U, `N=G+C+U`. The report field contracts are: + +| Field under `derived_metrics.lineage` unless stated | Type of envelope value | Exact meaning | +|---|---|---| +| `grounded_record_count` | integer | G, targets with complete nonempty roots. | +| `closed_record_count` | integer | C, targets with complete known-empty roots. | +| `unresolved_record_count` | integer | U, targets with incomplete/invalid ancestry. | +| `records_with_resolved_external_ancestry` | integer | G+C, including known-empty sets. | +| `resolved_lineage_coverage` | number or null | `(G+C)/N`. | +| `external_ancestry_coverage` | number or null | `G/N`. | +| `resolved_parent_edge_coverage` | number or null | Accepted resolved original target references / declared original target references. Exact zero declared count yields 1.0. | +| `distinct_external_root_count` | integer or null | Unique roots in the union over G, zero when the complete partition has G=0. | +| `top_shared_ancestors` | detail object or null | Bounded `RootContribution` rows, descending incidence then canonical key; despite the inherited name, roots with incidence one remain eligible. | +| `ancestry_concentration_hhi` | number or null | `sum((M_a/G)**2)`, where `M_a=sum(1/len(A_r))` over complete grounded targets containing a. | +| `effective_external_root_count` | number or null | Reciprocal HHI. | +| `lineage_depth` | integer or null | Exact whole-target maximum structural depth. | + +`no_declared_parents: bool` accompanies executed reference coverage. It is true +only when the exact total is zero. With lineage not requested, retain schema +1.0's immediate-validation observation semantics and its null ratio for zero +references; do not manufacture an executed coverage value. + +Root incidence shares divide by N; normalized fractional weights divide by G. +Each target in G contributes total fractional mass one. Multiple paths or alias +declarations do not multiply mass. Concentration is computed from all complete +root contributions before report-detail caps or redaction. If U>0, quantities +restricted to G carry partial status and explicit G/N coverage, even when their +conditional values are exact. No completeness of excluded targets is implied. + +Existing closure fields stay at +`derived_metrics.closure_exposure.lineage.{lower_bound,upper_bound,interval_width}` +with values `C/N`, `(C+U)/N`, `U/N`. G/C/U counts must cover all targets before +these values are emitted. A resource-aborted partial traversal provides no such +partition and therefore no numeric bounds. A fully classified unresolved record +may contribute to U and the conservative bound without having an exact root set. + +When N=0, counts are zero, version remains explicitly declared or null, and all +population ratios/bounds are null with `EMPTY_TARGET_SCOPE`. Distinct roots are +zero after successful empty analysis; HHI/effective roots are null with +`NO_RESOLVED_EXTERNAL_ROOTS`. When N>0 and G=0, those concentration fields remain +null with that reason, including all-closed and all-unresolved cases. No infinity, +NaN, fake version or zero-valued unavailable concentration is allowed. + +`proxy_signals.shared_ancestry_dependence` retains the current proxy vocabulary: + +| Condition | `status` | `level` | +|---|---|---| +| Some exact root incidence >=2, U=0 | `available` | `present` | +| Some exact root incidence >=2, U>0 | `partial` | `present` | +| N>0, U=0, every exact root incidence <=1 | `available` | `not_present` | +| N=0, incomplete evidence without a witness, or resource failure | `unavailable` | `indeterminate` | + +The proxy cites actual root/incidence and coverage fields. Topological allocation +does not certify causal contribution, semantic error, independent information, +biological relatedness, universal diversity, integrity or collapse. T4 retains +its theory-guided operationalization label; T6 is an engineering graph-validity +rule. Existing unavailable scientific conclusions remain explicit. + +### Step 6 direct calculation interfaces + +`metrics.bounds.lineage_closure_exposure(result)` returns immutable +`LineageClosureExposureBounds`. Its only stored field, `source`, is a revalidated +`LineageAnalysisResult`, excluded from the wrapper representation. Scope, target +G/C/U counts, denominator, coverage, unresolved reasons and input diagnostics +remain accessible without copying another record or root table. `lower_bound`, +`upper_bound` and `interval_width` expose the three ratios as floats or null. + +The interval's `status` is `available` for a completed nonempty target partition, +including U>0 and the all-unresolved interval `[0,1]`. This status describes the +conservative interval. Input execution status, errors and unresolved reasons +retain their separate meaning. Empty scope yields unavailable values with +`EMPTY_TARGET_SCOPE`; an aborted root traversal yields unavailable values with +`LINEAGE_RESOURCE_LIMIT_EXCEEDED`. The original direct-bound functions preserve +their formulas, classifications and input-only dependency behavior. + +`lineage.ancestry.shared_ancestry_dependence(result)` returns immutable +`SharedAncestryDependence`, retaining one revalidated `source` and deriving the +status/level table above. A positive witness is the first already ranked +`RootContribution` with incidence at least two. The proxy exposes actual +incidence and coverage field references, target scope and source diagnostics. +It adds no calibrated severity, risk threshold or causal conclusion. Both calls +consume already computed ancestry and perform no graph traversal, ingestion, +report rendering or network/file access. Step 7 serializes these explicitly +supplied results without invoking either calculation automatically. + +## 5. Schema 1.1 placement, detail bounds and privacy + +Step 7 changes every current output to report schema 1.1, including requests +without lineage. Strict 1.0 readers must adopt the new schema. Preserve the twelve +top-level sections, existing envelopes and all non-lineage numerical meanings. +Root and packaged report schemas must agree. No migration-loader framework is +introduced. + +The implemented `assemble_report` keywords are `lineage`, `lineage_bounds` and +`shared_ancestry`. Bounds and proxy handoffs require their matching explicit +lineage source. Assembly checks exact target/context scope and a private +computation-owner input signature against retained declarations, policy and +diagnostics, rejecting stale same-identity results. This consistency check does +not authenticate a hostile Python caller. It does not rerun graph traversal or +the root/concentration kernel, and the private signature is never reported. + +Without a lineage result, execution is `not_requested` and immediate-validation +observations retain their limited meanings. An explicit existing +`FamilyFailure(CapabilityKey.LINEAGE, ...)` can report a failure before graph +admission; it supplies no completed graph or root observations. Root-resource +failures with a typed result retain completed graph/depth/reference observations +and consumed-work counters. A separate family failure cannot contradict a +supplied lineage result. Step 8 owns CLI dispatch and graph-admission exception +orchestration. + +New observed fields are `observed_facts.lineage.graph_scope` (the public scope +counts/version inventory), `cycle_analysis`, `depth_summary`, +`unresolved_record_details`, and `resource_usage`. Existing declared/resolved/ +unresolved parent counts remain in their current observed locations. +`cycle_status` remains `acyclic` or `cyclic` when a completed graph check can +establish that value; its scope and reasons disclose rejected declarations. +An acyclic accepted-edge subgraph never certifies validity of rejected input. +Incomplete graph traversal gives null/unavailable, not `acyclic`. + +Every bounded collection uses this exact detail object: + +```text +{ + items: tuple[T, ...] | null, + total_count: int, + returned_count: int, + omitted_count: int, + limit: 100, + detail_status: "complete" | "truncated" | "omitted", + omission_reasons: tuple["diagnostic_limit" | "redacted_identity_details", ...] +} +``` + +`total_count=returned_count+omitted_count`. For visible detail, `items` is an array, +`returned_count=len(items)`, and at most 100 items appear. Complete detail has zero +omissions/reasons, including an empty array. Truncated detail has positive +omissions and `diagnostic_limit`. Privacy omission uses `items:null`, returned +zero, omitted equal to total, status omitted and reason +`redacted_identity_details`. Applying privacy to a previously capped collection +retains that diagnostic-limit reason when applicable. These presentation counts +do not change analytical status or denominators. + +`cycle_analysis` includes `detected: bool`, `cycle_count: int`, +`counting_method: "cyclic_strongly_connected_components"`, total/target member and +affected counts, and three bounded collections: `components`, +`cycle_member_record_keys`, `affected_record_keys`. Component rows contain +`component_index: int` (one-based canonical order), `member_count: int`, +`target_member_count: int`, `witness_record_keys: tuple[RecordKey,...] | None`, +`witness_edge_count: int | None`, and +`witness_reason: Literal["diagnostic_limit"] | None`. A returned witness is closed +and has at most 65 keys; omitted witness edge count is null. The detail object +around components provides the privacy omission shape for the entire row list. + +`unresolved_record_details` rows contain only `record_key` and sorted unique +`reason_codes`, ordered by canonical key. Root rows are the seven fields in +`RootContribution`. `depth_summary` contains `depth_resolved_record_count`, +`maximum_resolved_target_depth` and the explicit target count. No arbitrary +message, URI, path or content payload is admitted into these shapes. + +Standard mode preserves declared structural identities and record IDs. +Redacted mode supports record-ID `hash` (default), `omit` and explicit `preserve`; +record-ID overrides require redacted mode. In redacted/hash, +dataset IDs, root keys, witness keys and dynamic identities use the existing +scoped HMAC mechanism consistently; sort/rank before privacy transformation. +In record-ID omit mode, each identity-bearing collection uses its whole-list +omission variant. Never remove a required `record_key` while retaining an invalid +row. Aggregate values, counts, severity and execution status remain visible. +Redacted/preserve retains only declared record-ID fields while dataset versions +and other protected identities still use their existing transformation. Both +renderers consume the same schema-valid safe result. + +## 6. Resource and invocation interfaces + +`LineageLimits` has four required positive integer fields with these defaults; +the existing `ResourceLimits` exposes the corresponding configuration fields: + +| `LineageLimits` field | `ResourceLimits` / JSON `resource_limits` field | Default | +|---|---|---| +| `max_nodes` | `max_lineage_nodes` | 200000 | +| `max_edges` | `max_lineage_edges` | 1000000 | +| `max_root_memberships` | `max_lineage_root_memberships` | 1000000 | +| `max_root_union_visits` | `max_lineage_root_union_visits` | 10000000 | + +Defaults apply only to explicitly requested lineage. A supplied override is an +exact integer greater than zero; bool, null, fractional, infinite and nonpositive +values are invalid. Existing input byte/row/parent-list/depth limits remain +independent and retain their existing optional semantics. + +Check cumulative node admission across all batches before graph materialization, +unique-edge admission before insertion, stored membership before storing each +new `(record, root)` association, and union work before each candidate root visit. +A union visit is processing one parent's candidate root into a child's union, +including a candidate already present there. An anchor self-membership consumes +one stored membership and zero union visits. Shared immutable set storage may +reduce actual memory but does not reduce the logical membership accounting. + +`LineageResourceUsage` contains `admitted_node_count`, `admitted_edge_count`, +`stored_root_membership_count`, `root_union_visit_count`, `limits: LineageLimits`, +and `exhausted_limit: str | None`. On exhaustion, include +`attempted_value: int | None` for the rejected next admission/visit, stop dependent +work, and emit a typed error with `LINEAGE_RESOURCE_LIMIT_EXCEEDED`. Distinguish +these operational reason codes from the existing `ErrorCode` namespace; add the +smallest needed error constant during implementation without renaming existing +errors. A successfully completed earlier stage may retain its exact observations, +but no truncated partition, root distribution or bound is reported as exact. +Independently completed non-lineage families survive. + +Iteration order for budgeting is canonical nodes, canonical parent keys and +canonical root keys. Budget exhaustion must be deterministic for permuted input. +Limits bound defined work units, not peak RSS, loader allocations, decompression +or scientific thresholds. Step 9 measures actual time/RSS and may propose an +explicitly documented limit adjustment. It must not silently raise limits to +turn a failing check into a pass. + +The inert configuration field is `ResolvedConfig.lineage: bool = False`, +serialized as top-level `lineage: false` when explicit. The dedicated input role +is `FileRole.LINEAGE_CONTEXT` with value `lineage_context`; it permits repeated +existing local input declarations. Preserve all existing multi-role Python/config +behavior. Config resolution neither reads inputs nor starts analysis. + +`audit --lineage` and `example --lineage` enable computation. Repeatable +`--lineage-records PATH` supplies context to audit only with lineage enabled. +`validate` may ingest a configured/context input role for input/reference +validation, but rejects any true lineage execution request. Ordinary metric calls +select the primary version explicitly. Context does not request comparison, +trajectories, simulations or network access. Existing CLI comparison stays one +explicit earlier input; broader Python/config multiplicity stays unchanged. + +Context files may contain multiple context versions. Context versions must be +disjoint from primary and comparison versions, and duplicate composite keys +across repeated context inputs are rejected. A same-version ancestor must be +loaded alongside its target in the primary input. Config context paths resolve +relative to the config document; CLI context paths resolve relative to the +working directory. The existing CSV/JSONL/optional Parquet and privacy rules +apply unchanged. See [CLI commands](cli.md#explicit-lineage-and-context). + +The installed Hero selects v2 as its eight-record target and loads v1 as the +existing comparison/ancestor input. Plain `example` retains `not_requested` +lineage. Explicit `example --lineage` produces G=8, C=0, U=0, five supporting +roots, HHI=1/4, effective roots=4, lineage bounds [0, 0] and the witnessed shared +ancestry proxy. Direct bounds remain [1/2, 1/2]. Step 9 supplies the separate +scale measurements; Step 10 supplies candidate verification. + +## 7. Execution and error states + +Input observability stays separate from actual computation. Completed/partial +execution names performed operations. All other execution states have explicit +reasons. Use these rules in order: + +| Situation | Lineage execution | Value handling | +|---|---|---| +| No explicit request | `not_requested` | No graph execution claim; ordinary validation evidence may remain. | +| Fatal admission/resource failure | `failed` | No exact result from incomplete stages; earlier completed observations and other families may survive. | +| Invalid graph/input, with useful unaffected target results | `partial` | Preserve errors, complete subset values and precise target coverage. | +| Invalid graph/input, with no usable target ancestry result | `failed` | Preserve valid diagnostics; exact root metrics unavailable. | +| No lineage error, with U>0 | `partial` | Complete partition/bounds may be available; G-only metrics disclose partial coverage. | +| No lineage error and U=0, including a successful empty analysis | `completed` | Defined values available; empty-denominator metrics remain unavailable. | + +A disconnected loaded cycle remains a lineage error even with complete target +ancestry. Missing parents promoted by strict mode follow the error rows; ordinary +missing-parent warnings follow the unresolved-coverage row. A known all-closed +target can complete successfully while HHI remains unavailable. Root uncertainty +does not erase independently valid graph observations. + +Existing run-status behavior is preserved: fatal errors fail the run; other +errors yield partial when useful evidence survives and failed otherwise. +Warnings alone retain their existing run-status behavior. Any surviving lineage +error produces a non-success CLI audit exit under the existing error policy. +Redaction and capped detail do not themselves turn completed analysis into +partial execution. + +## 8. Direct verification and completion boundary + +Independent fixture expectations cover the frozen Hero, rational multi-root +allocation, missing and unknown branches separately, carryover aliases, known +empty versus unknown ancestry, empty targets, disconnected cycles, affected +descendants and target/context isolation. They describe future behavior and are +not successful runtime results in Step 1. + +Implementation checks protect those behaviors directly. New graph boundaries +receive bounded adversarial/resource cases. Candidate verification retains the +supported OS/Python/dependency and packaging gates. The actual 100k sparse lineage +measurement runs on the designated reference profile; ordinary step checks do +not repeat a release matrix. Historical behavior is consolidated under P5-D11, +without new source-body chains or evidence-of-evidence frameworks. + +No automatic longitudinal analysis, new independent-input evidence schema, +causal weighting, semantic inference, HTML implementation or theory-source change +is authorized by this contract. diff --git a/docs/privacy.md b/docs/privacy.md index 55d21d9..e7e67ca 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -1,6 +1,6 @@ # Privacy and Local Input Boundaries -Status: Phase 4 development milestone (`0.1.0.dev3`). `PRIVACY_AND_DATA_HANDLING.md` remains authoritative. +Status: Phase 5 dev4 candidate (`0.1.0.dev4`, report schema `1.1`). `PRIVACY_AND_DATA_HANDLING.md` remains authoritative. Package import does not contact a network, open user audit files, require optional PyArrow or start a service. Runtime operates on explicitly supplied local files and declarations. No telemetry, background worker, plugin, cloud client, database, model download or LLM service exists. CI/package installation acquire dependencies separately; CI uses synthetic fixtures only. @@ -64,7 +64,7 @@ Scopes, state IDs, source declarations, weights, content digests and original di ## Explicit safe report views -These APIs implement approved P4-D05 and P4-D08, with PR-015 privacy and +These APIs implement approved P4-D05, P4-D08 and P5-D08, with PR-015 privacy and PR-016 reproducibility ownership. `assemble_report` continues to assemble standard internal evidence. Its `CanonicalReport` validates structure and evidence semantics; validation alone is not a privacy transformation. After calculation and assembly, @@ -91,6 +91,8 @@ denominators, evidence classes, availability and error severity remain unchanged Pseudonyms do not reorder parallel simulation arrays. Record-ID modes are `hash` (the redacted default), `preserve` and `omit`. +Standard mode preserves record IDs; explicit record-ID overrides require +redacted mode. An explicit `preserve` applies only to declared record-ID fields, never to a matching string embedded in a diagnostic, path or other narrative. `omit` removes record-level linkage while retaining aggregate counts and disclosure of exclusion @@ -98,6 +100,33 @@ reasons. An omitted duplicate-group identity list uses the existing `redacted_identity_details` contract. These omissions describe a privacy choice, not missing input evidence, and do not turn an available metric into unavailable. +Lineage adds root identities, target and loaded version labels, cycle witnesses, +cycle-member and affected-record lists, and unresolved-record details to the +same protection boundary. Redacted/hash uses existing dataset/record identity +domains consistently across roots, witnesses, scopes and diagnostics. Ranking +and the 100-row presentation cap are applied before identity transformation. +The complete root distribution determines concentration before this cap. + +In redacted/omit, each identity-bearing lineage collection is omitted as a whole: +`items: null`, `returned_count: 0`, `omitted_count: total_count` and +`detail_status: "omitted"`. Its reasons include `redacted_identity_details` and +retain `diagnostic_limit` if the original collection was already capped. No row +survives with a required identity removed. Counts, resource usage, numerical +values, evidence classes and execution/error status remain visible. Explicit +redacted/preserve retains only declared record-ID fields; dataset versions and +other protected labels remain pseudonymized. + +The private lineage input signature binds retained declarations to the supplied +calculation result so stale same-key evidence cannot silently enter a report. +It is an internal consistency check, excluded from report serialization, and +provides no authenticity or anonymity guarantee. Assembly and rendering do not +rerun ancestry calculations. Explicit `audit --lineage` and `example --lineage` +invoke the accepted lineage kernels before assembly. Repeatable local context +inputs use the same loader, source-inventory and output-protection boundaries as +primary inputs. Context paths are included in publication collision checks. +The dedicated `lineage_context` role does not enable content-reference reads or +network access. `validate` can ingest this role while remaining input-only. + Identifier protection uses domain-separated HMAC-SHA-256 over unambiguous canonical encodings. A fresh secret gives run-scoped consistency and separates otherwise identical runs. An explicitly supplied local secret file permits @@ -189,3 +218,11 @@ privacy mode appropriate to the destination before sharing a report. Redacted output still contains aggregate counts and linkable input/configuration digests. Large reports consume memory during in-memory construction and rendering; per-file input limits do not bound total report memory or size. + +Lineage additionally enforces positive finite configuration limits for cumulative +nodes, unique edges, record/root memberships and union visits. These guards bound +defined graph work after table loading; they do not cap total process RSS or +decompression memory. A limit failure preserves available earlier observations +and independent metrics, with a nonzero exit and unavailable dependent values. +Context identities, roots, unresolved-record lists and cycle witnesses are +protected in reports and stderr under the same selected privacy mode. diff --git a/docs/release_process.md b/docs/release_process.md index 074aae5..7b6ecca 100644 --- a/docs/release_process.md +++ b/docs/release_process.md @@ -1,75 +1,138 @@ # Development and Release Verification -`0.1.0.dev3` is the Phase 4 report/CLI development milestone. Completion requires recorded candidate evidence and does not authorize stable v0.1 publication, a tag, main merge or Phase 5 implementation. The three `PHASE_4_*` milestone reports distinguish actual acceptance, remaining limits and unexecuted work. Prior phase records remain historical evidence. +## Current Phase 5 boundary -## Choose the gate by impact +Phase 5 Step 10 verifies development version `0.1.0.dev4`, with executable +report schema 1.1 for every report. The implemented lineage, CLI/context inputs, +bounded mutation checks and scale measurements are retained. Candidate status, +exact source, actual jobs and delivery limits are recorded in +[Phase 5 completion](../PHASE_5_COMPLETION.md). +Stable v0.1 publication, a tag and main merge require +separate authorization. -| Change or milestone | Required scope | -|---|---| -| Documentation or nonauthoritative metadata | Formatting/static checks and affected document consistency | -| Isolated implementation | Affected unit tests, dependency neighbors and applicable mathematical/security checks | -| Schema, public API, mathematics, provenance, privacy or package boundary | Relevant integration, compatibility and broader regression appropriate to impact | -| PR candidate | Complete canonical regression and required supported-environment matrix | -| Release candidate or phase delivery candidate | Complete regression, supported matrix, build/install, canonical examples, reproducibility, artifact integrity and applicable security checks | - -Documentation that participates in executable authority receives the applicable authority checks. Administrative successors after an accepted code candidate do not automatically repeat the full matrix when executable and authoritative bytes are unchanged. Preserve the accepted source identity and state exactly what changed. If an executable or authoritative change invalidates prior evidence, rerun the affected gate before acceptance. - -Keep three concerns distinct: the canonical suite protects current supported behavior; release verification covers environment/package/reproducibility boundaries; Git and archived records preserve superseded evidence. Preserve historical behavioral guarantees without automatically adding test-body migrations, registries or evidence-of-evidence layers. A new verification mechanism needs a concrete failure that existing controls cannot adequately detect. New governance mechanisms additionally require the reason Git/archive evidence is insufficient, maintenance cost and an exit or consolidation path. - -## Supported candidate profiles - -| Profile | Environment and evidence | -|---|---| -| Core, current dependencies | Ubuntu and Windows, Python 3.11 and 3.12; resolved versions retained; PyArrow genuinely absent | -| Core, minimum jointly compatible dependencies | Same OS/Python matrix, NumPy 2.0.0 and pandas 2.2.2; PyArrow absent | -| Real Parquet | Ubuntu/Python 3.12 with actual PyArrow; real roundtrip, row-limit and invalid-file regressions | -| Security | No-network/import, declarative mapping, safe input/output paths, privacy/redaction and protected-owner boundaries | -| Hero and mathematics | Frozen mathematical cases, literal standard/redacted report goldens and installed Hero behavior | -| Package and delivery | Wheel/sdist integrity, strict metadata checks, clean installed execution outside the checkout, resource equality and tracked-source archive | - -The package still declares `numpy>=2.0` and `pandas>=2.2`. NumPy 2.0.0 with pandas 2.2.0 cannot resolve because that pandas version requires NumPy below 2. The retained minimum profile uses pandas 2.2.2. This covers that jointly compatible pair, without claiming every intervening version or all optional-dependency minimums. Current profiles record the versions actually resolved at execution time. - -Retain Python/platform/dependency details, `pip freeze`, `pip check`, commands, exit codes, JUnit identities and workflow logs. Core absence and real-PyArrow presence must be checked explicitly. A skipped, failed, duplicate or uncollected required test cannot stand in for an executed passing test. Do not add repeated jobs/subsets into a fictitious distinct-test total. - -## Local checks and builds - -From a prepared full Git checkout: +Use one current verification path: ```bash -python -m pip install ".[test,release]" -python scripts/check_spec_consistency.py --phase 4 --step 11 -python scripts/check_traceability.py --phase 4 --step 11 -python scripts/release_check.py --phase 4 --step 11 --diff +python scripts/check_spec_consistency.py +python scripts/check_traceability.py +python scripts/release_check.py ``` -The Phase 4 Step 11 candidate runs `python -m pytest -p no:cacheprovider -q` in all eight core matrix cells and the real-Parquet cell, retaining the complete suite including the 100,000-record report case. The release job reuses the exact same-commit Ubuntu/Python 3.12 current-core and real-Parquet results rather than running both suites again. The Hero/mathematics and security roles run separately. Full Git history is needed for the bounded retained historical checks. Scope guards protect the fixed authority files, supported runtime behavior, schemas, Hero resources and mathematical/report oracles. The active manifest cannot expand its own permissions. - -Build into a fresh evidence directory: +Historical `--phase` and `--step` dispatch is retired. Earlier source forms, +migrations and receipts remain recoverable from Git and archived phase records. +The existing source check advances its single reference to the completed Step 9 +commit; no new Phase 5 hash registry is needed. During Step 10 it permits changes +only to `pyproject.toml` and the package root version declaration for dev4. +It retains the exact product file inventory and +rejects unrelated runtime, package-metadata and canonical Hero changes. +The frozen Phase 0 specification hashes are unchanged. Sixteen frozen +specifications and seven packaged resource copies remain protected. The source +inventory excludes only the generated `src/recursive_integrity_toolkit.egg-info/` +metadata directory; similarly named or nested paths are not excluded, and +symlink rejection and shipped-package inventory checks remain in force. + +## Risk-sensitive gates + +Ordinary PR checks use the focused job for the active step. Documentation-only +changes require formatting and affected consistency checks. As implementation +scope changes, update the focused selection to affected owners and dependency +neighbors. It does not establish candidate acceptance. + +Dispatch `ci.yml` with `gate=candidate` for a stable candidate. When the default +branch does not yet contain the dispatch-capable workflow, add the +`verification:candidate` label to its PR to select the same jobs. Only that label +event selects the matrix; subsequent commits use the ordinary affected gate. +Remove and re-add the label to select a changed candidate. A label is a scheduling +choice and does not grant source authority, merge permission or release approval. +The workflow runs: + +| Role | Required execution | +|---|---| +| Core | Ubuntu/Windows, Python 3.11/3.12, current/minimum compatible dependencies; PyArrow absent | +| Optional Parquet | Ubuntu/Python 3.12, real PyArrow with roundtrip, row-limit, invalid-file and both context-input regressions | +| Canonical Hero and mathematics | Frozen mathematical cases, report goldens and packaged Hero behavior | +| Security | No-network, input/path, privacy/redaction and output-publication boundaries | +| Reference performance | One Ubuntu/Python 3.12 profile, including required actual 100k workloads | +| Package/delivery | Wheel/sdist integrity, clean installed execution, reproducibility and tracked-source archive | + +Compatibility cells run complete canonical regression with +`--ignore=tests/performance`. The designated performance job runs +`tests/performance` once, avoiding nine repetitions of the costly 100k report +workload. The current sparse-lineage benchmark executes a real 100k reverse chain +through file loading, complete input/generation validation and ancestry analysis. +Separate bounded deep-chain and high-fan-in cases measure the full lineage CLI +through JSON/Markdown. The canonical Hero is measured afresh with lineage enabled. + +The existing minimum compatible pair remains NumPy 2.0.0/pandas 2.2.2. Record +actual resolved dependencies, `pip check`, Python/platform, commands, results +and material limitations. A skipped, failed, duplicate or uncollected required +test cannot stand in for a passing result. No raw test-count growth target is used. + +`RIT_TEST_PARQUET=1` collects all five real-PyArrow cases and missing PyArrow is a +failure. Core profiles set it to `0`, verify PyArrow is absent and collect the +complete non-Parquet suite. Selecting a supported profile never uses skip/xfail +to represent success. + +Hero and security are reusable candidate roles and may also be dispatched +explicitly for a relevant change. Delivery requires core, Parquet, performance, +Hero and security success. A failing command cannot be masked by evidence upload. + +## Package and candidate checks ```bash python -m build --outdir /absolute/evidence/dist python -m twine check --strict /absolute/evidence/dist/* -python scripts/release_check.py --phase 4 --step 11 --dist /absolute/evidence/dist +python scripts/release_check.py --junit /absolute/evidence/core.xml +python scripts/release_check.py --junit /absolute/evidence/parquet.xml --require-parquet +python scripts/release_check.py --dist /absolute/evidence/dist +python scripts/release_check.py --candidate /absolute/evidence ``` -The candidate build role performs two default builds from sdist with the same recorded build environment and `SOURCE_DATE_EPOCH`, comparing wheel bytes and sdist payloads. The package checks compare wheel/sdist module and resource bytes with the tested tree. Installed checks must import the installed artifact, with checkout source excluded. Keep import/input-only checks with NumPy/pandas/PyArrow blocked separate from numerical/audit checks with required dependencies available. Both operate under the declared no-network checks. Exercise installed `example`, `audit`, `validate`, help/version and the documented Python examples. Wheel and sdist must include the six exact Hero files and report schema, without adding Python modules. - -A source archive contains one project root and the complete tracked tree. Compare archived member bytes with the accepted checkout. Exclude Git internals, environments, caches, generated bytecode, private inputs and full theory PDFs. Wheel, sdist and source archive have different documented scopes. Retain artifact hashes with actual source commit/tree identity in the external receipt; committed documents need not encode their own eventual hash. - -## Performance evidence and limits - -Keep complete-report timing separate from the CLI's pre-publication `run.duration_seconds`. Use untraced wall time for the Hero under-five-second target and record hardware/environment. Allocation tracing is a separate observation with measured overhead. RSS describes fresh-process peak resident size and includes interpreter/native allocation; it is not a delta or a Python-only allocation measure. - -The accepted Step 10 reference-container 100,000-record metadata run took 1,362.4708 seconds, peaked at 7.12 GiB RSS and produced approximately 443 MB JSON and 165 MB Markdown. That run checked independent aggregate expectations through actual published outputs. It measures the declared synthetic workload without certifying other data shapes, common-laptop throughput, general linear scaling or deferred lineage behavior. Its full-scale Python allocation peak was not measured; a separately labeled bounded run characterized tracing overhead. - -The Step 10 observation supplies the comparison baseline; the Step 11 full candidate matrix also records fresh complete-suite performance observations. For later changes, reuse accepted observations when product/report bytes and relevant conditions are unchanged. Repeat expensive performance measurements only for a concrete impact or required gate, with comparable dataset, tracing mode, environment and resources. Preserve all attempts and explain target misses. A timeout is a resource guard and a failed attempt, never a successful throughput result. The [performance README](../tests/performance/README.md) defines the generator, expectations and measurement scope. - -## Acceptance and handoff - -The active CI workflow calls the reusable build/delivery workflow after core and Parquet success. Together with the separately dispatched Hero and security workflows, this supplies four verification roles from three dispatchable workflows. There is no additional push-triggered full matrix or standalone performance job. Record the actual immutable candidate commit and required workflow roles. Completion can be claimed only after their required checks pass and deliverable identities are known. If the final record changes only nonauthoritative documentation, verify those changes and reuse the identified code-candidate evidence under the policy above. Changes to executable/authoritative content require the corresponding checks. Do not claim a matrix was rerun when it was reused. - -The external receipt identifies candidate/final source, actual workflow attempts, commands, artifacts, hashes, failed attempts, fixes and limitations. Hosted artifacts have finite retention. If a hosted binary cannot be retrieved and verified, disclose that gap and distinguish locally reproduced bytes from the hosted artifact. Archive previous evidence without expanding permanent current execution merely to preserve old source forms. - -GitHub evaluates PR path filters against the complete three-dot diff. A documentation-only successor on an existing code PR can therefore trigger the matrix despite `paths-ignore`. After every required candidate gate passes, an acceptance-record successor may use `[skip ci]` when an exact diff confirms that only nonauthoritative documentation changed and executable, workflow, test and authority bytes are unchanged. Retain the accepted candidate identity and results, run focused document/source checks, and record evidence reuse. This cannot excuse a failed or outstanding product gate. Required checks may remain pending on the skipped head, so this handoff keeps the PR draft and unmerged; a later merge must satisfy its applicable checks. See GitHub's [path-filter semantics](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/trigger-a-workflow#git-diff-comparisons) and [skip instructions](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/skip-workflow-runs). - -Stop at the approved development milestone. Stable release, merge, tag, publication and the next phase remain separate actions. +Delivery reuses same-commit core/Parquet XML instead of rerunning those suites. +Candidate validation checks complete current canonical test identities against +collection, exact committed source, wheel/sdist module and resource bytes, safe +archive handling and clean installed behavior outside the checkout. Synthetic +archive/XML negatives test corruption detection; they never count as a real +build or installed execution. + +Build twice from the same committed source and recorded environment with +`SOURCE_DATE_EPOCH`; compare wheel bytes and every sdist file payload. Sdist archive +timestamps may differ. Keep installed import/input-only checks with numerical +and optional imports blocked separate from actual mathematical/audit execution. +Installed checks cover the existing public APIs, CLI, privacy, safe output and +both ordinary and explicitly enabled lineage Hero runs. The lineage smoke checks +the frozen G/C/U partition, root incidence/concentration, direct and lineage +bounds, proxy status, privacy and schema outside the checkout with network +blocked. Schema and Hero resources must match their canonical source bytes. + +The source archive contains one project root and the complete tracked tree. +Exclude Git internals, environments, caches, generated bytecode, private inputs +and full theory PDFs. Wheel, sdist and source archives retain their distinct scopes. + +## Performance and evidence reuse + +Measure complete output publication separately from report +`run.duration_seconds`. Untraced Hero wall time supports the under-five-second +target on the stated reference environment. Traced allocation and process peak +RSS are separate observations. Do not substitute a small workload or extrapolation +for required actual 100k execution. See [performance details](../tests/performance/README.md). + +Phase 4's metadata-only 100k run took about 1,362.47 seconds and 7.12 GiB peak RSS. +It did not execute general ancestry. Step 9's actual 100k ancestry API run took +123.36 seconds and 1.71 GiB peak RSS, excluding full audit serialization and +ordinary metrics. Its 1,000-record full CLI run took 11.15 seconds. Those scopes +are distinct from the Phase 4 workload and do not imply a comparable speedup. +See [Step 9 results](../PHASE_5_STEP_9.md) for environment and all attempts. +Comparable regression review uses the approved >20% time or >50% +memory thresholds. Operational timeouts are resource guards, not product SLAs. + +Reuse unchanged successful candidate evidence for nonauthoritative administrative +successors and disclose that reuse. Repeat affected gates when executable or +authoritative content changes. Preserve failed attempts and material limitations +in the existing completion/validation record; do not generate another permanent +receipt framework. A hosted artifact that cannot be retrieved is distinct from +locally reproduced bytes and must be described accurately. + +The historical Phase 4 procedure and full evidence are recoverable at +`1db3b1a460c233242ff37fbe45b8fac74d6101ef` and its phase completion records. +Stop at the authorized step. Verification never merges, tags or publishes by itself. diff --git a/docs/report_schema.md b/docs/report_schema.md index 1dab399..d6a2653 100644 --- a/docs/report_schema.md +++ b/docs/report_schema.md @@ -1,6 +1,6 @@ # Report Schema -Status: Phase 4 development report contract (`0.1.0.dev3`). Schema version: `1.0`. +Status: Phase 5 dev4 development report contract (`0.1.0.dev4`). Schema version: `1.1`. The authoritative public shape is `schemas/report.schema.json`, Draft 2020-12. Runtime validation is implemented in `result.py` using the standard library. `report_schema()` returns a detached copy of the same declarative contract. No runtime schema package, filesystem lookup or remote resolver is used. The `$schema` and `$id` identifiers are descriptive; all references are local `$defs` references. @@ -10,11 +10,17 @@ The installed CLI publishes `report.json` and `report.md` from the same validate privacy view. `audit` computes explicitly requested supported families; `validate` keeps analytical sections empty; `example` runs the packaged explicit Hero pair. Default `simulations` is `{}`. Existing Python scenario results may be supplied -explicitly to assembly, but CLI simulation execution is unsupported. General -lineage, ancestry, broader longitudinal orchestration and HTML remain deferred. +explicitly to assembly, but CLI simulation execution is unsupported. Python +callers can supply completed lineage, lineage bounds and shared-root proxy +results explicitly. `audit --lineage` and `example --lineage` compute these +families explicitly. Input inventories and diagnostic locations accept the +dedicated `lineage_context` role. Graph scope counts loaded context separately +from the primary target; ordinary metric denominators remain scoped to their +selected primary/comparison versions. Broader longitudinal orchestration and +HTML remain deferred. Capability eligibility and execution are separate. An input Level 4 report can -retain deferred lineage, and a completed explicit pair has partial longitudinal +retain `not_requested` lineage, and a completed explicit pair has partial longitudinal execution because additional change families remain deferred. This alone does not create an error. Unavailable values retain null plus their required reasons, while actual zero and empty measured sets remain values. @@ -26,7 +32,7 @@ retains input identities when privacy permits them. Warning codes, counts, capability effects and representative locations are preserved. Distinct contexts are retained, and only equal diagnostic entries are deduplicated in encounter order. This bounded representation prevents repeated full-dataset identity copies -without changing schema version, analytical values or unknown-state meaning. +without changing analytical values or unknown-state meaning. `run.duration_seconds` for CLI runs measures input/calculation work before report assembly and publication. It must not be treated as the complete report runtime. @@ -74,7 +80,7 @@ Every section is present in empty, validation-only and error-only reports. Empty Input capability status is `available`, `partial`, `unavailable` or `experimental`. Execution status is independently `completed`, `partial`, `not_requested`, `deferred` or `failed`. Executed work names its operations in `execution_scope`; all noncompleted statuses carry `execution_reason_codes`. A nonempty matrix contains exactly the seven approved capability keys. Its copy at `observability.capabilities` is structurally equal on input and shares the same immutable object in the canonical result. A mismatch is rejected. -`observability` may be empty before assessment; otherwise all six fields are required. Maximum level 0–5 must match its exact registered label. Level 4 input eligibility can coexist with lineage execution deferred. Phase 4 cannot claim completed or partially executed Phase 5 lineage analysis. Existing declared/resolved edge observations and the explicitly named earlier-version ordering certificate retain their limited Phase 2 provenance. +`observability` may be empty before assessment; otherwise all six fields are required. Maximum level 0–5 must match its exact registered label. Level 4 input eligibility can coexist with lineage execution `not_requested`. Supplied lineage results retain their actual `completed`, `partial` or `failed` execution status. Without an explicit result, existing declared/resolved edge observations and the explicitly named earlier-version ordering certificate retain their limited validation provenance. ## Analytical field registry @@ -102,8 +108,13 @@ Input capability status is `available`, `partial`, `unavailable` or `experimenta | `observed_facts.lineage.declared_parent_edge_count` | integer | `edges` | `observed_fact` | `PR-008` | `PR-008.declared_parent_edge_count` | 3 | Null only with unavailable reasons | | `observed_facts.lineage.resolved_parent_edge_count` | integer | `edges` | `observed_fact` | `PR-008` | `PR-008.resolved_parent_edge_count` | 3 | Null only with unavailable reasons | | `observed_facts.lineage.unresolved_parent_edge_count` | integer | `edges` | `observed_fact` | `PR-008` | `PR-008.unresolved_parent_edge_count` | 3 | Null only with unavailable reasons | -| `observed_facts.lineage.cycle_status` | enum | `status` | `observed_fact` | `T6` | `T6.graph_cycle_check` | 3 | Unavailable only: Phase 5 deferred | +| `observed_facts.lineage.cycle_status` | enum | `status` | `observed_fact` | `T6` | `T6.graph_cycle_check` | 3 | Null when graph analysis is unavailable | | `observed_facts.lineage.ordering_certificate` | ordering_certificate | `certificate` | `observed_fact` | `PR-008` | `PR-008.earlier_version_certificate` | 3 | Null only with unavailable reasons | +| `observed_facts.lineage.graph_scope` | lineage_scope | `scope` | `observed_fact` | `T4` | `T4.graph_scope` | 3 | Explicit target and loaded scope | +| `observed_facts.lineage.cycle_analysis` | cycle_analysis | `cycles` | `observed_fact` | `T6` | `T6.cyclic_strongly_connected_components` | 3 | Null only with unavailable reasons | +| `observed_facts.lineage.depth_summary` | depth_summary | `edges` | `observed_fact` | `PR-009` | `PR-009.resolved_depth_summary` | 3 | Resolved subset retained | +| `observed_facts.lineage.unresolved_record_details` | unresolved_record_details | `records` | `observed_fact` | `T4` | `T4.unresolved_record_details` | 3 | Null when target partition is unavailable | +| `observed_facts.lineage.resource_usage` | lineage_resource_usage | `work_units` | `observed_fact` | `PR-015` | `PR-015.lineage_resource_usage` | 3 | Exact consumed work and explicit limits | | `observed_facts.state_counts.by_version.*` | state_count[] | `records` | `observed_fact` | `T1` | `T1.state_counts` | 1 | Null only with unavailable reasons | | `derived_metrics.support.by_version.*.support_size` | integer | `states` | `derived_metric` | `T1` | `F-002` | 1 | Null only with unavailable reasons | | `derived_metrics.support.by_version.*.weighted_support_size` | integer | `states` | `derived_metric` | `T1` | `F-002` | 1 | Null only with unavailable reasons | @@ -139,20 +150,24 @@ Input capability status is `available`, `partial`, `unavailable` or `experimenta | `derived_metrics.closure_exposure.direct.lower_bound` | ratio | `ratio` | `derived_metric` | `T3` | `F-009` | 2 | Null only with unavailable reasons | | `derived_metrics.closure_exposure.direct.upper_bound` | ratio | `ratio` | `derived_metric` | `T3` | `F-010` | 2 | Null only with unavailable reasons | | `derived_metrics.closure_exposure.direct.interval_width` | ratio | `ratio` | `derived_metric` | `T3` | `T3.upper_minus_lower` | 2 | Null only with unavailable reasons | -| `derived_metrics.closure_exposure.lineage.lower_bound` | ratio | `ratio` | `derived_metric` | `T3` | `T3.lineage_closure` | 3 | Unavailable only: Phase 5 deferred | -| `derived_metrics.closure_exposure.lineage.upper_bound` | ratio | `ratio` | `derived_metric` | `T3` | `T3.lineage_closure` | 3 | Unavailable only: Phase 5 deferred | -| `derived_metrics.closure_exposure.lineage.interval_width` | ratio | `ratio` | `derived_metric` | `T3` | `T3.lineage_closure` | 3 | Unavailable only: Phase 5 deferred | +| `derived_metrics.closure_exposure.lineage.lower_bound` | ratio | `ratio` | `derived_metric` | `T3` | `T3.lineage_closure` | 3 | Null for empty or unavailable partition | +| `derived_metrics.closure_exposure.lineage.upper_bound` | ratio | `ratio` | `derived_metric` | `T3` | `T3.lineage_closure` | 3 | Null for empty or unavailable partition | +| `derived_metrics.closure_exposure.lineage.interval_width` | ratio | `ratio` | `derived_metric` | `T3` | `T3.lineage_closure` | 3 | Null for empty or unavailable partition | +| `derived_metrics.lineage.grounded_record_count` | integer | `records` | `derived_metric` | `T4` | `T4.grounded_record_count` | 3 | Null for unavailable partition | +| `derived_metrics.lineage.closed_record_count` | integer | `records` | `derived_metric` | `T4` | `T4.closed_record_count` | 3 | Null for unavailable partition | +| `derived_metrics.lineage.unresolved_record_count` | integer | `records` | `derived_metric` | `T4` | `T4.unresolved_record_count` | 3 | Null for unavailable partition | +| `derived_metrics.lineage.records_with_resolved_external_ancestry` | integer | `records` | `derived_metric` | `T4` | `T4.records_with_resolved_external_ancestry` | 3 | Null for unavailable partition | | `derived_metrics.lineage.resolved_parent_edge_coverage` | ratio | `ratio` | `derived_metric` | `PR-008` | `PR-008.resolved_edges_over_declared` | 3 | Null only with unavailable reasons | -| `derived_metrics.lineage.resolved_lineage_coverage` | ratio | `ratio` | `derived_metric` | `T4` | `T4.resolved_records_over_scope` | 3 | Unavailable only: Phase 5 deferred | -| `derived_metrics.lineage.external_ancestry_coverage` | ratio | `ratio` | `derived_metric` | `T4` | `T4.external_roots_over_scope` | 3 | Unavailable only: Phase 5 deferred | -| `derived_metrics.lineage.distinct_external_root_count` | integer | `roots` | `derived_metric` | `T4` | `T4.root_set_union` | 3 | Unavailable only: Phase 5 deferred | -| `derived_metrics.lineage.top_shared_ancestors` | ancestor_incidence[] | `records` | `derived_metric` | `T4` | `T4.incidence_ranking` | 3 | Unavailable only: Phase 5 deferred | -| `derived_metrics.lineage.ancestry_concentration_hhi` | ratio | `ratio` | `derived_metric` | `T4` | `F-012` | 3 | Unavailable only: Phase 5 deferred | -| `derived_metrics.lineage.effective_external_root_count` | number | `roots` | `derived_metric` | `T4` | `F-013` | 3 | Unavailable only: Phase 5 deferred | -| `derived_metrics.lineage.lineage_depth` | integer | `edges` | `derived_metric` | `PR-009` | `PR-009.maximum_resolved_parent_depth` | 3 | Unavailable only: Phase 5 deferred | +| `derived_metrics.lineage.resolved_lineage_coverage` | ratio | `ratio` | `derived_metric` | `T4` | `T4.resolved_records_over_scope` | 3 | Null for empty or unavailable partition | +| `derived_metrics.lineage.external_ancestry_coverage` | ratio | `ratio` | `derived_metric` | `T4` | `T4.external_roots_over_scope` | 3 | Null for empty or unavailable partition | +| `derived_metrics.lineage.distinct_external_root_count` | integer | `roots` | `derived_metric` | `T4` | `T4.root_set_union` | 3 | Complete-root subset with explicit coverage | +| `derived_metrics.lineage.top_shared_ancestors` | root_contribution_details | `records` | `derived_metric` | `T4` | `T4.incidence_ranking` | 3 | Null for unavailable root analysis | +| `derived_metrics.lineage.ancestry_concentration_hhi` | ratio | `ratio` | `derived_metric` | `T4` | `F-012` | 3 | Null without resolved external roots | +| `derived_metrics.lineage.effective_external_root_count` | number | `roots` | `derived_metric` | `T4` | `F-013` | 3 | Null without resolved external roots | +| `derived_metrics.lineage.lineage_depth` | integer | `edges` | `derived_metric` | `PR-009` | `PR-009.maximum_resolved_parent_depth` | 3 | Null for incomplete whole-target depth | | `proxy_signals.support_contraction` | proxy | `signal` | `proxy_signal` | `T1` | `T1.support_contraction` | 4 | No scalar value; typed class fields | | `proxy_signals.tail_fragility` | proxy | `signal` | `proxy_signal` | `T2` | `T2.tail_fragility` | 1 | No scalar value; typed class fields | -| `proxy_signals.shared_ancestry_dependence` | proxy | `signal` | `proxy_signal` | `T4` | `T4.shared_ancestry_dependence` | 3 | Unavailable only: Phase 5 deferred | +| `proxy_signals.shared_ancestry_dependence` | proxy | `signal` | `proxy_signal` | `T4` | `T4.shared_ancestry_dependence` | 3 | Presence may retain partial coverage; absence requires complete ancestry | | `proxy_signals.provenance_uncertainty` | proxy | `signal` | `proxy_signal` | `T3` | `T3.provenance_uncertainty` | 2 | No scalar value; typed class fields | | `simulations.closed_resampling` | scenario | `scenario` | `simulation` | `T1` | `T1.closed_resampling` | 5 | No scalar value; typed class fields | | `simulations.tail_extinction` | scenario | `scenario` | `simulation` | `T2` | `T2.tail_extinction` | 1 | No scalar value; typed class fields | @@ -188,7 +203,7 @@ The following tables enumerate non-envelope public fields and containers. A `$de | `run` | object | required | PR-016 | Not nullable | | `run.run_id` | string | required | PR-016 | Not nullable | | `run.toolkit_version` | string | required | PR-016 | Not nullable | -| `run.report_schema_version` | constant "1.0" | required | PR-013 | Not nullable | +| `run.report_schema_version` | constant "1.1" | required | PR-013 | Not nullable | | `run.started_at` | string or null | required | PR-016 | Explicitly not applicable/unavailable as described above | | `run.completed_at` | string or null | required | PR-016 | Explicitly not applicable/unavailable as described above | | `run.duration_seconds` | number or null | required | PR-016 | Explicitly not applicable/unavailable as described above | @@ -231,7 +246,7 @@ The following tables enumerate non-envelope public fields and containers. A `$de | `inputs` | object | required | PR-002 | Not nullable | | `inputs.artifacts` | object[] | optional | PR-002 | Not nullable | | `inputs.artifacts[]` | object | required | PR-002 | Not nullable | -| `inputs.artifacts[].role` | enum: records_primary, records_compare, provenance_manifest, schema_mapping, config, version_order, embedding_data, external_reference | required | PR-002 | Not nullable | +| `inputs.artifacts[].role` | enum: records_primary, records_compare, lineage_context, provenance_manifest, schema_mapping, config, version_order, embedding_data, external_reference | required | PR-002 | Not nullable | | `inputs.artifacts[].path` | string or null | required | PR-002 | Explicitly not applicable/unavailable as described above | | `inputs.artifacts[].path_redacted` | boolean | required | PR-002 | Not nullable | | `inputs.artifacts[].format` | enum: csv, jsonl, parquet, json, toml, npy | required | PR-002 | Not nullable | @@ -323,6 +338,11 @@ The following tables enumerate non-envelope public fields and containers. A `$de | `observed_facts.lineage.unresolved_parent_edge_count` | object | optional | Enclosing registered owner | Not nullable | | `observed_facts.lineage.cycle_status` | object | optional | Enclosing registered owner | Not nullable | | `observed_facts.lineage.ordering_certificate` | object | optional | Enclosing registered owner | Not nullable | +| `observed_facts.lineage.graph_scope` | object | optional | T4 | Not nullable | +| `observed_facts.lineage.cycle_analysis` | object | optional | T6 | Not nullable | +| `observed_facts.lineage.depth_summary` | object | optional | PR-009 | Not nullable | +| `observed_facts.lineage.unresolved_record_details` | object | optional | T4 | Not nullable | +| `observed_facts.lineage.resource_usage` | object | optional | PR-015 | Not nullable | | `observed_facts.state_counts` | object | optional | Enclosing registered owner | Not nullable | | `observed_facts.state_counts.by_version` | object | optional | Enclosing registered owner | Not nullable | | `observed_facts.state_counts.by_version.*` | object | required | Enclosing registered owner | Not nullable | @@ -387,6 +407,10 @@ The following tables enumerate non-envelope public fields and containers. A `$de | `derived_metrics.closure_exposure.lineage.upper_bound` | object | optional | T3 | Not nullable | | `derived_metrics.closure_exposure.lineage.interval_width` | object | optional | T3 | Not nullable | | `derived_metrics.lineage` | object | optional | Enclosing registered owner | Not nullable | +| `derived_metrics.lineage.grounded_record_count` | object | optional | T4 | Not nullable | +| `derived_metrics.lineage.closed_record_count` | object | optional | T4 | Not nullable | +| `derived_metrics.lineage.unresolved_record_count` | object | optional | T4 | Not nullable | +| `derived_metrics.lineage.records_with_resolved_external_ancestry` | object | optional | T4 | Not nullable | | `derived_metrics.lineage.resolved_parent_edge_coverage` | object | optional | Enclosing registered owner | Not nullable | | `derived_metrics.lineage.resolved_lineage_coverage` | object | optional | Enclosing registered owner | Not nullable | | `derived_metrics.lineage.external_ancestry_coverage` | object | optional | Enclosing registered owner | Not nullable | @@ -413,7 +437,7 @@ The following tables enumerate non-envelope public fields and containers. A `$de | `simulations` | object | required | Enclosing registered owner | Not nullable | | `simulations.closed_resampling` | object | optional | Enclosing registered owner | Not nullable | | `simulations.tail_extinction` | object | optional | Enclosing registered owner | Not nullable | -| `simulations.external_reopening` | forbidden in Phase 4 | optional | Enclosing registered owner | Not nullable | +| `simulations.external_reopening` | forbidden in the current schema | optional | Enclosing registered owner | Not nullable | ### `unavailable_conclusions` @@ -741,7 +765,7 @@ All emitted scenario evidence remains `simulation` and `experimental`. The commo `simulations.closed_resampling` retains direct `expected_diversity`, `sampled_paths`, `support_trajectories` and `extinction_events` paths. Analytic expectation requires the supplied expectation vector, initial diversity, contraction factor and underflow steps; its vector length is horizon plus one. Sampled paths require seed, replicate count, NumPy/RNG identity and scheduling metadata. Path count must equal replicate count. Optional support trajectories and events may only preserve evidence already supplied by the accepted owner; their presence does not authorize the adapter or renderer to calculate them. -`simulations.external_reopening` is a registered future field, explicitly rejected by this Phase 4 schema and constructor. `external_reference_loss` has no approved public trace and is unregistered. Neither is executed. The lack of implementation belongs in capability execution reasons and unavailable conclusions. +`simulations.external_reopening` is a registered future field, explicitly rejected by the current schema and constructor. `external_reference_loss` has no approved public trace and is unregistered. Neither is executed. The lack of implementation belongs in capability execution reasons and unavailable conclusions. ## Path reconciliation and unchanged meaning @@ -753,7 +777,7 @@ All emitted scenario evidence remains `simulation` and `experimental`. The commo | Existing `TailSelectionResult.tail_membership` | Public `derived_metrics.tail.tail_states`, plus `tail_rule` and typed selection metadata | Frozen planned public path; explicit adapter mapping, no numerical change | | Existing duplicate group object | `observed_facts.content.exact_duplicate_groups` | Frozen PR-006 planned public path | | Reporting operations-applied description | `inputs.schema_mapping.operations` with the exact accepted operation enum | Frozen PR-003 planned path and Phase 2 dispatcher | -| Old `cycle_detected` and cycle-status detail sketches | Registered `observed_facts.lineage.cycle_status` is an unavailable envelope | Registry §32 and P4-D01; graph operations remain Phase 5 | +| Old `cycle_detected` and cycle-status detail sketches | Registered `observed_facts.lineage.cycle_status` plus `cycle_analysis` | P5-D08 and the approved lineage contract; explicit completed graph observations | | Reporting simplified proxy example uses `status: present` | `level: present`, independent availability status | Full reporting §9.3/§16.6 | | Older privacy enum includes debug | Public Phase 4 privacy mode is standard or redacted | Approved Phase 4 Step 4 boundary | @@ -909,8 +933,14 @@ The following additional fields are part of the registered envelopes, with the o | `derived_metrics.lineage.external_ancestry_coverage` | `method` | string | optional | | `derived_metrics.lineage.distinct_external_root_count` | `method` | string | optional | | `derived_metrics.lineage.top_shared_ancestors` | `method` | string | optional | -| `derived_metrics.lineage.top_shared_ancestors` | `value[].record_key` | record_key | required | -| `derived_metrics.lineage.top_shared_ancestors` | `value[].incidence_count` | integer | required | +| `derived_metrics.lineage.top_shared_ancestors` | `value.items[].record_key` | record_key | required on visible rows | +| `derived_metrics.lineage.top_shared_ancestors` | `value.items[].incidence_count` | integer | required on visible rows | +| `derived_metrics.lineage.top_shared_ancestors` | `value.items[].incidence_share` | ratio | required on visible rows | +| `derived_metrics.lineage.top_shared_ancestors` | `value.items[].incidence_denominator` | integer | required on visible rows | +| `derived_metrics.lineage.top_shared_ancestors` | `value.items[].fractional_mass` | number | required on visible rows | +| `derived_metrics.lineage.top_shared_ancestors` | `value.items[].normalized_weight` | ratio | required on visible rows | +| `derived_metrics.lineage.top_shared_ancestors` | `value.items[].weight_denominator` | integer | required on visible rows | +| `derived_metrics.lineage.resolved_parent_edge_coverage` | `no_declared_parents` | boolean | required for executed lineage | | `derived_metrics.lineage.ancestry_concentration_hhi` | `method` | string | optional | | `derived_metrics.lineage.effective_external_root_count` | `method` | string | optional | | `derived_metrics.lineage.lineage_depth` | `method` | string | optional | @@ -970,7 +1000,7 @@ Schema validation is separate from evidence assembly, privacy selection, renderi ## Explicit evidence assembly -`recursive_integrity_toolkit.reports.assembly` retains report schema version `1.0`, the twelve sections, their order and registered field ownership. `weighted_source_type_masses` is a category-to-mass object (`category_mass_map`). +`recursive_integrity_toolkit.reports.assembly` emits report schema version `1.1`, retaining the twelve sections, their order and registered field ownership. Strict schema-1.0 readers must explicitly adopt 1.1; no migration loader is supplied. `weighted_source_type_masses` is a category-to-mass object (`category_mass_map`). ### Public Python interface @@ -985,6 +1015,9 @@ def assemble_report( tail: TailSelectionResult | None = None, comparison: SupportComparison | None = None, closure: DirectClosureExposureBounds | None = None, + lineage: LineageAnalysisResult | None = None, + lineage_bounds: LineageClosureExposureBounds | None = None, + shared_ancestry: SharedAncestryDependence | None = None, expected_diversity: ExpectedDiversityResult | None = None, resampling: ResamplingSimulation | None = None, extinction: tuple[ExtinctionProbabilityResult, ...] = (), @@ -1019,6 +1052,9 @@ Assembly validates result types, finite values, metadata ownership, accepted met | `tail` | `TailSelectionResult` | The declared rule, its parameters, supplied membership, support/share and rarity ranking | | `comparison` | `SupportComparison` | Supplied pair support delta, retention, loss/added counts, original support differences and diversity delta with explicit comparison basis | | `closure` | `DirectClosureExposureBounds` | Supplied lower bound, upper bound and interval width, with operationalization, coverage and separate confidence disclosure | +| `lineage` | `LineageAnalysisResult` | Supplied target/context scope, graph/cycle/depth observations, original-reference coverage, G/C/U partition, root metrics, bounded details, resource usage and execution status | +| `lineage_bounds` | `LineageClosureExposureBounds` | Explicit lineage closure interval from the same lineage source; direct bounds retain their own slot | +| `shared_ancestry` | `SharedAncestryDependence` | Explicit descriptive shared-root signal, independent availability, coverage and limitations | | `expected_diversity` | `ExpectedDiversityResult` | Experimental analytic closed-resampling expectation and supplied scenario metadata | | `resampling` | `ResamplingSimulation` | Experimental sampled paths, support trajectories, extinction events and supplied scenario metadata | | `extinction` | Tuple of `ExtinctionProbabilityResult` | Explicitly supplied selected-state one-step extinction marginals under one shared scenario basis | @@ -1026,6 +1062,41 @@ Assembly validates result types, finite values, metadata ownership, accepted met Empirical frequencies remain derived metrics. Explicit supplied probabilities are observed declarations under `observed_facts.supplied_state_probabilities.by_version`, with an explicit probability basis and no invented record-count denominator. Scenario probabilities remain simulations. Product metadata, capability decisions, warnings, errors and recommendations retain their product ownership without an additional scientific evidence class. +### Lineage scope, bounded details and source handoffs + +The complete field shapes are fixed in [the lineage contract](lineage_contract.md#5-schema-11-placement-detail-bounds-and-privacy). +Graph scope, cycle observations and resource usage describe all loaded records; +root/depth population metrics and original-reference counts describe only the +selected primary target. Context records support ancestry without changing N. +G/C/U counts cover all targets, independently of representation eligibility. +G-only concentration and root observations retain partial status and explicit +G/N coverage when U is positive. A complete conservative interval can remain +available even when its source execution is partial or failed for invalid input. + +Each of the five identity-bearing collections uses `items`, `total_count`, +`returned_count`, `omitted_count`, `limit: 100`, `detail_status` and +`omission_reasons`: top roots, unresolved records, cycle components, cycle +members and affected records. Visible detail is complete or truncated with +`diagnostic_limit`; whole-list privacy omission uses null items and +`redacted_identity_details`, retaining an existing limit reason. Counts always +satisfy `total_count = returned_count + omitted_count`. Component witnesses +contain at most 64 edges and 65 closed-path keys. Details preserve the owner's +rank/order, and their cap never changes full-distribution metrics or execution +status. + +Assembly binds the selected target and complete loaded identities to the bundle. +A private input signature additionally checks relevant retained declarations, +field presence, policy and diagnostics so changed grounding under identical +record keys cannot reuse an old result. Bounds and proxy results must match the +explicit lineage source. The signature remains internal and provides no +authenticity guarantee. These checks do not rerun graph or metric kernels. + +A typed root-resource failure retains completed graph/depth/reference results, +resource counters and diagnostics, while root counts, concentration and intervals +remain unavailable. A pre-result graph-admission error can use the existing +lineage `FamilyFailure`; it supplies no completed graph observations. Supplying +both a lineage result and a separate lineage family failure is rejected. + ### Scope, coverage and representation Empirical result scopes must use identities and versions retained in the validated bundle. Included and excluded identities, scope identifiers, denominator meanings and representation descriptors remain attached to their fields. Representation exclusions do not shrink input inventory, provenance denominators or unrelated result families. A missing representation permits input/provenance reporting and an explicit metadata recommendation; assembly does not select a topic field, hash representation or missing-state sentinel. @@ -1063,7 +1134,7 @@ The schema has one scalar/table slot per version and weighting mode, plus single Scenario model/version, assumptions, supplied/effective distributions, normalization disclosure, numerical policy, horizon, sample size, seed, RNG identity, replicate schedule, paths and underflow disclosures remain attached to the applicable result. Analytic expectations, sampled paths and analytic one-step probabilities retain their distinct method labels. Mathematical scopes may differ from the bundle's empirical scope without upgrading its observability assessment. -No scenario runs during assembly. Omitted scenario arguments leave `simulations` empty. A supplied extinction marginal alone does not select a tail or create a tail-fragility signal. External reopening remains rejected by the Phase 4 schema, and external-reference loss remains unregistered. Supplied scenarios cannot establish an empirical intervention effect or a calibrated production-failure forecast. +No scenario runs during assembly. Omitted scenario arguments leave `simulations` empty. A supplied extinction marginal alone does not select a tail or create a tail-fragility signal. External reopening remains rejected by the current schema, and external-reference loss remains unregistered. Supplied scenarios cannot establish an empirical intervention effect or a calibrated production-failure forecast. ### Input eligibility, execution and failures @@ -1074,7 +1145,7 @@ The seven capability input statuses, original reason codes, requirements and inp | `ingestion` | The retained bundle-validation operation is identified explicitly; retained ingestion errors produce partial/failed execution according to surviving input evidence. | | `content_diagnostics` | Names supplied distribution, duplicate and tail operations. With no supplied result or family error it is `not_requested`; supplied operations are `completed` unless retained family errors make execution `partial`. An error with no supplied operation is `failed`. | | `provenance` | Names supplied provenance-composition/direct-bound operations under the same completed/partial/failed rules. Independent validation coverage remains available even when no calculation is requested. | -| `lineage` | Always `deferred` in Phase 4, including Level 4 input eligibility. Retained immediate-parent validation errors remain errors without asserting graph execution. | +| `lineage` | Defaults to `not_requested`; an explicit source result retains its `completed`, `partial` or `failed` execution. A bound family failure before an analytical result reports `failed`. Immediate-parent validation alone never claims graph execution. | | `dataset_longitudinal` | A supplied pair records its limited support/diversity operation with `partial` execution and an explicit deferred-change-families reason. Without a supplied pair it is `not_requested`, or `failed` when an explicitly bound failure exists. | | `model_longitudinal` | `deferred`, with the missing implementation/evidence boundary retained. | | `intervention_simulation` | Names supplied analytic expectation, sampled path or extinction-marginal operations; no supplied scenario means `not_requested`. Explicitly bound failures remain failed/partial execution. | @@ -1096,12 +1167,13 @@ The current deterministic signals are restricted to these existing bases. Compar | `support_contraction` | A usable supplied explicit-pair support delta is negative. Basis fields cite the delta and supplied support differences/comparison details. | Restricted to the pair's selected scope, representation and declared common state meaning; no production failure or model-performance claim. | | `tail_fragility` | A usable supplied tail-support count is positive under the supplied explicit tail-selection rule. | Declared tail membership, with no hidden threshold, importance claim or calibrated production forecast. | | `provenance_uncertainty` | A supplied usable direct interval width is positive, or a cited usable row/required-field/grounding coverage is below one. | Cites actual fields with matching scopes and preserves their distinct denominator meanings; completeness does not establish truth or source independence. | +| `shared_ancestry_dependence` | An explicitly supplied proxy reports a complete external root with incidence at least two. An absent witness supports `not_present` only when nonempty target ancestry is complete. | Partial coverage can support `present`; incomplete evidence without a witness remains `indeterminate` and unavailable. No calibrated risk or causal conclusion. | -Zero/positive support delta and an empty selected tail can produce `not_present` under their own usable basis. Missing, unavailable or zero-denominator evidence is not converted into zero or a negative finding. When a signal has no usable basis it is omitted. Each emitted proxy retains basis paths, deterministic trigger text, scope, representation where applicable, coverage and limitations. No `shared_ancestry_dependence` proxy is generated in this phase. +Zero/positive support delta and an empty selected tail can produce `not_present` under their own usable basis. Missing, unavailable or zero-denominator evidence is not converted into zero or a negative finding. The existing non-lineage signals are omitted without a usable basis; an explicitly supplied shared-ancestry proxy retains its unavailable state and reasons. Each emitted proxy retains basis paths, deterministic trigger text, scope, representation where applicable, coverage and limitations. An omitted shared-ancestry argument does not request that proxy automatically. -The required unavailable-conclusion safeguards include model-performance decline, causal ancestor effect, universal integrity and universal collapse prediction. Applicable additional entries explain production failure, complete-pipeline closure, deferred lineage analysis/closure/external ancestry and empirical intervention effects. Each uses its registered owner and method, unavailable evidence class/status, specific reasons, blocking evidence, required next metadata, related capability and theory/product limit. Missing input, invalid evidence, incompatible declarations, deferred implementation and conclusions outside product scope remain distinguishable. Unavailable does not establish that a conclusion is false. +The required unavailable-conclusion safeguards include model-performance decline, causal ancestor effect, universal integrity and universal collapse prediction. Applicable additional entries explain production failure, complete-pipeline closure, unrequested or failed lineage analysis, incomplete ancestry coverage, unsupplied or unavailable lineage intervals, and empirical intervention effects. Each uses its registered owner and method, unavailable evidence class/status, specific reasons, blocking evidence, required next metadata, related capability and theory/product limit. Missing input, invalid evidence, incompatible declarations, deferred implementation and conclusions outside product scope remain distinguishable. Unavailable does not establish that a conclusion is false. -Lineage validation observations remain bounded by their source method. Retained reference-entry counts preserve the classifier's multiplicity and denominator. The earlier-version ordering certificate is described as its sufficient declared-order certificate; it is not reported as general graph-cycle traversal. Root counts, HHI, effective roots, lineage depth, graph traversal, lineage closure and ancestry proxies remain deferred with explicit disclosures. +Lineage validation observations remain bounded by their source method. Without an explicit analysis, retained reference-entry counts preserve the classifier's multiplicity and denominator, including null coverage when no reference exists. Executed lineage uses original target-reference multiplicity and coverage 1.0 for an exact zero total, with `no_declared_parents: true`. The earlier-version ordering certificate remains its limited declared-order observation. Supplied graph, root, concentration, depth, bounds and proxy results retain their independent scopes, availability and calculation owners. Recommendations are deterministic requests for evidence: missing matching provenance rows, usable required fields, unresolved external grounding, an explicitly missing representation, missing chronology, unresolved composite parent references and versioned model outcomes. Their priorities follow uncertainty reduction, the next available evidence requirement, error resolution and optional enrichment. Recommendations refer to the missing fields or retained classifier reasons; the absence of a single top-level representation summary alone does not assert that representation was never declared. @@ -1130,6 +1202,7 @@ error severity are preserved. Input-inventory and normalized configuration hashe remain linkable reproducibility metadata; raw per-record content digests are protected when they serve as state or group identities. +Standard mode preserves record IDs and does not accept record-ID overrides. In redacted mode, record IDs default to `hash`; `preserve` affects only the declared record-ID field, and `omit` removes identity-bearing lists. Duplicate groups with omitted identities retain `record_count` and the frozen @@ -1139,6 +1212,13 @@ reasons remain disclosed with their protected scope association under `run.identifier_protection.limitations`. Per-record linkage is intentionally omitted. Redaction does not introduce an unavailable analytical conclusion. +Lineage roots, witnesses, unresolved records and loaded/target version labels +use the same identifier domains as existing record keys and scopes. Each +identity-bearing lineage detail collection uses whole-list omission in +redacted/omit, preserving totals and aggregate analytical values. The +100-item cap and canonical ranking precede hashing. JSON and Markdown render +the same schema-valid safe result with identical metric values and omissions. + The existing optional `run.identifier_protection` object records `HMAC-SHA-256`, `run` or `cross_run` stability, record-ID mode and limitations. Fresh secrets separate runs; an explicitly supplied checked local secret file diff --git a/docs/theory_traceability.md b/docs/theory_traceability.md index 4bd2dd6..3464a5c 100644 --- a/docs/theory_traceability.md +++ b/docs/theory_traceability.md @@ -1,6 +1,40 @@ # Theory Traceability -Status: Phase 3 mathematical development milestone. Final acceptance is recorded in the three Phase 3 milestone reports. +## Current Phase 5 implementation + +Development version `0.1.0.dev4` implements the approved [lineage contract](lineage_contract.md) +and preserves the existing mathematical definitions and direct calculations. + +| Owner | Current implementation | Independent behavioral coverage | +|---|---|---| +| PR-008 | Retained immediate-parent evidence and canonical graph adjacency with target/context isolation | Parent batch, graph and context-input suites; missing, ambiguous, future, duplicate and malformed references | +| T6 | Iterative cyclic-component detection and affected-path propagation | Self-cycle, multi-node SCC, disjoint components, bounded witnesses and unaffected branches | +| PR-009 | Structural depth in `lineage/cycles.py`; generation remains separate in input validation | Same-version chains, missing/unknown inputs, cycles, grounded resets and declaration mismatch | +| T4 | Strict external ancestry, G/C/U, root incidence, fractional mass, HHI and effective roots | Frozen Hero and hand-authored partial/carryover/diamond/all-closed/all-unresolved oracles | +| T3 | Explicit lineage closure bounds from the full target partition | Rational interval, unavailable and denominator cases alongside preserved direct bounds | +| PR-012, PR-013, PR-015, PR-016 | Schema 1.1, evidence metadata, privacy and deterministic lineage reporting | Typed/wire semantic negatives, forged/stale results, standard/redacted output and input permutation cases | +| PR-011 and package boundary | Explicit CLI dispatch, context formats, installed ordinary/lineage Hero | Context isolation, input-only validation, clean installs and real optional Parquet | + +The Hero's eight v2 targets resolve to five external roots, HHI 1/4, effective +roots 4 and lineage bounds [0,0], while direct bounds remain [1/2,1/2]. The partial +oracle fixes N/G/C/U at 4/2/1/1, incidence 2 and 1, fractional masses 3/2 and 1/2, +HHI 5/8, effective roots 8/5 and lineage bounds [1/4,1/2]. These expectations were +specified independently of production calculations. Root incidence uses N; +fractional concentration normalizes over G. No confidence discount is introduced. + +Seven bounded Step 9 mutations were detected by current tests, covering wrong +denominators, duplicate memberships, carryover roots, unknown-state propagation, +cycle propagation, interval order and HHI inversion. [Step 9](../PHASE_5_STEP_9.md) +records those experiments; [completion](../PHASE_5_COMPLETION.md) records the +candidate. No new theorem, formula identifier, calibrated risk score, Phase 6A +longitudinal orchestration or Phase 6B reopening is claimed. + +The sections below retain historical implementation explanations. Current checks +protect owner/layer boundaries and current behavior; they no longer require +historical test-body or source-binding migrations to execute. Frozen specifications +and mathematical/Hero expectations remain unchanged. + +Historical Phase 3 status: mathematical development milestone. Final acceptance is recorded in the three Phase 3 milestone reports. The controlling maps are: diff --git a/pyproject.toml b/pyproject.toml index 678b009..df3615a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "recursive-integrity-toolkit" -version = "0.1.0.dev3" +version = "0.1.0.dev4" description = "Local-first, auditable research toolkit for recursive data integrity" readme = "README.md" requires-python = ">=3.11" diff --git a/schemas/config.schema.json b/schemas/config.schema.json index 043c95b..9ee329c 100644 --- a/schemas/config.schema.json +++ b/schemas/config.schema.json @@ -9,7 +9,7 @@ "inputs": { "type": "object", "description": "Explicit local input-role declarations. Detailed cardinality and filesystem validation are performed by the Phase 2 input layer.", - "propertyNames": {"enum": ["records_primary", "records_compare", "provenance_manifest", "schema_mapping", "config", "version_order", "embedding_data", "external_reference"]}, + "propertyNames": {"enum": ["records_primary", "records_compare", "lineage_context", "provenance_manifest", "schema_mapping", "config", "version_order", "embedding_data", "external_reference"]}, "additionalProperties": true }, "representation": { @@ -31,6 +31,7 @@ "privacy_mode": {"type": "string", "enum": ["standard", "redacted", "debug"], "default": "standard"}, "strict_mode": {"type": "boolean", "default": false, "description": "Promote only explicitly configured warnings in later validation steps."}, "strict_warning_codes": {"type": "array", "items": {"type": "string", "minLength": 1}, "uniqueItems": true}, + "lineage": {"type": "boolean", "default": false, "description": "Explicit lineage execution request for audit or example. Input-only validate rejects true."}, "resource_limits": { "type": "object", "properties": { @@ -38,7 +39,11 @@ "max_rows": {"type": "integer", "minimum": 1}, "max_content_bytes": {"type": "integer", "minimum": 1}, "max_parent_list_length": {"type": "integer", "minimum": 1}, - "max_json_depth": {"type": "integer", "minimum": 1} + "max_json_depth": {"type": "integer", "minimum": 1}, + "max_lineage_nodes": {"type": "integer", "minimum": 1, "default": 200000}, + "max_lineage_edges": {"type": "integer", "minimum": 1, "default": 1000000}, + "max_lineage_root_memberships": {"type": "integer", "minimum": 1, "default": 1000000}, + "max_lineage_root_union_visits": {"type": "integer", "minimum": 1, "default": 10000000} }, "additionalProperties": false }, diff --git a/schemas/report.schema.json b/schemas/report.schema.json index c3fce78..df4b687 100644 --- a/schemas/report.schema.json +++ b/schemas/report.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://recursive-integrity-toolkit.example/schemas/report.schema.json", - "title": "Recursive Integrity Toolkit canonical report 1.0", - "description": "Phase 4 Step 2 public contract. Product metadata and five analytical evidence classes remain separate.", + "title": "Recursive Integrity Toolkit canonical report 1.1", + "description": "Phase 5 public contract with bounded lineage evidence. Product metadata and five analytical evidence classes remain separate.", "type": "object", "properties": { "run": { @@ -19,7 +19,7 @@ "pattern": "^[^\\u0000]*$" }, "report_schema_version": { - "const": "1.0" + "const": "1.1" }, "started_at": { "anyOf": [ @@ -294,6 +294,7 @@ "enum": [ "records_primary", "records_compare", + "lineage_context", "provenance_manifest", "schema_mapping", "config", @@ -6752,7 +6753,12 @@ "const": "observed_fact" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "T6.graph_cycle_check" @@ -7011,7 +7017,7 @@ } ] }, - "ordering_certificate": { + "graph_scope": { "type": "object", "properties": { "value": { @@ -7019,10 +7025,34 @@ { "type": "object", "properties": { - "method": { - "const": "declared_earlier_version_order" + "target_dataset_version": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] }, - "version_order": { + "target_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "loaded_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "context_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "loaded_dataset_versions": { "type": "array", "items": { "type": "string", @@ -7031,15 +7061,14 @@ }, "minItems": 0, "uniqueItems": true - }, - "all_resolved_edges_follow_order": { - "type": "boolean" } }, "required": [ - "method", - "version_order", - "all_resolved_edges_follow_order" + "target_dataset_version", + "target_record_count", + "loaded_record_count", + "context_record_count", + "loaded_dataset_versions" ], "additionalProperties": false }, @@ -7049,7 +7078,7 @@ ] }, "unit": { - "const": "certificate" + "const": "scope" }, "evidence_class": { "const": "observed_fact" @@ -7063,11 +7092,11 @@ ] }, "method_id": { - "const": "PR-008.earlier_version_certificate" + "const": "T4.graph_scope" }, "owner_ids": { "const": [ - "PR-008" + "T4" ] }, "theory_map_ids": { @@ -7310,10 +7339,34 @@ "value": { "type": "object", "properties": { - "method": { - "const": "declared_earlier_version_order" + "target_dataset_version": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] }, - "version_order": { + "target_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "loaded_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "context_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "loaded_dataset_versions": { "type": "array", "items": { "type": "string", @@ -7322,15 +7375,14 @@ }, "minItems": 0, "uniqueItems": true - }, - "all_resolved_edges_follow_order": { - "type": "boolean" } }, "required": [ - "method", - "version_order", - "all_resolved_edges_follow_order" + "target_dataset_version", + "target_record_count", + "loaded_record_count", + "context_record_count", + "loaded_dataset_versions" ], "additionalProperties": false } @@ -7338,71 +7390,3357 @@ } } ] - } - }, - "required": [], - "additionalProperties": false - }, - "state_counts": { - "type": "object", - "properties": { - "by_version": { + }, + "cycle_analysis": { "type": "object", - "propertyNames": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "additionalProperties": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "array", - "items": { + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "detected": { + "type": "boolean" + }, + "cycle_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "counting_method": { + "const": "cyclic_strongly_connected_components" + }, + "cycle_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "target_cycle_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "affected_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "target_affected_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "components": { "type": "object", "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_index": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "member_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "target_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "witness_record_keys": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 2, + "maxItems": 65 + }, + { + "type": "null" + } + ] + }, + "witness_edge_count": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 64 + }, + { + "type": "null" + } + ] + }, + "witness_reason": { + "anyOf": [ + { + "const": "diagnostic_limit" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "component_index", + "member_count", + "target_member_count", + "witness_record_keys", + "witness_edge_count", + "witness_reason" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] }, - "state_count": { + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { "type": "integer", "minimum": 0, "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true } }, "required": [ - "state_id", - "state_count" + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" ], - "additionalProperties": false + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_index": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "member_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "target_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "witness_record_keys": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 2, + "maxItems": 65 + }, + { + "type": "null" + } + ] + }, + "witness_edge_count": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 64 + }, + { + "type": "null" + } + ] + }, + "witness_reason": { + "anyOf": [ + { + "const": "diagnostic_limit" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "component_index", + "member_count", + "target_member_count", + "witness_record_keys", + "witness_edge_count", + "witness_reason" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] }, - "minItems": 0 - }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "records" - }, - "evidence_class": { - "const": "observed_fact" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "T1.state_counts" + "cycle_member_record_keys": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + }, + "affected_record_keys": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + } + }, + "required": [ + "detected", + "cycle_count", + "counting_method", + "cycle_member_count", + "target_cycle_member_count", + "affected_record_count", + "target_affected_record_count", + "components", + "cycle_member_record_keys", + "affected_record_keys" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "cycles" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T6.cyclic_strongly_connected_components" + }, + "owner_ids": { + "const": [ + "T6" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "detected": { + "type": "boolean" + }, + "cycle_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "counting_method": { + "const": "cyclic_strongly_connected_components" + }, + "cycle_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "target_cycle_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "affected_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "target_affected_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "components": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_index": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "member_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "target_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "witness_record_keys": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 2, + "maxItems": 65 + }, + { + "type": "null" + } + ] + }, + "witness_edge_count": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 64 + }, + { + "type": "null" + } + ] + }, + "witness_reason": { + "anyOf": [ + { + "const": "diagnostic_limit" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "component_index", + "member_count", + "target_member_count", + "witness_record_keys", + "witness_edge_count", + "witness_reason" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_index": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "member_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "target_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "witness_record_keys": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 2, + "maxItems": 65 + }, + { + "type": "null" + } + ] + }, + "witness_edge_count": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 64 + }, + { + "type": "null" + } + ] + }, + "witness_reason": { + "anyOf": [ + { + "const": "diagnostic_limit" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "component_index", + "member_count", + "target_member_count", + "witness_record_keys", + "witness_edge_count", + "witness_reason" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + }, + "cycle_member_record_keys": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + }, + "affected_record_keys": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + } + }, + "required": [ + "detected", + "cycle_count", + "counting_method", + "cycle_member_count", + "target_cycle_member_count", + "affected_record_count", + "target_affected_record_count", + "components", + "cycle_member_record_keys", + "affected_record_keys" + ], + "additionalProperties": false + } + } + } + } + ] + }, + "depth_summary": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "depth_resolved_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "maximum_resolved_target_depth": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "target_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "depth_resolved_record_count", + "maximum_resolved_target_depth", + "target_record_count" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "edges" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "PR-009.resolved_depth_summary" + }, + "owner_ids": { + "const": [ + "PR-009" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "depth_resolved_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "maximum_resolved_target_depth": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "target_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "depth_resolved_record_count", + "maximum_resolved_target_depth", + "target_record_count" + ], + "additionalProperties": false + } + } + } + } + ] + }, + "unresolved_record_details": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "record_key", + "reason_codes" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "record_key", + "reason_codes" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "records" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T4.unresolved_record_details" + }, + "owner_ids": { + "const": [ + "T4" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "record_key", + "reason_codes" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "record_key", + "reason_codes" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + } + } + } + } + ] + }, + "resource_usage": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "admitted_node_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "admitted_edge_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "stored_root_membership_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "root_union_visit_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limits": { + "type": "object", + "properties": { + "max_nodes": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_edges": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_root_memberships": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_root_union_visits": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "max_nodes", + "max_edges", + "max_root_memberships", + "max_root_union_visits" + ], + "additionalProperties": false + }, + "exhausted_limit": { + "anyOf": [ + { + "type": "string", + "enum": [ + "max_nodes", + "max_edges", + "max_root_memberships", + "max_root_union_visits" + ] + }, + { + "type": "null" + } + ] + }, + "attempted_value": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "admitted_node_count", + "admitted_edge_count", + "stored_root_membership_count", + "root_union_visit_count", + "limits", + "exhausted_limit", + "attempted_value" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "work_units" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "PR-015.lineage_resource_usage" + }, + "owner_ids": { + "const": [ + "PR-015" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "admitted_node_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "admitted_edge_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "stored_root_membership_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "root_union_visit_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limits": { + "type": "object", + "properties": { + "max_nodes": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_edges": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_root_memberships": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_root_union_visits": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "max_nodes", + "max_edges", + "max_root_memberships", + "max_root_union_visits" + ], + "additionalProperties": false + }, + "exhausted_limit": { + "anyOf": [ + { + "type": "string", + "enum": [ + "max_nodes", + "max_edges", + "max_root_memberships", + "max_root_union_visits" + ] + }, + { + "type": "null" + } + ] + }, + "attempted_value": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "admitted_node_count", + "admitted_edge_count", + "stored_root_membership_count", + "root_union_visit_count", + "limits", + "exhausted_limit", + "attempted_value" + ], + "additionalProperties": false + } + } + } + } + ] + }, + "ordering_certificate": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "method": { + "const": "declared_earlier_version_order" + }, + "version_order": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "all_resolved_edges_follow_order": { + "type": "boolean" + } + }, + "required": [ + "method", + "version_order", + "all_resolved_edges_follow_order" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "certificate" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "PR-008.earlier_version_certificate" + }, + "owner_ids": { + "const": [ + "PR-008" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "method": { + "const": "declared_earlier_version_order" + }, + "version_order": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "all_resolved_edges_follow_order": { + "type": "boolean" + } + }, + "required": [ + "method", + "version_order", + "all_resolved_edges_follow_order" + ], + "additionalProperties": false + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + }, + "state_counts": { + "type": "object", + "properties": { + "by_version": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "additionalProperties": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "state_id", + "state_count" + ], + "additionalProperties": false + }, + "minItems": 0 + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "records" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T1.state_counts" }, "owner_ids": { "const": [ @@ -8533,134 +11871,445 @@ "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + } + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + } + }, + "support_delta": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "integer", + "minimum": -1.7976931348623157e+308, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "states" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "F-005" + }, + "owner_ids": { + "const": [ + "T1" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true + "minItems": 1 }, "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minItems": 1 } - }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } - }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } + } + }, + "else": { + "properties": { + "value": { + "type": "integer", + "minimum": -1.7976931348623157e+308, + "maximum": 1.7976931348623157e+308 } - ] + } } }, - "required": [], - "additionalProperties": false - } + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] }, - "support_delta": { + "support_retention_ratio": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "integer", - "minimum": -1.7976931348623157e+308, - "maximum": 1.7976931348623157e+308 + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, { "type": "null" @@ -8668,7 +12317,7 @@ ] }, "unit": { - "const": "states" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -8682,7 +12331,7 @@ ] }, "method_id": { - "const": "F-005" + "const": "F-006" }, "owner_ids": { "const": [ @@ -8927,9 +12576,17 @@ "else": { "properties": { "value": { - "type": "integer", - "minimum": -1.7976931348623157e+308, - "maximum": 1.7976931348623157e+308 + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] } } } @@ -8955,23 +12612,15 @@ } ] }, - "support_retention_ratio": { + "support_loss_count": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 }, { "type": "null" @@ -8979,7 +12628,7 @@ ] }, "unit": { - "const": "ratio" + "const": "states" }, "evidence_class": { "const": "derived_metric" @@ -8993,7 +12642,7 @@ ] }, "method_id": { - "const": "F-006" + "const": "T1.support_set_difference" }, "owner_ids": { "const": [ @@ -9238,17 +12887,9 @@ "else": { "properties": { "value": { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 } } } @@ -9274,7 +12915,7 @@ } ] }, - "support_loss_count": { + "support_added_count": { "type": "object", "properties": { "value": { @@ -9577,15 +13218,20 @@ } ] }, - "support_added_count": { + "extinct_states": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true }, { "type": "null" @@ -9593,7 +13239,7 @@ ] }, "unit": { - "const": "states" + "const": "set_of_states" }, "evidence_class": { "const": "derived_metric" @@ -9607,7 +13253,7 @@ ] }, "method_id": { - "const": "T1.support_set_difference" + "const": "T1.earlier_minus_later" }, "owner_ids": { "const": [ @@ -9852,9 +13498,14 @@ "else": { "properties": { "value": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true } } } @@ -9880,7 +13531,7 @@ } ] }, - "extinct_states": { + "added_states": { "type": "object", "properties": { "value": { @@ -9915,7 +13566,7 @@ ] }, "method_id": { - "const": "T1.earlier_minus_later" + "const": "T1.later_minus_earlier" }, "owner_ids": { "const": [ @@ -10193,7 +13844,7 @@ } ] }, - "added_states": { + "retained_states": { "type": "object", "properties": { "value": { @@ -10228,7 +13879,7 @@ ] }, "method_id": { - "const": "T1.later_minus_earlier" + "const": "T1.support_intersection" }, "owner_ids": { "const": [ @@ -10476,50 +14127,276 @@ "type": "array", "items": { "type": "string", - "minLength": 0, + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + } + } + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] + }, + "comparison_details": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "earlier_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "later_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "version_order": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 2, + "uniqueItems": true + }, + "version_order_source": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "earlier_state_semantics": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "later_state_semantics": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "harmonized_state_semantics": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "compatibility_method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "earlier_representation": { + "$ref": "#/$defs/representation" + }, + "later_representation": { + "$ref": "#/$defs/representation" + }, + "harmonized_representation": { + "$ref": "#/$defs/representation" + }, + "original_earlier_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "original_later_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "harmonized_earlier_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "harmonized_later_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "retention_denominator": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "retention_denominator_basis": { + "type": "string", + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } - } - ] - }, - "retained_states": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "state_mapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "target_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "source_state", + "target_state" + ], + "additionalProperties": false + }, + "minItems": 0 + }, + "mapping_effect": { + "type": "array", + "items": { + "type": "object", + "properties": { + "dataset_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "original_support_size": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "harmonized_support_size": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "dataset_version", + "original_support_size", + "harmonized_support_size" + ], + "additionalProperties": false + }, + "minItems": 0 + }, + "collision_groups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "target_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "source_states": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 2, + "uniqueItems": true + } + }, + "required": [ + "target_state", + "source_states" + ], + "additionalProperties": false + }, + "minItems": 0 + } }, - "minItems": 0, - "uniqueItems": true + "required": [ + "earlier_version", + "later_version", + "version_order", + "version_order_source", + "earlier_state_semantics", + "later_state_semantics", + "harmonized_state_semantics", + "compatibility_method", + "earlier_representation", + "later_representation", + "harmonized_representation", + "original_earlier_support", + "original_later_support", + "harmonized_earlier_support", + "harmonized_later_support", + "retention_denominator", + "retention_denominator_basis", + "state_mapping", + "mapping_effect", + "collision_groups" + ], + "additionalProperties": false }, { "type": "null" @@ -10527,7 +14404,7 @@ ] }, "unit": { - "const": "set_of_states" + "const": "comparison" }, "evidence_class": { "const": "derived_metric" @@ -10541,7 +14418,7 @@ ] }, "method_id": { - "const": "T1.support_intersection" + "const": "T1.explicit_pair_basis" }, "owner_ids": { "const": [ @@ -10786,822 +14663,602 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } - } - ] - }, - "comparison_details": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "object", - "properties": { - "earlier_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "later_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "version_order": { - "type": "array", - "items": { + "type": "object", + "properties": { + "earlier_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "later_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "version_order": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 2, + "uniqueItems": true + }, + "version_order_source": { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 2, - "uniqueItems": true - }, - "version_order_source": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "earlier_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "later_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "harmonized_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "compatibility_method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "earlier_representation": { - "$ref": "#/$defs/representation" - }, - "later_representation": { - "$ref": "#/$defs/representation" - }, - "harmonized_representation": { - "$ref": "#/$defs/representation" - }, - "original_earlier_support": { - "type": "array", - "items": { + "earlier_state_semantics": { "type": "string", - "minLength": 0, + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - }, - "original_later_support": { - "type": "array", - "items": { + "later_state_semantics": { "type": "string", - "minLength": 0, + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - }, - "harmonized_earlier_support": { - "type": "array", - "items": { + "harmonized_state_semantics": { "type": "string", - "minLength": 0, + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - }, - "harmonized_later_support": { - "type": "array", - "items": { + "compatibility_method": { "type": "string", - "minLength": 0, + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - }, - "retention_denominator": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "earlier_representation": { + "$ref": "#/$defs/representation" + }, + "later_representation": { + "$ref": "#/$defs/representation" + }, + "harmonized_representation": { + "$ref": "#/$defs/representation" + }, + "original_earlier_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" }, - { - "type": "null" - } - ] - }, - "retention_denominator_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "state_mapping": { - "type": "array", - "items": { - "type": "object", - "properties": { - "source_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "target_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - } + "minItems": 0, + "uniqueItems": true + }, + "original_later_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" }, - "required": [ - "source_state", - "target_state" - ], - "additionalProperties": false + "minItems": 0, + "uniqueItems": true }, - "minItems": 0 - }, - "mapping_effect": { - "type": "array", - "items": { - "type": "object", - "properties": { - "dataset_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "original_support_size": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - { - "type": "null" - } - ] + "harmonized_earlier_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "harmonized_later_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "retention_denominator": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 }, - "harmonized_support_size": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - { - "type": "null" - } - ] + { + "type": "null" } + ] + }, + "retention_denominator_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "state_mapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "target_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "source_state", + "target_state" + ], + "additionalProperties": false + }, + "minItems": 0 + }, + "mapping_effect": { + "type": "array", + "items": { + "type": "object", + "properties": { + "dataset_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "original_support_size": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "harmonized_support_size": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "dataset_version", + "original_support_size", + "harmonized_support_size" + ], + "additionalProperties": false }, - "required": [ - "dataset_version", - "original_support_size", - "harmonized_support_size" - ], - "additionalProperties": false + "minItems": 0 }, - "minItems": 0 - }, - "collision_groups": { - "type": "array", - "items": { - "type": "object", - "properties": { - "target_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "source_states": { - "type": "array", - "items": { + "collision_groups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "target_state": { "type": "string", "minLength": 0, "pattern": "^[^\\u0000]*$" }, - "minItems": 2, - "uniqueItems": true - } + "source_states": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 2, + "uniqueItems": true + } + }, + "required": [ + "target_state", + "source_states" + ], + "additionalProperties": false }, - "required": [ - "target_state", - "source_states" - ], - "additionalProperties": false - }, - "minItems": 0 - } - }, - "required": [ - "earlier_version", - "later_version", - "version_order", - "version_order_source", - "earlier_state_semantics", - "later_state_semantics", - "harmonized_state_semantics", - "compatibility_method", - "earlier_representation", - "later_representation", - "harmonized_representation", - "original_earlier_support", - "original_later_support", - "harmonized_earlier_support", - "harmonized_later_support", - "retention_denominator", - "retention_denominator_basis", - "state_mapping", - "mapping_effect", - "collision_groups" - ], - "additionalProperties": false - }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "comparison" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "T1.explicit_pair_basis" - }, - "owner_ids": { - "const": [ - "T1" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" - } - ] - }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "denominator_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" + "minItems": 0 + } + }, + "required": [ + "earlier_version", + "later_version", + "version_order", + "version_order_source", + "earlier_state_semantics", + "later_state_semantics", + "harmonized_state_semantics", + "compatibility_method", + "earlier_representation", + "later_representation", + "harmonized_representation", + "original_earlier_support", + "original_later_support", + "harmonized_earlier_support", + "harmonized_later_support", + "retention_denominator", + "retention_denominator_basis", + "state_mapping", + "mapping_effect", + "collision_groups" + ], + "additionalProperties": false + } } - ] - }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" + } }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - } - }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ { "if": { "properties": { "status": { - "const": "unavailable" + "enum": [ + "available", + "partial" + ] } } }, "then": { "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 + "representation": { + "$ref": "#/$defs/representation" } } - }, - "else": { + } + } + ] + } + }, + "required": [], + "additionalProperties": false + }, + "diversity": { + "type": "object", + "properties": { + "by_version": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "additionalProperties": { + "type": "object", + "properties": { + "gini_simpson_diversity": { + "type": "object", "properties": { "value": { - "type": "object", - "properties": { - "earlier_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "later_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "version_order": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 2, - "uniqueItems": true - }, - "version_order_source": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "earlier_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "later_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + { + "type": "null" + } + ] + }, + "unit": { + "const": "dimensionless" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "F-003" + }, + "owner_ids": { + "const": [ + "T1" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" }, - "harmonized_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "compatibility_method": { + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "earlier_representation": { - "$ref": "#/$defs/representation" - }, - "later_representation": { - "$ref": "#/$defs/representation" - }, - "harmonized_representation": { - "$ref": "#/$defs/representation" - }, - "original_earlier_support": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "original_later_support": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "harmonized_earlier_support": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "harmonized_later_support": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "retention_denominator": { - "anyOf": [ + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "retention_denominator_basis": { + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "state_mapping": { - "type": "array", - "items": { - "type": "object", - "properties": { - "source_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "target_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - } - }, - "required": [ - "source_state", - "target_state" - ], - "additionalProperties": false + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, - "minItems": 0 - }, - "mapping_effect": { - "type": "array", - "items": { - "type": "object", - "properties": { - "dataset_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "original_support_size": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - { - "type": "null" - } - ] - }, - "harmonized_support_size": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - { - "type": "null" - } - ] - } - }, - "required": [ - "dataset_version", - "original_support_size", - "harmonized_support_size" - ], - "additionalProperties": false + "reason_codes": { + "minItems": 1 }, - "minItems": 0 - }, - "collision_groups": { - "type": "array", - "items": { - "type": "object", - "properties": { - "target_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 }, - "source_states": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 2, - "uniqueItems": true + { + "exclusiveMaximum": 0 } - }, - "required": [ - "target_state", - "source_states" - ], - "additionalProperties": false - }, - "minItems": 0 + ] + } + } + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } } }, - "required": [ - "earlier_version", - "later_version", - "version_order", - "version_order_source", - "earlier_state_semantics", - "later_state_semantics", - "harmonized_state_semantics", - "compatibility_method", - "earlier_representation", - "later_representation", - "harmonized_representation", - "original_earlier_support", - "original_later_support", - "harmonized_earlier_support", - "harmonized_later_support", - "retention_denominator", - "retention_denominator_basis", - "state_mapping", - "mapping_effect", - "collision_groups" - ], - "additionalProperties": false - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } } - } + ] }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } - } - ] - } - }, - "required": [], - "additionalProperties": false - }, - "diversity": { - "type": "object", - "properties": { - "by_version": { - "type": "object", - "propertyNames": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "additionalProperties": { - "type": "object", - "properties": { - "gini_simpson_diversity": { + "simpson_concentration": { "type": "object", "properties": { "value": { @@ -11639,7 +15296,7 @@ ] }, "method_id": { - "const": "F-003" + "const": "F-004" }, "owner_ids": { "const": [ @@ -11920,7 +15577,7 @@ } ] }, - "simpson_concentration": { + "weighted_gini_simpson_diversity": { "type": "object", "properties": { "value": { @@ -11958,7 +15615,7 @@ ] }, "method_id": { - "const": "F-004" + "const": "F-003" }, "owner_ids": { "const": [ @@ -12239,7 +15896,7 @@ } ] }, - "weighted_gini_simpson_diversity": { + "weighted_simpson_concentration": { "type": "object", "properties": { "value": { @@ -12277,7 +15934,7 @@ ] }, "method_id": { - "const": "F-003" + "const": "F-004" }, "owner_ids": { "const": [ @@ -12558,23 +16215,42 @@ } ] }, - "weighted_simpson_concentration": { + "state_frequencies": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_frequency": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - { - "exclusiveMaximum": 0 - } - ] + "required": [ + "state_id", + "state_frequency" + ], + "additionalProperties": false + }, + "minItems": 0 }, { "type": "null" @@ -12582,7 +16258,7 @@ ] }, "unit": { - "const": "dimensionless" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -12596,7 +16272,7 @@ ] }, "method_id": { - "const": "F-004" + "const": "F-001" }, "owner_ids": { "const": [ @@ -12841,17 +16517,36 @@ "else": { "properties": { "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_frequency": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - { - "exclusiveMaximum": 0 - } - ] + "required": [ + "state_id", + "state_frequency" + ], + "additionalProperties": false + }, + "minItems": 0 } } } @@ -12877,7 +16572,7 @@ } ] }, - "state_frequencies": { + "weighted_state_frequencies": { "type": "object", "properties": { "value": { @@ -13234,7 +16929,7 @@ } ] }, - "weighted_state_frequencies": { + "weighted_state_masses": { "type": "object", "properties": { "value": { @@ -13249,10 +16944,10 @@ "minLength": 0, "pattern": "^[^\\u0000]*$" }, - "state_frequency": { + "state_mass": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -13265,7 +16960,7 @@ }, "required": [ "state_id", - "state_frequency" + "state_mass" ], "additionalProperties": false }, @@ -13277,7 +16972,7 @@ ] }, "unit": { - "const": "ratio" + "const": "user_declared_weight_mass" }, "evidence_class": { "const": "derived_metric" @@ -13291,7 +16986,7 @@ ] }, "method_id": { - "const": "F-001" + "const": "T1.state_weight_mass" }, "owner_ids": { "const": [ @@ -13545,10 +17240,10 @@ "minLength": 0, "pattern": "^[^\\u0000]*$" }, - "state_frequency": { + "state_mass": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -13561,7 +17256,7 @@ }, "required": [ "state_id", - "state_frequency" + "state_mass" ], "additionalProperties": false }, @@ -13591,42 +17286,107 @@ } ] }, - "weighted_state_masses": { + "distribution_basis": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "array", - "items": { - "type": "object", - "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "state_mass": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - } + "type": "object", + "properties": { + "input_basis": { + "type": "string", + "enum": [ + "empirical_assignments", + "explicit_counts_divided_by_included_records", + "weighted_record_mass", + "explicit_probability_vector" + ] }, - "required": [ - "state_id", - "state_mass" - ], - "additionalProperties": false + "analyzed_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "frequency_denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "supplied_probability_total": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "probability_residual": { + "anyOf": [ + { + "type": "number", + "minimum": -1.7976931348623157e+308, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "numerical_policy": { + "$ref": "#/$defs/numerical_policy" + } }, - "minItems": 0 + "required": [ + "input_basis", + "analyzed_record_count", + "frequency_denominator", + "denominator_basis", + "supplied_probability_total", + "probability_residual", + "numerical_policy" + ], + "additionalProperties": false }, { "type": "null" @@ -13634,7 +17394,7 @@ ] }, "unit": { - "const": "user_declared_weight_mass" + "const": "basis" }, "evidence_class": { "const": "derived_metric" @@ -13648,7 +17408,7 @@ ] }, "method_id": { - "const": "T1.state_weight_mass" + "const": "T1.validated_distribution_basis" }, "owner_ids": { "const": [ @@ -13893,36 +17653,101 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "object", - "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "state_mass": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - } + "type": "object", + "properties": { + "input_basis": { + "type": "string", + "enum": [ + "empirical_assignments", + "explicit_counts_divided_by_included_records", + "weighted_record_mass", + "explicit_probability_vector" + ] + }, + "analyzed_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "frequency_denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "supplied_probability_total": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "probability_residual": { + "anyOf": [ + { + "type": "number", + "minimum": -1.7976931348623157e+308, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] }, - "required": [ - "state_id", - "state_mass" - ], - "additionalProperties": false + "numerical_policy": { + "$ref": "#/$defs/numerical_policy" + } }, - "minItems": 0 + "required": [ + "input_basis", + "analyzed_record_count", + "frequency_denominator", + "denominator_basis", + "supplied_probability_total", + "probability_residual", + "numerical_policy" + ], + "additionalProperties": false } } } @@ -13947,507 +17772,649 @@ } } ] - }, - "distribution_basis": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "object", - "properties": { - "input_basis": { - "type": "string", - "enum": [ - "empirical_assignments", - "explicit_counts_divided_by_included_records", - "weighted_record_mass", - "explicit_probability_vector" - ] - }, - "analyzed_record_count": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - "frequency_denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "denominator_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "supplied_probability_total": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "probability_residual": { - "anyOf": [ - { - "type": "number", - "minimum": -1.7976931348623157e+308, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "numerical_policy": { - "$ref": "#/$defs/numerical_policy" - } - }, - "required": [ - "input_basis", - "analyzed_record_count", - "frequency_denominator", - "denominator_basis", - "supplied_probability_total", - "probability_residual", - "numerical_policy" - ], - "additionalProperties": false - }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "basis" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "T1.validated_distribution_basis" - }, - "owner_ids": { - "const": [ - "T1" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] + } + }, + "required": [], + "additionalProperties": false + } + }, + "gini_simpson_diversity_delta": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": -1, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "dimensionless" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "F-018" + }, + "owner_ids": { + "const": [ + "T1" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, - "coverage_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" - } - ] + "reason_codes": { + "minItems": 1 }, - "denominator": { - "anyOf": [ + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": -1, + "maximum": 1, + "oneOf": [ { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] - }, - "denominator_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" - } + } + } + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" ] - }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + }, + "tail": { + "type": "object", + "properties": { + "tail_support_size": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } + { + "type": "null" + } + ] + }, + "unit": { + "const": "states" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T2.declared_tail_rule" + }, + "owner_ids": { + "const": [ + "T2" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 - } - } + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "else": { - "properties": { - "value": { - "type": "object", - "properties": { - "input_basis": { - "type": "string", - "enum": [ - "empirical_assignments", - "explicit_counts_divided_by_included_records", - "weighted_record_mass", - "explicit_probability_vector" - ] - }, - "analyzed_record_count": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - "frequency_denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "denominator_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "supplied_probability_total": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "probability_residual": { - "anyOf": [ - { - "type": "number", - "minimum": -1.7976931348623157e+308, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "numerical_policy": { - "$ref": "#/$defs/numerical_policy" - } - }, - "required": [ - "input_basis", - "analyzed_record_count", - "frequency_denominator", - "denominator_basis", - "supplied_probability_total", - "probability_residual", - "numerical_policy" - ], - "additionalProperties": false - } - } + { + "exclusiveMaximum": 0 } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 } - ] + } + }, + "else": { + "properties": { + "value": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + } } }, - "required": [], - "additionalProperties": false - } + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] }, - "gini_simpson_diversity_delta": { + "tail_record_share": { "type": "object", "properties": { "value": { "anyOf": [ { "type": "number", - "minimum": -1, + "minimum": 0, "maximum": 1, "oneOf": [ { @@ -14464,7 +18431,7 @@ ] }, "unit": { - "const": "dimensionless" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -14478,11 +18445,11 @@ ] }, "method_id": { - "const": "F-018" + "const": "T2.tail_record_share" }, "owner_ids": { "const": [ - "T1" + "T2" ] }, "theory_map_ids": { @@ -14724,7 +18691,7 @@ "properties": { "value": { "type": "number", - "minimum": -1, + "minimum": 0, "maximum": 1, "oneOf": [ { @@ -14758,23 +18725,59 @@ } } ] - } - }, - "required": [], - "additionalProperties": false - }, - "tail": { - "type": "object", - "properties": { - "tail_support_size": { + }, + "rarity_ranking": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "state_frequency": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "rarity_rank": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "in_tail": { + "type": "boolean" + } + }, + "required": [ + "state_id", + "state_count", + "state_frequency", + "rarity_rank", + "in_tail" + ], + "additionalProperties": false + }, + "minItems": 0 }, { "type": "null" @@ -14782,7 +18785,7 @@ ] }, "unit": { - "const": "states" + "const": "ordinal_rank" }, "evidence_class": { "const": "derived_metric" @@ -14796,7 +18799,7 @@ ] }, "method_id": { - "const": "T2.declared_tail_rule" + "const": "T2.frequency_count_unicode_order" }, "owner_ids": { "const": [ @@ -15041,9 +19044,52 @@ "else": { "properties": { "value": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "state_frequency": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "rarity_rank": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "in_tail": { + "type": "boolean" + } + }, + "required": [ + "state_id", + "state_count", + "state_frequency", + "rarity_rank", + "in_tail" + ], + "additionalProperties": false + }, + "minItems": 0 } } } @@ -15069,23 +19115,20 @@ } ] }, - "tail_record_share": { + "tail_states": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true }, { "type": "null" @@ -15093,7 +19136,7 @@ ] }, "unit": { - "const": "ratio" + "const": "set_of_states" }, "evidence_class": { "const": "derived_metric" @@ -15107,7 +19150,7 @@ ] }, "method_id": { - "const": "T2.tail_record_share" + "const": "T2.declared_tail_rule" }, "owner_ids": { "const": [ @@ -15349,20 +19392,17 @@ } } }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "else": { + "properties": { + "value": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true } } } @@ -15388,58 +19428,107 @@ } ] }, - "rarity_ranking": { + "tail_rule": { + "type": "string", + "enum": [ + "singleton_count", + "count_at_or_below", + "frequency_at_or_below", + "state_list" + ] + }, + "selection": { + "$ref": "#/$defs/tail_selection" + } + }, + "required": [], + "additionalProperties": false + }, + "provenance": { + "type": "object", + "properties": { + "source_type_shares": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "array", - "items": { - "type": "object", - "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "state_count": { - "type": "integer", - "minimum": 1, - "maximum": 1.7976931348623157e+308 - }, - "state_frequency": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "rarity_rank": { - "type": "integer", - "minimum": 1, - "maximum": 1.7976931348623157e+308 - }, - "in_tail": { - "type": "boolean" - } + "type": "object", + "properties": { + "human": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "required": [ - "state_id", - "state_count", - "state_frequency", - "rarity_rank", - "in_tail" - ], - "additionalProperties": false + "synthetic": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "mixed": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "sensor": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "unknown": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - "minItems": 0 + "required": [ + "human", + "synthetic", + "mixed", + "sensor", + "unknown" + ], + "additionalProperties": false }, { "type": "null" @@ -15447,7 +19536,7 @@ ] }, "unit": { - "const": "ordinal_rank" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -15461,11 +19550,11 @@ ] }, "method_id": { - "const": "T2.frequency_count_unicode_order" + "const": "F-007" }, "owner_ids": { "const": [ - "T2" + "PR-005" ] }, "theory_map_ids": { @@ -15706,91 +19795,170 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "object", - "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "state_count": { - "type": "integer", - "minimum": 1, - "maximum": 1.7976931348623157e+308 - }, - "state_frequency": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "rarity_rank": { - "type": "integer", - "minimum": 1, - "maximum": 1.7976931348623157e+308 - }, - "in_tail": { - "type": "boolean" - } + "type": "object", + "properties": { + "human": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "required": [ - "state_id", - "state_count", - "state_frequency", - "rarity_rank", - "in_tail" - ], - "additionalProperties": false + "synthetic": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "mixed": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "sensor": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "unknown": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - "minItems": 0 - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" + "required": [ + "human", + "synthetic", + "mixed", + "sensor", + "unknown" + ], + "additionalProperties": false } } } } ] }, - "tail_states": { + "weighted_source_type_shares": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "type": "object", + "properties": { + "human": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "synthetic": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "mixed": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "sensor": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "unknown": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - "minItems": 0, - "uniqueItems": true + "required": [ + "human", + "synthetic", + "mixed", + "sensor", + "unknown" + ], + "additionalProperties": false }, { "type": "null" @@ -15798,7 +19966,7 @@ ] }, "unit": { - "const": "set_of_states" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -15812,11 +19980,11 @@ ] }, "method_id": { - "const": "T2.declared_tail_rule" + "const": "F-007" }, "owner_ids": { "const": [ - "T2" + "PR-005" ] }, "theory_map_ids": { @@ -16057,59 +20225,89 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "type": "object", + "properties": { + "human": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "synthetic": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "mixed": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "sensor": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "unknown": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - "minItems": 0, - "uniqueItems": true - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" + "required": [ + "human", + "synthetic", + "mixed", + "sensor", + "unknown" + ], + "additionalProperties": false } } } } ] }, - "tail_rule": { - "type": "string", - "enum": [ - "singleton_count", - "count_at_or_below", - "frequency_at_or_below", - "state_list" - ] - }, - "selection": { - "$ref": "#/$defs/tail_selection" - } - }, - "required": [], - "additionalProperties": false - }, - "provenance": { - "type": "object", - "properties": { - "source_type_shares": { + "weighted_source_type_masses": { "type": "object", "properties": { "value": { @@ -16120,7 +20318,7 @@ "human": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16133,7 +20331,7 @@ "synthetic": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16146,7 +20344,7 @@ "mixed": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16159,7 +20357,7 @@ "sensor": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16172,7 +20370,7 @@ "unknown": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16198,7 +20396,7 @@ ] }, "unit": { - "const": "ratio" + "const": "user_declared_weight_mass" }, "evidence_class": { "const": "derived_metric" @@ -16212,7 +20410,7 @@ ] }, "method_id": { - "const": "F-007" + "const": "PR-005.source_weight_mass" }, "owner_ids": { "const": [ @@ -16462,7 +20660,7 @@ "human": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16475,7 +20673,7 @@ "synthetic": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16488,7 +20686,7 @@ "mixed": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16501,7 +20699,7 @@ "sensor": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16514,7 +20712,7 @@ "unknown": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16539,88 +20737,23 @@ } ] }, - "weighted_source_type_shares": { + "total_weight": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "object", - "properties": { - "human": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "synthetic": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "mixed": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "sensor": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "unknown": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + { + "exclusiveMaximum": 0 } - }, - "required": [ - "human", - "synthetic", - "mixed", - "sensor", - "unknown" - ], - "additionalProperties": false + ] }, { "type": "null" @@ -16628,7 +20761,7 @@ ] }, "unit": { - "const": "ratio" + "const": "user_declared_weight_mass" }, "evidence_class": { "const": "derived_metric" @@ -16642,7 +20775,7 @@ ] }, "method_id": { - "const": "F-007" + "const": "PR-005.total_weight" }, "owner_ids": { "const": [ @@ -16878,179 +21011,49 @@ }, "reason_codes": { "minItems": 1 - }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "object", - "properties": { - "human": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "synthetic": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "mixed": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "sensor": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "unknown": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - } - }, - "required": [ - "human", - "synthetic", - "mixed", - "sensor", - "unknown" - ], - "additionalProperties": false - } - } - } - } - ] - }, - "weighted_source_type_masses": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "object", - "properties": { - "human": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "synthetic": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "mixed": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "sensor": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } + } + ] + }, + "missing_provenance_weight": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "unknown": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + { + "exclusiveMaximum": 0 } - }, - "required": [ - "human", - "synthetic", - "mixed", - "sensor", - "unknown" - ], - "additionalProperties": false + ] }, { "type": "null" @@ -17072,7 +21075,7 @@ ] }, "method_id": { - "const": "PR-005.source_weight_mass" + "const": "PR-005.missing_provenance_weight" }, "owner_ids": { "const": [ @@ -17317,89 +21320,24 @@ "else": { "properties": { "value": { - "type": "object", - "properties": { - "human": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "synthetic": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "mixed": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "sensor": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "unknown": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + { + "exclusiveMaximum": 0 } - }, - "required": [ - "human", - "synthetic", - "mixed", - "sensor", - "unknown" - ], - "additionalProperties": false + ] } } } } ] }, - "total_weight": { + "missing_provenance_share": { "type": "object", "properties": { "value": { @@ -17407,7 +21345,7 @@ { "type": "number", "minimum": 0, - "maximum": 1.7976931348623157e+308, + "maximum": 1, "oneOf": [ { "minimum": 0 @@ -17423,7 +21361,7 @@ ] }, "unit": { - "const": "user_declared_weight_mass" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -17437,11 +21375,11 @@ ] }, "method_id": { - "const": "PR-005.total_weight" + "const": "PR-004.one_minus_row_coverage" }, "owner_ids": { "const": [ - "PR-005" + "PR-004" ] }, "theory_map_ids": { @@ -17684,7 +21622,7 @@ "value": { "type": "number", "minimum": 0, - "maximum": 1.7976931348623157e+308, + "maximum": 1, "oneOf": [ { "minimum": 0 @@ -17699,7 +21637,7 @@ } ] }, - "missing_provenance_weight": { + "weighted_missing_provenance_share": { "type": "object", "properties": { "value": { @@ -17707,7 +21645,7 @@ { "type": "number", "minimum": 0, - "maximum": 1.7976931348623157e+308, + "maximum": 1, "oneOf": [ { "minimum": 0 @@ -17723,7 +21661,7 @@ ] }, "unit": { - "const": "user_declared_weight_mass" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -17737,7 +21675,7 @@ ] }, "method_id": { - "const": "PR-005.missing_provenance_weight" + "const": "PR-005.missing_provenance_weight_share" }, "owner_ids": { "const": [ @@ -17984,7 +21922,7 @@ "value": { "type": "number", "minimum": 0, - "maximum": 1.7976931348623157e+308, + "maximum": 1, "oneOf": [ { "minimum": 0 @@ -17993,623 +21931,623 @@ "exclusiveMaximum": 0 } ] - } - } - } - } - ] - }, - "missing_provenance_share": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + }, + "closure_exposure": { + "type": "object", + "properties": { + "direct": { + "type": "object", + "properties": { + "lower_bound": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "F-009" + }, + "owner_ids": { + "const": [ + "T3" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ { - "minimum": 0 + "$ref": "#/$defs/representation" }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "ratio" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "PR-004.one_minus_row_coverage" - }, - "owner_ids": { - "const": [ - "PR-004" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + "coverage_reason": { + "anyOf": [ { - "minimum": 0 + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, { - "exclusiveMaximum": 0 + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ - { + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ { - "type": "null" + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } } ] }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ + "upper_bound": { + "type": "object", + "properties": { + "value": { + "anyOf": [ { - "minimum": 0 + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "denominator_reason": { - "anyOf": [ - { + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "enum": [ + "available", + "partial", + "unavailable" + ] }, - { - "type": "null" - } - ] - }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - } - }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } - }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 + "method_id": { + "const": "F-010" + }, + "owner_ids": { + "const": [ + "T3" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" ] - } - } - } - } - ] - }, - "weighted_missing_provenance_share": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ { - "minimum": 0 + "$ref": "#/$defs/representation" }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "ratio" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "PR-005.missing_provenance_weight_share" - }, - "owner_ids": { - "const": [ - "PR-005" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + "coverage_reason": { + "anyOf": [ { - "minimum": 0 + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] }, - { - "type": "null" - } - ] - }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ + "denominator_reason": { + "anyOf": [ { - "minimum": 0 + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "denominator_reason": { - "anyOf": [ - { + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - { - "type": "null" - } - ] - }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - } - }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" } }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 + "then": { + "properties": { + "value": { + "type": "null" }, - { - "exclusiveMaximum": 0 + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 } - ] + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } } } - } - } - ] - } - }, - "required": [], - "additionalProperties": false - }, - "closure_exposure": { - "type": "object", - "properties": { - "direct": { - "type": "object", - "properties": { - "lower_bound": { + ] + }, + "interval_width": { "type": "object", "properties": { "value": { @@ -18647,7 +22585,7 @@ ] }, "method_id": { - "const": "F-009" + "const": "T3.upper_minus_lower" }, "owner_ids": { "const": [ @@ -18909,7 +22847,20 @@ } ] }, - "upper_bound": { + "classification_basis": { + "const": "toolkit_operationalization" + }, + "confidence_disclosure": { + "const": "provenance_confidence_is_separate_and_does_not_discount_grounding" + } + }, + "required": [], + "additionalProperties": false + }, + "lineage": { + "type": "object", + "properties": { + "lower_bound": { "type": "object", "properties": { "value": { @@ -18947,7 +22898,7 @@ ] }, "method_id": { - "const": "F-010" + "const": "T3.lineage_closure" }, "owner_ids": { "const": [ @@ -19209,7 +23160,7 @@ } ] }, - "interval_width": { + "upper_bound": { "type": "object", "properties": { "value": { @@ -19247,7 +23198,7 @@ ] }, "method_id": { - "const": "T3.upper_minus_lower" + "const": "T3.lineage_closure" }, "owner_ids": { "const": [ @@ -19509,20 +23460,7 @@ } ] }, - "classification_basis": { - "const": "toolkit_operationalization" - }, - "confidence_disclosure": { - "const": "provenance_confidence_is_separate_and_does_not_discount_grounding" - } - }, - "required": [], - "additionalProperties": false - }, - "lineage": { - "type": "object", - "properties": { - "lower_bound": { + "interval_width": { "type": "object", "properties": { "value": { @@ -19552,7 +23490,12 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "T3.lineage_closure" @@ -19626,813 +23569,1682 @@ "representation": { "anyOf": [ { - "$ref": "#/$defs/representation" + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [], + "additionalProperties": false + }, + "lineage": { + "type": "object", + "properties": { + "resolved_parent_edge_coverage": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] + { + "type": "null" + } + ] + }, + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "PR-008.resolved_edges_over_declared" + }, + "owner_ids": { + "const": [ + "PR-008" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" }, - "coverage_reason": { - "anyOf": [ + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "denominator": { - "anyOf": [ + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "denominator_reason": { - "anyOf": [ + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "no_declared_parents": { + "type": "boolean" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } + } + ] + }, + "resolved_lineage_coverage": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + { + "type": "null" } + ] + }, + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T4.resolved_records_over_scope" + }, + "owner_ids": { + "const": [ + "T4" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } - }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - } - } - } + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" } ] }, - "upper_bound": { - "type": "object", - "properties": { - "value": { - "anyOf": [ + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "unit": { - "const": "ratio" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "const": "unavailable" - }, - "method_id": { - "const": "T3.lineage_closure" + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "owner_ids": { - "const": [ - "T3" + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } ] }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" - } - ] - }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "denominator_reason": { - "anyOf": [ + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } + } + ] + }, + "external_ancestry_coverage": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true + { + "type": "null" + } + ] + }, + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T4.external_roots_over_scope" + }, + "owner_ids": { + "const": [ + "T4" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" }, - "weighting": { - "$ref": "#/$defs/weighting" + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "input_basis": { + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "redaction": { - "$ref": "#/$defs/redaction" + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "method": { + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } + } + ] + }, + "distinct_external_root_count": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" } + ] + }, + "unit": { + "const": "roots" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T4.root_set_union" + }, + "owner_ids": { + "const": [ + "T4" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } - }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - } - } - } + "$ref": "#/$defs/representation" + }, + { + "type": "null" } ] }, - "interval_width": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "ratio" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "const": "unavailable" - }, - "method_id": { - "const": "T3.lineage_closure" - }, - "owner_ids": { - "const": [ - "T3" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "denominator_reason": { - "anyOf": [ + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + { + "type": "null" } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + } + } + } + ] + }, + "top_shared_ancestors": { + "type": "object", + "properties": { + "value": { + "anyOf": [ { - "if": { - "properties": { - "status": { - "const": "unavailable" - } + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "incidence_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "incidence_share": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "incidence_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "fractional_mass": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "normalized_weight": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "weight_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "record_key", + "incidence_count", + "incidence_share", + "incidence_denominator", + "fractional_mass", + "normalized_weight", + "weight_denominator" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true } }, - "then": { - "properties": { - "value": { - "type": "null" + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } }, - "reason_codes": { - "minItems": 1 + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } }, - "required_evidence": { - "minItems": 1 + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "incidence_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "incidence_share": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "incidence_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "fractional_mass": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "normalized_weight": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "weight_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "record_key", + "incidence_count", + "incidence_share", + "incidence_denominator", + "fractional_mass", + "normalized_weight", + "weight_denominator" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 }, - { - "exclusiveMaximum": 0 + "omission_reasons": { + "maxItems": 0 } - ] + } } - } - } - } - ] - } - }, - "required": [], - "additionalProperties": false - } - }, - "required": [], - "additionalProperties": false - }, - "lineage": { - "type": "object", - "properties": { - "resolved_parent_edge_coverage": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 }, { - "exclusiveMaximum": 0 + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } } ] }, @@ -20442,7 +25254,7 @@ ] }, "unit": { - "const": "ratio" + "const": "records" }, "evidence_class": { "const": "derived_metric" @@ -20456,11 +25268,11 @@ ] }, "method_id": { - "const": "PR-008.resolved_edges_over_declared" + "const": "T4.incidence_ranking" }, "owner_ids": { "const": [ - "PR-008" + "T4" ] }, "theory_map_ids": { @@ -20701,15 +25513,287 @@ "else": { "properties": { "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "incidence_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "incidence_share": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "incidence_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "fractional_mass": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "normalized_weight": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "weight_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "record_key", + "incidence_count", + "incidence_share", + "incidence_denominator", + "fractional_mass", + "normalized_weight", + "weight_denominator" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ { - "minimum": 0 + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "incidence_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "incidence_share": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "incidence_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "fractional_mass": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "normalized_weight": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "weight_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "record_key", + "incidence_count", + "incidence_share", + "incidence_denominator", + "fractional_mass", + "normalized_weight", + "weight_denominator" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } }, { - "exclusiveMaximum": 0 + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } } ] } @@ -20718,23 +25802,15 @@ } ] }, - "resolved_lineage_coverage": { + "grounded_record_count": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 }, { "type": "null" @@ -20742,16 +25818,21 @@ ] }, "unit": { - "const": "ratio" + "const": "records" }, "evidence_class": { "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { - "const": "T4.resolved_records_over_scope" + "const": "T4.grounded_record_count" }, "owner_ids": { "const": [ @@ -20996,40 +26077,24 @@ "else": { "properties": { "value": { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 } } } } ] }, - "external_ancestry_coverage": { + "closed_record_count": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 }, { "type": "null" @@ -21037,16 +26102,21 @@ ] }, "unit": { - "const": "ratio" + "const": "records" }, "evidence_class": { "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { - "const": "T4.external_roots_over_scope" + "const": "T4.closed_record_count" }, "owner_ids": { "const": [ @@ -21291,24 +26361,16 @@ "else": { "properties": { "value": { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 } } } } ] }, - "distinct_external_root_count": { + "unresolved_record_count": { "type": "object", "properties": { "value": { @@ -21324,16 +26386,21 @@ ] }, "unit": { - "const": "roots" + "const": "records" }, "evidence_class": { "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { - "const": "T4.root_set_union" + "const": "T4.unresolved_record_count" }, "owner_ids": { "const": [ @@ -21587,32 +26654,15 @@ } ] }, - "top_shared_ancestors": { + "records_with_resolved_external_ancestry": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "array", - "items": { - "type": "object", - "properties": { - "record_key": { - "$ref": "#/$defs/record_key" - }, - "incidence_count": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - } - }, - "required": [ - "record_key", - "incidence_count" - ], - "additionalProperties": false - }, - "minItems": 0 + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 }, { "type": "null" @@ -21626,10 +26676,15 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { - "const": "T4.incidence_ranking" + "const": "T4.records_with_resolved_external_ancestry" }, "owner_ids": { "const": [ @@ -21874,26 +26929,9 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "object", - "properties": { - "record_key": { - "$ref": "#/$defs/record_key" - }, - "incidence_count": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - } - }, - "required": [ - "record_key", - "incidence_count" - ], - "additionalProperties": false - }, - "minItems": 0 + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 } } } @@ -21930,7 +26968,12 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "F-012" @@ -22225,7 +27268,12 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "F-013" @@ -22512,7 +27560,12 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "PR-009.maximum_resolved_parent_depth" @@ -23342,7 +28395,12 @@ "const": "proxy_signal" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "T4.shared_ancestry_dependence" @@ -23514,7 +28572,7 @@ "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 1, + "minItems": 0, "uniqueItems": true }, "required_evidence": { @@ -23524,14 +28582,19 @@ "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 1, + "minItems": 0, "uniqueItems": true }, "signal": { "const": "shared_ancestry_dependence" }, "level": { - "const": "indeterminate" + "type": "string", + "enum": [ + "present", + "not_present", + "indeterminate" + ] }, "basis_fields": { "type": "array", @@ -29503,6 +34566,9 @@ }, "scenario_requested": { "type": "boolean" + }, + "lineage_requested": { + "type": "boolean" } }, "required": [], diff --git a/scripts/check_spec_consistency.py b/scripts/check_spec_consistency.py index 85fe9d4..9f9c13e 100644 --- a/scripts/check_spec_consistency.py +++ b/scripts/check_spec_consistency.py @@ -1,18 +1,22 @@ -"""Check Phase 2 specification and repository presence without analytical work.""" +"""Check current specifications, schema shape and canonical packaged resources. +Historical phase/step gates remain recoverable in Git. This command performs no +product calculations and does not certify a test run or release candidate. +""" from __future__ import annotations +import argparse import json from pathlib import Path +import runpy +import sys ROOT = Path(__file__).resolve().parents[1] REQUIRED = { "README.md", "LICENSE", "NOTICE", "pyproject.toml", "PHASE_0_APPROVAL.md", - "PHASE_1_PLAN.md", "REPOSITORY_ARCHITECTURE.md", "DEPENDENCY_STRATEGY.md", + "REPOSITORY_ARCHITECTURE.md", "DEPENDENCY_STRATEGY.md", "DATA_AND_PROVENANCE_SPEC.md", "OBSERVABILITY_AND_REPORTING.md", - "THEORY_TO_CODE_TRACEABILITY.md", "PHASE_1_COMPLETION.md", - "PHASE_2_COMPLETION.md", "PHASE_2_VALIDATION_REPORT.md", - "PHASE_2_ARCHITECTURE_COMPLIANCE_REPORT.md", + "THEORY_TO_CODE_TRACEABILITY.md", "PHASE_5_PLAN.md", "PHASE_5_DECISIONS.md", } SCHEMAS = {"report.schema.json", "config.schema.json", "schema_mapping.schema.json", "version_order.schema.json", "normalized_manifest.schema.json"} @@ -21,188 +25,34 @@ WORKFLOWS = {"ci.yml", "golden.yml", "security.yml", "release.yml"} -def main() -> int: - missing = sorted(name for name in REQUIRED if not (ROOT / name).is_file()) +def audit(root: Path = ROOT) -> dict: + missing = sorted(name for name in REQUIRED if not (root / name).is_file()) if missing: - raise SystemExit(f"Missing required files: {missing}") - schema_root = ROOT / "schemas" - if {path.name for path in schema_root.glob("*.json")} != SCHEMAS: - raise SystemExit("Schema file set differs from approved architecture") - for path in sorted(schema_root.glob("*.json")): + raise ValueError(f"Missing required files: {missing}") + if {path.name for path in (root / "schemas").glob("*.json")} != SCHEMAS: + raise ValueError("Schema file set differs from the supported architecture") + for path in sorted((root / "schemas").glob("*.json")): schema = json.loads(path.read_text(encoding="utf-8")) if not isinstance(schema, dict) or schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema": - raise SystemExit(f"Unexpected schema dialect or root: {path.name}") - if {p.name for p in (ROOT / "examples/hero").iterdir() if p.is_file()} != HERO: - raise SystemExit("Hero file set differs from approved architecture") - if {p.name for p in (ROOT / ".github/workflows").glob("*.yml")} != WORKFLOWS: - raise SystemExit("Workflow file set differs from approved architecture") - print(f"required root files: {len(REQUIRED)}") - print(f"schemas parsed: {len(SCHEMAS)}") - print(f"hero files found: {len(HERO)}") - print(f"workflows found: {len(WORKFLOWS)}") - print("specification structure: PASS") + raise ValueError(f"Unexpected schema dialect or root: {path.name}") + if {path.name for path in (root / "examples/hero").iterdir() if path.is_file()} != HERO: + raise ValueError("Canonical Hero file set differs") + if {path.name for path in (root / ".github/workflows").glob("*.yml")} != WORKFLOWS: + raise ValueError("Workflow file set differs") + checker = runpy.run_path(str(ROOT / "scripts/release_check.py"), run_name="current_specification_checks") + return {"schemas_parsed": len(SCHEMAS), "hero_files": len(HERO), + **checker["verify_frozen_specifications"](root), **checker["verify_resources"](root)} + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + if any(arg in {"--phase", "--step"} or arg.startswith(("--phase=", "--step=")) for arg in argv): + raise SystemExit("Historical phase dispatch has been retired. Run this current check without --phase/--step; prior gates are recoverable from Git history.") + argparse.ArgumentParser(description=__doc__).parse_args(argv) + print(json.dumps(audit(), indent=2)) + print("Current specification consistency: PASS") return 0 -def phase4_main(step: int = 1) -> int: - """Validate Step 1 control and preserve the inherited structure checks.""" - import runpy - - if type(step) is not int or step != 1: - raise ValueError("Only authorized Phase 4 Step 1 specification checking is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_specification_control") - control["audit_phase4"](step=step) - result = main() - print("Phase 4 Step 1: approved governance and frozen specification structure: PASS") - return result - - -def phase4_step2_main(step: int = 2) -> int: - """Check the authorized canonical contract and the frozen repository layout.""" - import runpy - - if type(step) is not int or step != 2: - raise ValueError("Only authorized Phase 4 Step 2 specification checking is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step2_specification_control") - control["audit_phase4_step2"](step=step) - result = main() - print("Phase 4 Step 2: canonical report schema and preserved specification structure: PASS") - return result - - -def phase4_step3_main(step: int = 3) -> int: - """Check authorized evidence assembly against the frozen public contract.""" - import runpy - - if type(step) is not int or step != 3: - raise ValueError("Only authorized Phase 4 Step 3 specification checking is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step3_specification_control") - control["audit_phase4_step3"](step=step) - result = main() - print("Phase 4 Step 3: explicit evidence assembly and preserved specification structure: PASS") - return result - - -def phase4_step4_main(step: int = 4) -> int: - """Check authorized evidence assembly against the frozen public contract.""" - import runpy - - if type(step) is not int or step != 4: - raise ValueError("Only authorized Phase 4 Step 4 specification checking is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step4_specification_control") - control["audit_phase4_step4"](step=step) - result = main() - print("Phase 4 Step 4: privacy views and safe diagnostics and preserved specification structure: PASS") - return result - - -def phase4_step5_main(step: int = 5) -> int: - """Check pure renderers against the frozen report and repository contracts.""" - import runpy - - if type(step) is not int or step != 5: - raise ValueError("Only authorized Phase 4 Step 5 specification checking is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step5_specification_control") - control["audit_phase4_step5"](step=step) - result = main() - print("Phase 4 Step 5: JSON and Markdown renderers and preserved specification structure: PASS") - return result - - -def phase4_step6_main(step: int = 6) -> int: - """Check safe output publication against the frozen report and repository contracts.""" - import runpy - - if type(step) is not int or step != 6: - raise ValueError("Only authorized Phase 4 Step 6 specification checking is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step6_specification_control") - control["audit_phase4_step6"](step=step) - result = main() - print("Phase 4 Step 6: safe JSON and Markdown publication and preserved specification structure: PASS") - return result - - -def phase4_step7_main(step: int = 7) -> int: - """Check CLI orchestration against the frozen report and repository contracts.""" - import runpy - - if type(step) is not int or step != 7: - raise ValueError("Only authorized Phase 4 Step 7 specification checking is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step7_specification_control") - control["audit_phase4_step7"](step=step) - result = main() - print("Phase 4 Step 7: local audit and input-only validation and preserved specification structure: PASS") - return result - - -def phase4_step8_main(step: int = 8) -> int: - """Check CLI orchestration against the frozen report and repository contracts.""" - import runpy - - if type(step) is not int or step != 8: - raise ValueError("Only authorized Phase 4 Step 8 specification checking is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step8_specification_control") - control["audit_phase4_step8"](step=step) - result = main() - print("Phase 4 Step 8: local audit and input-only validation and preserved specification structure: PASS") - return result - - -def phase4_current_main(step: int = 11) -> int: - """Check CLI orchestration against the frozen report and repository contracts.""" - import runpy - - if type(step) is not int or step != 11: - raise ValueError("Only authorized Phase 4 Step 11 specification checking is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_current_specification_control") - control["audit_phase4_current"](step=step) - result = main() - print("Phase 4 Step 11: final documentation and preserved specification structure: PASS") - return result - - -def cli_main(argv: list[str] | None = None) -> int: - """Keep the argument-free historical gate and add explicit Phase 4 dispatch.""" - import argparse - - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--phase", type=int, choices=(3, 4)) - parser.add_argument("--step", type=int) - args = parser.parse_args(argv) - if args.phase is None and args.step is None: - return main() - if args.phase == 3 and args.step == 11: - return main() - if args.phase == 4 and args.step == 1: - return phase4_main(step=args.step) - if args.phase == 4 and args.step == 2: - return phase4_step2_main(step=args.step) - if args.phase == 4 and args.step == 3: - return phase4_step3_main(step=args.step) - if args.phase == 4 and args.step == 4: - return phase4_step4_main(step=args.step) - if args.phase == 4 and args.step == 5: - return phase4_step5_main(step=args.step) - if args.phase == 4 and args.step == 6: - return phase4_step6_main(step=args.step) - if args.phase == 4 and args.step == 7: - return phase4_step7_main(step=args.step) - if args.phase == 4 and args.step == 8: - return phase4_step8_main(step=args.step) - if args.phase == 4 and args.step == 11: - return phase4_current_main(step=args.step) - parser.error("Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5 or --phase 4 --step 6 or --phase 4 --step 7 or --phase 4 --step 8 or --phase 4 --step 11") - return 2 - - if __name__ == "__main__": - raise SystemExit(cli_main()) + raise SystemExit(main()) diff --git a/scripts/check_traceability.py b/scripts/check_traceability.py index 098d4cd..e78a575 100644 --- a/scripts/check_traceability.py +++ b/scripts/check_traceability.py @@ -1,5749 +1,88 @@ -"""Check inherited Phase 2 rules and approved Phase 3 Step 8 boundaries. +"""Check current module ownership and local-only architectural boundaries. -The Theory Owner authorized synchronized checker maintenance for each explicitly -approved Phase 2 step on 2026-09-16. Step 9 adds explicit local bundle orchestration only. Earlier -validation and explicit local content-read definitions stay intact. -General graphs, cycles, roots, ancestors, metrics and later layers stay protected. - -This script inspects source syntax without importing the package or loading -hero data. Definition and import allowlists are structural checks; behavioral -correctness remains covered by the unit and security tests. +Behavioral tests own formulas and product outcomes. This static check retains +owner declarations, dependency boundaries, input-only validation and pure +rendering. It does not pin function bodies or replay historical source forms. """ - from __future__ import annotations +import argparse import ast +import json from pathlib import Path - +import sys ROOT = Path(__file__).resolve().parents[1] -PACKAGE = ROOT / "src" / "recursive_integrity_toolkit" -BOOTSTRAP = {"__init__.py", "__main__.py", "cli.py"} -STEP1_CORE = {"models.py", "errors.py", "config.py"} -STEP2_IO = {"io/loaders.py", "utils/hashing.py", "utils/paths.py"} -STEP3_MAPPING = {"io/schema_mapping.py"} -STEP8_OBSERVABILITY = {"observability/levels.py"} -ALLOWED_FUNCTIONS = { - "__init__.py": set(), - "__main__.py": set(), - "cli.py": {"build_parser", "main"}, - "config.py": { - "_bool_value", "_enum_value", "_frozen_mapping", "_invalid", - "_optional_positive_int", "_optional_string", "_parse_input_source", - "_parse_inputs", "_parse_representation", "_parse_resource_limits", - "_parse_simulation", "_parse_version_order", "_string_mapping", - "_string_sequence", "load_config", "resolve_config", - }, - "errors.py": {"IngestionError.__init__", "ToolkitError.__init__", "MappingError.__init__"}, - "io/loaders.py": { - "_check_csv_quotes", "_check_depth", "_failure", "_finite_float", "_headers", - "_limits", "_parse_csv", "_parse_jsonl", "_parse_parquet", "_read_source", - "_reject_constant", "_row_limit", "_select_format", "_text", - "_unique_object", "inventory_source", "load_table", - }, - "models.py": { - "ObservabilityAssessment.__post_init__", "RecordKey.__post_init__", - "RecordKey.__str__", "RecordKey.parse", "ValidationCoverage.__post_init__", - "_validate_identifier", - }, - "utils/hashing.py": {"sha256_bytes"}, - "utils/paths.py": {"local_input_path"}, - "io/schema_mapping.py": { - "_error", "_copy_value", "_unique_object", "_reject_constant", "_finite_float", - "_json_value", "_name", "_validate_operation", "_validate_document", - "compile_mapping", "parse_mapping_json", "load_mapping", "_select", - "_coalesce", "_apply_operation", "map_row", - }, -} -ALLOWED_CLASSES = { - "config.py": {"RepresentationConfig", "ResourceLimits", "ScenarioConfig", "ResolvedConfig"}, - "errors.py": { - "ConfigurationError", "ErrorCode", "IngestionError", "InputError", - "SchemaError", "SecurityError", "ToolkitError", "WarningCode", "MappingError", - }, - "models.py": { - "AuditBundle", "Capability", "CapabilityKey", "CapabilityStatus", "FileFormat", - "FileInventoryEntry", "InputSource", "LoadedTable", "FileRole", - "ObservabilityAssessment", "ParentResolutionStatus", "PrivacyMode", "RawRow", - "RecordKey", "ValidationCoverage", "ValidationMessage", "ValidationSeverity", - }, - "io/schema_mapping.py": {"MappingPlan", "FieldMappingTrace", "MappingNotice", "MappedRow"}, -} -ALLOWED_CORE_IMPORTS = { - "config.py": {".errors", ".models", "__future__", "dataclasses", "json", "pathlib", "tomllib", "typing"}, - "errors.py": {"enum", "__future__"}, - "io/loaders.py": { - "..config", "..errors", "..models", "..utils.hashing", "..utils.paths", - "__future__", "csv", "io", "json", "math", "os", "pathlib", "pyarrow", - "pyarrow.parquet", "stat", "threading", - }, - "models.py": {"__future__", "dataclasses", "typing", "enum", "pathlib"}, - "utils/hashing.py": {"__future__", "hashlib"}, - "utils/paths.py": {"__future__", "re", "os", "pathlib", "..errors"}, - "io/schema_mapping.py": { - "..config", "..models", "..errors", "..utils.hashing", ".loaders", "__future__", - "dataclasses", "datetime", "json", "math", "pathlib", "re", - }, -} - -# Step 4 permission covers row-local normalization, base validation and ordering. -# Step 5 join definitions are authorized individually below, never by wildcard. -STEP4_ROWS = {"io/normalization.py", "io/validation.py", "utils/ordering.py"} -ALLOWED_FUNCTIONS.update({ - "io/normalization.py": { - "_policy", "_location", "_csv_quotes", "_input", "_state", "_timestamp", - "_parent_list", "_field_value", "_freeze", "normalize_row", "normalize_table", - }, - "io/validation.py": { - "_fail", "canonical_fields", "validate_canonical_values", "validate_unique_keys", - }, - "utils/ordering.py": {"_record_order_key", "stable_record_order"}, -}) -ALLOWED_FUNCTIONS["errors.py"].add("CanonicalValidationError.__init__") -ALLOWED_CLASSES["errors.py"].add("CanonicalValidationError") -ALLOWED_CLASSES["models.py"].update({ - "SourceType", "ProvenanceConfidence", "ExternalGrounding", "Transformation", - "ContentMode", "NormalizationOptions", "RowLocation", "CanonicalRow", -}) -ALLOWED_CORE_IMPORTS.update({ - "io/normalization.py": { - "__future__", "math", "re", "datetime", "zoneinfo", "types", "json", "..config", - "..errors", "..models", "..utils.ordering", ".schema_mapping", ".loaders", ".validation", - }, - "io/validation.py": {"__future__", "math", "datetime", "..errors", "..models"}, - "utils/ordering.py": {"__future__", "..models"}, -}) - -# Step 5 changes no module-level allowlist. Only these exact definitions are added. -ALLOWED_FUNCTIONS["io/validation.py"].update({ - "_validate_present_fields", "_join_location", "_join_fields", "_join_identity", - "assess_provenance_row", "_join_message", "_join_options", "join_provenance", -}) -ALLOWED_FUNCTIONS["models.py"].update({"ValidationCoverage.ratio", "ProvenanceJoinResult.has_errors"}) -ALLOWED_CLASSES["models.py"].update({"ProvenanceAssessment", "ProvenanceMatch", "ProvenanceJoinResult"}) -ALLOWED_CORE_IMPORTS["io/validation.py"].add("types") - -# Step 6 adds only chronology, immediate references and generation validation. -# The executable module set is unchanged; no future module is opened. -ALLOWED_FUNCTIONS["io/validation.py"].update({ - "_version_failure", "_version_labels", "_version_map", "_version_timestamp", - "resolve_version_order", "_checked_order", "_parent_keys", "parse_parent_ids", - "_reference_key", "_parent_notice", "_parent_lookup", "_resolve_parent_list", - "resolve_parent_references", "validate_generation_declarations", -}) -ALLOWED_FUNCTIONS["models.py"].add("GenerationValidationResult.has_errors") -ALLOWED_CLASSES["models.py"].update({ - "VersionOrderResult", "ParentReference", "ParentValidationResult", - "GenerationAssessment", "GenerationValidationResult", -}) -ALLOWED_CORE_IMPORTS["io/validation.py"].add("json") - -# Step 7 opens no new module. Content paths and explicit reads stay in IO helpers. -ALLOWED_FUNCTIONS["utils/paths.py"].update({ - "_content_error", "_content_lexical", "_content_base", "_inside_base", - "resolve_content_reference", "_open_content_fd", -}) -ALLOWED_FUNCTIONS["io/loaders.py"].add("load_content_reference") -ALLOWED_CORE_IMPORTS["utils/paths.py"].add("stat") - -# Step 8 opens exactly one classifier. No analytical or IO permission is added. -ALLOWED_FUNCTIONS["observability/levels.py"] = { - "_invalid", "_cap", "_records", "_row_key", "_content_keys", "_representation", - "_content_capability", "_provenance_capability", "_lineage_capability", - "_compatibility", "_longitudinal_capability", "_distribution", - "_scenario_capability", "classify_observability", -} -ALLOWED_CLASSES["models.py"].add("ScenarioParameters") -ALLOWED_CORE_IMPORTS["observability/levels.py"] = { - "__future__", "math", "types", "..config", "..errors", "..io.validation", "..models", -} - - -# Step 9 delegates to existing input layers. Exact call sites keep all reads -# within explicit invocation; no eager orchestration or later-layer permission. -ALLOWED_FUNCTIONS["io/validation.py"].update({ - "_bundle_source", "_bundle_source_key", "_bundle_control", "_bundle_plain", - "_bundle_document", "_bundle_setup", "_bundle_incomplete_provenance", - "_bundle_table", "_bundle_order", "_bundle_message_key", "_bundle_messages", - "_bundle_error_message", "validate_bundle", "_bundle_row_key", "_bundle_inventory_key", -}) -ALLOWED_FUNCTIONS["models.py"].add("BundleValidationResult.has_errors") -ALLOWED_CLASSES["models.py"].update({"RowMappingEvidence", "BundleValidationResult"}) -ALLOWED_CORE_IMPORTS["io/validation.py"].update({ - "pathlib", "dataclasses", "tomllib", "..config", "..utils.paths", "..utils.hashing", - ".loaders", ".normalization", ".schema_mapping", "..observability.levels", -}) -PIPELINE_CALL_SITES = { - "_read_source": {"_bundle_control"}, - "load_table": {"_bundle_table"}, - "inventory_source": {"validate_bundle"}, - "load_content_reference": {"validate_bundle"}, - "parse_mapping_json": {"validate_bundle"}, - "map_row": {"_bundle_table"}, - "normalize_row": {"_bundle_table"}, - "classify_observability": {"validate_bundle"}, - "_bundle_control": {"_bundle_setup", "validate_bundle"}, - "_bundle_table": {"validate_bundle"}, - "_bundle_setup": {"validate_bundle"}, - "validate_bundle": set(), -} -PIPELINE_IMPORT_NAMES = { - ".loaders": {"_limits", "_read_source", "_select_format", "_text", "_check_depth", - "_finite_float", "_reject_constant", "_unique_object", "load_table", - "inventory_source", "load_content_reference"}, - ".normalization": {"_field_value", "_input", "_policy", "_state", "normalize_row"}, - ".schema_mapping": {"map_row", "parse_mapping_json"}, - "..observability.levels": {"classify_observability"}, -} - -# Preserve every import and file restriction from the Phase 1 checker. -FORBIDDEN_IMPORTS = { - "numpy", "pandas", "pyarrow", "networkx", "scipy", "sklearn", "torch", - "tensorflow", "transformers", -} +FORBIDDEN_IMPORTS = {"socket", "urllib", "http", "ftplib", "requests", "httpx", + "aiohttp", "subprocess", "networkx", "scipy", "sklearn", "torch", + "tensorflow", "transformers"} FORBIDDEN_FILES = {"collapse_score.py", "integrity_score.py", "universal_score.py"} - - - -# Phase 3 Step 1: declarations and validation contracts only. -PHASE3_ACTIVE_STEP = 11 -PHASE3_CONTRACT_CLASSES = { - "CalculationEvidenceClass", "CalculationStatus", "CalculationReason", "NumericalPolicy", - "RepresentationDescriptor", "RecordStateAssignment", "CalculationScope", "WeightingOptions", - "TailSelectionOptions", "CalculationMetadata", "ScalarCalculation", "ExplicitPairContext", - "ClosedResamplingMetadata", -} -PHASE3_CONTRACT_FUNCTIONS = { - "_calculation_text", "_calculation_strings", "_calculation_integer", "_calculation_number", - "NumericalPolicy.__post_init__", "RepresentationDescriptor.__post_init__", - "RecordStateAssignment.__post_init__", "CalculationScope.__post_init__", - "WeightingOptions.__post_init__", "TailSelectionOptions.__post_init__", - "CalculationMetadata.__post_init__", "ScalarCalculation.__post_init__", - "ExplicitPairContext.__post_init__", "ClosedResamplingMetadata.__post_init__", -} -ALLOWED_FUNCTIONS["models.py"].update(PHASE3_CONTRACT_FUNCTIONS) -ALLOWED_CLASSES["models.py"].update(PHASE3_CONTRACT_CLASSES) -ALLOWED_CORE_IMPORTS["models.py"].add("math") - - -# Step 2 opens only literal-field representation, not any mathematical metric. -PHASE3_FIELD_MODULES = {"representations/base.py", "representations/field.py"} -ALLOWED_FUNCTIONS["representations/base.py"] = { - "RepresentationResult.selected_count", "RepresentationResult.included_count", - "RepresentationResult.excluded_count", "RepresentationProtocol.__call__", - "_representation_error", "_literal_text", "_selected_records", "_selected_key", -} -ALLOWED_CLASSES["representations/base.py"] = { - "RepresentationSelection", "RepresentationResult", "RepresentationProtocol", -} -ALLOWED_FUNCTIONS["representations/field.py"] = { - "_selection", "select_field_representation", "assign_field_states", -} -REPRESENTATION_IMPORT_NAMES = { - "representations/base.py": { - "__future__": {"annotations"}, "dataclasses": {"dataclass", "field"}, - "types": {"MappingProxyType"}, "typing": {"Protocol"}, "..config": {"RepresentationConfig"}, - "..errors": {"CanonicalValidationError", "ErrorCode"}, "..io.validation": {"validate_canonical_values"}, - "..models": {"CalculationReason", "CalculationScope", "CalculationStatus", "CanonicalRow", - "FileRole", "RecordKey", "RecordStateAssignment", "RepresentationDescriptor", - "RowLocation", "ValidationCoverage", "ValidationMessage"}, - }, - "representations/field.py": { - "__future__": {"annotations"}, "..config": {"RepresentationConfig"}, - "..errors": {"ErrorCode", "WarningCode"}, - "..models": {"CalculationReason", "CalculationScope", "CalculationStatus", "CanonicalRow", - "RecordKey", "RecordStateAssignment", "RepresentationDescriptor", - "ValidationCoverage", "ValidationMessage", "ValidationSeverity"}, - ".base": {"RepresentationResult", "RepresentationSelection", "_literal_text", - "_representation_error", "_selected_records"}, - }, -} -ALLOWED_CORE_IMPORTS["representations/base.py"] = set(REPRESENTATION_IMPORT_NAMES["representations/base.py"]) -ALLOWED_CORE_IMPORTS["representations/field.py"] = set(REPRESENTATION_IMPORT_NAMES["representations/field.py"]) - - -def _phase3_representation_boundary(tree: ast.Module, relative: str) -> None: - """Reject hidden computation, I/O, aliases and user callback execution.""" - direct_calls = { - "representations/base.py": { - "dataclass", "field", "len", "type", "bool", "any", "set", "tuple", "dict", "sorted", "str", - "CanonicalValidationError", "RowLocation", "_representation_error", "_literal_text", - "validate_canonical_values", - }, - "representations/field.py": { - "type", "any", "len", "tuple", "sorted", "_representation_error", "_literal_text", - "_selected_records", "_selection", "RepresentationDescriptor", "RepresentationSelection", - "ValidationMessage", "RecordStateAssignment", "CalculationScope", - "RepresentationResult", "ValidationCoverage", - }, - } - method_calls = { - "representations/base.py": {("value", "encode"), ("version", "strip"), - ("seen", "add"), ("loaded", "add"), ("selected", "append")}, - "representations/field.py": {("values", "get"), ("value", "encode"), - ("assignments", "append"), ("states", "append"), - ("included", "append"), ("excluded", "append")}, - } - for top in tree.body: - doc = (isinstance(top, ast.Expr) and isinstance(top.value, ast.Constant) - and isinstance(top.value.value, str)) - if not doc and not isinstance(top, (ast.ImportFrom, ast.FunctionDef, ast.ClassDef)): - raise SystemExit("Unexpected eager representation operation") - for node in ast.walk(tree): - if isinstance(node, (ast.Lambda, ast.AsyncFunctionDef, ast.Await, ast.Yield, ast.YieldFrom)): - raise SystemExit("Unexpected callback in field representations") - if isinstance(node, ast.Import): - raise SystemExit("Import must use exact representation symbols") - if isinstance(node, ast.ImportFrom): - module = "." * node.level + (node.module or "") - allowed = REPRESENTATION_IMPORT_NAMES[relative].get(module, set()) - if any(a.name not in allowed or a.asname is not None for a in node.names): - raise SystemExit("Import symbol outside representation scope") - if isinstance(node, ast.BinOp): - difference = (relative == "representations/base.py" and isinstance(node.op, ast.Sub) - and isinstance(node.left, ast.Call) and isinstance(node.left.func, ast.Name) - and node.left.func.id == "set" and len(node.left.args) == 1 - and isinstance(node.left.args[0], ast.Name) and node.left.args[0].id == "dataset_versions" - and isinstance(node.right, ast.Name) and node.right.id == "loaded") - if not isinstance(node.op, ast.BitOr) and not difference: - raise SystemExit("Unexpected metric arithmetic in representations") - if isinstance(node, ast.Call): - direct = isinstance(node.func, ast.Name) and node.func.id in direct_calls[relative] - method = (isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) - and (node.func.value.id, node.func.attr) in method_calls[relative]) - if not direct and not method: - raise SystemExit("Unexpected call in pure field representations") - - - -# Step 3 opens only exact record-form states and PR-006 duplicate counts. -PHASE3_EXACT_MODULES = {"representations/content_hash.py", "metrics/duplicates.py"} -ALLOWED_FUNCTIONS["representations/content_hash.py"] = { - "exact_content_bytes", "_payload_snapshot", "assign_content_states", -} -ALLOWED_CLASSES["representations/content_hash.py"] = {"ExactContentRepresentation"} -ALLOWED_FUNCTIONS["metrics/duplicates.py"] = { - "ExactDuplicateGroup.representative", "ExactDuplicateGroup.record_count", "detect_exact_duplicates", -} -ALLOWED_CLASSES["metrics/duplicates.py"] = {"ExactDuplicateGroup", "ExactDuplicateResult"} -EXACT_IMPORT_NAMES = { - "representations/content_hash.py": { - "__future__": {"annotations"}, "dataclasses": {"dataclass", "field"}, - "types": {"MappingProxyType"}, "..errors": {"ErrorCode"}, - "..models": {"CalculationReason", "CalculationScope", "CalculationStatus", "CanonicalRow", - "ContentMode", "RecordKey", "RecordStateAssignment", "RepresentationDescriptor", "ValidationCoverage"}, - "..utils.hashing": {"sha256_bytes"}, - ".base": {"RepresentationResult", "RepresentationSelection", "_literal_text", - "_representation_error", "_selected_records"}, - }, - "metrics/duplicates.py": { - "__future__": {"annotations"}, "dataclasses": {"dataclass", "field"}, - "..models": {"CalculationEvidenceClass", "CalculationMetadata", "CalculationScope", "CalculationStatus", - "CanonicalRow", "ContentMode", "RecordKey", "RepresentationDescriptor", "ScalarCalculation", - "ValidationCoverage"}, - "..representations.content_hash": {"assign_content_states"}, - }, -} -for _exact_module, _exact_imports in EXACT_IMPORT_NAMES.items(): - ALLOWED_CORE_IMPORTS[_exact_module] = set(_exact_imports) - - -def _phase3_exact_boundary(tree: ast.Module, relative: str) -> None: - """Exact imports and calls; only PR-006 counting arithmetic is admitted.""" - direct_calls = { - "representations/content_hash.py": { - "dataclass", "field", "type", "len", "set", "tuple", "_literal_text", "_representation_error", - "_selected_records", "_payload_snapshot", "exact_content_bytes", "sha256_bytes", - "RepresentationDescriptor", "RepresentationSelection", "RepresentationResult", - "RecordStateAssignment", "CalculationScope", "ValidationCoverage", "ExactContentRepresentation", - }, - "metrics/duplicates.py": { - "dataclass", "field", "assign_content_states", "sorted", "tuple", "len", - "ExactDuplicateGroup", "CalculationMetadata", "ScalarCalculation", "ExactDuplicateResult", - }, - } - methods = { - "representations/content_hash.py": { - ("text", "strip"), ("text", "encode"), ("resolved_content", "items"), - ("assignments", "append"), ("snapshots", "append"), ("keys", "append"), ("states", "append"), - }, - "metrics/duplicates.py": { - ("members", "setdefault"), ("bucket", "append"), ("groups", "append"), ("counts", "append"), - }, - } - expected_subtractions = { - "representations/content_hash.py": {ast.dump(ast.parse(expr, mode="eval").body) for expr in - ("set(payloads) - selected_keys", "selected_keys - set(payloads)")}, - "metrics/duplicates.py": {ast.dump(ast.parse("len(keys) - 1", mode="eval").body)}, - } - for top in tree.body: - doc = (isinstance(top, ast.Expr) and isinstance(top.value, ast.Constant) - and isinstance(top.value.value, str)) - if not doc and not isinstance(top, (ast.ImportFrom, ast.FunctionDef, ast.ClassDef)): - raise SystemExit("Unexpected eager exact-content operation") - for node in ast.walk(tree): - if isinstance(node, (ast.Lambda, ast.AsyncFunctionDef, ast.Await, ast.Yield, ast.YieldFrom, ast.Import)): - raise SystemExit("Unexpected dynamic or import operation in exact-content scope") - if isinstance(node, ast.ImportFrom): - module = "." * node.level + (node.module or "") - allowed = EXACT_IMPORT_NAMES[relative].get(module, set()) - if any(a.name not in allowed or a.asname is not None for a in node.names): - raise SystemExit("Import symbol outside exact-content scope") - if isinstance(node, ast.BinOp) and not isinstance(node.op, ast.BitOr): - if ast.dump(node) not in expected_subtractions[relative]: - raise SystemExit("Unexpected arithmetic outside exact duplicate definitions") - if isinstance(node, ast.AugAssign): - expected = ast.parse("duplicates += len(keys) - 1").body[0] - if relative != "metrics/duplicates.py" or ast.dump(node) != ast.dump(expected): - raise SystemExit("Unexpected augmented arithmetic outside PR-006") - if isinstance(node, ast.Call): - direct = isinstance(node.func, ast.Name) and node.func.id in direct_calls[relative] - method = (isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) - and (node.func.value.id, node.func.attr) in methods[relative]) - if not direct and not method: - raise SystemExit("Unexpected call in pure exact-content scope") - - - -# Step 4 opens only the reviewed single-scope F-001 through F-004 kernels. -PHASE3_DISTRIBUTION_MODULES = {"metrics/diversity.py"} -ALLOWED_FUNCTIONS["metrics/diversity.py"] = { - "_invalid", "_text", "_key", "_context", "_number", "_pairs", "_probabilities", - "_metadata", "_metrics", "distribution_from_counts", "distribution_from_probabilities", - "_assignments", "_weights", "calculate_state_distribution", -} -ALLOWED_CLASSES["metrics/diversity.py"] = {"StateFrequency", "DistributionMetrics", "StateDistributionResult"} -ALLOWED_CORE_IMPORTS["metrics/diversity.py"] = { - "__future__", "dataclasses", "math", "types", "..errors", "..models", "..representations.base", -} -DISTRIBUTION_SOURCE_SHA256 = "34b1b77d00b9a2169aa7680d0edf77b05d5aec0bdc26b3bd141d38853f9728a9" - - -def _phase3_distribution_boundary(tree: ast.Module) -> None: - """Pin the Step 9 additive body and compare in this interpreter, not across AST formats.""" - import hashlib - raw = (PACKAGE / "metrics/diversity.py").read_bytes() - if hashlib.sha256(raw).hexdigest() != DISTRIBUTION_SOURCE_SHA256: - raise SystemExit("Unexpected distribution source outside the reviewed Step 9 body") - if ast.dump(tree, include_attributes=False) != ast.dump(ast.parse(raw), include_attributes=False): - raise SystemExit("Unexpected distribution AST outside the reviewed Step 9 body") - - -ALLOWED_FUNCTIONS["metrics/diversity.py"].update({'_pair_distribution', '_pair_scalar_check', 'compare_support', '_pair_metadata_check', '_pair_metadata', '_harmonize_distribution'}) -ALLOWED_CLASSES["metrics/diversity.py"].update({'SupportComparison'}) -ALLOWED_CORE_IMPORTS["metrics/diversity.py"].add("..representations.compatibility") - - -# Step 5 opens only declared provenance composition and direct classification. -PHASE3_PROVENANCE_MODULES = {"metrics/provenance.py"} -ALLOWED_FUNCTIONS["metrics/provenance.py"] = set(['_checked_join', '_composition', '_direct', '_invalid', '_key', '_keys', '_messages', '_metadata', '_scalar', '_text', '_weighted', '_weights', 'classify_direct_grounding', 'summarize_provenance']) -ALLOWED_CLASSES["metrics/provenance.py"] = set(['DeclaredComposition', 'DirectGroundingAssignment', 'DirectGroundingBasis', 'ProvenanceCompositionResult', 'WeightedSourceComposition']) -ALLOWED_CORE_IMPORTS["metrics/provenance.py"] = { - "__future__", "dataclasses", "math", "types", "..errors", "..models", "..io.validation", -} -PROVENANCE_AST_SHA256 = "554e40896edf5c0e30432015ad357a82e3b3419b6f7529d4e22a09894b4569c2" - - -def _phase3_provenance_boundary(tree: ast.Module) -> None: - """Pin the reviewed Step 5 body, never learn a new body at gate execution. - - Independent rational tests cover mathematics; mutation tests cover change - control. Only empty Python-version type_params metadata is omitted. - """ - import hashlib - for node in ast.walk(tree): - if "type_params" in node._fields: - if getattr(node, "type_params", []): - raise SystemExit("Unexpected generic declaration in provenance metrics") - node._fields = tuple(name for name in node._fields if name != "type_params") - try: - serialized = ast.dump(tree, include_attributes=False, show_empty=True) - except TypeError: - serialized = ast.dump(tree, include_attributes=False) - if hashlib.sha256(serialized.encode("utf-8")).hexdigest() != PROVENANCE_AST_SHA256: - raise SystemExit("Unexpected provenance implementation outside reviewed Step 5 body") - - - - -# Step 6 opens only direct F-009/F-010 bounds, with no lineage branch. -PHASE3_BOUNDS_MODULES = {"metrics/bounds.py"} -ALLOWED_FUNCTIONS["metrics/bounds.py"] = {'_metric', 'closure_exposure_bounds', '_messages', '_confidence', '_text', '_invalid', '_key', '_scope', '_coverage', '_count', '_basis', 'DirectClosureExposureBounds.status', 'direct_closure_exposure', '_observed'} -ALLOWED_CLASSES["metrics/bounds.py"] = {'DirectClosureExposureBounds'} -ALLOWED_CORE_IMPORTS["metrics/bounds.py"] = {"__future__", "dataclasses", "math", "..errors", "..models", ".provenance"} -BOUNDS_SOURCE_SHA256 = "01693a92ab9c0a78cba5addf7bd778fca5d14051599d7878dd682ddc5c9707a4" - - -def _phase3_bounds_boundary(tree: ast.Module) -> None: - """Pin reviewed bytes, then compare ASTs in the same interpreter. - - The digest is a reviewed literal. It is never reset from inspected input. - Both actual file edits and injected executable nodes are rejected. - """ - import hashlib - raw = (PACKAGE / "metrics/bounds.py").read_bytes() - if hashlib.sha256(raw).hexdigest() != BOUNDS_SOURCE_SHA256: - raise SystemExit("Direct bounds source differs from the reviewed Step 6 body") - if ast.dump(tree, include_attributes=False) != ast.dump(ast.parse(raw), include_attributes=False): - raise SystemExit("Unexpected direct bounds executable syntax outside Step 6") - - -# Step 7 opens only explicit observed-tail diagnostics and analytic F-014. -PHASE3_TAIL_MODULES = {"metrics/tail.py"} -ALLOWED_FUNCTIONS["metrics/tail.py"] = {'_state', '_metadata', 'select_tail', '_distribution', 'one_step_extinction_probability', '_rarity_key', '_invalid', '_options'} -ALLOWED_CLASSES["metrics/tail.py"] = {'TailSelectionResult', 'RarityEntry', 'ExtinctionProbabilityResult'} -ALLOWED_CORE_IMPORTS["metrics/tail.py"] = {"__future__", "dataclasses", "math", "..errors", "..models", ".diversity"} -TAIL_SOURCE_SHA256 = "b7618dea6a28fd69aa951d7ae5170c772bc2bea2999906ed61bca9079b8d5cf7" - - -def _phase3_tail_boundary(tree: ast.Module) -> None: - """Fixed reviewed bytes and same-interpreter syntax; no later T2 behaviors.""" - import hashlib - raw = (PACKAGE / "metrics/tail.py").read_bytes() - if hashlib.sha256(raw).hexdigest() != TAIL_SOURCE_SHA256: - raise SystemExit("Tail source differs from the reviewed Step 7 body") - if ast.dump(tree, include_attributes=False) != ast.dump(ast.parse(raw), include_attributes=False): - raise SystemExit("Unexpected tail/scenario executable syntax outside Step 7") - - -# Step 8 opens only explicit closed resampling, never T5 reopening. -PHASE3_RESAMPLING_MODULES = {"metrics/resampling.py"} -ALLOWED_FUNCTIONS["metrics/resampling.py"] = set(['_diversity', '_generation', '_inputs', '_integer', '_invalid', '_metadata', '_resources', '_sample_counts', '_state', 'expected_diversity_after_steps', 'simulate_closed_resampling']) -ALLOWED_CLASSES["metrics/resampling.py"] = set(['ExpectedDiversityResult', 'ResamplingInput', 'ResamplingReplicate', 'ResamplingSimulation', 'SampledGeneration']) -ALLOWED_CORE_IMPORTS["metrics/resampling.py"] = {"__future__", "dataclasses", "math", "types", "..errors", "..models", ".diversity", "numpy"} -RESAMPLING_SOURCE_SHA256 = "0896b80c26ad0b9865e24957202b54c2ab9217921d1c19e84ab22d84928f2e40" - - -def _phase3_resampling_boundary(tree: ast.Module) -> None: - """Fixed reviewed source and exact AST; lazy NumPy only in the sampler.""" - import hashlib - raw = (PACKAGE / "metrics/resampling.py").read_bytes() - if hashlib.sha256(raw).hexdigest() != RESAMPLING_SOURCE_SHA256: - raise SystemExit("Resampling source differs from the reviewed Step 8 body") - if ast.dump(tree, include_attributes=False) != ast.dump(ast.parse(raw), include_attributes=False): - raise SystemExit("Unexpected resampling syntax outside Step 8") - - -def _phase3_contract_boundary(tree: ast.Module) -> None: - """Reject calculation or execution inside the newly authorized declarations.""" - permitted = {"type", "len", "set", "any", "ValueError", "TypeError", "field", "dataclass", - "WeightingOptions", "_calculation_text", "_calculation_strings", - "_calculation_integer", "_calculation_number"} - roots = {name.split(".")[0] for name in PHASE3_CONTRACT_FUNCTIONS} | PHASE3_CONTRACT_CLASSES - for top in tree.body: - if not isinstance(top, (ast.FunctionDef, ast.ClassDef)) or top.name not in roots: - continue - for node in ast.walk(top): - if isinstance(node, (ast.Lambda, ast.AsyncFunctionDef, ast.Await, ast.Yield, ast.YieldFrom)): - raise SystemExit("Unexpected callback in Phase 3 contracts") - if isinstance(node, ast.BinOp) and not isinstance(node.op, (ast.BitOr, ast.BitAnd)): - raise SystemExit("Unexpected metric arithmetic in Phase 3 contracts") - if isinstance(node, ast.Call): - direct = isinstance(node.func, ast.Name) and node.func.id in permitted - finite = (isinstance(node.func, ast.Attribute) and node.func.attr == "isfinite" - and isinstance(node.func.value, ast.Name) and node.func.value.id == "math" - and isinstance(top, ast.FunctionDef) and top.name == "_calculation_number") - if not (direct or finite): - raise SystemExit("Unexpected call in Phase 3 contracts") - for node in ast.walk(tree): - if isinstance(node, ast.Import) and any(alias.name == "math" for alias in node.names): - if len(node.names) != 1 or node.names[0].asname is not None or not any( - isinstance(fn, ast.FunctionDef) and fn.name == "_calculation_number" - and any(child is node for child in ast.walk(fn)) for fn in tree.body): - raise SystemExit("Unexpected math import outside finite-type validation") - - -def _definitions(tree: ast.AST, prefix: str = "") -> tuple[list[str], list[str]]: - """Collect qualified definitions without executing inspected source.""" - functions: list[str] = [] - classes: list[str] = [] - for node in ast.iter_child_nodes(tree): - child_prefix = prefix - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - name = f"{prefix}.{node.name}" if prefix else node.name - if isinstance(node, ast.AsyncFunctionDef): - raise SystemExit(f"Async implementation is outside Step 8: {name}") - if isinstance(node, ast.ClassDef): - classes.append(name) - else: - functions.append(name) - child_prefix = name - child_functions, child_classes = _definitions(node, child_prefix) - functions.extend(child_functions) - classes.extend(child_classes) - return functions, classes - - -# Step 9 opens only explicit pair compatibility; no metric dispatch here. -PHASE3_PAIR_MODULES = {"representations/compatibility.py"} -ALLOWED_FUNCTIONS["representations/compatibility.py"] = {'_scope', '_mapping', '_descriptor', 'validate_representation_compatibility', '_invalid', '_context', '_text'} -ALLOWED_CLASSES["representations/compatibility.py"] = {'StateMappingDeclaration', 'RepresentationCompatibility'} -ALLOWED_CORE_IMPORTS["representations/compatibility.py"] = {'..models', 'types', '..io.validation', '..errors', '__future__', 'dataclasses'} -PAIR_SOURCE_SHA256 = "c4ce8e30d8026dd7ac1e10ac09e7b70a23ae2c4fc451f2e37890b919965d4509" - - -def _phase3_pair_boundary(tree: ast.Module) -> None: - """Exact reviewed declaration checks; reject source changes and injected bodies.""" - import hashlib - raw = (PACKAGE / "representations/compatibility.py").read_bytes() - if hashlib.sha256(raw).hexdigest() != PAIR_SOURCE_SHA256: - raise SystemExit("Unexpected compatibility source outside reviewed Step 9") - if ast.dump(tree, include_attributes=False) != ast.dump(ast.parse(raw), include_attributes=False): - raise SystemExit("Unexpected compatibility AST outside reviewed Step 9") - - -def main() -> int: - paths = sorted(PACKAGE.rglob("*.py")) - if len(paths) != 40: - raise SystemExit(f"Expected 40 package modules, found {len(paths)}") - relative_paths = {path.relative_to(PACKAGE).as_posix() for path in paths} - missing = (BOOTSTRAP | STEP1_CORE | STEP2_IO | STEP3_MAPPING | STEP4_ROWS | STEP8_OBSERVABILITY | PHASE3_FIELD_MODULES | PHASE3_EXACT_MODULES | PHASE3_DISTRIBUTION_MODULES | PHASE3_PROVENANCE_MODULES | PHASE3_BOUNDS_MODULES | PHASE3_TAIL_MODULES | PHASE3_RESAMPLING_MODULES | PHASE3_PAIR_MODULES) - relative_paths - if missing: - raise SystemExit(f"Required startup or Step 8 modules missing: {sorted(missing)}") - - imported_roots: set[str] = set() - forbidden_locations: list[str] = [] - placeholder_count = 0 - +DYNAMIC_CALLS = {"eval", "exec", "compile", "__import__"} +INPUT_ONLY = {"io/validation.py", "io/normalization.py", "io/schema_mapping.py", "io/loaders.py", + "observability/levels.py"} +RENDERERS = {"reports/json_report.py", "reports/markdown_report.py"} + + +def _module_name(node: ast.ImportFrom, relative: str) -> str: + """Resolve relative imports sufficiently to check the package layer.""" + if not node.level: + return node.module or "" + parent = ["recursive_integrity_toolkit", *Path(relative).parts[:-1]] + prefix = parent[:len(parent) - node.level + 1] + return ".".join(prefix + ([node.module] if node.module else [])) + + +def audit(root: Path = ROOT) -> dict: + package = root / "src/recursive_integrity_toolkit" + paths = sorted(package.rglob("*.py")) + if not paths: + raise ValueError("Package source is missing") for path in paths: - relative = path.relative_to(PACKAGE).as_posix() - if path.name in FORBIDDEN_FILES: - raise SystemExit(f"Forbidden module found: {path}") + relative = path.relative_to(package).as_posix() + if path.is_symlink() or path.name in FORBIDDEN_FILES: + raise ValueError(f"Aliased or prohibited module: {relative}") tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - if relative == "models.py": - _phase3_contract_boundary(tree) - if relative in PHASE3_FIELD_MODULES: - _phase3_representation_boundary(tree, relative) - if relative in PHASE3_EXACT_MODULES: - _phase3_exact_boundary(tree, relative) - if relative in PHASE3_DISTRIBUTION_MODULES: - _phase3_distribution_boundary(tree) - if relative in PHASE3_PROVENANCE_MODULES: - _phase3_provenance_boundary(tree) - if relative in PHASE3_BOUNDS_MODULES: - _phase3_bounds_boundary(tree) - if relative in PHASE3_TAIL_MODULES: - _phase3_tail_boundary(tree) - if relative in PHASE3_RESAMPLING_MODULES: - _phase3_resampling_boundary(tree) - if relative in PHASE3_PAIR_MODULES: - _phase3_pair_boundary(tree) doc = ast.get_docstring(tree) or "" if "Owner IDs:" not in doc or "Current phase status:" not in doc: - raise SystemExit(f"Owner or phase metadata missing: {path}") - - if relative not in BOOTSTRAP | STEP1_CORE | STEP2_IO | STEP3_MAPPING | STEP4_ROWS | STEP8_OBSERVABILITY | PHASE3_FIELD_MODULES | PHASE3_EXACT_MODULES | PHASE3_DISTRIBUTION_MODULES | PHASE3_PROVENANCE_MODULES | PHASE3_BOUNDS_MODULES | PHASE3_TAIL_MODULES | PHASE3_RESAMPLING_MODULES | PHASE3_PAIR_MODULES: - if not ( - len(tree.body) == 1 - and isinstance(tree.body[0], ast.Expr) - and isinstance(tree.body[0].value, ast.Constant) - and isinstance(tree.body[0].value.value, str) - ): - raise SystemExit(f"Protected non-Step-8 executable body found: {path}") - placeholder_count += 1 - else: - functions, classes = _definitions(tree) - if set(functions) != ALLOWED_FUNCTIONS[relative] or len(functions) != len(set(functions)): - raise SystemExit(f"Unexpected Step 9 functions in {relative}: {functions}") - if set(classes) != ALLOWED_CLASSES.get(relative, set()) or len(classes) != len(set(classes)): - raise SystemExit(f"Unexpected Step 9 classes in {relative}: {classes}") - + raise ValueError(f"Owner or implementation status missing: {relative}") for node in ast.walk(tree): - if relative == "io/validation.py": - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id in PIPELINE_CALL_SITES: - if not any(isinstance(fn, ast.FunctionDef) and fn.name in PIPELINE_CALL_SITES[node.func.id] - and any(child is node for child in ast.walk(fn)) for fn in tree.body): - raise SystemExit(f"Unexpected bundle read or orchestration call site: {node.func.id}") - if isinstance(node, ast.ImportFrom): - imported_path = "." * node.level + (node.module or "") - if imported_path in PIPELINE_IMPORT_NAMES: - if any(alias.name not in PIPELINE_IMPORT_NAMES[imported_path] or alias.asname is not None for alias in node.names): - raise SystemExit(f"Unexpected adjacent-layer import in {relative}") - if not any(isinstance(fn, ast.FunctionDef) and fn.name in ALLOWED_FUNCTIONS["io/validation.py"] - and (fn.name.startswith("_bundle_") or fn.name == "validate_bundle") - and any(child is node for child in ast.walk(fn)) for fn in tree.body): - raise SystemExit(f"Unexpected eager adjacent-layer import in {relative}") - if relative in STEP8_OBSERVABILITY: - if isinstance(node, ast.BinOp) and isinstance(node.op, (ast.Mult, ast.Div, ast.FloorDiv, ast.Mod, ast.Pow, ast.MatMult)): - raise SystemExit(f"Unexpected analytical arithmetic in {relative}") - if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) and node.func.value.id == "math": - if node.func.attr not in {"isfinite", "fsum", "isclose"} or not any( - isinstance(fn, ast.FunctionDef) and fn.name == "_distribution" - and any(child is node for child in ast.walk(fn)) for fn in tree.body - ): - raise SystemExit(f"Unexpected math use outside input validation in {relative}") - if relative in STEP2_IO: - if isinstance(node, ast.Lambda): - raise SystemExit(f"Unexpected dynamic callback in {relative}") - if isinstance(node, ast.Call): - blocked_calls = {"eval", "exec", "compile", "__import__", "globals", "locals", "vars", "input", "breakpoint"} - blocked_methods = {"write_text", "write_bytes", "mkdir", "unlink", "remove", "system", "popen", "connect", "urlopen", "urlretrieve", "send", "sendall"} - if (isinstance(node.func, ast.Name) and node.func.id in blocked_calls) or ( - isinstance(node.func, ast.Attribute) and node.func.attr in blocked_methods - ): - raise SystemExit(f"Unexpected executable, write or network capability in {relative}") - if relative in STEP3_MAPPING | STEP4_ROWS | STEP8_OBSERVABILITY | PHASE3_FIELD_MODULES | PHASE3_EXACT_MODULES | PHASE3_DISTRIBUTION_MODULES | PHASE3_PROVENANCE_MODULES | PHASE3_BOUNDS_MODULES | PHASE3_TAIL_MODULES | PHASE3_RESAMPLING_MODULES | PHASE3_PAIR_MODULES | {"models.py"}: - if isinstance(node, ast.Lambda): - raise SystemExit(f"Unexpected dynamic callback in {relative}") - if isinstance(node, ast.Call): - dangerous = {"eval", "exec", "compile", "__import__", "getattr", "setattr", - "delattr", "globals", "locals", "vars", "open", "input", "breakpoint"} - io_methods = {"open", "read_text", "read_bytes", "write_text", "write_bytes", - "mkdir", "unlink", "system", "popen", "connect", "urlopen"} - if (isinstance(node.func, ast.Name) and node.func.id in dangerous) or ( - isinstance(node.func, ast.Attribute) and node.func.attr in io_methods - ): - raise SystemExit(f"Unexpected executable or IO capability in {relative}") - if isinstance(node.func, ast.Name) and node.func.id == "_read_source": - if not ((relative in STEP3_MAPPING and any( - isinstance(fn, ast.FunctionDef) and fn.name == "load_mapping" - and any(child is node for child in ast.walk(fn)) for fn in tree.body - )) or (relative == "io/validation.py" and any( - isinstance(fn, ast.FunctionDef) and fn.name == "_bundle_control" - and any(child is node for child in ast.walk(fn)) for fn in tree.body - ))): - raise SystemExit(f"Unexpected file read outside explicit mapping loader in {relative}") - imports: set[str] = set() + imports = [] if isinstance(node, ast.Import): - imports.update(alias.name for alias in node.names) - imported_roots.update(alias.name.split(".")[0] for alias in node.names) + imports = [alias.name for alias in node.names] elif isinstance(node, ast.ImportFrom): - imports.add("." * node.level + (node.module or "")) - if node.module: - imported_roots.add(node.module.split(".")[0]) - for imported in imports: - root = imported.lstrip(".").split(".")[0] - if root in FORBIDDEN_IMPORTS: - lazy_parquet = ( - relative == "io/loaders.py" - and imported in {"pyarrow", "pyarrow.parquet"} - and any( - isinstance(function, ast.FunctionDef) - and function.name == "_parse_parquet" - and any(child is node for child in ast.walk(function)) - for function in tree.body - ) - ) - lazy_numpy = ( - relative == "metrics/resampling.py" and imported == "numpy" - and isinstance(node, ast.Import) - and len(node.names) == 1 and node.names[0].name == "numpy" and node.names[0].asname == "np" - and any(isinstance(fn, ast.FunctionDef) and fn.name == "simulate_closed_resampling" - and any(child is node for child in ast.walk(fn)) for fn in tree.body) - ) - if not (lazy_parquet or lazy_numpy): - forbidden_locations.append(f"{relative}: {imported}") - if relative in STEP1_CORE | STEP2_IO | STEP3_MAPPING | STEP4_ROWS | STEP8_OBSERVABILITY | PHASE3_FIELD_MODULES | PHASE3_EXACT_MODULES | PHASE3_DISTRIBUTION_MODULES | PHASE3_PROVENANCE_MODULES | PHASE3_BOUNDS_MODULES | PHASE3_TAIL_MODULES | PHASE3_RESAMPLING_MODULES | PHASE3_PAIR_MODULES: - unexpected = imports - ALLOWED_CORE_IMPORTS[relative] - if unexpected: - raise SystemExit(f"Import outside Step 9 contracts in {relative}: {sorted(unexpected)}") - - if forbidden_locations: - raise SystemExit(f"Dependency outside the approved scope: {forbidden_locations}") - - print(f"package modules checked: {len(paths)}") - print(f"allowed startup functions: {sorted(ALLOWED_FUNCTIONS['cli.py'])}") - print(f"authorized Step 2 contract modules: {sorted(STEP1_CORE)}") - print(f"authorized Step 2 file modules: {sorted(STEP2_IO)}") - print(f"authorized Step 3 mapping modules: {sorted(STEP3_MAPPING)}") - print(f"authorized Step 4 row modules: {sorted(STEP4_ROWS)}") - print(f"authorized Step 8 observability modules: {sorted(STEP8_OBSERVABILITY)}") - print(f"protected docstring-only modules: {placeholder_count}") - print("owner metadata: PASS") - print("Phase 3 Step 11: retained Step 9 explicit-pair and inherited definition/import boundaries: PASS") - print("no-algorithm phase boundary: PASS") - return 0 - - -def phase4_main(step: int = 1) -> int: - """Apply the approved Step 1 control before the frozen Phase 3 AST gate.""" - import runpy - - if type(step) is not int or step != 1: - raise ValueError("Only authorized Phase 4 Step 1 traceability is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_traceability_control") - control["audit_phase4"](step=step) - result = main() - print("Phase 4 Step 1: approved governance and frozen Phase 3 traceability: PASS") - return result - - -PHASE4_STEP2_REPORT_SCHEMA_SHA256 = "34b7edb8e8672fca3f3d4f6fe647b4d8967acb6f0435544ba449c1c44f7bc1f6" -PHASE4_STEP2_RESULT_IMPORTS = ["Import(names=[alias(name='math')])", - "Import(names=[alias(name='re')])", - "Import(names=[alias(name='sys')])", - "ImportFrom(module='__future__', names=[alias(name='annotations')], level=0)", - "ImportFrom(module='collections.abc', names=[alias(name='Mapping')], level=0)", - "ImportFrom(module='dataclasses', names=[alias(name='dataclass')], level=0)", - "ImportFrom(module='enum', names=[alias(name='StrEnum')], level=0)", - "ImportFrom(module='types', names=[alias(name='MappingProxyType')], level=0)"] -PHASE4_STEP2_RESULT_FUNCTIONS = ['CanonicalReport.__init__', - 'CanonicalReport.from_dict', - 'CanonicalReport.to_dict', - '_array', - '_base_envelope', - '_build_contract', - '_check', - '_check_nullable_reason', - '_enum', - '_equality_key', - '_fail', - '_field', - '_freeze', - '_integer', - '_json_input', - '_map', - '_matches', - '_metric', - '_nullable', - '_number', - '_object', - '_observed', - '_ref', - '_resolved_contract', - '_same', - '_semantic_checks', - '_strings', - '_text', - '_thaw', - '_walk', - 'report_schema', - 'validate_report'] -PHASE4_STEP2_RESULT_CLASSES = ['CanonicalReport', - 'ExecutionStatus', - 'FieldDefinition', - 'ReportEvidenceClass', - 'ReportStatus', - 'ReportValidationError', - 'RunStatus'] -PHASE4_STEP2_RESULT_CALLS = {'': ['_build_contract', '_freeze', 'tuple'], - 'CanonicalReport': ['dataclass'], - 'CanonicalReport.__init__': ['MappingProxyType', - 'MappingProxyType', - '_freeze', - 'object.__setattr__', - 'validate_report'], - 'CanonicalReport.from_dict': ['CanonicalReport'], - 'CanonicalReport.to_dict': ['_thaw'], - 'FieldDefinition': ['dataclass'], - '_base_envelope': ['_array', - '_array', - '_enum', - '_enum', - '_enum', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_number', - '_number', - '_ref', - '_ref', - '_strings', - '_strings', - '_strings', - '_strings', - '_text', - '_text'], - '_build_contract': ['FieldDefinition', - 'FieldDefinition', - 'FieldDefinition', - 'FieldDefinition', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_array', - '_base_envelope', - '_base_envelope', - '_base_envelope', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_enum', - '_field_entries.append', - '_field_entries.append', - '_field_entries.append', - '_field_entries.append', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_integer', - '_map', - '_map', - '_map', - '_map', - '_map', - '_map', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_metric', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_nullable', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_number', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_object', - '_observed', - '_observed', - '_observed', - '_observed', - '_observed', - '_observed', - '_observed', - '_observed', - '_observed', - '_observed', - '_observed', - '_observed', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_ref', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_strings', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - '_text', - 'list', - 'list', - 'props.update', - 'props.update', - 'props.update', - 'props.update', - 'tuple', - 'unavailable_owners.items', - 'unavailable_variants.append'], - '_check': ['_check', - '_check', - '_check', - '_check', - '_check', - '_check', - '_check', - '_equality_key', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_matches', - '_matches', - '_matches', - '_same', - '_same', - 'any', - 'any', - 'any', - 'enumerate', - 'isinstance', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'math.isfinite', - 're.search', - 'reference.removeprefix', - 'reference.startswith', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'sum', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'value.is_integer', - 'value.items'], - '_check_nullable_reason': ['_fail'], - '_enum': ['list'], - '_equality_key': ['_equality_key', - '_equality_key', - 'frozenset', - 'tuple', - 'type', - 'type', - 'type', - 'type', - 'value.items'], - '_fail': ['ReportValidationError'], - '_field': ['FieldDefinition', - '_base_envelope', - '_field_entries.append', - '_nullable', - '_object', - '_ref', - '_ref', - '_ref', - '_text', - '_text', - 'properties.update', - "result['allOf'].append", - 'tuple'], - '_freeze': ['MappingProxyType', '_freeze', '_freeze', 'tuple', 'type', 'type', 'value.items'], - '_json_input': ['_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_json_input', - '_json_input', - 'active.add', - 'active.remove', - 'enumerate', - 'id', - 'id', - 'id', - 'math.isfinite', - 'set', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'value.encode', - 'value.items'], - '_map': ['_text'], - '_matches': ['_check'], - '_metric': ['_field'], - '_object': ['list', 'list'], - '_observed': ['_field'], - '_resolved_contract': ['_matches', - '_resolved_contract', - '_resolved_contract', - 'contract.get', - 'contract.get', - "contract['$ref'].removeprefix"], - '_same': ['_same', - '_same', - 'all', - 'all', - 'isinstance', - 'isinstance', - 'len', - 'len', - 'set', - 'set', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'zip'], - '_semantic_checks': ['_check_nullable_reason', - '_check_nullable_reason', - '_check_nullable_reason', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_fail', - '_walk', - 'all', - 'all', - 'any', - 'any', - 'any', - 'any', - 'any', - 'bool', - 'contract.get', - 'int', - 'item.get', - 'item.get', - 'item.get', - 'item.get', - 'item.get', - "item['owner_ids'][0].startswith", - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'math.isclose', - 'math.isclose', - 'observability.get', - "payload['capabilities'].items", - "payload['derived_metrics'].get", - "payload['derived_metrics'].get", - "payload['derived_metrics'].get('closure_exposure', {}).get", - "payload['simulations'].items", - 'set', - 'set', - 'set', - 'set', - 'type', - 'type', - 'type'], - '_strings': ['_array', '_text'], - '_thaw': ['_thaw', '_thaw', 'isinstance', 'type', 'type', 'value.items'], - '_walk': ['_resolved_contract', - '_walk', - '_walk', - 'contract.get', - 'contract.get', - 'enumerate', - 'isinstance', - 'properties.get', - 'type', - 'type', - 'value.items'], - 'report_schema': ['_freeze', '_thaw'], - 'validate_report': ['_check', '_json_input', '_semantic_checks']} -PHASE4_STEP2_RESULT_ARITHMETIC = {'_build_contract': ["'PR-004.' + name", - "'PR-004.' + name", - "'PR-006.' + name", - "'PR-008.' + name", - "'derived_metrics.closure_exposure.direct.' + name", - "'derived_metrics.closure_exposure.lineage.' + name", - "'derived_metrics.diversity.by_version.*.' + name", - "'derived_metrics.diversity.by_version.*.' + name", - "'derived_metrics.lineage.' + name", - "'derived_metrics.provenance.' + name", - "'derived_metrics.support.' + name", - "'derived_metrics.support.by_version.*.' + name", - "'observed_facts.content.' + name", - "'observed_facts.lineage.' + name", - "'observed_facts.provenance.' + name", - "'observed_facts.provenance.' + name", - "'observed_facts.provenance.' + name", - "'observed_facts.provenance.' + name", - "'proxy_signals.' + name", - "'simulations.' + name", - "'unavailable_conclusions[].' + name", - '-1', - '-sys.float_info.max', - "owner + '.'", - "owner + '.'", - "owner + '.'", - "owner + '.'", - "owner + '.'", - "owner + '.' + name", - "owner + '.' + name", - "owner + '.' + name", - "owner + '.' + name", - "owner + '.' + name", - "owner + '.unavailable_conclusion'", - "owner + '.unavailable_conclusion'", - "required += ('by_state',)"], - '_check': ["not _same(value, schema['const'])", - "not any((_matches(value, item) for item in schema['anyOf']))", - "not any((_same(value, choice) for choice in schema['enum']))", - "not reference.startswith('#/$defs/')", - 'not valid_type[expected]', - "path + '.'", - "path + '.' + key", - "path + '.'", - "path + '.'", - "path + f'[{index}]'"], - '_json_input': ['not finite', "path + '.'", "path + '.'", 'set | None'], - '_number': ['-sys.float_info.max', 'int | float', 'int | float'], - '_object': ['tuple | list', 'tuple | list | None'], - '_ref': ["'#/$defs/' + name"], - '_semantic_checks': ['-1', - 'included + excluded', - 'included_ids & excluded_ids', - "item['ratio'] * item['denominator']", - "not capability['execution_reason_codes']", - "not capability['execution_scope']", - "not item['reason']", - "not item['reason_codes']", - "not item['required_evidence']", - "not math.isclose(interval['interval_width']['value'], upper - lower, rel_tol=1e-12, " - 'abs_tol=1e-12)', - "not math.isclose(item['ratio'] * item['denominator'], item['numerator'], " - 'rel_tol=1e-12, abs_tol=1e-12)', - 'not present', - "parameters['simulation_horizon'] + 1", - "parameters['simulation_horizon'] + 1", - 'upper - lower'], - '_walk': ['path + (index,)', 'path + (key,)']} -PHASE4_STEP2_RESULT_MUTATIONS = [('CanonicalReport.__init__', "object.__setattr__(self, 'sections', frozen)")] -PHASE4_STEP2_RESULT_DECLARATIONS = [('', - "CAPABILITY_KEYS = ('ingestion', 'content_diagnostics', 'provenance', 'lineage', 'dataset_longitudinal', " - "'model_longitudinal', 'intervention_simulation')"), - ('', 'FIELD_REGISTRY = tuple(_field_entries)'), - ('', - "LEVEL_LABELS = ('ingest_observability', 'content_or_representation_observability', " - "'provenance_observability', 'lineage_observability', 'longitudinal_dataset_observability', " - "'experimental_intervention_or_scenario_observability')"), - ('', - "RUN_NULLABLE_FIELDS = ('started_at', 'completed_at', 'duration_seconds', 'python_version', 'platform', " - "'command', 'config_hash', 'random_seed')"), - ('', - "SECTION_ORDER = ('run', 'inputs', 'observability', 'capabilities', 'observed_facts', 'derived_metrics', " - "'proxy_signals', 'simulations', 'unavailable_conclusions', 'recommended_next_metadata', 'warnings', " - "'errors')"), - ('', '_REPORT_SCHEMA = _build_contract()'), - ('', '_REPORT_SCHEMA = _freeze(_REPORT_SCHEMA)'), - ('', '_field_entries: list[FieldDefinition] = []'), - ('', 'del _field_entries'), - ('CanonicalReport', 'sections: Mapping'), - ('ExecutionStatus', "COMPLETED = 'completed'"), - ('ExecutionStatus', "DEFERRED = 'deferred'"), - ('ExecutionStatus', "FAILED = 'failed'"), - ('ExecutionStatus', "NOT_REQUESTED = 'not_requested'"), - ('ExecutionStatus', "PARTIAL = 'partial'"), - ('FieldDefinition', 'evidence_class: str'), - ('FieldDefinition', 'method_id: str'), - ('FieldDefinition', 'minimum_level: int'), - ('FieldDefinition', 'owner: str'), - ('FieldDefinition', 'path: str'), - ('FieldDefinition', 'requires_representation: bool = False'), - ('FieldDefinition', "status_boundary: str = 'current'"), - ('FieldDefinition', 'unit: str'), - ('FieldDefinition', 'value_type: str'), - ('ReportEvidenceClass', "DERIVED_METRIC = 'derived_metric'"), - ('ReportEvidenceClass', "OBSERVED_FACT = 'observed_fact'"), - ('ReportEvidenceClass', "PROXY_SIGNAL = 'proxy_signal'"), - ('ReportEvidenceClass', "SIMULATION = 'simulation'"), - ('ReportEvidenceClass', "UNAVAILABLE_CONCLUSION = 'unavailable_conclusion'"), - ('ReportStatus', "AVAILABLE = 'available'"), - ('ReportStatus', "EXPERIMENTAL = 'experimental'"), - ('ReportStatus', "PARTIAL = 'partial'"), - ('ReportStatus', "UNAVAILABLE = 'unavailable'"), - ('RunStatus', "COMPLETE = 'complete'"), - ('RunStatus', "FAILED = 'failed'"), - ('RunStatus', "PARTIAL = 'partial'")] -PHASE4_STEP2_RESULT_TRAVERSAL = [('', 'del _field_entries'), - ('_walk', '(yield (path, value, contract))'), - ('_walk', "(yield from _walk(item, path + (index,), contract['items']))"), - ('_walk', '(yield from _walk(item, path + (key,), child))')] -PHASE4_STEP2_RESULT_ATTRIBUTES = {'CanonicalReport.__init__': ['object.__setattr__'], - 'CanonicalReport.to_dict': ['self.sections'], - '_build_contract': ['_field_entries.append', - '_field_entries.append', - '_field_entries.append', - '_field_entries.append', - 'item.value', - 'props.update', - 'props.update', - 'props.update', - 'props.update', - 'sys.float_info', - 'sys.float_info', - 'sys.float_info.max', - 'sys.float_info.max', - 'unavailable_owners.items', - 'unavailable_variants.append'], - '_check': ['math.isfinite', - 're.search', - 'reference.removeprefix', - 'reference.startswith', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'schema.get', - 'value.is_integer', - 'value.items'], - '_equality_key': ['value.items'], - '_field': ['_field_entries.append', 'properties.update', "result['allOf'].append"], - '_freeze': ['value.items'], - '_integer': ['sys.float_info', 'sys.float_info.max'], - '_json_input': ['active.add', 'active.remove', 'math.isfinite', 'value.encode', 'value.items'], - '_number': ['sys.float_info', 'sys.float_info', 'sys.float_info.max', 'sys.float_info.max'], - '_resolved_contract': ['contract.get', 'contract.get', "contract['$ref'].removeprefix"], - '_semantic_checks': ['contract.get', - 'item.get', - 'item.get', - 'item.get', - 'item.get', - 'item.get', - "item['owner_ids'][0].startswith", - 'math.isclose', - 'math.isclose', - 'observability.get', - "payload['capabilities'].items", - "payload['derived_metrics'].get", - "payload['derived_metrics'].get", - "payload['derived_metrics'].get('closure_exposure', {}).get", - "payload['simulations'].items"], - '_thaw': ['value.items'], - '_walk': ['contract.get', 'contract.get', 'properties.get', 'value.items']} -PHASE4_STEP2_RESULT_HEADERS = [('CanonicalReport', (), (), ('dataclass(frozen=True, slots=True, init=False)',)), - ('CanonicalReport.__init__', - "arguments(posonlyargs=[], args=[arg(arg='self'), arg(arg='payload', annotation=Name(id='dict', " - 'ctx=Load()))], kwonlyargs=[], kw_defaults=[], defaults=[])', - (), - 'None'), - ('CanonicalReport.from_dict', - "arguments(posonlyargs=[], args=[arg(arg='cls'), arg(arg='payload', annotation=Name(id='dict', " - 'ctx=Load()))], kwonlyargs=[], kw_defaults=[], defaults=[])', - ('classmethod',), - "'CanonicalReport'"), - ('CanonicalReport.to_dict', - "arguments(posonlyargs=[], args=[arg(arg='self')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - 'dict'), - ('ExecutionStatus', ('StrEnum',), (), ()), - ('FieldDefinition', (), (), ('dataclass(frozen=True, slots=True)',)), - ('ReportEvidenceClass', ('StrEnum',), (), ()), - ('ReportStatus', ('StrEnum',), (), ()), - ('ReportValidationError', ('ValueError',), (), ()), - ('RunStatus', ('StrEnum',), (), ()), - ('_array', - "arguments(posonlyargs=[], args=[arg(arg='item', annotation=Name(id='dict', ctx=Load()))], " - "kwonlyargs=[arg(arg='minimum', annotation=Name(id='int', ctx=Load())), arg(arg='unique', " - "annotation=Name(id='bool', ctx=Load()))], kw_defaults=[Constant(value=0), Constant(value=False)], " - 'defaults=[])', - (), - 'dict'), - ('_base_envelope', - "arguments(posonlyargs=[], args=[arg(arg='evidence', annotation=Name(id='str', ctx=Load())), " - "arg(arg='unit', annotation=Name(id='str', ctx=Load())), arg(arg='owner', annotation=Name(id='str', " - "ctx=Load())), arg(arg='method', annotation=Name(id='str', ctx=Load()))], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - 'dict'), - ('_build_contract', - 'arguments(posonlyargs=[], args=[], kwonlyargs=[], kw_defaults=[], defaults=[])', - (), - 'dict'), - ('_check', - "arguments(posonlyargs=[], args=[arg(arg='value'), arg(arg='schema', annotation=Name(id='dict', " - "ctx=Load())), arg(arg='path', annotation=Name(id='str', ctx=Load()))], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - 'None'), - ('_check_nullable_reason', - "arguments(posonlyargs=[], args=[arg(arg='value', annotation=Name(id='dict', ctx=Load())), arg(arg='name', " - "annotation=Name(id='str', ctx=Load())), arg(arg='reason_name', annotation=Name(id='str', ctx=Load())), " - "arg(arg='path', annotation=Name(id='str', ctx=Load()))], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - 'None'), - ('_enum', - "arguments(posonlyargs=[], args=[], vararg=arg(arg='values', annotation=Name(id='str', ctx=Load())), " - 'kwonlyargs=[], kw_defaults=[], defaults=[])', - (), - 'dict'), - ('_equality_key', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_fail', - "arguments(posonlyargs=[], args=[arg(arg='path', annotation=Name(id='str', ctx=Load())), arg(arg='rule', " - "annotation=Name(id='str', ctx=Load()))], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - 'None'), - ('_field', - "arguments(posonlyargs=[], args=[arg(arg='path', annotation=Name(id='str', ctx=Load())), arg(arg='value', " - "annotation=Name(id='dict', ctx=Load()))], kwonlyargs=[arg(arg='owner', annotation=Name(id='str', " - "ctx=Load())), arg(arg='evidence', annotation=Name(id='str', ctx=Load())), arg(arg='unit', " - "annotation=Name(id='str', ctx=Load())), arg(arg='method', annotation=Name(id='str', ctx=Load())), " - "arg(arg='level', annotation=Name(id='int', ctx=Load())), arg(arg='value_type', annotation=Name(id='str', " - "ctx=Load())), arg(arg='representation', annotation=Name(id='bool', ctx=Load())), arg(arg='boundary', " - "annotation=Name(id='str', ctx=Load()))], kw_defaults=[None, None, None, None, None, None, " - "Constant(value=False), Constant(value='current')], defaults=[])", - (), - 'dict'), - ('_freeze', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_integer', - "arguments(posonlyargs=[], args=[], kwonlyargs=[arg(arg='minimum', annotation=Name(id='int', " - 'ctx=Load()))], kw_defaults=[Constant(value=0)], defaults=[])', - (), - 'dict'), - ('_json_input', - "arguments(posonlyargs=[], args=[arg(arg='value'), arg(arg='path', annotation=Name(id='str', ctx=Load())), " - "arg(arg='active', annotation=BinOp(left=Name(id='set', ctx=Load()), op=BitOr(), " - "right=Constant(value=None)))], kwonlyargs=[], kw_defaults=[], defaults=[Constant(value='$'), " - 'Constant(value=None)])', - (), - 'None'), - ('_map', - "arguments(posonlyargs=[], args=[arg(arg='values', annotation=Name(id='dict', ctx=Load()))], " - 'kwonlyargs=[], kw_defaults=[], defaults=[])', - (), - 'dict'), - ('_matches', - "arguments(posonlyargs=[], args=[arg(arg='value'), arg(arg='schema', annotation=Name(id='dict', " - 'ctx=Load()))], kwonlyargs=[], kw_defaults=[], defaults=[])', - (), - 'bool'), - ('_metric', - "arguments(posonlyargs=[], args=[arg(arg='path', annotation=Name(id='str', ctx=Load())), arg(arg='value', " - "annotation=Name(id='dict', ctx=Load())), arg(arg='owner', annotation=Name(id='str', ctx=Load())), " - "arg(arg='unit', annotation=Name(id='str', ctx=Load())), arg(arg='method', annotation=Name(id='str', " - "ctx=Load())), arg(arg='level', annotation=Name(id='int', ctx=Load())), arg(arg='value_type', " - "annotation=Name(id='str', ctx=Load()))], kwonlyargs=[arg(arg='representation', annotation=Name(id='bool', " - "ctx=Load())), arg(arg='boundary', annotation=Name(id='str', ctx=Load()))], " - "kw_defaults=[Constant(value=False), Constant(value='current')], defaults=[])", - (), - 'dict'), - ('_nullable', - "arguments(posonlyargs=[], args=[arg(arg='schema', annotation=Name(id='dict', ctx=Load()))], " - 'kwonlyargs=[], kw_defaults=[], defaults=[])', - (), - 'dict'), - ('_number', - "arguments(posonlyargs=[], args=[], kwonlyargs=[arg(arg='minimum', annotation=BinOp(left=Name(id='int', " - "ctx=Load()), op=BitOr(), right=Name(id='float', ctx=Load()))), arg(arg='maximum', " - "annotation=BinOp(left=Name(id='int', ctx=Load()), op=BitOr(), right=Name(id='float', ctx=Load())))], " - "kw_defaults=[UnaryOp(op=USub(), operand=Attribute(value=Attribute(value=Name(id='sys', ctx=Load()), " - "attr='float_info', ctx=Load()), attr='max', ctx=Load())), Attribute(value=Attribute(value=Name(id='sys', " - "ctx=Load()), attr='float_info', ctx=Load()), attr='max', ctx=Load())], defaults=[])", - (), - 'dict'), - ('_object', - "arguments(posonlyargs=[], args=[arg(arg='properties', annotation=Name(id='dict', ctx=Load())), " - "arg(arg='required', annotation=BinOp(left=BinOp(left=Name(id='tuple', ctx=Load()), op=BitOr(), " - "right=Name(id='list', ctx=Load())), op=BitOr(), right=Constant(value=None)))], kwonlyargs=[], " - 'kw_defaults=[], defaults=[Constant(value=None)])', - (), - 'dict'), - ('_observed', - "arguments(posonlyargs=[], args=[arg(arg='path', annotation=Name(id='str', ctx=Load())), arg(arg='value', " - "annotation=Name(id='dict', ctx=Load())), arg(arg='owner', annotation=Name(id='str', ctx=Load())), " - "arg(arg='unit', annotation=Name(id='str', ctx=Load())), arg(arg='method', annotation=Name(id='str', " - "ctx=Load())), arg(arg='level', annotation=Name(id='int', ctx=Load())), arg(arg='value_type', " - "annotation=Name(id='str', ctx=Load()))], kwonlyargs=[arg(arg='boundary', annotation=Name(id='str', " - "ctx=Load())), arg(arg='representation', annotation=Name(id='bool', ctx=Load()))], " - "kw_defaults=[Constant(value='current'), Constant(value=False)], defaults=[])", - (), - 'dict'), - ('_ref', - "arguments(posonlyargs=[], args=[arg(arg='name', annotation=Name(id='str', ctx=Load()))], kwonlyargs=[], " - 'kw_defaults=[], defaults=[])', - (), - 'dict'), - ('_resolved_contract', - "arguments(posonlyargs=[], args=[arg(arg='value'), arg(arg='contract')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - ''), - ('_same', - "arguments(posonlyargs=[], args=[arg(arg='left'), arg(arg='right')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - 'bool'), - ('_semantic_checks', - "arguments(posonlyargs=[], args=[arg(arg='payload', annotation=Name(id='dict', ctx=Load()))], " - 'kwonlyargs=[], kw_defaults=[], defaults=[])', - (), - 'None'), - ('_strings', - "arguments(posonlyargs=[], args=[], kwonlyargs=[arg(arg='minimum', annotation=Name(id='int', " - 'ctx=Load()))], kw_defaults=[Constant(value=0)], defaults=[])', - (), - 'dict'), - ('_text', - "arguments(posonlyargs=[], args=[], kwonlyargs=[arg(arg='empty', annotation=Name(id='bool', ctx=Load()))], " - 'kw_defaults=[Constant(value=False)], defaults=[])', - (), - 'dict'), - ('_thaw', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_walk', - "arguments(posonlyargs=[], args=[arg(arg='value'), arg(arg='path'), arg(arg='contract')], kwonlyargs=[], " - 'kw_defaults=[], defaults=[Tuple(elts=[], ctx=Load()), Constant(value=None)])', - (), - ''), - ('report_schema', - 'arguments(posonlyargs=[], args=[], kwonlyargs=[], kw_defaults=[], defaults=[])', - (), - 'dict'), - ('validate_report', - "arguments(posonlyargs=[], args=[arg(arg='payload', annotation=Name(id='dict', ctx=Load()))], " - 'kwonlyargs=[], kw_defaults=[], defaults=[])', - (), - 'None')] - - -def phase4_step2_result_boundary(path: Path) -> None: - """Inspect only the supplied canonical-result source without executing it. - - Fixed reviewed inventories constrain definitions, imports, call targets and - validation arithmetic. They are never regenerated from inspected source. - The release controller separately freezes the other 39 runtime modules. - """ - try: - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - except (OSError, UnicodeError, SyntaxError) as exc: - raise ValueError("Phase 4 Step 2 result source cannot be inspected") from exc - doc = ast.get_docstring(tree) or "" - if "Owner IDs:" not in doc or "Current phase status:" not in doc: - raise ValueError("Phase 4 Step 2 result ownership metadata is missing") - allowed_top = (ast.Import, ast.ImportFrom, ast.Assign, ast.AnnAssign, - ast.FunctionDef, ast.ClassDef, ast.Delete) - for index, node in enumerate(tree.body): - if index == 0 and isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str): - continue - if not isinstance(node, allowed_top): - raise ValueError("Phase 4 Step 2 result has an unapproved module statement") - imports = [] - functions = [] - classes = [] - calls = {} - arithmetic = {} - mutations = [] - declarations = [] - traversal = [] - attributes = {} - headers = [] - call_names = {node.func.id for node in ast.walk(tree) - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)} - forbidden = (ast.Lambda, ast.AsyncFunctionDef, ast.Await, ast.With, - ast.AsyncWith, ast.Global, ast.Nonlocal, ast.NamedExpr) - forbidden_names = {"eval", "exec", "compile", "__import__", "getattr", - "setattr", "delattr", "globals", "locals", "vars", - "open", "input", "breakpoint", "print", "exit", "quit"} - forbidden_attributes = {"__builtins__", "__globals__", "__code__", "__subclasses__", - "__class__", "__dict__", "__getattribute__", "__getattr__", - "open", "read", "read_text", "read_bytes", "write", "write_text", - "write_bytes", "mkdir", "unlink", "system", "popen", "connect", - "connect_ex", "getaddrinfo", "urlopen", "urlretrieve", "send", "sendall"} - stack = [(tree, "")] - while stack: - node, scope = stack.pop() - if isinstance(node, forbidden): - raise ValueError("Phase 4 Step 2 result has a callback or executable control outside scope") - if isinstance(node, (ast.FunctionDef, ast.ClassDef)): - scope = node.name if scope == "" else scope + "." + node.name - if isinstance(node, ast.FunctionDef): - functions.append(scope) - headers.append((scope, ast.dump(node.args, include_attributes=False), - tuple(ast.unparse(item) for item in node.decorator_list), - ast.unparse(node.returns) if node.returns is not None else "")) - else: - classes.append(scope) - headers.append((scope, tuple(ast.unparse(item) for item in node.bases), - tuple(ast.unparse(item) for item in node.keywords), - tuple(ast.unparse(item) for item in node.decorator_list))) - if isinstance(node, (ast.Import, ast.ImportFrom)): - if scope != "" or node not in tree.body: - raise ValueError("Phase 4 Step 2 result imports must be explicit and at module scope") - imports.append(ast.dump(node, include_attributes=False)) - if isinstance(node, ast.Name) and node.id in forbidden_names: - raise ValueError("Phase 4 Step 2 result names a forbidden execution capability") - if scope != "" and ((isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store) and node.id in call_names) - or (isinstance(node, ast.arg) and node.arg in call_names)): - raise ValueError("Phase 4 Step 2 result shadows an approved call target") - if isinstance(node, ast.Attribute): - attributes.setdefault(scope, []).append(ast.unparse(node)) - if isinstance(node, ast.Attribute) and node.attr in forbidden_attributes: - raise ValueError("Phase 4 Step 2 result names a forbidden IO or dynamic capability") - if isinstance(node, ast.Call): - if not isinstance(node.func, (ast.Name, ast.Attribute)): - raise ValueError("Phase 4 Step 2 result invokes a dynamic call target") - calls.setdefault(scope, []).append(ast.unparse(node.func)) - if isinstance(node.func, ast.Attribute) and node.func.attr == "__setattr__": - mutations.append((scope, ast.unparse(node))) - if isinstance(node, (ast.BinOp, ast.UnaryOp, ast.AugAssign)): - arithmetic.setdefault(scope, []).append(ast.unparse(node)) - if (scope == "" or scope in classes) and isinstance(node, (ast.Assign, ast.AnnAssign, ast.Delete)): - declarations.append((scope, ast.unparse(node))) - if isinstance(node, (ast.Yield, ast.YieldFrom, ast.Delete)): - traversal.append((scope, ast.unparse(node))) - if isinstance(node, ast.Attribute) and isinstance(node.ctx, (ast.Store, ast.Del)): - raise ValueError("Phase 4 Step 2 result has an unapproved attribute mutation") - stack.extend((child, scope) for child in ast.iter_child_nodes(node)) - if sorted(imports) != PHASE4_STEP2_RESULT_IMPORTS: - raise ValueError("Phase 4 Step 2 result imports differ from the reviewed stdlib inventory") - if sorted(functions) != PHASE4_STEP2_RESULT_FUNCTIONS or sorted(classes) != PHASE4_STEP2_RESULT_CLASSES: - raise ValueError("Phase 4 Step 2 result definitions differ from the approved contract") - if {name: sorted(values) for name, values in calls.items()} != PHASE4_STEP2_RESULT_CALLS: - raise ValueError("Phase 4 Step 2 result call targets differ from the reviewed inventory") - if {name: sorted(values) for name, values in arithmetic.items()} != PHASE4_STEP2_RESULT_ARITHMETIC: - raise ValueError("Phase 4 Step 2 result arithmetic exceeds reviewed semantic validation") - if sorted(mutations) != PHASE4_STEP2_RESULT_MUTATIONS: - raise ValueError("Phase 4 Step 2 result mutation differs from reviewed immutable construction") - if sorted(declarations) != PHASE4_STEP2_RESULT_DECLARATIONS: - raise ValueError("Phase 4 Step 2 result module declarations differ from the approved contract") - if sorted(traversal) != PHASE4_STEP2_RESULT_TRAVERSAL: - raise ValueError("Phase 4 Step 2 result generator or teardown differs from reviewed local traversal") - if {name: sorted(values) for name, values in attributes.items()} != PHASE4_STEP2_RESULT_ATTRIBUTES: - raise ValueError("Phase 4 Step 2 result attributes exceed the reviewed pure contract") - if sorted(headers) != PHASE4_STEP2_RESULT_HEADERS: - raise ValueError("Phase 4 Step 2 result definition headers differ from the reviewed contract") - - -def phase4_step2_main(step: int = 2) -> int: - """Validate Step 2 control, reviewed schema bytes and the result AST only.""" - import hashlib - import runpy - - if type(step) is not int or step != 2: - raise ValueError("Only authorized Phase 4 Step 2 traceability is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step2_traceability_control") - control["audit_phase4_step2"](step=step) - schema_path = ROOT / "schemas/report.schema.json" - if hashlib.sha256(schema_path.read_bytes()).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Phase 4 Step 2 report schema differs from the reviewed contract") - phase4_step2_result_boundary(PACKAGE / "result.py") - print("Phase 4 Step 2: canonical results and reviewed local schema: PASS") - return 0 - - - - -# Step 3 reviewed pure adapters. These independent inventories are never -# regenerated by the active gate from inspected source. -PHASE4_STEP3_ASSEMBLY_IMPORTS = ["ImportFrom(module='__future__', names=[alias(name='annotations')], level=0)", - "ImportFrom(module='dataclasses', names=[alias(name='dataclass')], level=0)", - "ImportFrom(module='math', names=[alias(name='isfinite'), alias(name='isclose')], level=0)", - "ImportFrom(module='metrics.bounds', names=[alias(name='DirectClosureExposureBounds')], level=2)", - "ImportFrom(module='metrics.diversity', names=[alias(name='DistributionMetrics'), " - "alias(name='StateDistributionResult'), alias(name='SupportComparison'), alias(name='StateFrequency')], " - 'level=2)', - "ImportFrom(module='metrics.duplicates', names=[alias(name='ExactDuplicateResult'), " - "alias(name='ExactDuplicateGroup')], level=2)", - "ImportFrom(module='metrics.provenance', names=[alias(name='ProvenanceCompositionResult'), " - "alias(name='DeclaredComposition'), alias(name='WeightedSourceComposition'), " - "alias(name='DirectGroundingBasis'), alias(name='DirectGroundingAssignment')], level=2)", - "ImportFrom(module='metrics.resampling', names=[alias(name='ExpectedDiversityResult'), " - "alias(name='ResamplingSimulation'), alias(name='ResamplingInput'), alias(name='ResamplingReplicate'), " - "alias(name='SampledGeneration')], level=2)", - "ImportFrom(module='metrics.tail', names=[alias(name='TailSelectionResult'), " - "alias(name='ExtinctionProbabilityResult'), alias(name='RarityEntry')], level=2)", - "ImportFrom(module='models', names=[alias(name='BundleValidationResult'), " - "alias(name='CalculationEvidenceClass'), alias(name='CalculationMetadata'), " - "alias(name='CalculationReason'), alias(name='CalculationScope'), alias(name='CalculationStatus'), " - "alias(name='CapabilityKey'), alias(name='CanonicalRow'), alias(name='NumericalPolicy'), " - "alias(name='RecordKey'), alias(name='RepresentationDescriptor'), alias(name='ScalarCalculation'), " - "alias(name='ValidationCoverage'), alias(name='ValidationMessage'), alias(name='ValidationSeverity'), " - "alias(name='WeightingOptions'), alias(name='Capability'), alias(name='CapabilityStatus'), " - "alias(name='ObservabilityAssessment'), alias(name='FileInventoryEntry'), alias(name='FileRole'), " - "alias(name='FileFormat'), alias(name='VersionOrderResult'), alias(name='ProvenanceJoinResult'), " - "alias(name='ProvenanceMatch'), alias(name='ProvenanceAssessment'), alias(name='ExplicitPairContext'), " - "alias(name='RowMappingEvidence'), alias(name='RowLocation'), alias(name='GenerationValidationResult'), " - "alias(name='RecordStateAssignment'), alias(name='TailSelectionOptions')], level=2)", - "ImportFrom(module='pathlib', names=[alias(name='PosixPath'), alias(name='WindowsPath')], level=0)", - "ImportFrom(module='representations.compatibility', names=[alias(name='RepresentationCompatibility'), " - "alias(name='StateMappingDeclaration')], level=2)", - "ImportFrom(module='result', names=[alias(name='CanonicalReport'), alias(name='FIELD_REGISTRY'), " - "alias(name='LEVEL_LABELS'), alias(name='SECTION_ORDER'), alias(name='ReportValidationError')], level=2)", - "ImportFrom(module='types', names=[alias(name='MappingProxyType')], level=0)"] - -PHASE4_STEP3_ASSEMBLY_FUNCTIONS = ['FamilyFailure.__post_init__', - '_base', - '_bundle_check', - '_bundle_scope', - '_capabilities', - '_closed_simulation', - '_closure', - '_comparison', - '_coverage', - '_definition', - '_diagnostics', - '_disclosure_base', - '_disclosure_recommend', - '_disclosure_scope', - '_disclosure_signal', - '_disclosure_unavailable', - '_disclosures', - '_distribution', - '_distribution_exclusions', - '_duplicates', - '_envelope', - '_extinction', - '_inputs', - '_key', - '_lineage_observations', - '_message_family', - '_metadata', - '_metadata_text', - '_number', - '_policy', - '_provenance', - '_provenance_consistency', - '_representation', - '_require', - '_scalar', - '_scope', - '_simulation_inputs', - '_table', - '_tail', - '_typed', - 'assemble_report'] - -PHASE4_STEP3_ASSEMBLY_CLASSES = ['FamilyFailure', 'ReportAssemblyError'] - -PHASE4_STEP3_ASSEMBLY_MUTATIONS = [] - -PHASE4_STEP3_ASSEMBLY_DECLARATIONS = [('FamilyFailure', 'capability: CapabilityKey'), ('FamilyFailure', 'messages: tuple[ValidationMessage, ...]')] - -PHASE4_STEP3_ASSEMBLY_TRAVERSAL = [] - -PHASE4_STEP3_ASSEMBLY_HEADERS = [('FamilyFailure', (), (), ('dataclass(frozen=True, slots=True)',)), - ('FamilyFailure.__post_init__', - "arguments(posonlyargs=[], args=[arg(arg='self')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('ReportAssemblyError', ('ReportValidationError',), (), ()), - ('_base', - "arguments(posonlyargs=[], args=[arg(arg='path'), arg(arg='scope')], " - "kwonlyargs=[arg(arg='representation'), arg(arg='coverage'), arg(arg='denominator'), " - "arg(arg='assumptions'), arg(arg='limitations'), arg(arg='status'), arg(arg='reasons'), " - "arg(arg='required')], kw_defaults=[Constant(value=None), Constant(value=None), Constant(value=None), " - "Tuple(elts=[], ctx=Load()), Tuple(elts=[], ctx=Load()), Constant(value='available'), Tuple(elts=[], " - 'ctx=Load()), Tuple(elts=[], ctx=Load())], defaults=[])', - (), - ''), - ('_bundle_check', - "arguments(posonlyargs=[], args=[arg(arg='bundle')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_bundle_scope', - "arguments(posonlyargs=[], args=[arg(arg='bundle')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_capabilities', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='bundle'), arg(arg='operations'), " - "arg(arg='failures')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_closed_simulation', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='value')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - ''), - ('_closure', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='value'), arg(arg='bundle'), " - "arg(arg='provenance')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_comparison', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='value'), arg(arg='bundle')], kwonlyargs=[], " - 'kw_defaults=[], defaults=[])', - (), - ''), - ('_coverage', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_definition', - "arguments(posonlyargs=[], args=[arg(arg='path')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_diagnostics', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='messages'), arg(arg='family')], " - 'kwonlyargs=[], kw_defaults=[], defaults=[Constant(value=None)])', - (), - ''), - ('_disclosure_base', - "arguments(posonlyargs=[], args=[arg(arg='owner'), arg(arg='evidence_class'), arg(arg='method'), " - "arg(arg='scope'), arg(arg='basis')], kwonlyargs=[], kw_defaults=[], defaults=[Constant(value=None)])", - (), - ''), - ('_disclosure_recommend', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='priority'), arg(arg='metadata'), " - "arg(arg='scope'), arg(arg='unlock'), arg(arg='reason')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_disclosure_scope', - "arguments(posonlyargs=[], args=[arg(arg='payload')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_disclosure_signal', - "arguments(posonlyargs=[], args=[arg(arg='name'), arg(arg='owner'), arg(arg='basis'), arg(arg='paths'), " - "arg(arg='present'), arg(arg='rule'), arg(arg='limitations')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_disclosure_unavailable', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='name'), arg(arg='owner'), " - "arg(arg='capability'), arg(arg='statement'), arg(arg='reasons'), arg(arg='blockers'), " - "arg(arg='next_metadata'), arg(arg='limit')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_disclosures', - "arguments(posonlyargs=[], args=[arg(arg='payload')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_distribution', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='value'), arg(arg='bundle')], " - "kwonlyargs=[arg(arg='weighted'), arg(arg='coverage')], kw_defaults=[Constant(value=False), " - 'Constant(value=None)], defaults=[])', - (), - ''), - ('_distribution_exclusions', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='value')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - ''), - ('_duplicates', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='value'), arg(arg='bundle')], kwonlyargs=[], " - 'kw_defaults=[], defaults=[])', - (), - ''), - ('_envelope', - "arguments(posonlyargs=[], args=[arg(arg='path'), arg(arg='value'), arg(arg='scope')], kwonlyargs=[], " - "kw_defaults=[], kwarg=arg(arg='kwargs'), defaults=[])", - (), - ''), - ('_extinction', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='values')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - ''), - ('_inputs', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='bundle')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - ''), - ('_key', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_lineage_observations', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='bundle')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - ''), - ('_message_family', - "arguments(posonlyargs=[], args=[arg(arg='message')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_metadata', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[arg(arg='name'), arg(arg='scope'), " - "arg(arg='representation'), arg(arg='owner'), arg(arg='evidence'), arg(arg='unit'), arg(arg='formula'), " - "arg(arg='weighting')], kw_defaults=[None, None, None, None, None, None, Constant(value=None), " - 'Constant(value=None)], defaults=[])', - (), - ''), - ('_metadata_text', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_number', - "arguments(posonlyargs=[], args=[arg(arg='value'), arg(arg='name')], kwonlyargs=[arg(arg='integer'), " - "arg(arg='minimum'), arg(arg='maximum')], kw_defaults=[Constant(value=False), Constant(value=None), " - 'Constant(value=None)], defaults=[])', - (), - ''), - ('_policy', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_provenance', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='value'), arg(arg='bundle')], kwonlyargs=[], " - 'kw_defaults=[], defaults=[])', - (), - ''), - ('_provenance_consistency', - "arguments(posonlyargs=[], args=[arg(arg='value'), arg(arg='bundle')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - ''), - ('_representation', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_require', - "arguments(posonlyargs=[], args=[arg(arg='condition'), arg(arg='message')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - ''), - ('_scalar', - "arguments(posonlyargs=[], args=[arg(arg='path'), arg(arg='value'), arg(arg='scope'), " - "arg(arg='representation'), arg(arg='bundle')], kwonlyargs=[arg(arg='name'), arg(arg='denominator'), " - "arg(arg='coverage'), arg(arg='weighting'), arg(arg='limitations'), arg(arg='formula')], " - 'kw_defaults=[Constant(value=None), Constant(value=None), Constant(value=None), Constant(value=None), ' - 'Tuple(elts=[], ctx=Load()), Constant(value=None)], defaults=[])', - (), - ''), - ('_scope', - "arguments(posonlyargs=[], args=[arg(arg='value'), arg(arg='bundle')], kwonlyargs=[], kw_defaults=[], " - 'defaults=[])', - (), - ''), - ('_simulation_inputs', - "arguments(posonlyargs=[], args=[arg(arg='value')], kwonlyargs=[], kw_defaults=[], defaults=[])", - (), - ''), - ('_table', - "arguments(posonlyargs=[], args=[arg(arg='path'), arg(arg='value'), arg(arg='metadata'), arg(arg='scope'), " - "arg(arg='representation'), arg(arg='bundle')], kwonlyargs=[arg(arg='name'), arg(arg='status'), " - "arg(arg='reasons'), arg(arg='denominator'), arg(arg='coverage'), arg(arg='weighting'), " - "arg(arg='limitations'), arg(arg='formula')], kw_defaults=[None, None, Tuple(elts=[], ctx=Load()), " - 'Constant(value=None), Constant(value=None), Constant(value=None), Tuple(elts=[], ctx=Load()), ' - 'Constant(value=None)], defaults=[])', - (), - ''), - ('_tail', - "arguments(posonlyargs=[], args=[arg(arg='payload'), arg(arg='value'), arg(arg='bundle')], kwonlyargs=[], " - 'kw_defaults=[], defaults=[])', - (), - ''), - ('_typed', - "arguments(posonlyargs=[], args=[arg(arg='value'), arg(arg='expected'), arg(arg='name')], kwonlyargs=[], " - 'kw_defaults=[], defaults=[])', - (), - ''), - ('assemble_report', - "arguments(posonlyargs=[], args=[arg(arg='bundle', annotation=Name(id='BundleValidationResult', " - "ctx=Load()))], kwonlyargs=[arg(arg='run', annotation=Name(id='dict', ctx=Load())), " - "arg(arg='distributions', annotation=Subscript(value=Name(id='tuple', ctx=Load()), " - "slice=Tuple(elts=[BinOp(left=Name(id='StateDistributionResult', ctx=Load()), op=BitOr(), " - "right=Name(id='DistributionMetrics', ctx=Load())), Constant(value=Ellipsis)], ctx=Load()), ctx=Load())), " - "arg(arg='provenance', annotation=BinOp(left=Name(id='ProvenanceCompositionResult', ctx=Load()), " - "op=BitOr(), right=Constant(value=None))), arg(arg='duplicates', " - "annotation=BinOp(left=Name(id='ExactDuplicateResult', ctx=Load()), op=BitOr(), " - "right=Constant(value=None))), arg(arg='tail', annotation=BinOp(left=Name(id='TailSelectionResult', " - "ctx=Load()), op=BitOr(), right=Constant(value=None))), arg(arg='comparison', " - "annotation=BinOp(left=Name(id='SupportComparison', ctx=Load()), op=BitOr(), right=Constant(value=None))), " - "arg(arg='closure', annotation=BinOp(left=Name(id='DirectClosureExposureBounds', ctx=Load()), op=BitOr(), " - "right=Constant(value=None))), arg(arg='expected_diversity', " - "annotation=BinOp(left=Name(id='ExpectedDiversityResult', ctx=Load()), op=BitOr(), " - "right=Constant(value=None))), arg(arg='resampling', annotation=BinOp(left=Name(id='ResamplingSimulation', " - "ctx=Load()), op=BitOr(), right=Constant(value=None))), arg(arg='extinction', " - "annotation=Subscript(value=Name(id='tuple', ctx=Load()), " - "slice=Tuple(elts=[Name(id='ExtinctionProbabilityResult', ctx=Load()), Constant(value=Ellipsis)], " - "ctx=Load()), ctx=Load())), arg(arg='family_errors', annotation=Subscript(value=Name(id='tuple', " - "ctx=Load()), slice=Tuple(elts=[Name(id='FamilyFailure', ctx=Load()), Constant(value=Ellipsis)], " - 'ctx=Load()), ctx=Load()))], kw_defaults=[None, Tuple(elts=[], ctx=Load()), Constant(value=None), ' - 'Constant(value=None), Constant(value=None), Constant(value=None), Constant(value=None), ' - 'Constant(value=None), Constant(value=None), Tuple(elts=[], ctx=Load()), Tuple(elts=[], ctx=Load())], ' - 'defaults=[])', - (), - 'CanonicalReport')] - -PHASE4_STEP3_ASSEMBLY_LITERAL_DECLARATIONS = [('_metadata_text', "allowed = ('sampled_path; sequential_binomial_complement_v1',)"), - ('_metadata_text', 'allowed = methods.get(name, ())'), - ('_metadata_text', - "assumptions = ('Approved direct grounding partition; uncertainty remains unresolved.',)"), - ('_metadata_text', - "assumptions = ('Categorical multinomial sampling; positive explicit n; one-step horizon; no external " - "reopening.' if value.evidence_class is CalculationEvidenceClass.SIMULATION else 'Explicit rule over " - "positive observed unweighted count support.',)"), - ('_metadata_text', - "assumptions = ('Earlier and later scopes explicitly selected and independently validated.', 'State " - "identity uses the declared common basis, including any disclosed map.')"), - ('_metadata_text', - "assumptions = ('Exact explicit single-version Phase 2 join scope; no representation exclusions.',)"), - ('_metadata_text', - "assumptions = ('Fixed finite declared state space and constant positive integer resample size.', 'X_t " - "conditional on p_t is Multinomial(n,p_t); p_(t+1)=X_t/n.', 'No mutation, migration, independent real data " - "or external corrective input.')"), - ('_metadata_text', - "assumptions = ('One explicitly selected version and declared representation.', 'No implicit pooling, " - "probability repair, confidence weighting or sampling.')"), - ('_metadata_text', - "assumptions = ('exact_utf8_v1 and SHA-256; equal digests verified against exact bytes', 'one explicitly " - "selected version; unweighted records')"), - ('_metadata_text', - "basis = ('empirical_assignments', 'explicit_counts_divided_by_included_records', 'weighted_record_mass', " - "'explicit_probability_vector')"), - ('_metadata_text', - "limitations = ('Exact record form does not establish semantic identity or independent origin.',)"), - ('_metadata_text', - "limitations = ('Experimental conditional simulation; no calibrated production-failure probability.', " - "'Diversity contraction holds in expectation, not monotonically on every sampled path.', 'Simulated steps " - "are not record generations, training epochs or dataset releases.', 'No external-reference loss, " - "reopening, lineage, risk score or audit workflow is implemented.', 'Floating-point and pseudorandom " - "sampling are numerical realizations of the declared model.')"), - ('_metadata_text', - "limitations = ('Observed/supplied support only; no permanent extinction or causal/model-performance " - "verdict.', 'A coarsened comparison cannot recover distinctions lost through its mapping.')"), - ('_metadata_text', - "limitations = ('Representation-bound; does not establish functional failure or semantic completeness.',)"), - ('_metadata_text', - "limitations = ('Representation-bound; no calibrated production-failure or universal risk conclusion.',)"), - ('_metadata_text', - "limitations = ('Supplied declarations only; no truth or source-independence certification.',)"), - ('_metadata_text', - "limitations = ('Toolkit operationalization relative to supplied metadata, without lineage or truth " - "certification.',)"), - ('_metadata_text', - "methods = {'support_size': basis, 'gini_simpson_diversity': basis, 'simpson_concentration': basis, " - "'state_frequency': basis + ('empirical_assignments; n_i/N',), 'state_count': tuple(('Definitions 7.1; ' + " - "item for item in basis)) + ('count included record-state assignments',), 'state_mass': ('Definitions 9.5; " - "canonical record weights summed within each state',), 'support_delta': ('later support_size minus earlier " - "support_size',), 'support_loss_count': ('cardinality of earlier support minus later support',), " - "'support_added_count': ('cardinality of later support minus earlier support',), " - "'support_retention_ratio': ('intersection support size / earlier positive-mass support size',), " - "'gini_simpson_diversity_delta': ('later Gini-Simpson diversity minus earlier diversity',), " - "'extinct_states': ('earlier support minus later support',), 'added_states': ('later support minus earlier " - "support',), 'retained_states': ('earlier support intersect later support',), 'source_type_counts': " - "('Definitions 3.3/11.1; count declared canonical categories',), 'provenance_confidence_counts': " - "('Definitions 3.3/11.1; count declared canonical categories',), 'source_type_shares': ('category count / " - "all selected valid records',), 'source_type_field_coverage': ('Definitions 3.13; nonmissing valid field / " - "all selected valid records',), 'provenance_confidence_field_coverage': ('Definitions 3.13; nonmissing " - "valid field / all selected valid records',), 'provenance_row_coverage': ('Reuse Phase 2 coverage; " - "Definitions 3.10-3.12',), 'provenance_required_field_coverage': ('Reuse Phase 2 coverage; Definitions " - "3.10-3.12',), 'grounding_field_coverage': ('Reuse Phase 2 coverage; Definitions 3.10-3.12',), " - "'analyzed_record_count': ('All selected valid records',), 'records_with_matching_rows': ('Phase 2 " - "matched-row inventory',), 'missing_provenance_count': ('Phase 2 missing-row inventory',), " - "'missing_provenance_share': ('Definitions 11.4; missing rows / all selected valid records',), " - "'known_open_count': ('toolkit_operationalization; Definitions 3.7-3.9; P3-D08',), 'known_closed_count': " - "('toolkit_operationalization; Definitions 3.7-3.9; P3-D08',), 'unresolved_grounding_count': " - "('toolkit_operationalization; Definitions 3.7-3.9; P3-D08',), 'weighted_source_type_masses': " - "('Definitions 19; sum explicit weights by source',), 'weighted_source_type_shares': ('F-007 weighted " - "variant; category mass / all selected record weight mass',), 'total_weight': ('Definitions 19; all " - "selected weights',), 'missing_provenance_weight': ('Definitions 19; missing-row mass',), " - "'weighted_missing_provenance_share': ('Definitions 11.4/19; missing-row mass / all selected record weight " - "mass',), 'duplicate_record_count': ('sum_group_size_minus_one; DEFINITIONS_AND_UNITS:8.3',), " - "'duplicate_group_count': ('count_groups_of_size_greater_than_one; DEFINITIONS_AND_UNITS:8.4',), " - "'rarity_rank': ('ascending frequency, then count, then Unicode state ID; 1-based ordinal',), " - "'tail_support_size': tuple(('Definitions 10.1-10.6; explicit ' + rule for rule in ('singleton_count', " - "'count_at_or_below', 'frequency_at_or_below', 'state_list'))), 'tail_membership': tuple(('Definitions " - "10.1-10.6; explicit ' + rule for rule in ('singleton_count', 'count_at_or_below', " - "'frequency_at_or_below', 'state_list'))), 'tail_record_share': tuple(('Definitions 10.1-10.6; explicit ' " - "+ rule + '; selected counts / included records' for rule in ('singleton_count', 'count_at_or_below', " - "'frequency_at_or_below', 'state_list'))), 'direct_closure_exposure_lower_bound': ('known_closed_count / " - "total_record_count', 'no usable required-provenance row'), 'direct_closure_exposure_upper_bound': " - "('(known_closed_count + unresolved_grounding_count) / total_record_count', 'no usable required-provenance " - "row'), 'direct_closure_exposure_interval_width': ('upper_bound - lower_bound = unresolved_grounding_count " - "/ total_record_count', 'no usable required-provenance row'), 'one_step_extinction_probability': ('F-014; " - "(1-p_i)^n; analytic one-step closed multinomial',), 'expected_gini_simpson_diversity': " - "('analytic_expectation; D0*(1-1/n)**t; t=0..steps; constant n',)}"), - ('_metadata_text', 'name = value.metric_name'), - ('_metadata_text', - "pair_names = ('support_delta', 'support_loss_count', 'support_added_count', 'support_retention_ratio', " - "'gini_simpson_diversity_delta', 'extinct_states', 'added_states', 'retained_states')")] - -PHASE4_STEP3_ASSEMBLY_LITERAL_CONTAINERS = [('_bundle_check', "('source_type', 'external_grounding', 'provenance_confidence')"), - ('_capabilities', - "('Input eligibility is distinct from executed analysis; Phase 5 lineage remains deferred.',)"), - ('_capabilities', "('partial', ['R_LONGITUDINAL_FAMILIES_DEFERRED'])"), - ('_capabilities', "['R_INPUT_VALIDATION_ERROR']"), - ('_capabilities', "['R_LONGITUDINAL_FAMILIES_DEFERRED']"), - ('_capabilities', - "{'content': 'record_coverage', 'representation': 'representation_coverage', 'row': " - "'provenance_row_coverage', 'required_fields': 'provenance_required_field_coverage', 'grounding': " - "'grounding_field_coverage'}"), - ('_closed_simulation', - "('Fixed finite declared state space and constant positive integer resample size.', 'X_t conditional on " - "p_t is Multinomial(n,p_t); p_(t+1)=X_t/n.', 'No mutation, migration, independent real data or external " - "corrective input.')"), - ('_closed_simulation', "('gini_simpson_diversity', 'F-003', 'ratio')"), - ('_closed_simulation', "('state_count', None, 'sampled_records')"), - ('_closed_simulation', "('state_frequency', 'F-001', 'ratio')"), - ('_closed_simulation', "('support_size', 'F-002', 'states')"), - ('_closed_simulation', - "(('state_count', None, 'sampled_records'), ('state_frequency', 'F-001', 'ratio'), ('support_size', " - "'F-002', 'states'), ('gini_simpson_diversity', 'F-003', 'ratio'))"), - ('_closed_simulation', - "['Extinction events are not separately inferred by the assembly adapter; inspect the supplied sampled " - "support paths.']"), - ('_comparison', "('earlier', 'later')"), - ('_comparison', "('earlier_to_later', 'later_to_earlier')"), - ('_comparison', "('extinct_states', 'added_states', 'retained_states')"), - ('_comparison', "{'observed_facts': {}, 'derived_metrics': {}}"), - ('_comparison', "{'observed_facts': {}, 'derived_metrics': {}}"), - ('_disclosure_base', "('T1', 'T2', 'T3', 'T4', 'T5', 'T6')"), - ('_disclosure_recommend', "['PR-014']"), - ('_disclosure_scope', - "{'dataset_versions': [], 'record_count': None, 'excluded_record_count': None, 'denominator_basis': " - "'no_selected_record_scope_supplied', 'scope_id': 'report_inputs'}"), - ('_disclosure_unavailable', - "['Unavailable means the supplied evidence does not support this conclusion; it does not establish that " - "the conclusion is false.']"), - ('_disclosures', - "('R_PARENT_UNRESOLVED', 'R_PARENT_AMBIGUOUS', 'R_PARENT_DECLARATION_MISSING', 'R_PARENT_INVALID')"), - ('_disclosures', "('available', 'partial')"), - ('_disclosures', "('available', 'partial')"), - ('_disclosures', "('available', 'partial')"), - ('_disclosures', "('available', 'partial')"), - ('_disclosures', "('external_ancestry', 'T4', 'External ancestry results are deferred to Phase 5.')"), - ('_disclosures', - "('grounding_field_coverage', 'external_grounding', 'records whose grounding remains unresolved', 'a " - "narrower direct closure exposure interval when the additional evidence resolves grounding')"), - ('_disclosures', "('lineage_analysis', 'PR-014', 'General lineage graph analysis is deferred to Phase 5.')"), - ('_disclosures', "('lineage_closure_exposure', 'T3', 'Lineage closure exposure is deferred to Phase 5.')"), - ('_disclosures', - "('provenance_required_field_coverage', 'required_provenance_fields', 'records lacking usable required " - "provenance', 'usable declared provenance without discarding independently valid calculations')"), - ('_disclosures', - "('provenance_row_coverage', 'provenance_required_field_coverage', 'grounding_field_coverage')"), - ('_disclosures', "('support', 'diversity', 'tail')"), - ('_disclosures', "('support', 'diversity', 'tail')"), - ('_disclosures', - "(('lineage_analysis', 'PR-014', 'General lineage graph analysis is deferred to Phase 5.'), " - "('lineage_closure_exposure', 'T3', 'Lineage closure exposure is deferred to Phase 5.'), " - "('external_ancestry', 'T4', 'External ancestry results are deferred to Phase 5.'))"), - ('_disclosures', - "(('provenance_required_field_coverage', 'required_provenance_fields', 'records lacking usable required " - "provenance', 'usable declared provenance without discarding independently valid calculations'), " - "('grounding_field_coverage', 'external_grounding', 'records whose grounding remains unresolved', 'a " - "narrower direct closure exposure interval when the additional evidence resolves grounding'))"), - ('_disclosures', "['A bounded operational definition and evidence matched to that definition.']"), - ('_disclosures', "['A controlled empirical intervention design with measured comparable outcomes.']"), - ('_disclosures', - "['A separately defined bounded outcome and appropriately validated predictive evidence.']"), - ('_disclosures', - "['A stated production-failure criterion and corresponding versioned outcome measurements.']"), - ('_disclosures', - "['An explicit pipeline boundary and evidence for its relevant external inputs and transformations.']"), - ('_disclosures', - "['An identified ancestor, measured outcomes and a controlled or otherwise justified causal design.']"), - ('_disclosures', - "['Direct closure exposure describes the selected declared grounding scope, without tracing every pipeline " - "input.']"), - ('_disclosures', - "['Each coverage keeps its own field meaning and denominator.', 'This signal reports unresolved supplied " - "provenance evidence and does not estimate factual truth or source independence.', 'A not_present signal " - "is limited to the cited fields and does not certify universal integrity.']"), - ('_disclosures', "['No accepted versioned model-outcome analysis is supplied.']"), - ('_disclosures', "['No operational failure outcome is measured by these structural fields.']"), - ('_disclosures', "['No universal collapse prediction is defined by the approved toolkit.']"), - ('_disclosures', - "['Only existing reference validation observations can be reported; no graph traversal, root tracing or " - "ancestry metric executes.']"), - ('_disclosures', - "['Phase 4 does not compute ancestry; topology alone would not identify a causal contribution.']"), - ('_disclosures', - "['Preserve explicit composite parent references, chronology and external-grounding metadata for the " - "future Phase 5 analysis.']"), - ('_disclosures', "['R_CAUSAL_EVIDENCE_MISSING', 'R_LINEAGE_EXECUTION_DEFERRED']"), - ('_disclosures', "['R_CONTROLLED_EMPIRICAL_DESIGN_MISSING']"), - ('_disclosures', "['R_LINEAGE_EXECUTION_DEFERRED']"), - ('_disclosures', "['R_MODEL_EVIDENCE_MISSING']"), - ('_disclosures', "['R_OUTCOME_EVIDENCE_MISSING']"), - ('_disclosures', "['R_OUTSIDE_PRODUCT_SCOPE']"), - ('_disclosures', "['R_PIPELINE_BOUNDARY_EVIDENCE_MISSING']"), - ('_disclosures', "['R_UNIVERSAL_OPERATIONAL_DEFINITION_ABSENT']"), - ('_disclosures', "['Supplied scenario outputs are conditional mathematical or stochastic results.']"), - ('_disclosures', - "['Tail membership depends on the explicit rule and selected representation.', 'This signal is not a " - "calibrated forecast of the production pipeline.', 'No simulation is run by this signal, and tail " - "membership does not establish importance or harm.']"), - ('_disclosures', - "['The signal is restricted to the selected versions and their declared common state meaning.', 'An " - 'observed support decrease does not establish model-performance decline, production failure or universal ' - "collapse.']"), - ('_disclosures', "['This product defines no universal integrity scalar or operational test.']"), - ('_disclosures', "['Versioned model evaluation outcomes with comparable tasks and evaluation conditions.']"), - ('_disclosures', "['derived_metrics.support.support_delta']"), - ('_disclosures', "['derived_metrics.tail.tail_support_size', 'derived_metrics.tail.selection']"), - ('_disclosures', - "['eligibility for an explicitly requested comparison after state-meaning compatibility is supplied']"), - ('_disclosures', "['evidence needed by a future model-longitudinal implementation']"), - ('_disclosures', "['more complete declared provenance coverage']"), - ('_disclosures', "['more complete immediate-parent validation; future Phase 5 analysis remains deferred']"), - ('_disclosures', "['representation-dependent support, diversity and tail diagnostics']"), - ('_distribution', "('empirical_assignments', 'explicit_counts_divided_by_included_records')"), - ('_distribution', - "('empirical_assignments', 'explicit_counts_divided_by_included_records', 'weighted_record_mass', " - "'explicit_probability_vector')"), - ('_distribution_exclusions', "('derived_metrics', 'diversity')"), - ('_distribution_exclusions', "('derived_metrics', 'support')"), - ('_distribution_exclusions', "('state_counts', 'supplied_state_probabilities')"), - ('_distribution_exclusions', "(('derived_metrics', 'support'), ('derived_metrics', 'diversity'))"), - ('_duplicates', "('usable_exact_content',)"), - ('_inputs', "('R_CALC_EMPTY_SCOPE',)"), - ('_inputs', "('Validation coverage describes supplied declarations without certifying their truth.',)"), - ('_inputs', "('nonempty_validated_record_scope',)"), - ('_inputs', - "['Input hashes identify supplied bytes and do not certify authenticity.', 'Only allowlisted evidence " - "fields are assembled; raw content, extras, notes and embeddings are not exported.']"), - ('_inputs', "['R_INVENTORY_ONLY_ROLE']"), - ('_lineage_observations', - "('Counts preserve validation reference-entry multiplicity; they do not describe an ancestry graph.',)"), - ('_lineage_observations', - "('Immediate-reference validation coverage does not establish resolved ancestry or external roots.',)"), - ('_lineage_observations', "('No general graph-cycle traversal executes in Phase 4.',)"), - ('_lineage_observations', "('Phase_5_graph_analysis',)"), - ('_lineage_observations', "('R_GRAPH_EXECUTION_DEFERRED',)"), - ('_lineage_observations', "('R_NO_DECLARED_PARENT_ENTRIES',)"), - ('_lineage_observations', - "('This retained earlier-version ordering certificate is not general graph-cycle traversal.',)"), - ('_lineage_observations', "('declared_parent_reference_entries',)"), - ('_message_family', - "('E_PARENT_AMBIGUOUS', 'E_PARENT_FORMAT', 'E_PARENT_FUTURE_VERSION', 'E_LINEAGE_CYCLE', " - "'W_GENERATION_MISMATCH')"), - ('_message_family', "('parent_ids', 'generation')"), - ('_message_family', "('representation', 'topic', 'embedding_cluster', 'content', 'content_ref')"), - ('_message_family', "('source_type', 'provenance_confidence', 'external_grounding')"), - ('_message_family', "('version_order', 'pair_order', 'representation_compatibility')"), - ('_message_family', "['content_diagnostics']"), - ('_message_family', "['dataset_longitudinal']"), - ('_message_family', "['ingestion']"), - ('_message_family', "['lineage']"), - ('_message_family', "['provenance']"), - ('_metadata_text', - "('(known_closed_count + unresolved_grounding_count) / total_record_count', 'no usable required-provenance " - "row')"), - ('_metadata_text', "('All selected valid records',)"), - ('_metadata_text', "('Approved direct grounding partition; uncertainty remains unresolved.',)"), - ('_metadata_text', "('Definitions 11.4/19; missing-row mass / all selected record weight mass',)"), - ('_metadata_text', "('Definitions 11.4; missing rows / all selected valid records',)"), - ('_metadata_text', "('Definitions 19; all selected weights',)"), - ('_metadata_text', "('Definitions 19; missing-row mass',)"), - ('_metadata_text', "('Definitions 19; sum explicit weights by source',)"), - ('_metadata_text', "('Definitions 3.13; nonmissing valid field / all selected valid records',)"), - ('_metadata_text', "('Definitions 3.13; nonmissing valid field / all selected valid records',)"), - ('_metadata_text', "('Definitions 3.3/11.1; count declared canonical categories',)"), - ('_metadata_text', "('Definitions 3.3/11.1; count declared canonical categories',)"), - ('_metadata_text', "('Definitions 9.5; canonical record weights summed within each state',)"), - ('_metadata_text', - "('Earlier and later scopes explicitly selected and independently validated.', 'State identity uses the " - "declared common basis, including any disclosed map.')"), - ('_metadata_text', "('Exact explicit single-version Phase 2 join scope; no representation exclusions.',)"), - ('_metadata_text', "('Exact record form does not establish semantic identity or independent origin.',)"), - ('_metadata_text', - "('Experimental conditional simulation; no calibrated production-failure probability.', 'Diversity " - "contraction holds in expectation, not monotonically on every sampled path.', 'Simulated steps are not " - "record generations, training epochs or dataset releases.', 'No external-reference loss, reopening, " - "lineage, risk score or audit workflow is implemented.', 'Floating-point and pseudorandom sampling are " - "numerical realizations of the declared model.')"), - ('_metadata_text', "('F-007 weighted variant; category mass / all selected record weight mass',)"), - ('_metadata_text', "('F-014; (1-p_i)^n; analytic one-step closed multinomial',)"), - ('_metadata_text', - "('Fixed finite declared state space and constant positive integer resample size.', 'X_t conditional on " - "p_t is Multinomial(n,p_t); p_(t+1)=X_t/n.', 'No mutation, migration, independent real data or external " - "corrective input.')"), - ('_metadata_text', - "('Observed/supplied support only; no permanent extinction or causal/model-performance verdict.', 'A " - "coarsened comparison cannot recover distinctions lost through its mapping.')"), - ('_metadata_text', - "('One explicitly selected version and declared representation.', 'No implicit pooling, probability " - "repair, confidence weighting or sampling.')"), - ('_metadata_text', "('Phase 2 matched-row inventory',)"), - ('_metadata_text', "('Phase 2 missing-row inventory',)"), - ('_metadata_text', - "('Representation-bound; does not establish functional failure or semantic completeness.',)"), - ('_metadata_text', - "('Representation-bound; no calibrated production-failure or universal risk conclusion.',)"), - ('_metadata_text', "('Reuse Phase 2 coverage; Definitions 3.10-3.12',)"), - ('_metadata_text', "('Reuse Phase 2 coverage; Definitions 3.10-3.12',)"), - ('_metadata_text', "('Reuse Phase 2 coverage; Definitions 3.10-3.12',)"), - ('_metadata_text', "('Supplied declarations only; no truth or source-independence certification.',)"), - ('_metadata_text', - "('Toolkit operationalization relative to supplied metadata, without lineage or truth certification.',)"), - ('_metadata_text', "('analytic_expectation; D0*(1-1/n)**t; t=0..steps; constant n',)"), - ('_metadata_text', "('ascending frequency, then count, then Unicode state ID; 1-based ordinal',)"), - ('_metadata_text', "('cardinality of earlier support minus later support',)"), - ('_metadata_text', "('cardinality of later support minus earlier support',)"), - ('_metadata_text', "('category count / all selected valid records',)"), - ('_metadata_text', "('count included record-state assignments',)"), - ('_metadata_text', "('count_groups_of_size_greater_than_one; DEFINITIONS_AND_UNITS:8.4',)"), - ('_metadata_text', "('earlier support intersect later support',)"), - ('_metadata_text', "('earlier support minus later support',)"), - ('_metadata_text', - "('empirical_assignments', 'explicit_counts_divided_by_included_records', 'weighted_record_mass', " - "'explicit_probability_vector')"), - ('_metadata_text', "('empirical_assignments; n_i/N',)"), - ('_metadata_text', - "('exact_utf8_v1 and SHA-256; equal digests verified against exact bytes', 'one explicitly selected " - "version; unweighted records')"), - ('_metadata_text', "('intersection support size / earlier positive-mass support size',)"), - ('_metadata_text', "('known_closed_count / total_record_count', 'no usable required-provenance row')"), - ('_metadata_text', "('later Gini-Simpson diversity minus earlier diversity',)"), - ('_metadata_text', "('later support minus earlier support',)"), - ('_metadata_text', "('later support_size minus earlier support_size',)"), - ('_metadata_text', "('sampled_path; sequential_binomial_complement_v1',)"), - ('_metadata_text', "('singleton_count', 'count_at_or_below', 'frequency_at_or_below', 'state_list')"), - ('_metadata_text', "('singleton_count', 'count_at_or_below', 'frequency_at_or_below', 'state_list')"), - ('_metadata_text', "('singleton_count', 'count_at_or_below', 'frequency_at_or_below', 'state_list')"), - ('_metadata_text', "('state_count', 'state_frequency', 'support_size', 'gini_simpson_diversity')"), - ('_metadata_text', "('sum_group_size_minus_one; DEFINITIONS_AND_UNITS:8.3',)"), - ('_metadata_text', - "('support_delta', 'support_loss_count', 'support_added_count', 'support_retention_ratio', " - "'gini_simpson_diversity_delta', 'extinct_states', 'added_states', 'retained_states')"), - ('_metadata_text', "('toolkit_operationalization; Definitions 3.7-3.9; P3-D08',)"), - ('_metadata_text', "('toolkit_operationalization; Definitions 3.7-3.9; P3-D08',)"), - ('_metadata_text', "('toolkit_operationalization; Definitions 3.7-3.9; P3-D08',)"), - ('_metadata_text', - "('upper_bound - lower_bound = unresolved_grounding_count / total_record_count', 'no usable " - "required-provenance row')"), - ('_provenance_consistency', "('confirmed', 'log_derived', 'estimated', 'unknown')"), - ('_provenance_consistency', "('human', 'synthetic', 'mixed', 'sensor', 'unknown')"), - ('_provenance_consistency', "('known_closed', 'valid_direct_grounding_no', ())"), - ('_provenance_consistency', "('known_open', 'valid_direct_grounding_yes', ())"), - ('_provenance_consistency', "('unresolved_grounding', 'declared_unknown_grounding', ())"), - ('_provenance_consistency', "('unresolved_grounding', 'missing_provenance_row', ())"), - ('_scalar', "('valid_nonempty_input_for_declared_scope',)"), - ('_table', "('valid_nonempty_input_for_declared_scope',)"), - ('assemble_report', "['existing_bundle_validation']")] - -PHASE4_STEP3_ASSEMBLY_CALLS = {'FamilyFailure': ['dataclass'], - 'FamilyFailure.__post_init__': ['ReportAssemblyError', 'ReportAssemblyError', 'any', 'type', 'type', 'type'], - '_base': ['_definition', - 'dict.fromkeys', - 'dict.fromkeys', - 'dict.fromkeys', - 'dict.fromkeys', - 'field.owner.startswith', - 'list', - 'list', - 'list', - 'list'], - '_bundle_check': ['_coverage', - '_coverage', - '_coverage', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - 'all', - 'all', - 'all', - 'all', - 'all', - 'all', - 'all', - 'all', - 'all', - 'all', - 'assessment.capabilities.items', - 'capability.coverage_details.values', - 'len', - 'len', - 'len', - 'len', - 'len', - 'row.values.get', - 'row.values.get', - 'set', - 'set', - 'set', - 'set', - 'set', - 'set', - 'sum', - 'sum', - 'sum', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type'], - '_bundle_scope': ['_key', 'len', 'list'], - '_capabilities': ['_coverage', - '_coverage', - '_coverage', - '_coverage', - '_require', - '_require', - '_typed', - '_typed', - 'any', - 'dict.fromkeys', - 'dict.fromkeys', - 'dict.fromkeys', - "entry['notes'].append", - "entry['notes'].append", - 'list', - 'list', - 'list', - 'list', - 'list', - 'list', - 'list', - 'list', - 'list', - 'matrix.values', - 'original.coverage_details.items', - 'reasons.append', - 'set', - 'set', - 'type'], - '_closed_simulation': ['WeightingOptions', - 'WeightingOptions', - '_base', - '_metadata', - '_metadata', - '_policy', - '_representation', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_scope', - '_simulation_inputs', - '_typed', - '_typed', - 'dict.fromkeys', - 'generations.append', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'list', - 'list', - 'list', - 'list', - 'list', - 'list', - 'list', - 'list', - 'paths.append', - 'range', - 'range', - 'set', - 'set', - 'target.update', - 'target.update', - 'trajectories.append', - 'tuple', - 'type', - 'type', - 'zip'], - '_closure': ['_coverage', - '_number', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_scalar', - '_scope', - '_typed', - 'len', - "payload['derived_metrics'].setdefault", - 'set', - 'set', - 'sum'], - '_comparison': ['_distribution', - '_distribution', - '_envelope', - '_representation', - '_representation', - '_representation', - '_representation', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_scalar', - '_scalar', - '_scope', - '_table', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - '_typed', - 'effects.append', - 'len', - 'len', - 'len', - 'list', - 'list', - 'list', - 'list', - 'list', - 'list', - 'list', - 'mapping.state_mapping.items', - 'order.index', - 'order.index', - "payload['derived_metrics'].setdefault", - "payload['derived_metrics'].setdefault", - 'support.values', - 'type', - 'type', - 'zip'], - '_coverage': ['_number', '_number', '_require', '_typed'], - '_definition': ['ReportAssemblyError'], - '_diagnostics': ['_key', - '_key', - '_message_family', - '_require', - '_require', - '_require', - '_typed', - '_typed', - "payload['errors'].append", - "payload['inputs']['limitations'].append", - "payload['warnings'].append", - 'type', - 'type', - 'type', - 'type', - 'type'], - '_disclosure_recommend': ["payload['recommended_next_metadata'].append"], - '_disclosure_scope': ["payload['inputs'].get"], - '_disclosure_signal': ['_disclosure_base', 'dict.fromkeys', 'list', 'list', 'list', 'result.update'], - '_disclosure_unavailable': ['_disclosure_base', - '_disclosure_scope', - "payload['unavailable_conclusions'].append", - 'result.update'], - '_disclosures': ['_disclosure_recommend', - '_disclosure_recommend', - '_disclosure_recommend', - '_disclosure_recommend', - '_disclosure_recommend', - '_disclosure_recommend', - '_disclosure_signal', - '_disclosure_signal', - '_disclosure_signal', - '_disclosure_unavailable', - '_disclosure_unavailable', - '_disclosure_unavailable', - '_disclosure_unavailable', - '_disclosure_unavailable', - '_disclosure_unavailable', - '_disclosure_unavailable', - '_disclosure_unavailable', - 'any', - 'any', - 'any', - 'basis.append', - 'basis.append', - 'content_cap.get', - 'derived.get', - 'derived.get', - 'derived.get', - "derived.get('closure_exposure', {}).get", - 'direct.get', - 'list', - 'longitudinal_cap.get', - 'paths.append', - 'paths.append', - "payload['capabilities'].get", - "payload['capabilities'].get", - "payload['capabilities'].get", - "payload['observed_facts'].get", - 'provenance.get', - 'provenance.get', - 'provenance.get', - 'provenance.get', - 'reasons.extend', - 'set', - 'set', - 'support.get', - 'tail.get'], - '_distribution': ['WeightingOptions', - '_envelope', - '_number', - '_number', - '_number', - '_number', - '_policy', - '_representation', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_scalar', - '_scope', - '_table', - '_table', - '_table', - '_table', - '_typed', - '_typed', - '_typed', - 'all', - 'all', - 'all', - 'all', - 'isclose', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - "payload['derived_metrics'].setdefault", - "payload['derived_metrics'].setdefault", - "payload['derived_metrics'].setdefault('diversity', {}).setdefault", - "payload['derived_metrics'].setdefault('diversity', {}).setdefault('by_version', " - '{}).setdefault', - "payload['derived_metrics'].setdefault('support', {}).setdefault", - "payload['derived_metrics'].setdefault('support', {}).setdefault('by_version', " - '{}).setdefault', - "payload['observed_facts'].setdefault", - "payload['observed_facts'].setdefault", - "payload['observed_facts'].setdefault('state_counts', {}).setdefault", - "payload['observed_facts'].setdefault('supplied_state_probabilities', {}).setdefault", - 'set', - 'sum', - 'sum', - 'type', - 'type', - 'type', - 'type', - 'type'], - '_distribution_exclusions': ['_key', - '_require', - '_require', - '_require', - 'all', - 'exclusions.append', - 'len', - 'len', - "payload['observed_facts'].get", - "payload['observed_facts'].get(family, {}).get", - "payload['observed_facts'].get(family, {}).get('by_version', {}).get", - 'payload[section].get', - 'payload[section].get(family, {}).get', - "payload[section].get(family, {}).get('by_version', {}).get", - "payload[section].get(family, {}).get('by_version', {}).get(version, " - '{}).values', - 'set', - 'type', - 'type', - 'type'], - '_duplicates': ['_coverage', - '_envelope', - '_key', - '_representation', - '_require', - '_require', - '_require', - '_require', - '_require', - '_scalar', - '_scope', - '_typed', - '_typed', - 'enumerate', - 'groups.append', - 'len', - 'len', - 'len', - 'len', - 'len', - "payload['observed_facts'].setdefault", - 'seen.intersection', - 'seen.update', - 'set', - 'set', - 'set', - 'set', - 'str', - 'sum', - 'tuple'], - '_envelope': ['_base'], - '_extinction': ['ScalarCalculation', - 'WeightingOptions', - '_base', - '_metadata', - '_policy', - '_representation', - '_require', - '_require', - '_require', - '_require', - '_require', - '_scope', - '_typed', - '_typed', - 'target.update', - "target['initial_distribution'].append", - "target['parameters']['state_order'].append", - 'type'], - '_inputs': ['_bundle_scope', - '_coverage', - '_diagnostics', - '_diagnostics', - '_diagnostics', - '_diagnostics', - '_envelope', - '_envelope', - '_key', - '_key', - '_require', - '_require', - '_require', - 'affected.append', - 'any', - 'dict.fromkeys', - 'enumerate', - 'hashes.add', - 'iter', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'list', - 'list', - 'list', - 'list', - 'messages_by_location.get', - 'messages_by_location.get', - 'messages_by_location.get', - 'messages_by_location.setdefault', - 'messages_by_location.setdefault((message.file_role, message.file_path), []).append', - 'next', - 'operations.append', - "payload['inputs']['artifacts'].append", - "payload['inputs']['file_hashes'].append", - 'records_by_version.get', - 'records_by_version.setdefault', - 'records_by_version.setdefault(row.record_key.dataset_version, []).append', - 'set', - 'set', - 'set', - 'sorted', - 'str', - 'str', - 'str', - 'tuple', - 'tuple', - 'tuple', - 'type', - 'unmapped.update', - 'versions_by_location.get', - 'versions_by_location.setdefault', - 'versions_by_location.setdefault((row.location.file_role, row.location.file_path), set()).add'], - '_key': ['_typed'], - '_lineage_observations': ['_coverage', '_envelope', '_envelope', '_envelope', '_envelope', 'list'], - '_metadata': ['CalculationMetadata', - 'WeightingOptions', - 'WeightingOptions', - '_metadata_text', - '_require', - '_require', - '_require', - '_require', - '_require', - '_typed', - '_typed', - 'type', - 'type'], - '_metadata_text': ['_require', - '_require', - '_require', - '_require', - 'all', - 'all', - 'methods.get', - 'name.startswith', - 'tuple', - 'tuple', - 'tuple', - 'tuple', - 'type', - 'type', - 'type', - 'type'], - '_number': ['_require', '_require', '_require', '_require', 'isfinite', 'type', 'type'], - '_policy': ['NumericalPolicy', '_typed'], - '_provenance': ['WeightingOptions', - '_coverage', - '_coverage', - '_provenance_consistency', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_scalar', - '_scalar', - '_scalar', - '_scalar', - '_scope', - '_table', - '_table', - '_table', - '_table', - '_table', - '_typed', - '_typed', - '_typed', - '_typed', - 'all', - 'dict', - 'dict', - 'dict', - 'len', - 'len', - 'len', - 'len', - "payload['derived_metrics'].setdefault", - "payload['observed_facts'].setdefault", - 'set', - 'sum', - 'type', - 'type', - 'zip'], - '_provenance_consistency': ['_coverage', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_typed', - '_typed', - 'all', - 'any', - 'dict', - 'expected.values', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'len', - 'match.provenance.values.get', - 'match.provenance.values.get', - 'set', - 'set', - 'set', - 'set', - 'set', - 'set', - 'set', - 'set', - 'sum', - 'sum', - 'sum', - 'sum', - 'sum', - 'sum', - 'tuple', - 'tuple', - 'tuple', - 'tuple', - 'type', - 'type', - 'type', - 'zip'], - '_representation': ['RepresentationDescriptor', '_typed'], - '_require': ['ReportAssemblyError'], - '_scalar': ['CalculationEvidenceClass', - 'ScalarCalculation', - '_definition', - '_envelope', - '_metadata', - '_representation', - '_scope', - '_typed', - 'field.method_id.startswith', - 'path.rsplit', - 'tuple', - 'tuple'], - '_scope': ['CalculationScope', - '_key', - '_key', - '_require', - '_require', - '_typed', - 'len', - 'len', - 'list', - 'set', - 'set', - 'set'], - '_simulation_inputs': ['_number', - '_policy', - '_representation', - '_require', - '_require', - '_require', - '_scope', - '_typed', - 'all', - 'len', - 'len', - 'len', - 'type', - 'type', - 'type', - 'type', - 'type'], - '_table': ['CalculationEvidenceClass', - '_definition', - '_envelope', - '_metadata', - '_representation', - '_require', - '_require', - '_require', - '_scope', - 'all', - 'field.method_id.startswith', - 'tuple', - 'tuple', - 'type', - 'type', - 'type'], - '_tail': ['_require', - '_require', - '_scalar', - '_scope', - '_table', - '_typed', - '_typed', - 'list', - 'list', - 'ranking.append', - 'set'], - '_typed': ['_require', 'type'], - 'assemble_report': ['CanonicalReport.from_dict', - 'FamilyFailure', - '_bundle_check', - '_capabilities', - '_closed_simulation', - '_closed_simulation', - '_closure', - '_comparison', - '_coverage', - '_diagnostics', - '_diagnostics', - '_diagnostics', - '_diagnostics', - '_disclosures', - '_distribution', - '_distribution', - '_distribution', - '_distribution_exclusions', - '_duplicates', - '_extinction', - '_inputs', - '_lineage_observations', - '_provenance', - '_representation', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_require', - '_tail', - '_typed', - 'all', - 'all', - 'any', - 'dict', - 'empirical_representations.append', - 'empirical_representations.append', - 'empirical_representations.append', - 'empirical_representations.append', - 'enumerate', - 'failures.setdefault', - 'failures.setdefault(failure.capability.value, []).extend', - 'len', - 'len', - 'len', - 'len', - "operations['content_diagnostics'].append", - "operations['content_diagnostics'].append", - "operations['content_diagnostics'].append", - "operations['dataset_longitudinal'].append", - "operations['intervention_simulation'].append", - "operations['intervention_simulation'].append", - "operations['intervention_simulation'].append", - "operations['provenance'].append", - "operations['provenance'].append", - "payload['inputs']['limitations'].append", - 'run.get', - 'run.get', - 'set', - 'set', - 'set', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type', - 'type']} - -PHASE4_STEP3_ASSEMBLY_ARITHMETIC = {'FamilyFailure.__post_init__': ['not self.messages'], - '_capabilities': ["bundle.observability.limitations + ('Input eligibility is distinct from executed " - "analysis; Phase 5 lineage remains deferred.',)", - 'not completed', - 'reasons + family_codes'], - '_closed_simulation': ["target['limitations'] + ['Extinction events are not separately inferred by the " - "assembly adapter; inspect the supplied sampled support paths.']", - 'value.simulation_horizon + 1'], - '_closure': ["'derived_metrics.closure_exposure.direct.' + name"], - '_comparison': ["'derived_metrics.support.' + name", - "'derived_metrics.support.' + name", - 'context.earlier_scope.excluded_record_keys + context.later_scope.excluded_record_keys', - 'context.earlier_scope.included_record_keys + context.later_scope.included_record_keys', - "context.earlier_scope.scope_id + ' -> '", - "context.earlier_scope.scope_id + ' -> ' + context.later_scope.scope_id", - 'earlier + later', - 'value.limitations + compatibility.limitations'], - '_diagnostics': ["'Validation notice ' + message.code", - "'Validation notice ' + message.code + ': '", - "'Validation notice ' + message.code + ': ' + message.message"], - '_disclosure_signal': ["list(basis['limitations']) + limitations", "owner + '.'", "owner + '.' + name"], - '_disclosure_unavailable': ["owner + '.unavailable_conclusion'"], - '_disclosures': ["'observed_facts.provenance.' + key", - "not any((key in derived for key in ('support', 'diversity', 'tail')))", - 'not basis', - "set(content_cap.get('reason_codes', ())) | set(longitudinal_cap.get('reason_codes', ()))"], - '_distribution': ["'derived_metrics.' + family", - "'derived_metrics.' + family + '.by_version.*.'", - "'derived_metrics.' + family + '.by_version.*.' + prefix", - "'derived_metrics.' + family + '.by_version.*.' + prefix + name", - "'derived_metrics.diversity.by_version.*.' + prefix", - "'derived_metrics.diversity.by_version.*.' + prefix + 'state_frequencies'", - 'not weighted', - 'not weighted', - 'not weighted', - "prefix + 'state_frequencies'", - "prefix + 'support_size'", - 'prefix + name', - 'prefix + name'], - '_duplicates': ["'exact_duplicate_group_' + str(index + 1)", - "'observed_facts.content.' + name", - "group['record_count'] - 1", - 'index + 1', - 'not seen.intersection(group.record_keys)'], - '_inputs': ["'observed_facts.provenance.' + name", - "'validated_version:' + version", - 'bundle.records + (() if bundle.provenance is None else bundle.provenance)', - 'messages + uncertain', - 'messages + uncertain', - "tuple(messages_by_location.get((artifact.role, '[redacted]'), ())) + " - 'tuple(messages_by_location.get((artifact.role, None), ()))'], - '_lineage_observations': ["'observed_facts.lineage.' + name", 'coverage.denominator - coverage.numerator'], - '_metadata_text': ["'Definitions 10.1-10.6; explicit ' + rule", - "'Definitions 10.1-10.6; explicit ' + rule", - "'Definitions 10.1-10.6; explicit ' + rule", - "'Definitions 10.1-10.6; explicit ' + rule + '; selected counts / included records'", - "'Definitions 7.1; ' + item", - "basis + ('empirical_assignments; n_i/N',)", - "tuple(('Definitions 7.1; ' + item for item in basis)) + ('count included record-state " - "assignments',)"], - '_number': ["name + ' is above its accepted bound'", - "name + ' is below its accepted bound'", - "name + ' must be a built-in number'", - "name + ' must be finite'"], - '_provenance': ["'derived_metrics.provenance.' + name", - "'derived_metrics.provenance.' + name", - "'observed_facts.provenance.' + field", - "'observed_facts.provenance.' + field", - "'observed_facts.provenance.' + field + '_counts'", - "'observed_facts.provenance.' + field + '_field_coverage'", - "'observed_facts.provenance.' + name", - "'observed_facts.provenance.' + name", - "'observed_facts.provenance.' + name", - "field + '_counts'", - "field + '_counts'", - "field + '_field_coverage'", - "field + '_field_coverage'"], - '_provenance_consistency': ['len(keys) - len(missing)', - 'not row.required_fields_valid', - 'not value.scope.excluded_record_keys'], - '_require': ['not condition'], - '_scalar': ['-1', 'value.metadata.limitations + tuple(limitations)'], - '_scope': ['value.included_record_keys + value.excluded_record_keys'], - '_table': ['metadata.limitations + tuple(limitations)'], - '_tail': ["'derived_metrics.tail.' + name", "'derived_metrics.tail.' + name"], - '_typed': ["name + ' requires its exact accepted result type'"], - 'assemble_report': ["'supplied_declared_tail:' + tail.scope.scope_id", - "'supplied_direct_closure_interval:' + closure.scope.scope_id", - "'supplied_distribution:' + (supplied.unweighted.scope.scope_id if type(supplied) is " - 'StateDistributionResult else supplied.scope.scope_id)', - "'supplied_exact_duplicates:' + duplicates.scope.scope_id", - "'supplied_provenance_composition:' + provenance.scope.scope_id", - 'DirectClosureExposureBounds | None', - 'ExactDuplicateResult | None', - 'ExpectedDiversityResult | None', - 'ProvenanceCompositionResult | None', - 'ResamplingSimulation | None', - 'StateDistributionResult | DistributionMetrics', - 'SupportComparison | None', - 'TailSelectionResult | None', - 'supplied.unweighted.scope.included_record_keys + ' - 'supplied.unweighted.scope.excluded_record_keys']} - -PHASE4_STEP3_ASSEMBLY_ATTRIBUTES = {'FamilyFailure.__post_init__': ['ValidationSeverity.ERROR', - 'ValidationSeverity.FATAL', - 'item.severity', - 'self.capability', - 'self.messages', - 'self.messages', - 'self.messages'], - '_base': ['dict.fromkeys', - 'dict.fromkeys', - 'dict.fromkeys', - 'dict.fromkeys', - 'field.evidence_class', - 'field.method_id', - 'field.owner', - 'field.owner', - 'field.owner', - 'field.owner.startswith', - 'field.unit'], - '_bundle_check': ['assessment.capabilities', - 'assessment.capabilities', - 'assessment.capabilities', - 'assessment.capabilities.items', - 'assessment.maximum_level', - 'assessment.maximum_level', - 'bundle.generation', - 'bundle.generation', - 'bundle.inventory', - 'bundle.inventory', - 'bundle.inventory', - 'bundle.mapping_traces', - 'bundle.mapping_traces', - 'bundle.observability', - 'bundle.observability', - 'bundle.provenance', - 'bundle.provenance', - 'bundle.provenance', - 'bundle.provenance', - 'bundle.provenance', - 'bundle.provenance', - 'bundle.provenance_join', - 'bundle.provenance_join', - 'bundle.records', - 'bundle.records', - 'bundle.records', - 'bundle.records', - 'bundle.validation_messages', - 'bundle.validation_messages', - 'bundle.version_order', - 'bundle.version_order', - 'bundle.version_order', - 'bundle.version_order.loaded_versions', - 'bundle.version_order.order', - 'capability.coverage', - 'capability.coverage', - 'capability.coverage_details', - 'capability.coverage_details', - 'capability.coverage_details', - 'capability.coverage_details.values', - 'capability.notes', - 'capability.reason_codes', - 'capability.requirements_met', - 'capability.requirements_missing', - 'capability.status', - 'coverage.denominator', - 'coverage.numerator', - 'item.file_format', - 'item.path', - 'item.role', - 'join.grounding_field_coverage', - 'join.matches', - 'join.matches', - 'join.matches', - 'join.matches', - 'join.matches', - 'join.matches', - 'join.matches', - 'join.matches', - 'join.matches', - 'join.missing_record_keys', - 'join.missing_record_keys', - 'join.provenance_required_field_coverage', - 'join.provenance_row_coverage', - 'join.provenance_supplied', - 'join.scope_record_keys', - 'join.scope_record_keys', - 'join.scope_record_keys', - 'join.scope_record_keys', - 'join.scope_record_keys', - 'join.scope_record_keys', - 'join.scope_record_keys', - 'join.selected_dataset_versions', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance.grounding_known', - 'match.provenance.required_fields_valid', - 'match.record_key', - 'match.record_key', - 'match.record_key', - 'match.record_key', - 'row.grounding_known', - 'row.location', - 'row.location', - 'row.record_key', - 'row.record_key', - 'row.record_key', - 'row.record_key', - 'row.required_fields_valid', - 'row.values', - 'row.values', - 'row.values', - 'row.values', - 'row.values.get', - 'row.values.get'], - '_bundle_scope': ['bundle.records', - 'bundle.records', - 'bundle.version_order', - 'bundle.version_order.loaded_versions', - 'row.record_key'], - '_capabilities': ['CapabilityKey.DATASET_LONGITUDINAL', - 'CapabilityKey.DATASET_LONGITUDINAL', - 'CapabilityKey.INGESTION', - 'CapabilityKey.INTERVENTION_SIMULATION', - 'CapabilityKey.LINEAGE', - 'CapabilityKey.LINEAGE', - 'CapabilityKey.LINEAGE', - 'CapabilityKey.LINEAGE', - 'CapabilityKey.MODEL_LONGITUDINAL', - 'CapabilityKey.MODEL_LONGITUDINAL', - 'CapabilityKey.PROVENANCE', - 'bundle.observability', - 'bundle.observability', - 'bundle.observability', - 'bundle.observability', - 'bundle.observability', - 'bundle.observability.basis', - 'bundle.observability.capabilities', - 'bundle.observability.limitations', - 'bundle.observability.maximum_level', - 'bundle.observability.maximum_level', - 'bundle.provenance_join', - 'bundle.provenance_join', - 'bundle.provenance_join', - 'bundle.provenance_join.grounding_field_coverage', - 'bundle.provenance_join.provenance_required_field_coverage', - 'bundle.provenance_join.provenance_row_coverage', - 'bundle.records', - 'dict.fromkeys', - 'dict.fromkeys', - 'dict.fromkeys', - "entry['notes'].append", - "entry['notes'].append", - 'key.value', - 'key.value', - 'key.value', - 'matrix.values', - 'original.coverage', - 'original.coverage', - 'original.coverage', - 'original.coverage', - 'original.coverage_details', - 'original.coverage_details', - 'original.coverage_details', - 'original.coverage_details.items', - 'original.notes', - 'original.reason_codes', - 'original.requirements_met', - 'original.requirements_missing', - 'original.status', - 'original.status', - 'original.status.value', - 'reasons.append'], - '_closed_simulation': ['CalculationEvidenceClass.SIMULATION', - 'CalculationEvidenceClass.SIMULATION', - 'CalculationEvidenceClass.SIMULATION', - 'dict.fromkeys', - 'generation.gini_simpson_diversity', - 'generation.state_counts', - 'generation.state_counts', - 'generation.state_counts', - 'generation.state_counts', - 'generation.state_counts', - 'generation.state_frequencies', - 'generation.state_frequencies', - 'generation.step', - 'generation.step', - 'generation.support', - 'generation.support', - 'generation.support', - 'generation.support_size', - 'generation.support_size', - 'generation.support_size', - 'generations.append', - 'inputs.effective_distribution', - 'inputs.effective_distribution', - 'inputs.input_basis', - 'inputs.numerical_policy', - 'inputs.representation', - 'inputs.representation', - 'inputs.representation', - 'inputs.scope', - 'inputs.scope', - 'inputs.scope', - 'paths.append', - 'replicate.generations', - 'replicate.generations', - 'replicate.replicate_index', - 'replicate.replicate_index', - 'target.update', - 'target.update', - 'trajectories.append', - 'value.assumptions', - 'value.assumptions', - 'value.contraction_factor', - 'value.evidence_class', - 'value.expected_diversity', - 'value.expected_diversity_metadata', - 'value.experimental', - 'value.initial_gini_simpson_diversity', - 'value.inputs', - 'value.limitations', - 'value.method', - 'value.method', - 'value.method_version', - 'value.method_version', - 'value.model_name', - 'value.model_name', - 'value.numerical_underflow_steps', - 'value.numpy_version', - 'value.random_seed', - 'value.replicate_schedule', - 'value.resample_size', - 'value.resample_size', - 'value.resample_size', - 'value.rng_name', - 'value.sampled_paths', - 'value.sampler_algorithm', - 'value.simulation_horizon', - 'value.simulation_horizon', - 'value.simulation_replicates', - 'value.simulation_replicates', - 'value.state_order', - 'value.state_order', - 'value.state_order', - 'value.state_order', - 'value.state_schedule', - 'value.trajectory_metadata', - 'value.trajectory_metadata'], - '_closure': ["payload['derived_metrics'].setdefault", - 'provenance.confidence', - 'provenance.confidence', - 'provenance.confidence', - 'provenance.confidence.counts', - 'provenance.confidence.field_coverage', - 'provenance.confidence.status', - 'provenance.direct_grounding', - 'provenance.direct_grounding', - 'provenance.direct_grounding', - 'provenance.direct_grounding.known_closed_count', - 'provenance.direct_grounding.known_closed_count.value', - 'provenance.direct_grounding.known_open_count', - 'provenance.direct_grounding.known_open_count.value', - 'provenance.direct_grounding.unresolved_grounding_count', - 'provenance.direct_grounding.unresolved_grounding_count.value', - 'provenance.grounding_field_coverage', - 'provenance.input_has_errors', - 'provenance.provenance_required_field_coverage', - 'provenance.provenance_row_coverage', - 'provenance.scope', - 'provenance.validation_messages', - 'value.classification_basis', - 'value.classification_basis', - 'value.confidence_counts', - 'value.confidence_disclosure', - 'value.confidence_disclosure', - 'value.confidence_field_coverage', - 'value.confidence_status', - 'value.denominator', - 'value.denominator', - 'value.denominator', - 'value.denominator_basis', - 'value.grounding_field_coverage', - 'value.grounding_field_coverage', - 'value.grounding_field_coverage', - 'value.input_has_errors', - 'value.interval_width', - 'value.known_closed_count', - 'value.known_closed_count', - 'value.known_closed_count', - 'value.known_open_count', - 'value.known_open_count', - 'value.known_open_count', - 'value.limitations', - 'value.lower_bound', - 'value.operationalization_label', - 'value.provenance_required_field_coverage', - 'value.provenance_row_coverage', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope.denominator_basis', - 'value.scope.included_record_keys', - 'value.unresolved_grounding_count', - 'value.unresolved_grounding_count', - 'value.unresolved_grounding_count', - 'value.upper_bound', - 'value.validation_messages'], - '_comparison': ['bundle.version_order', - 'compatibility.collision_groups', - 'compatibility.collision_groups', - 'compatibility.context', - 'compatibility.earlier_state_semantics', - 'compatibility.earlier_state_semantics', - 'compatibility.earlier_state_semantics', - 'compatibility.earlier_state_semantics', - 'compatibility.earlier_state_semantics', - 'compatibility.harmonized_representation', - 'compatibility.harmonized_representation', - 'compatibility.harmonized_representation', - 'compatibility.harmonized_representation', - 'compatibility.harmonized_representation', - 'compatibility.harmonized_state_semantics', - 'compatibility.harmonized_state_semantics', - 'compatibility.harmonized_state_semantics', - 'compatibility.harmonized_state_semantics', - 'compatibility.later_state_semantics', - 'compatibility.later_state_semantics', - 'compatibility.later_state_semantics', - 'compatibility.later_state_semantics', - 'compatibility.later_state_semantics', - 'compatibility.limitations', - 'compatibility.mapping', - 'compatibility.mapping', - 'compatibility.mapping', - 'compatibility.method', - 'compatibility.method', - 'compatibility.method', - 'context.earlier_representation', - 'context.earlier_representation', - 'context.earlier_representation', - 'context.earlier_representation', - 'context.earlier_representation', - 'context.earlier_scope', - 'context.earlier_scope', - 'context.earlier_scope', - 'context.earlier_scope', - 'context.earlier_scope', - 'context.earlier_scope', - 'context.earlier_scope.dataset_versions', - 'context.earlier_scope.excluded_record_keys', - 'context.earlier_scope.included_record_keys', - 'context.earlier_scope.scope_id', - 'context.later_representation', - 'context.later_representation', - 'context.later_representation', - 'context.later_representation', - 'context.later_representation', - 'context.later_scope', - 'context.later_scope', - 'context.later_scope', - 'context.later_scope', - 'context.later_scope', - 'context.later_scope', - 'context.later_scope.dataset_versions', - 'context.later_scope.excluded_record_keys', - 'context.later_scope.included_record_keys', - 'context.later_scope.scope_id', - 'context.version_order', - 'context.version_order', - 'context.version_order', - 'context.version_order.order', - 'context.version_order.order_source', - 'effects.append', - 'harmonized.denominator_basis', - 'harmonized.frequency_denominator', - 'harmonized.input_basis', - 'harmonized.weighting', - 'item.weighting', - 'item.weighting', - 'item.weighting.weighting_mode', - 'item.weighting.weighting_mode', - 'mapping.direction', - 'mapping.direction', - 'mapping.direction', - 'mapping.direction', - 'mapping.direction', - 'mapping.source_representation', - 'mapping.source_state_semantics', - 'mapping.state_mapping', - 'mapping.state_mapping', - 'mapping.state_mapping.items', - 'mapping.target_representation', - 'mapping.target_state_semantics', - 'order.index', - 'order.index', - 'original.denominator_basis', - 'original.frequency_denominator', - 'original.input_basis', - 'original.weighting', - "payload['derived_metrics'].setdefault", - "payload['derived_metrics'].setdefault", - 'scalar.reason_codes', - 'scalar.status', - 'scalar_scope.dataset_versions', - 'scalar_scope.denominator_basis', - 'scalar_scope.excluded_record_keys', - 'scalar_scope.included_record_keys', - 'scalar_scope.scope_id', - 'support.values', - 'value.added_states', - 'value.compatibility', - 'value.extinct_states', - 'value.gini_simpson_diversity_delta', - 'value.gini_simpson_diversity_delta', - 'value.gini_simpson_diversity_delta', - 'value.gini_simpson_diversity_delta.reason_codes', - 'value.gini_simpson_diversity_delta.status', - 'value.harmonized_earlier', - 'value.harmonized_earlier', - 'value.harmonized_earlier', - 'value.harmonized_earlier', - 'value.harmonized_earlier', - 'value.harmonized_earlier', - 'value.harmonized_earlier', - 'value.harmonized_earlier', - 'value.harmonized_earlier', - 'value.harmonized_earlier.representation', - 'value.harmonized_earlier.scope', - 'value.harmonized_earlier.support', - 'value.harmonized_earlier.support_size', - 'value.harmonized_earlier.support_size.value', - 'value.harmonized_earlier.weighting', - 'value.harmonized_later', - 'value.harmonized_later', - 'value.harmonized_later', - 'value.harmonized_later', - 'value.harmonized_later', - 'value.harmonized_later', - 'value.harmonized_later', - 'value.harmonized_later', - 'value.harmonized_later', - 'value.harmonized_later.representation', - 'value.harmonized_later.scope', - 'value.harmonized_later.support', - 'value.harmonized_later.support_size', - 'value.harmonized_later.support_size.value', - 'value.harmonized_later.weighting', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.mapping_effect', - 'value.mapping_effect', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier', - 'value.original_earlier.denominator_basis', - 'value.original_earlier.input_basis', - 'value.original_earlier.input_basis', - 'value.original_earlier.input_basis', - 'value.original_earlier.input_basis', - 'value.original_earlier.input_basis', - 'value.original_earlier.representation', - 'value.original_earlier.scope', - 'value.original_earlier.support', - 'value.original_earlier.support_size', - 'value.original_earlier.support_size.value', - 'value.original_earlier.weighting', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later', - 'value.original_later.denominator_basis', - 'value.original_later.input_basis', - 'value.original_later.input_basis', - 'value.original_later.representation', - 'value.original_later.scope', - 'value.original_later.support', - 'value.original_later.support_size', - 'value.original_later.support_size.value', - 'value.original_later.weighting', - 'value.reason_codes', - 'value.reason_codes', - 'value.reason_codes', - 'value.retained_states', - 'value.retention_denominator', - 'value.retention_denominator', - 'value.retention_denominator', - 'value.retention_denominator', - 'value.retention_denominator', - 'value.retention_denominator_basis', - 'value.set_metadata', - 'value.set_metadata', - 'value.status', - 'value.status', - 'value.status', - 'value.support_added_count', - 'value.support_delta', - 'value.support_delta', - 'value.support_delta', - 'value.support_delta.metadata', - 'value.support_delta.metadata', - 'value.support_delta.metadata.assumptions', - 'value.support_delta.metadata.scope', - 'value.support_loss_count', - 'value.support_retention_ratio'], - '_coverage': ['value.denominator', - 'value.denominator', - 'value.denominator', - 'value.denominator', - 'value.denominator_name', - 'value.numerator', - 'value.numerator', - 'value.numerator', - 'value.ratio'], - '_definition': ['definition.path'], - '_diagnostics': ['ValidationSeverity.INFO', - 'ValidationSeverity.WARNING', - 'message.code', - 'message.code', - 'message.code', - 'message.code', - 'message.field', - 'message.field', - 'message.field', - 'message.field', - 'message.file_role', - 'message.file_role', - 'message.file_role', - 'message.file_role', - 'message.file_role', - 'message.file_role', - 'message.file_role.value', - 'message.file_role.value', - 'message.line_number', - 'message.message', - 'message.message', - 'message.message', - 'message.message', - 'message.record_key', - 'message.record_key', - 'message.record_key', - 'message.record_key', - 'message.row_number', - 'message.row_number', - 'message.severity', - 'message.severity', - 'message.severity', - 'message.severity', - 'message.severity.value', - "payload['errors'].append", - "payload['inputs']['limitations'].append", - "payload['warnings'].append"], - '_disclosure_recommend': ["payload['recommended_next_metadata'].append"], - '_disclosure_scope': ["payload['inputs'].get"], - '_disclosure_signal': ['dict.fromkeys', 'result.update'], - '_disclosure_unavailable': ["payload['unavailable_conclusions'].append", 'result.update'], - '_disclosures': ['basis.append', - 'basis.append', - 'content_cap.get', - 'derived.get', - 'derived.get', - 'derived.get', - "derived.get('closure_exposure', {}).get", - 'direct.get', - 'longitudinal_cap.get', - 'paths.append', - 'paths.append', - "payload['capabilities'].get", - "payload['capabilities'].get", - "payload['capabilities'].get", - "payload['observed_facts'].get", - 'provenance.get', - 'provenance.get', - 'provenance.get', - 'provenance.get', - 'reasons.extend', - 'support.get', - 'tail.get'], - '_distribution': ['CalculationStatus.AVAILABLE', - 'CalculationStatus.AVAILABLE', - 'CalculationStatus.AVAILABLE', - 'CalculationStatus.AVAILABLE', - "payload['derived_metrics'].setdefault", - "payload['derived_metrics'].setdefault", - "payload['derived_metrics'].setdefault('diversity', {}).setdefault", - "payload['derived_metrics'].setdefault('diversity', {}).setdefault('by_version', " - '{}).setdefault', - "payload['derived_metrics'].setdefault('support', {}).setdefault", - "payload['derived_metrics'].setdefault('support', {}).setdefault('by_version', " - '{}).setdefault', - "payload['observed_facts'].setdefault", - "payload['observed_facts'].setdefault", - "payload['observed_facts'].setdefault('state_counts', {}).setdefault", - "payload['observed_facts'].setdefault('supplied_state_probabilities', {}).setdefault", - 'row.state_count', - 'row.state_count', - 'row.state_count', - 'row.state_count', - 'row.state_count', - 'row.state_frequency', - 'row.state_frequency', - 'row.state_frequency', - 'row.state_frequency', - 'row.state_id', - 'row.state_id', - 'row.state_id', - 'row.state_id', - 'row.state_id', - 'row.state_id', - 'row.state_id', - 'row.state_mass', - 'row.state_mass', - 'row.state_mass', - 'row.state_mass', - 'row.state_mass', - 'scalar.reason_codes', - 'scalar.status', - 'value.analyzed_record_count', - 'value.analyzed_record_count', - 'value.count_metadata', - 'value.count_metadata', - 'value.count_metadata', - 'value.count_metadata', - 'value.denominator_basis', - 'value.denominator_basis', - 'value.denominator_basis', - 'value.denominator_basis', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_denominator', - 'value.frequency_metadata', - 'value.frequency_metadata', - 'value.gini_simpson_diversity', - 'value.gini_simpson_diversity', - 'value.input_basis', - 'value.input_basis', - 'value.input_basis', - 'value.input_basis', - 'value.input_basis', - 'value.input_basis', - 'value.input_basis', - 'value.input_basis', - 'value.input_basis', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.mass_metadata', - 'value.mass_metadata', - 'value.mass_metadata', - 'value.mass_metadata', - 'value.numerical_policy', - 'value.probability_residual', - 'value.reason_codes', - 'value.reason_codes', - 'value.reason_codes', - 'value.reason_codes', - 'value.reason_codes', - 'value.representation', - 'value.representation', - 'value.representation', - 'value.representation', - 'value.representation', - 'value.representation', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope.dataset_versions', - 'value.scope.dataset_versions', - 'value.scope.denominator_basis', - 'value.scope.included_record_keys', - 'value.scope.included_record_keys', - 'value.simpson_concentration', - 'value.simpson_concentration', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.states', - 'value.status', - 'value.status', - 'value.status', - 'value.status', - 'value.status', - 'value.status', - 'value.status', - 'value.status', - 'value.status', - 'value.status', - 'value.supplied_probability_total', - 'value.support', - 'value.support', - 'value.support', - 'value.support_size', - 'value.support_size', - 'value.support_size', - 'value.support_size.value', - 'value.weighting', - 'value.weighting', - 'value.weighting', - 'value.weighting', - 'value.weighting', - 'value.weighting', - 'value.weighting.weighting_mode'], - '_distribution_exclusions': ['exclusions.append', - 'item.exclusion_reason', - 'item.exclusion_reason', - 'item.exclusion_reason.value', - 'item.record_key', - 'item.record_key', - 'item.state_id', - "payload['observed_facts'].get", - "payload['observed_facts'].get(family, {}).get", - "payload['observed_facts'].get(family, {}).get('by_version', {}).get", - 'payload[section].get', - 'payload[section].get(family, {}).get', - "payload[section].get(family, {}).get('by_version', {}).get", - "payload[section].get(family, {}).get('by_version', {}).get(version, " - '{}).values', - 'scope.dataset_versions', - 'scope.excluded_record_keys', - 'scope.excluded_record_keys', - 'value.excluded_assignments', - 'value.excluded_assignments', - 'value.excluded_assignments', - 'value.excluded_assignments', - 'value.excluded_assignments', - 'value.unweighted', - 'value.unweighted.scope'], - '_duplicates': ['CalculationEvidenceClass.OBSERVED_FACT', - 'CalculationStatus.AVAILABLE', - 'CalculationStatus.UNAVAILABLE', - 'group.record_keys', - 'group.record_keys', - 'group.record_keys', - 'group.record_keys', - 'group.record_keys', - 'group.record_keys', - 'group.record_keys', - 'group.record_keys', - 'groups.append', - 'item.value', - "payload['observed_facts'].setdefault", - 'seen.intersection', - 'seen.update', - 'status.value', - 'value.coverage', - 'value.duplicate_group_count', - 'value.duplicate_group_count', - 'value.duplicate_group_count', - 'value.duplicate_group_count', - 'value.duplicate_group_count', - 'value.duplicate_group_count.reason_codes', - 'value.duplicate_group_count.status', - 'value.duplicate_group_count.value', - 'value.duplicate_group_count.value', - 'value.duplicate_record_count', - 'value.duplicate_record_count', - 'value.duplicate_record_count', - 'value.duplicate_record_count.value', - 'value.duplicate_record_count.value', - 'value.evidence_class', - 'value.exact_duplicate_groups', - 'value.limitations', - 'value.limitations', - 'value.representation', - 'value.representation', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope.included_record_keys'], - '_extinction': ['CalculationEvidenceClass.SIMULATION', - 'CalculationEvidenceClass.SIMULATION', - 'CalculationStatus.AVAILABLE', - 'first.input_basis', - 'first.limitations', - 'first.numerical_policy', - 'first.numerical_policy', - 'first.one_step_extinction_probability', - 'first.one_step_extinction_probability.metadata', - 'first.one_step_extinction_probability.metadata.assumptions', - 'first.representation', - 'first.representation', - 'first.resample_size', - 'first.resample_size', - 'first.resample_size', - 'first.resample_size', - 'first.scope', - 'first.scope', - 'scalar.metadata', - 'scalar.metadata', - 'scalar.reason_codes', - 'scalar.status', - 'scalar.status', - 'scalar.value', - 'scalar.value', - 'target.update', - "target['initial_distribution'].append", - "target['parameters']['state_order'].append", - 'value.evaluation_method', - 'value.evidence_class', - 'value.experimental', - 'value.input_basis', - 'value.method', - 'value.model_name', - 'value.numerical_policy', - 'value.numerical_underflow', - 'value.one_step_extinction_probability', - 'value.random_seed', - 'value.representation', - 'value.representation', - 'value.resample_size', - 'value.scope', - 'value.scope', - 'value.simulation_horizon', - 'value.state_frequency', - 'value.state_frequency', - 'value.state_id', - 'value.state_id', - 'value.state_id', - 'value.state_id'], - '_inputs': ['FileRole.EMBEDDING_DATA', - 'FileRole.EXTERNAL_REFERENCE', - 'ValidationSeverity.ERROR', - 'ValidationSeverity.FATAL', - 'affected.append', - 'artifact.fields', - 'artifact.file_format', - 'artifact.file_format.value', - 'artifact.path', - 'artifact.path', - 'artifact.path', - 'artifact.role', - 'artifact.role', - 'artifact.role', - 'artifact.role', - 'artifact.role', - 'artifact.role', - 'artifact.role.value', - 'artifact.row_count', - 'artifact.sha256', - 'artifact.sha256', - 'artifact.size_bytes', - 'bundle.generation', - 'bundle.generation', - 'bundle.generation.messages', - 'bundle.inventory', - 'bundle.mapping_traces', - 'bundle.mapping_traces', - 'bundle.observability', - 'bundle.observability.validation_messages', - 'bundle.provenance', - 'bundle.provenance', - 'bundle.provenance_join', - 'bundle.records', - 'bundle.records', - 'bundle.validation_messages', - 'bundle.validation_messages', - 'bundle.version_order', - 'bundle.version_order', - 'bundle.version_order', - 'bundle.version_order.loaded_versions', - 'bundle.version_order.order', - 'bundle.version_order.order_source', - 'coverage.denominator', - 'coverage.denominator', - 'dict.fromkeys', - 'hashes.add', - 'join.grounding_field_coverage', - 'join.messages', - 'join.provenance_required_field_coverage', - 'join.provenance_row_coverage', - 'join.provenance_row_coverage', - 'join.provenance_row_coverage.denominator_name', - 'join.scope_record_keys', - 'join.scope_record_keys', - 'join.scope_record_keys', - 'join.selected_dataset_versions', - 'message.code', - 'message.file_path', - 'message.file_role', - 'message.severity', - 'messages_by_location.get', - 'messages_by_location.get', - 'messages_by_location.get', - 'messages_by_location.setdefault', - 'messages_by_location.setdefault((message.file_role, message.file_path), []).append', - 'operations.append', - "payload['inputs']['artifacts'].append", - "payload['inputs']['file_hashes'].append", - 'records_by_version.get', - 'records_by_version.setdefault', - 'records_by_version.setdefault(row.record_key.dataset_version, []).append', - 'row.location', - 'row.location', - 'row.location.file_path', - 'row.location.file_role', - 'row.record_key', - 'row.record_key', - 'row.record_key', - 'row.record_key.dataset_version', - 'row.record_key.dataset_version', - 'trace.fields', - 'trace.mapping_sha256', - 'trace.unmapped_fields', - 'unmapped.update', - 'versions_by_location.get', - 'versions_by_location.setdefault', - 'versions_by_location.setdefault((row.location.file_role, row.location.file_path), set()).add'], - '_key': ['value.dataset_version', 'value.record_id'], - '_lineage_observations': ['CapabilityKey.LINEAGE', - 'bundle.observability', - 'bundle.observability.capabilities', - 'bundle.version_order', - 'bundle.version_order.order', - 'capability.coverage', - 'capability.requirements_met', - 'coverage.denominator', - 'coverage.denominator', - 'coverage.denominator', - 'coverage.denominator', - 'coverage.numerator', - 'coverage.numerator'], - '_message_family': ['message.code', - 'message.field', - 'message.field', - 'message.field', - 'message.field', - 'message.file_role', - 'message.file_role', - 'message.file_role.value'], - '_metadata': ['value.assumptions', - 'value.assumptions', - 'value.evidence_class', - 'value.evidence_class', - 'value.formula_id', - 'value.formula_id', - 'value.limitations', - 'value.limitations', - 'value.method', - 'value.metric_name', - 'value.metric_name', - 'value.owner_id', - 'value.owner_id', - 'value.representation', - 'value.representation', - 'value.scope', - 'value.scope', - 'value.unit', - 'value.unit', - 'value.weighting', - 'value.weighting', - 'value.weighting', - 'value.weighting', - 'value.weighting', - 'value.weighting.weight_field', - 'value.weighting.weighting_mode'], - '_metadata_text': ['CalculationEvidenceClass.SIMULATION', - 'CalculationEvidenceClass.SIMULATION', - 'CalculationEvidenceClass.SIMULATION', - 'methods.get', - 'name.startswith', - 'value.assumptions', - 'value.assumptions', - 'value.assumptions', - 'value.assumptions', - 'value.evidence_class', - 'value.evidence_class', - 'value.evidence_class', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.method', - 'value.metric_name', - 'value.owner_id', - 'value.owner_id', - 'value.owner_id', - 'value.owner_id', - 'value.owner_id', - 'value.owner_id'], - '_policy': ['value.absolute_tolerance', - 'value.absolute_tolerance', - 'value.probability_mass_tolerance', - 'value.probability_mass_tolerance', - 'value.relative_tolerance', - 'value.relative_tolerance'], - '_provenance': ['CalculationReason.EMPTY_SCOPE', - 'CalculationReason.EMPTY_SCOPE', - 'CalculationStatus.AVAILABLE', - 'CalculationStatus.AVAILABLE', - 'CalculationStatus.UNAVAILABLE', - 'CalculationStatus.UNAVAILABLE', - 'bundle.provenance_join', - 'coverage.denominator', - 'coverage.ratio', - 'coverage.ratio', - 'coverage.ratio', - 'coverage.ratio', - 'grounding.assignments', - 'grounding.assignments', - 'grounding.assignments', - 'grounding.known_closed_count', - 'grounding.known_open_count', - 'grounding.limitations', - 'grounding.scope', - 'grounding.unresolved_grounding_count', - 'item.counts', - 'item.counts', - 'item.counts', - 'item.counts', - 'item.counts_metadata', - 'item.field_coverage', - 'item.field_coverage', - 'item.field_coverage', - 'item.field_coverage', - 'item.field_coverage', - 'item.field_coverage', - 'item.field_coverage', - 'item.field_coverage', - 'item.field_coverage.denominator', - 'item.field_coverage.ratio', - 'item.field_coverage.ratio', - 'item.field_coverage.ratio', - 'item.field_coverage.ratio', - 'item.field_coverage.ratio', - 'item.field_coverage.ratio', - 'item.field_coverage_metadata', - 'item.field_name', - 'item.reason_codes', - 'item.reason_codes', - 'item.record_key', - 'item.shares', - 'item.shares', - 'item.shares_metadata', - 'item.status', - 'item.status', - 'join.grounding_field_coverage', - 'join.provenance_required_field_coverage', - 'join.provenance_row_coverage', - 'match.provenance', - "payload['derived_metrics'].setdefault", - "payload['observed_facts'].setdefault", - 'value.analyzed_record_count', - 'value.analyzed_record_count', - 'value.analyzed_record_count.value', - 'value.confidence', - 'value.coverage_metadata', - 'value.coverage_metadata', - 'value.direct_grounding', - 'value.grounding_field_coverage', - 'value.grounding_field_coverage', - 'value.grounding_field_coverage.ratio', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.limitations', - 'value.missing_provenance_count', - 'value.missing_provenance_count', - 'value.missing_provenance_count.value', - 'value.missing_provenance_share', - 'value.provenance_required_field_coverage', - 'value.provenance_row_coverage', - 'value.records_with_matching_rows', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope.included_record_keys', - 'value.scope.included_record_keys', - 'value.source', - 'value.weighted_source', - 'value.weighted_source', - 'weighted.mass_metadata', - 'weighted.missing_provenance_weight', - 'weighted.reason_codes', - 'weighted.shares_metadata', - 'weighted.status', - 'weighted.total_weight', - 'weighted.total_weight', - 'weighted.total_weight.value', - 'weighted.weighted_missing_provenance_share', - 'weighted.weighted_source_type_masses', - 'weighted.weighted_source_type_shares'], - '_provenance_consistency': ['CalculationStatus.AVAILABLE', - 'CalculationStatus.UNAVAILABLE', - 'ValidationSeverity.ERROR', - 'ValidationSeverity.FATAL', - 'assignment.basis', - 'assignment.classification', - 'assignment.missing_required_fields', - 'bundle.provenance_join', - 'bundle.provenance_join', - 'bundle.provenance_join', - 'bundle.provenance_join.matches', - 'bundle.provenance_join.messages', - 'bundle.provenance_join.provenance_supplied', - 'bundle.records', - 'composition.counts', - 'composition.counts', - 'composition.counts', - 'composition.field_coverage', - 'composition.field_coverage', - 'composition.field_coverage', - 'composition.field_coverage.denominator', - 'composition.field_coverage.denominator_name', - 'composition.field_coverage.numerator', - 'composition.shares', - 'composition.status', - 'composition.status', - 'composition.unavailable_record_keys', - 'coverage.denominator', - 'coverage.denominator_name', - 'coverage.numerator', - 'expected.values', - 'grounding.assignments', - 'grounding.assignments', - 'grounding.assignments', - 'grounding.assignments', - 'grounding.assignments', - 'grounding.input_has_errors', - 'grounding.known_closed_count', - 'grounding.known_open_count', - 'grounding.unresolved_grounding_count', - 'grounding.validation_messages', - 'item.classification', - 'item.record_key', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance', - 'match.provenance.grounding_known', - 'match.provenance.required_fields_valid', - 'match.provenance.values', - 'match.provenance.values', - 'match.provenance.values.get', - 'match.provenance.values.get', - 'match.record_key', - 'match.record_key', - 'match.record_key', - 'match.record_key', - 'message.record_key', - 'message.record_key', - 'message.severity', - 'row.missing_required_fields', - 'row.record_key', - 'row.record_key', - 'row.record_key.dataset_version', - 'row.required_fields_valid', - 'row.values', - 'row.values', - 'scalar.value', - 'value.analyzed_record_count', - 'value.analyzed_record_count.value', - 'value.confidence', - 'value.direct_grounding', - 'value.grounding_field_coverage', - 'value.input_has_errors', - 'value.input_has_errors', - 'value.missing_provenance_count', - 'value.missing_provenance_count.value', - 'value.provenance_required_field_coverage', - 'value.provenance_row_coverage', - 'value.provenance_supplied', - 'value.records_with_matching_rows', - 'value.records_with_matching_rows.value', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.scope.dataset_versions', - 'value.scope.dataset_versions', - 'value.scope.denominator_basis', - 'value.scope.denominator_basis', - 'value.scope.excluded_record_keys', - 'value.scope.included_record_keys', - 'value.source', - 'value.validation_messages', - 'value.validation_messages'], - '_representation': ['value.binning_or_mapping_rule', - 'value.binning_or_mapping_rule', - 'value.field_name', - 'value.field_name', - 'value.missing_state_id', - 'value.missing_state_id', - 'value.missing_value_policy', - 'value.missing_value_policy', - 'value.normalization_profile', - 'value.normalization_profile', - 'value.representation_name', - 'value.representation_name', - 'value.representation_source', - 'value.representation_source', - 'value.representation_version', - 'value.representation_version'], - '_scalar': ['CalculationStatus.UNAVAILABLE', - 'field.evidence_class', - 'field.method_id', - 'field.method_id', - 'field.method_id.startswith', - 'field.owner', - 'field.unit', - 'path.rsplit', - 'reason.value', - 'value.metadata', - 'value.metadata', - 'value.metadata', - 'value.metadata', - 'value.metadata', - 'value.metadata', - 'value.metadata', - 'value.metadata.assumptions', - 'value.metadata.limitations', - 'value.metadata.method', - 'value.metadata.weighting', - 'value.metadata.weighting', - 'value.metadata.weighting.weight_field', - 'value.metadata.weighting.weighting_mode', - 'value.reason_codes', - 'value.reason_codes', - 'value.status', - 'value.status', - 'value.status', - 'value.status.value', - 'value.value', - 'value.value'], - '_scope': ['bundle.records', - 'bundle.version_order', - 'bundle.version_order.loaded_versions', - 'row.record_key', - 'value.dataset_versions', - 'value.dataset_versions', - 'value.dataset_versions', - 'value.denominator_basis', - 'value.denominator_basis', - 'value.excluded_record_keys', - 'value.excluded_record_keys', - 'value.excluded_record_keys', - 'value.excluded_record_keys', - 'value.included_record_keys', - 'value.included_record_keys', - 'value.included_record_keys', - 'value.included_record_keys', - 'value.scope_id', - 'value.scope_id'], - '_simulation_inputs': ['value.correction_applied', - 'value.correction_method', - 'value.effective_distribution', - 'value.effective_distribution', - 'value.effective_probability_total', - 'value.normalization_divisor', - 'value.numerical_policy', - 'value.probability_corrections', - 'value.probability_residual', - 'value.representation', - 'value.scope', - 'value.supplied_distribution', - 'value.supplied_distribution', - 'value.supplied_probability_total'], - '_table': ['CalculationStatus.UNAVAILABLE', - 'CalculationStatus.UNAVAILABLE', - 'field.evidence_class', - 'field.method_id', - 'field.method_id', - 'field.method_id.startswith', - 'field.owner', - 'field.unit', - 'metadata.assumptions', - 'metadata.limitations', - 'metadata.method', - 'metadata.weighting', - 'metadata.weighting', - 'metadata.weighting.weight_field', - 'metadata.weighting.weighting_mode', - 'reason.value', - 'status.value'], - '_tail': ['CalculationStatus.AVAILABLE', - 'item.in_tail', - 'item.in_tail', - 'item.rarity_rank', - 'item.state_count', - 'item.state_frequency', - 'item.state_id', - 'item.state_id', - 'ranking.append', - 'scalar.reason_codes', - 'scalar.status', - 'value.denominator', - 'value.denominator', - 'value.limitations', - 'value.limitations', - 'value.membership_metadata', - 'value.options', - 'value.options', - 'value.options', - 'value.options', - 'value.options', - 'value.options.count_threshold', - 'value.options.frequency_threshold', - 'value.options.rule', - 'value.options.rule', - 'value.options.state_ids', - 'value.ranking_metadata', - 'value.ranking_rule', - 'value.rarity_ranking', - 'value.rarity_ranking', - 'value.reason_codes', - 'value.reason_codes', - 'value.representation', - 'value.representation', - 'value.scope', - 'value.scope', - 'value.scope', - 'value.status', - 'value.status', - 'value.status', - 'value.tail_membership', - 'value.tail_membership', - 'value.tail_record_share', - 'value.tail_support_size'], - 'assemble_report': ['CanonicalReport.from_dict', - 'bundle.inventory', - 'bundle.observability', - 'bundle.observability.capabilities', - 'bundle.records', - 'bundle.records', - 'bundle.records', - 'bundle.records', - 'bundle.records', - 'bundle.records', - 'bundle.records', - 'bundle.version_order', - 'bundle.version_order.loaded_versions', - 'closure.scope', - 'closure.scope.scope_id', - 'closure.validation_messages', - 'duplicates.representation', - 'duplicates.scope', - 'duplicates.scope.scope_id', - 'empirical_representations.append', - 'empirical_representations.append', - 'empirical_representations.append', - 'empirical_representations.append', - 'failure.capability', - 'failure.capability', - 'failure.capability', - 'failure.capability.value', - 'failure.capability.value', - 'failure.messages', - 'failure.messages', - 'failure.messages', - 'failures.setdefault', - 'failures.setdefault(failure.capability.value, []).extend', - 'key.value', - "operations['content_diagnostics'].append", - "operations['content_diagnostics'].append", - "operations['content_diagnostics'].append", - "operations['dataset_longitudinal'].append", - "operations['intervention_simulation'].append", - "operations['intervention_simulation'].append", - "operations['intervention_simulation'].append", - "operations['provenance'].append", - "operations['provenance'].append", - "payload['inputs']['limitations'].append", - 'provenance.scope', - 'provenance.scope.scope_id', - 'provenance.validation_messages', - 'row.record_key', - 'row.record_key', - 'row.record_key.dataset_version', - 'run.get', - 'run.get', - 'supplied.coverage', - 'supplied.coverage', - 'supplied.coverage', - 'supplied.coverage', - 'supplied.coverage', - 'supplied.coverage.denominator', - 'supplied.coverage.numerator', - 'supplied.coverage.ratio', - 'supplied.coverage.ratio', - 'supplied.input_basis', - 'supplied.representation', - 'supplied.scope', - 'supplied.scope.scope_id', - 'supplied.selection_messages', - 'supplied.unweighted', - 'supplied.unweighted', - 'supplied.unweighted', - 'supplied.unweighted', - 'supplied.unweighted', - 'supplied.unweighted', - 'supplied.unweighted', - 'supplied.unweighted', - 'supplied.unweighted.representation', - 'supplied.unweighted.representation', - 'supplied.unweighted.scope', - 'supplied.unweighted.scope', - 'supplied.unweighted.scope', - 'supplied.unweighted.scope', - 'supplied.unweighted.scope', - 'supplied.unweighted.scope.excluded_record_keys', - 'supplied.unweighted.scope.included_record_keys', - 'supplied.unweighted.scope.included_record_keys', - 'supplied.unweighted.scope.scope_id', - 'supplied.weighted', - 'supplied.weighted', - 'supplied.weighted', - 'supplied.weighted', - 'supplied.weighted.representation', - 'supplied.weighted.scope', - 'tail.representation', - 'tail.scope', - 'tail.scope.scope_id']} - - -def phase4_step3_assembly_boundary(path: Path) -> None: - """Inspect only the supplied assembly source without executing it. - - Fixed reviewed inventories constrain definitions, imports, call targets and - validation arithmetic. They are never regenerated from inspected source. - The release controller separately freezes result and the other runtime modules. - """ - try: - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - except (OSError, UnicodeError, SyntaxError) as exc: - raise ValueError("Phase 4 Step 3 assembly source cannot be inspected") from exc - doc = ast.get_docstring(tree) or "" - if "Owner IDs:" not in doc or "Current phase status:" not in doc: - raise ValueError("Phase 4 Step 3 assembly ownership metadata is missing") - allowed_top = (ast.Import, ast.ImportFrom, ast.Assign, ast.AnnAssign, - ast.FunctionDef, ast.ClassDef) - for index, node in enumerate(tree.body): - if index == 0 and isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str): - continue - if not isinstance(node, allowed_top): - raise ValueError("Phase 4 Step 3 assembly has an unapproved module statement") - imports = [] - functions = [] - classes = [] - calls = {} - arithmetic = {} - mutations = [] - declarations = [] - traversal = [] - attributes = {} - headers = [] - literal_declarations = [] - literal_containers = [] - call_names = {node.func.id for node in ast.walk(tree) - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)} - forbidden = (ast.Lambda, ast.AsyncFunctionDef, ast.Await, ast.With, - ast.AsyncWith, ast.Global, ast.Nonlocal, ast.NamedExpr) - forbidden_names = {"eval", "exec", "compile", "__import__", "getattr", - "setattr", "delattr", "globals", "locals", "vars", - "open", "input", "breakpoint", "print", "exit", "quit", - "asdict", "astuple"} - forbidden_attributes = {"__builtins__", "__globals__", "__code__", "__subclasses__", - "__class__", "__dict__", "__getattribute__", "__getattr__", "__setattr__", - "__reduce__", "__reduce_ex__", "__mro__", "__bases__", - "open", "read", "read_text", "read_bytes", "write", "write_text", - "write_bytes", "mkdir", "unlink", "system", "popen", "connect", - "connect_ex", "getaddrinfo", "urlopen", "urlretrieve", "send", "sendall"} - stack = [(tree, "")] - while stack: - node, scope = stack.pop() - if isinstance(node, forbidden): - raise ValueError("Phase 4 Step 3 assembly has a callback or executable control outside scope") - if isinstance(node, (ast.FunctionDef, ast.ClassDef)): - scope = node.name if scope == "" else scope + "." + node.name - if isinstance(node, ast.FunctionDef): - functions.append(scope) - headers.append((scope, ast.dump(node.args, include_attributes=False), - tuple(ast.unparse(item) for item in node.decorator_list), - ast.unparse(node.returns) if node.returns is not None else "")) - else: - classes.append(scope) - headers.append((scope, tuple(ast.unparse(item) for item in node.bases), - tuple(ast.unparse(item) for item in node.keywords), - tuple(ast.unparse(item) for item in node.decorator_list))) - if isinstance(node, (ast.Import, ast.ImportFrom)): - if scope != "" or node not in tree.body: - raise ValueError("Phase 4 Step 3 assembly imports must be explicit and at module scope") - imports.append(ast.dump(node, include_attributes=False)) - if isinstance(node, ast.Name) and node.id in forbidden_names: - raise ValueError("Phase 4 Step 3 assembly names a forbidden execution capability") - if scope != "" and ((isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store) and node.id in call_names) - or (isinstance(node, ast.arg) and node.arg in call_names)): - raise ValueError("Phase 4 Step 3 assembly shadows an approved call target") - if isinstance(node, ast.Attribute): - attributes.setdefault(scope, []).append(ast.unparse(node)) - if isinstance(node, ast.Attribute) and node.attr in forbidden_attributes: - raise ValueError("Phase 4 Step 3 assembly names a forbidden IO or dynamic capability") - if isinstance(node, ast.Call): - if not isinstance(node.func, (ast.Name, ast.Attribute)): - raise ValueError("Phase 4 Step 3 assembly invokes a dynamic call target") - calls.setdefault(scope, []).append(ast.unparse(node.func)) - if isinstance(node.func, ast.Attribute) and node.func.attr == "__setattr__": - mutations.append((scope, ast.unparse(node))) - if isinstance(node, (ast.BinOp, ast.UnaryOp, ast.AugAssign)): - arithmetic.setdefault(scope, []).append(ast.unparse(node)) - if (scope == "" or scope in classes) and isinstance(node, (ast.Assign, ast.AnnAssign, ast.Delete)): - declarations.append((scope, ast.unparse(node))) - if scope == "_metadata_text" and isinstance(node, (ast.Assign, ast.AnnAssign)): - literal_declarations.append((scope, ast.unparse(node))) - if isinstance(node, (ast.Dict, ast.Tuple, ast.List, ast.Set)): - try: - literal = ast.literal_eval(node) - except (ValueError, TypeError, SyntaxError): - literal = None - if literal: - literal_containers.append((scope, ast.unparse(node))) - if isinstance(node, (ast.Yield, ast.YieldFrom, ast.Delete)): - traversal.append((scope, ast.unparse(node))) - if isinstance(node, ast.Attribute) and isinstance(node.ctx, (ast.Store, ast.Del)): - raise ValueError("Phase 4 Step 3 assembly has an unapproved attribute mutation") - stack.extend((child, scope) for child in ast.iter_child_nodes(node)) - if sorted(imports) != PHASE4_STEP3_ASSEMBLY_IMPORTS: - raise ValueError("Phase 4 Step 3 assembly imports differ from the reviewed import-safe typed-result inventory") - if sorted(functions) != PHASE4_STEP3_ASSEMBLY_FUNCTIONS or sorted(classes) != PHASE4_STEP3_ASSEMBLY_CLASSES: - raise ValueError("Phase 4 Step 3 assembly definitions differ from the approved contract") - if {name: sorted(values) for name, values in calls.items()} != PHASE4_STEP3_ASSEMBLY_CALLS: - raise ValueError("Phase 4 Step 3 assembly call targets differ from the reviewed inventory") - if {name: sorted(values) for name, values in arithmetic.items()} != PHASE4_STEP3_ASSEMBLY_ARITHMETIC: - raise ValueError("Phase 4 Step 3 assembly arithmetic exceeds reviewed semantic validation") - if sorted(mutations) != PHASE4_STEP3_ASSEMBLY_MUTATIONS: - raise ValueError("Phase 4 Step 3 assembly mutation differs from reviewed immutable construction") - if sorted(declarations) != PHASE4_STEP3_ASSEMBLY_DECLARATIONS: - raise ValueError("Phase 4 Step 3 assembly module declarations differ from the approved contract") - if sorted(traversal) != PHASE4_STEP3_ASSEMBLY_TRAVERSAL: - raise ValueError("Phase 4 Step 3 assembly generator or teardown differs from reviewed local traversal") - if {name: sorted(values) for name, values in attributes.items()} != PHASE4_STEP3_ASSEMBLY_ATTRIBUTES: - raise ValueError("Phase 4 Step 3 assembly attributes exceed the reviewed pure contract") - if sorted(literal_declarations) != PHASE4_STEP3_ASSEMBLY_LITERAL_DECLARATIONS: - raise ValueError("Phase 4 Step 3 assembly method registry differs from reviewed owner declarations") - if sorted(literal_containers) != PHASE4_STEP3_ASSEMBLY_LITERAL_CONTAINERS: - raise ValueError("Phase 4 Step 3 assembly literal evidence boundaries differ from reviewed declarations") - if sorted(headers) != PHASE4_STEP3_ASSEMBLY_HEADERS: - raise ValueError("Phase 4 Step 3 assembly definition headers differ from the reviewed contract") - - - -def phase4_step3_main(step: int = 3) -> int: - """Validate Step 3 control, the frozen canonical contract and pure assembly.""" - import hashlib - import runpy - - if type(step) is not int or step != 3: - raise ValueError("Only authorized Phase 4 Step 3 traceability is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step3_traceability_control") - control["audit_phase4_step3"](step=step) - schema_path = ROOT / "schemas/report.schema.json" - if hashlib.sha256(schema_path.read_bytes()).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Phase 4 Step 3 report schema differs from the frozen Step 2 contract") - phase4_step2_result_boundary(PACKAGE / "result.py") - phase4_step3_assembly_boundary(PACKAGE / "reports/assembly.py") - print("Phase 4 Step 3: frozen canonical results and reviewed evidence assembly: PASS") - return 0 - - -# Step 4 adds privacy after canonical assembly. Historical stage checkers above -# retain their original source. Fixed identities below are review data; none is -# derived from the current source when the boundary runs. -PHASE4_STEP4_INHERITED_AST_SHA256 = { - "src/recursive_integrity_toolkit/result.py": "4cbf8f28cb0aa04edc0a931c8a27b64c18ee1975b7e93a51edf62d34f6615ca9", - "src/recursive_integrity_toolkit/reports/assembly.py": "23b0574b56baa9b147a221742c53a37d995352d52d51f528bda0316eef30fd72", - "src/recursive_integrity_toolkit/utils/hashing.py": "d263ee776c8c2c9751106b959589cb7c49f5bbc3be0cbf7634717c88dccb8b98", - "src/recursive_integrity_toolkit/utils/logging.py": "35ad89e78bf13acf4ca8adea239b5c79db8001f5cbabbf2cfc7e27680e9c3e57", - "src/recursive_integrity_toolkit/config.py": "a85a8bab70aa2e2625be4943d1769ac33301d8a6725396925c7dbf44ac28f3b2", -} -PHASE4_STEP4_RUNTIME_AST_SHA256 = { - "src/recursive_integrity_toolkit/config.py": "92fb35fe0895f799c0b90c20d80f0262c92da8afb2b96e17154a870837ed71d5", - "src/recursive_integrity_toolkit/reports/assembly.py": "e96f66d5fcaa45e2eea53f19e78db345d703b4d565b4d635944682c728eb2329", - "src/recursive_integrity_toolkit/result.py": "a5f9f85b2e10128202feb1bc122f912315b0b46ccf1e2c67e0c7127229160a12", - "src/recursive_integrity_toolkit/utils/hashing.py": "6774dbc0177e876731da2d163f85acd583430dee1747a092528e0edaa869c0da", - "src/recursive_integrity_toolkit/utils/logging.py": "973a746221ea3d83fd1ab42a5687075bed6ad962a0c727cfa6350e94f71c94d3" -} -PHASE4_STEP4_ADDITIVE_BINDINGS = { - "src/recursive_integrity_toolkit/result.py": [ - "PrivacyMode", - "RecordIdMode", - "SafeReportView" - ], - "src/recursive_integrity_toolkit/reports/assembly.py": [ - "_PRIVACY_SAFE_TEXT", - "_privacy_contract", - "_privacy_alias", - "_privacy_text", - "_privacy_value", - "_privacy_run_fields", - "privacy_view", - "build_run_metadata" - ], - "src/recursive_integrity_toolkit/utils/hashing.py": [ - "canonical_json_bytes", - "sha256_canonical", - "_phase4_secret_file", - "IdentifierProtection" - ], - "src/recursive_integrity_toolkit/utils/logging.py": [ - "_TEMPLATES", - "_FIELDS", - "_EXCEPTION_TYPES", - "_severity", - "_code", - "_protection", - "_policy", - "safe_code", - "safe_diagnostic_text", - "safe_remediation", - "safe_field", - "safe_role", - "safe_effects", - "safe_record_key", - "_position", - "safe_diagnostic", - "format_diagnostic", - "emit_diagnostic" - ], - "src/recursive_integrity_toolkit/config.py": [ - "Phase4Options", - "_phase4_config_data", - "_phase4_option_path", - "_phase4_literal", - "resolve_phase4_options", - "phase4_config_summary", - "phase4_config_hash" - ] -} - - -def _phase4_step4_ast_digest(nodes: list) -> str: - """Hash a location-free AST consistently on supported Python 3.11/3.12.""" - import hashlib - import json - - def canonical(value): - if isinstance(value, ast.AST): - return [type(value).__name__, [[name, canonical(item)] - for name, item in ast.iter_fields(value) - if name != "type_params" or item]] - if isinstance(value, list): - return [canonical(item) for item in value] - return [type(value).__name__, repr(value)] - - # Python 3.12 adds empty type_params fields. Ignoring only empty ones makes - # the same non-generic source portable; nonempty type parameters remain data. - module = ast.Module(body=nodes, type_ignores=[]) - payload = json.dumps(canonical(module), ensure_ascii=True, separators=(",", ":")) - return hashlib.sha256(payload.encode("utf-8")).hexdigest() - - -def _phase4_step4_bound_names(node) -> set[str]: - """Return only explicit top-level definition or assignment targets.""" - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - return {node.name} - if isinstance(node, ast.Assign): - targets = node.targets - elif isinstance(node, ast.AnnAssign): - targets = [node.target] - else: - return set() - return {item.id for target in targets for item in ast.walk(target) - if isinstance(item, ast.Name) and isinstance(item.ctx, ast.Store)} - - -def phase4_step4_runtime_boundary(path: Path, relative: str) -> None: - """Check one reviewed Step 4 runtime module without importing its code. - - The complete reviewed AST constrains calls, imports, declarations, arithmetic, - sink use and control flow. A separate accepted-Step-3 identity freezes all - inherited non-import declarations after the explicitly named additions are - removed. Only the module ownership documentation is outside the AST digest. - """ - if relative not in PHASE4_STEP4_INHERITED_AST_SHA256: - raise ValueError("Phase 4 Step 4 runtime path is outside the approved five modules") - try: - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - except (OSError, UnicodeError, SyntaxError) as exc: - raise ValueError("Phase 4 Step 4 runtime source cannot be inspected") from exc - doc = ast.get_docstring(tree) or "" - if "Owner IDs:" not in doc or "Current phase status:" not in doc: - raise ValueError("Phase 4 Step 4 runtime ownership metadata is missing") - body = tree.body[1:] if ast.get_docstring(tree) is not None else tree.body - if _phase4_step4_ast_digest(body) != PHASE4_STEP4_RUNTIME_AST_SHA256.get(relative): - raise ValueError("Phase 4 Step 4 runtime differs from the reviewed privacy implementation") - additions = set(PHASE4_STEP4_ADDITIVE_BINDINGS.get(relative, ())) - inherited = [] - observed_additions = set() - for node in body: - if isinstance(node, (ast.Import, ast.ImportFrom)): - continue - names = _phase4_step4_bound_names(node) - if names & additions: - if not names <= additions or names & observed_additions: - raise ValueError("Phase 4 Step 4 additions rebind an inherited or duplicate name") - observed_additions.update(names) - else: - inherited.append(node) - if observed_additions != additions: - raise ValueError("Phase 4 Step 4 reviewed additive declaration is missing") - if _phase4_step4_ast_digest(inherited) != PHASE4_STEP4_INHERITED_AST_SHA256[relative]: - raise ValueError("Phase 4 Step 4 changed an inherited runtime helper or contract") - - -def phase4_step4_privacy_boundary(root: Path) -> None: - """Enforce the five explicit privacy modules and frozen public schema.""" - import hashlib - - for relative in sorted(PHASE4_STEP4_INHERITED_AST_SHA256): - phase4_step4_runtime_boundary(root / relative, relative) - schema = root / "schemas/report.schema.json" - if hashlib.sha256(schema.read_bytes()).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Phase 4 Step 4 changed the frozen public report schema") - - -def phase4_step4_main(step: int = 4) -> int: - """Validate Step 4 control, additive privacy and inherited helper identities.""" - import runpy - - if type(step) is not int or step != 4: - raise ValueError("Only authorized Phase 4 Step 4 traceability is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step4_traceability_control") - control["audit_phase4_step4"](step=step) - phase4_step4_privacy_boundary(ROOT) - print("Phase 4 Step 4: inherited contracts and reviewed privacy, metadata and diagnostics: PASS") - return 0 - - -# Step 5 opens only two rendering placeholders. These review identities are -# fixed checker data, independent of the source being inspected at gate time. -PHASE4_STEP5_RUNTIME_AST_SHA256 = { - "src/recursive_integrity_toolkit/reports/json_report.py": "04c46cdd644021b01b285dd6e62a2b026be27ac60607cc6117582d9581c0a283", - "src/recursive_integrity_toolkit/reports/markdown_report.py": "08ece6fc0ba3ea57dab335508978a974439c7f37aa84e4dcd652615fa6628e84", -} -PHASE4_STEP5_RUNTIME_OWNERS = { - "src/recursive_integrity_toolkit/reports/json_report.py": ["PR-013", "PR-016"], - "src/recursive_integrity_toolkit/reports/markdown_report.py": ["PR-013", "PR-018"], -} - - -def phase4_step5_runtime_boundary(path: Path, relative: str) -> None: - """Inspect the complete reviewed renderer AST without importing its code. - - A fixed identity covers imports, function signatures, literal declarations, - calls, effects and control flow. The two accepted placeholders contained no - inherited functions or assignments. Every other runtime module is frozen by - the separately verified accepted Step 4 file inventory. - """ - if relative not in PHASE4_STEP5_RUNTIME_AST_SHA256: - raise ValueError("Phase 4 Step 5 runtime path is outside the two approved renderers") - try: - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - except (OSError, UnicodeError, SyntaxError) as exc: - raise ValueError("Phase 4 Step 5 runtime source cannot be inspected") from exc - doc = ast.get_docstring(tree) or "" - if ("Owner IDs:" not in doc or "Current phase status:" not in doc - or not all(owner in doc for owner in PHASE4_STEP5_RUNTIME_OWNERS[relative])): - raise ValueError("Phase 4 Step 5 renderer ownership metadata is missing") - body = tree.body[1:] if ast.get_docstring(tree) is not None else tree.body - if _phase4_step4_ast_digest(body) != PHASE4_STEP5_RUNTIME_AST_SHA256[relative]: - raise ValueError("Phase 4 Step 5 runtime differs from the reviewed rendering implementation") - - -def phase4_step5_renderers_boundary(root: Path) -> None: - """Enforce both pure renderer implementations and the frozen public schema.""" - import hashlib - - for relative in sorted(PHASE4_STEP5_RUNTIME_AST_SHA256): - phase4_step5_runtime_boundary(root / relative, relative) - if hashlib.sha256((root / "schemas/report.schema.json").read_bytes()).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Phase 4 Step 5 changed the frozen public report schema") - - -def phase4_step5_main(step: int = 5) -> int: - """Validate explicit Step 5 authorization and reviewed renderer boundaries.""" - import runpy - - if type(step) is not int or step != 5: - raise ValueError("Only authorized Phase 4 Step 5 traceability is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), - run_name="phase4_step5_traceability_control") - control["audit_phase4_step5"](step=step) - phase4_step5_renderers_boundary(ROOT) - print("Phase 4 Step 5: reviewed JSON and Markdown rendering and frozen prior contracts: PASS") - return 0 - - -PHASE4_STEP6_RUNTIME_AST_SHA256 = {'src/recursive_integrity_toolkit/utils/logging.py': '1ce0d0c77339f4c63e9f091fa218cfcb0202c709b235643f550a8c3d4c0160b5', - 'src/recursive_integrity_toolkit/utils/paths.py': 'f9e77ed3638b520f81769ae2c09095572913b501383d53124d51cde2826a5efa'} - -PHASE4_STEP6_INHERITED_AST_SHA256 = {'src/recursive_integrity_toolkit/utils/logging.py': '973a746221ea3d83fd1ab42a5687075bed6ad962a0c727cfa6350e94f71c94d3', - 'src/recursive_integrity_toolkit/utils/paths.py': '0010c7bbf86e389ac2599eae023a89caaabe48b10a65cfaeddb01aa619112824'} - -PHASE4_STEP6_INHERITED_NODES = {'src/recursive_integrity_toolkit/utils/logging.py': 24, - 'src/recursive_integrity_toolkit/utils/paths.py': 17} - -def phase4_step6_runtime_boundary(path: Path, relative: str) -> None: - """Enforce reviewed output code and unchanged inherited input/diagnostic AST.""" - if relative not in PHASE4_STEP6_RUNTIME_AST_SHA256: - raise ValueError("Step 6 runtime is limited to output paths and diagnostics") - try: - tree = ast.parse(path.read_text(encoding="utf-8")) - except (OSError, UnicodeError, SyntaxError) as error: - raise ValueError("Step 6 output helper cannot be inspected") from error - doc = ast.get_docstring(tree) or "" - if "Owner IDs:" not in doc or "Current phase status:" not in doc or "PR-015" not in doc: - raise ValueError("Step 6 output ownership metadata is missing") - body = tree.body[1:] if ast.get_docstring(tree) is not None else tree.body - if _phase4_step4_ast_digest(body) != PHASE4_STEP6_RUNTIME_AST_SHA256[relative]: - raise ValueError("Step 6 output implementation differs from the reviewed AST") - if _phase4_step4_ast_digest(body[:PHASE4_STEP6_INHERITED_NODES[relative]]) != PHASE4_STEP6_INHERITED_AST_SHA256[relative]: - raise ValueError("Step 6 changed an inherited input or diagnostic helper") - - -def phase4_step6_output_boundary(root: Path) -> None: - for relative in sorted(PHASE4_STEP6_RUNTIME_AST_SHA256): - phase4_step6_runtime_boundary(root / relative, relative) - phase4_step5_renderers_boundary(root) - - -def phase4_step6_main(step: int = 6) -> int: - import runpy - if type(step) is not int or step != 6: - raise ValueError("Only authorized Phase 4 Step 6 traceability is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), run_name="phase4_step6_traceability_control") - control["audit_phase4_step6"](step=step) - phase4_step6_output_boundary(ROOT) - print("Phase 4 Step 6: safe publication and preserved input/report contracts: PASS") - return 0 - - -PHASE4_STEP7_RUNTIME_AST_SHA256 = {'src/recursive_integrity_toolkit/cli.py': 'c1791ddf073a9ca5ddf08268b7a66e1a67f359a1b8d97a636d37e914dbb0f40b', - 'src/recursive_integrity_toolkit/config.py': 'cbb96ab25130e4fe45a94cc03299aaa27015a496e38e7dcc6bec1c5d2f7e2573'} - -PHASE4_STEP7_CONFIG_INHERITED_AST_SHA256 = '92fb35fe0895f799c0b90c20d80f0262c92da8afb2b96e17154a870837ed71d5' - -PHASE4_STEP7_CONFIG_INHERITED_NODES = 40 - -def phase4_step7_runtime_boundary(path: Path, relative: str) -> None: - """Inspect fixed reviewed orchestration AST and unchanged inherited config.""" - if relative not in PHASE4_STEP7_RUNTIME_AST_SHA256: - raise ValueError("Step 7 runtime is limited to CLI and additive configuration") - try: - tree = ast.parse(path.read_text(encoding="utf-8")) - except (OSError, UnicodeError, SyntaxError) as error: - raise ValueError("Step 7 runtime cannot be inspected") from error - doc = ast.get_docstring(tree) or "" - if "Owner IDs:" not in doc or "Current phase status:" not in doc or "PR-016" not in doc: - raise ValueError("Step 7 ownership documentation is missing") - body = tree.body[1:] - if _phase4_step4_ast_digest(body) != PHASE4_STEP7_RUNTIME_AST_SHA256[relative]: - raise ValueError("Step 7 runtime differs from the reviewed orchestration") - if relative.endswith("/config.py") and _phase4_step4_ast_digest(body[:PHASE4_STEP7_CONFIG_INHERITED_NODES]) != PHASE4_STEP7_CONFIG_INHERITED_AST_SHA256: - raise ValueError("Step 7 changed inherited configuration behavior") - - -def phase4_step7_cli_boundary(root: Path) -> None: - for relative in sorted(PHASE4_STEP7_RUNTIME_AST_SHA256): - phase4_step7_runtime_boundary(root / relative, relative) - phase4_step6_output_boundary(root) - - -def phase4_step7_main(step: int = 7) -> int: - import runpy - if type(step) is not int or step != 7: - raise ValueError("Only authorized Phase 4 Step 7 traceability is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), run_name="phase4_step7_traceability_control") - control["audit_phase4_step7"](step=step) - phase4_step7_cli_boundary(ROOT) - print("Phase 4 Step 7: local audit/validate orchestration and frozen calculation/report owners: PASS") - return 0 - - -PHASE4_STEP8_RUNTIME_AST_SHA256 = {'src/recursive_integrity_toolkit/cli.py': 'c4bd7981153974857c03ebe79b7daca6cc68ab324980b8434e6ef8dd3d7fd9c8', - 'src/recursive_integrity_toolkit/config.py': 'df7f8d96997cf0bcae5c53d78a130f07ca847c3bbc609e8e6d5c507d926507fa'} - -PHASE4_STEP8_CONFIG_INHERITED_AST_SHA256 = 'cbb96ab25130e4fe45a94cc03299aaa27015a496e38e7dcc6bec1c5d2f7e2573' - -PHASE4_STEP8_CONFIG_INHERITED_NODES = 42 - -def phase4_step8_runtime_boundary(path: Path, relative: str) -> None: - """Inspect fixed reviewed orchestration AST and unchanged inherited config.""" - if relative not in PHASE4_STEP8_RUNTIME_AST_SHA256: - raise ValueError("Step 8 runtime is limited to CLI and additive configuration") - try: - tree = ast.parse(path.read_text(encoding="utf-8")) - except (OSError, UnicodeError, SyntaxError) as error: - raise ValueError("Step 8 runtime cannot be inspected") from error - doc = ast.get_docstring(tree) or "" - if "Owner IDs:" not in doc or "Current phase status:" not in doc or "PR-016" not in doc: - raise ValueError("Step 8 ownership documentation is missing") - body = tree.body[1:] - if _phase4_step4_ast_digest(body) != PHASE4_STEP8_RUNTIME_AST_SHA256[relative]: - raise ValueError("Step 8 runtime differs from the reviewed orchestration") - if relative.endswith("/config.py") and _phase4_step4_ast_digest(body[:PHASE4_STEP8_CONFIG_INHERITED_NODES]) != PHASE4_STEP8_CONFIG_INHERITED_AST_SHA256: - raise ValueError("Step 8 changed inherited configuration behavior") - - -def phase4_step8_cli_boundary(root: Path) -> None: - for relative in sorted(PHASE4_STEP8_RUNTIME_AST_SHA256): - phase4_step8_runtime_boundary(root / relative, relative) - phase4_step6_output_boundary(root) - - -def phase4_step8_main(step: int = 8) -> int: - import runpy - if type(step) is not int or step != 8: - raise ValueError("Only authorized Phase 4 Step 8 traceability is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), run_name="phase4_step8_traceability_control") - control["audit_phase4_step8"](step=step) - phase4_step8_cli_boundary(ROOT) - print("Phase 4 Step 8: explicit pair and packaged Hero orchestration and frozen calculation/report owners: PASS") - return 0 - - -def phase4_current_main(step: int = 11) -> int: - import runpy - if type(step) is not int or step != 11: - raise ValueError("Only authorized Phase 4 Step 11 traceability is available") - control = runpy.run_path(str(ROOT / "scripts/release_check.py"), run_name="phase4_current_traceability_control") - control["audit_phase4_current"](step=step) - # The accepted Step 10 product bytes remain frozen, including its assembly repair. - # Only the package version literal changes in the forty-module runtime. - print("Phase 4 Step 11: final dev3 candidate; 39 runtime modules and all schemas frozen; exact version literals: PASS") + module = _module_name(node, relative) + imports = [module, *(module + "." + alias.name for alias in node.names)] + for module in imports: + if module.split(".")[0] in FORBIDDEN_IMPORTS: + raise ValueError(f"Prohibited network/execution/dependency import in {relative}: {module}") + if relative in INPUT_ONLY and module.startswith(tuple("recursive_integrity_toolkit." + layer for layer in ("metrics", "lineage", "reports"))): + raise ValueError(f"Input-only module imports an analytical/report owner: {relative}") + if relative in RENDERERS and module.startswith(tuple("recursive_integrity_toolkit." + layer for layer in ("io", "metrics", "lineage", "representations"))): + raise ValueError(f"Renderer imports a calculation or input owner: {relative}") + if module.split(".")[0] == "pyarrow" and relative != "io/loaders.py": + raise ValueError(f"Optional Parquet dependency escaped its loader: {relative}") + if module.split(".")[0] == "numpy" and relative != "metrics/resampling.py": + raise ValueError(f"Numerical dependency escaped the explicit sampler: {relative}") + if isinstance(node, ast.Call): + name = node.func.id if isinstance(node.func, ast.Name) else node.func.attr if isinstance(node.func, ast.Attribute) else "" + regex_compile = (isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "re" and name == "compile") + if name in DYNAMIC_CALLS and not regex_compile: + raise ValueError(f"Dynamic execution is prohibited: {relative}") + if relative in RENDERERS and name in {"open", "read_text", "read_bytes", "write_text", "write_bytes"}: + raise ValueError(f"Renderer performs file IO: {relative}") + return {"owned_modules_checked": len(paths), "local_only_and_layer_boundaries": "PASS"} + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + if any(arg in {"--phase", "--step"} or arg.startswith(("--phase=", "--step=")) for arg in argv): + raise SystemExit("Historical phase dispatch has been retired. Run this current check without --phase/--step; prior gates are recoverable from Git history.") + argparse.ArgumentParser(description=__doc__).parse_args(argv) + print(json.dumps(audit(), indent=2)) return 0 -def cli_main(argv: list[str] | None = None) -> int: - """Select the active phase without changing historical checker semantics.""" - import argparse - - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--phase", type=int, choices=(3, 4)) - parser.add_argument("--step", type=int) - args = parser.parse_args(argv) - if args.phase is None and args.step is None: - return main() - if args.phase == 3 and args.step == PHASE3_ACTIVE_STEP: - return main() - if args.phase == 4 and args.step == 1: - return phase4_main(step=args.step) - if args.phase == 4 and args.step == 2: - return phase4_step2_main(step=args.step) - if args.phase == 4 and args.step == 3: - return phase4_step3_main(step=args.step) - if args.phase == 4 and args.step == 4: - return phase4_step4_main(step=args.step) - if args.phase == 4 and args.step == 5: - return phase4_step5_main(step=args.step) - if args.phase == 4 and args.step == 6: - return phase4_step6_main(step=args.step) - if args.phase == 4 and args.step == 7: - return phase4_step7_main(step=args.step) - if args.phase == 4 and args.step == 8: - return phase4_step8_main(step=args.step) - if args.phase == 4 and args.step == 11: - return phase4_current_main(step=args.step) - parser.error("Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5 or --phase 4 --step 6 or --phase 4 --step 7 or --phase 4 --step 8 or --phase 4 --step 11") - return 2 - - if __name__ == "__main__": - raise SystemExit(cli_main()) + raise SystemExit(main()) diff --git a/scripts/release_check.py b/scripts/release_check.py index 1935888..e2a0ed7 100644 --- a/scripts/release_check.py +++ b/scripts/release_check.py @@ -1,20 +1,20 @@ -"""Audit explicit phase boundaries and retain tested intermediate artifacts. +"""Current source and candidate checks for Recursive Integrity Toolkit. -Maintainer tooling only. Phase 2 authority and restoration guards stay frozen. -Phase 3 Step 11 verifies the final development milestone and its retained evidence. -Only the approved dev2 version literals change in the frozen runtime/metadata. -Publication, merge and later-phase implementation remain unauthorized. +Step 10 opens only the dev4 version declarations. Historical dispatch, source-body +migrations and phase registries are recoverable from the accepted Git commit. +Installed checks below retain their existing product, privacy and package cases. """ from __future__ import annotations import argparse -import ast +from functools import lru_cache import hashlib import importlib.metadata +import importlib.util +import io import json import os -from pathlib import Path -import re +from pathlib import Path, PurePosixPath import subprocess import sys import tarfile @@ -24,476 +24,95 @@ import zipfile ROOT = Path(__file__).resolve().parents[1] -STEP9_BASELINE = "e1790d60b88370651eb59dc31fbb2343e2daec1f" -PHASE2_FINAL = "78554993febb01609cb90814cc24cce2012bf7d7" -PHASE2_TREE = "706b99e27c2d284a7435ae03e2eb2376214bfa46" -PHASE2_TEST_TREE = "d787d9b06c41a91f8587f6b57a8af710419d114f" -PLAN_SHA256 = "e1c9a6776e3cd2511d37a6bbbdb4a9d5a33b66ff6b00bc00bf3e08a01e6576f1" -ACTIVE_PHASE, ACTIVE_STEP = 3, 11 -PHASE3_COMPLETE = True # Accepted implementation below; final handoff still requires all roles on HEAD. -PHASE3_IMPLEMENTATION_ACCEPTANCE = "9dc321c6997a231209653a0f86d0476824e96e3a" -PHASE3_IMPLEMENTATION_TREE = "a1086ca95075aff3ea8a83afae548806b2079dc9" -STEP10_ALLOWED = { - "README.md", "CHANGELOG.md", "docs/architecture.md", "docs/data_schema.md", - "docs/privacy.md", ".github/workflows/ci.yml", ".github/workflows/security.yml", - ".github/workflows/release.yml", ".github/workflows/golden.yml", - "tests/integration/test_ci_workflows.py", "tests/integration/test_license_notices.py", - "tests/integration/test_no_algorithms.py", "tests/integration/test_prohibited_structure.py", - "tests/integration/test_repository_structure.py", "scripts/check_spec_consistency.py", - "scripts/check_traceability.py", "scripts/release_check.py", "pyproject.toml", - "PHASE_2_COMPLETION.md", "PHASE_2_VALIDATION_REPORT.md", "PHASE_2_ARCHITECTURE_COMPLIANCE_REPORT.md", -} -PHASE3_G = { - "scripts/check_traceability.py", "scripts/check_spec_consistency.py", "scripts/release_check.py", - "tests/conftest.py", "tests/integration/test_no_algorithms.py", "tests/integration/test_ci_workflows.py", - "tests/integration/test_repository_structure.py", "tests/integration/test_owner_ids.py", - "tests/integration/test_package_import.py", "tests/integration/test_package_install.py", - "tests/integration/test_optional_dependency.py", "tests/integration/test_no_network.py", - "tests/unit/test_PR016_determinism.py", ".github/workflows/ci.yml", ".github/workflows/security.yml", - ".github/workflows/golden.yml", ".github/workflows/release.yml", "docs/architecture.md", - "docs/theory_traceability.md", "PHASE_3_DECISIONS.md", "PHASE_3_BASELINE.json", -} -STEP1_NEW = {"PHASE_3_PLAN.md", "PHASE_3_DECISIONS.md", "PHASE_3_BASELINE.json", - "tests/unit/test_phase3_contracts.py", "tests/golden/phase3_math_cases.json", "tests/golden/phase3_math_cases.md"} -STEP1_ALLOWED = PHASE3_G | STEP1_NEW | { - "src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py"} -STEP1_FINAL = "20487e356239d5b5642b6fff595b381d0faa2a4e" -STEP1_TREE = "7d9867fb7543bfd50a9b01bb48530a3411121d31" -STEP2_NEW = {"tests/unit/test_T1_representation.py", - "tests/fixtures/minimal_valid/phase3_field_records.jsonl", - "tests/fixtures/invalid_schema/phase3_representation_cases.json"} -STEP2_EXCEPTION = "tests/unit/test_phase3_contracts.py" -STEP2_ALLOWED = PHASE3_G | STEP2_NEW | { - "src/recursive_integrity_toolkit/representations/base.py", - "src/recursive_integrity_toolkit/representations/field.py", - "src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py", - "docs/data_schema.md", STEP2_EXCEPTION, -} -STEP2_FINAL = "9e5d4c4f38834c42a5b7c19de4e67f471ab9bbca" -STEP2_TREE = "096e038145bced4b01f2c0059659082a72501112" -STEP3_NEW = {"tests/fixtures/minimal_valid/phase3_exact_content.jsonl"} -STEP3_ALLOWED = PHASE3_G | STEP3_NEW | { - "src/recursive_integrity_toolkit/representations/content_hash.py", - "src/recursive_integrity_toolkit/metrics/duplicates.py", - "src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py", - "tests/unit/test_PR006_duplicates.py", "tests/unit/test_T1_representation.py", - "docs/data_schema.md", "docs/privacy.md", -} -STEP3_FINAL = "d2651ca755c82af2e2fec3963bcb6fe270a23be6" -STEP3_TREE = "7871e6253d770d1561d9f732dd5c0b0f842a4138" -STEP4_NEW = {"tests/fixtures/weighted/phase3_distribution_cases.json"} -STEP4_ALLOWED = PHASE3_G | STEP4_NEW | { - "src/recursive_integrity_toolkit/metrics/diversity.py", - "src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py", - "tests/unit/test_T1_support.py", "tests/unit/test_T1_diversity.py", - "tests/unit/test_phase3_contracts.py", -} -STEP4_FINAL = "b943a9712ed1e5b64e09c2c394077fdeed78114f" -STEP4_TREE = "e8f503b44cee0d3db059c03da6599fe44503f03b" -STEP5_NEW = {"tests/fixtures/provenance_partial/phase3_composition.json", - "tests/fixtures/provenance_unknown/phase3_grounding_crossed.json", - "tests/fixtures/weighted/phase3_source_weights.json"} -STEP5_ALLOWED = PHASE3_G | STEP5_NEW | { - "src/recursive_integrity_toolkit/metrics/provenance.py", - "src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py", - "tests/unit/test_PR004_coverage.py", "tests/unit/test_PR005_source_shares.py", "tests/unit/test_T3_provenance.py", -} -STEP5_FINAL = "4611f897d054b352733cc90543eef6f6d71cd811" -STEP5_TREE = "faffd5717dc1bf99e0081a122a338c65a68a13a7" -STEP6_NEW = {"tests/fixtures/provenance_unknown/phase3_direct_bounds.json"} -STEP6_ALLOWED = PHASE3_G | STEP6_NEW | { - "src/recursive_integrity_toolkit/metrics/bounds.py", - "src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py", - "tests/unit/test_T3_bounds.py", "tests/unit/test_T3_provenance.py", -} -STEP6_FINAL = "c7d5ceed2f154bd2ccffbe49961dbf1b65b9df63" -STEP6_TREE = "bc1878e10a9fd056afd196e85126af7ddb9555e5" -STEP7_NEW = {"tests/fixtures/minimal_valid/phase3_tail_cases.json"} -STEP7_ALLOWED = PHASE3_G | STEP7_NEW | { - "src/recursive_integrity_toolkit/metrics/tail.py", - "src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py", - "tests/unit/test_T2_tail.py", "tests/unit/test_phase3_contracts.py", -} -STEP7_FINAL = "d3d95bde50762e60b4dd03cb2bd988618c9a10fe" -STEP7_TREE = "5cb81772542ebfa87c634572068dbf9ee49f41d0" -STEP8_NEW = {"tests/fixtures/resampling/phase3_closed_cases.json"} -STEP8_ALLOWED = PHASE3_G | STEP8_NEW | { - "src/recursive_integrity_toolkit/metrics/resampling.py", - "src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py", - "tests/unit/test_T1_resampling.py", "tests/unit/test_T5_reopening.py", "tests/unit/test_phase3_contracts.py", -} -STEP8_FINAL = "a43cba0e81fd89c3b20a64737bfa89cd2bac4de9" -STEP8_TREE = "f7ecf2326770b362b51ba67357ec450e311a086a" -STEP9_NEW = {"tests/unit/test_T1_compatibility.py", - "tests/fixtures/representation_compatible/phase3_pair.json", - "tests/fixtures/representation_incompatible/phase3_pair.json"} -STEP9_ALLOWED = PHASE3_G | STEP9_NEW | { - "src/recursive_integrity_toolkit/representations/compatibility.py", - "src/recursive_integrity_toolkit/representations/field.py", - "src/recursive_integrity_toolkit/metrics/diversity.py", - "src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py", - "tests/unit/test_T1_support.py", "tests/unit/test_T1_diversity.py", "tests/unit/test_T1_representation.py", -} -STEP9_FINAL = "03fe1e7e99c8170e6be7714649ab0d926e01b6d6" -STEP9_TREE = "765d28e6ff4715fa6d59f44760f1a937489d7120" -PHASE3_STEP10_NEW = {'tests/golden/test_phase3_math.py', 'tests/integration/test_phase3_metric_pipeline.py'} -PHASE3_STEP10_ALLOWED = PHASE3_G | PHASE3_STEP10_NEW | {'tests/golden/phase3_math_cases.md', 'tests/performance/test_hero_runtime.py', 'tests/golden/phase3_math_cases.json', 'tests/performance/test_metadata_100k.py'} -STEP10_FINAL = "150a2a105e01883672ef0c2300b41b0de3be352e" -STEP10_TREE = "b609cb75913fc6352729086819a24795caae1dd8" -STEP10_CONTROL_SHA256 = "8c48a575005b6b0ed78bba8dae21703b1edd6a42bd4a548ae57b083d4f5cc9bc" -PHASE3_STEP11_NEW = {"PHASE_3_COMPLETION.md", "PHASE_3_VALIDATION_REPORT.md", - "PHASE_3_ARCHITECTURE_COMPLIANCE_REPORT.md"} -STEP11_TEST_EXCEPTIONS = {"tests/integration/test_phase3_metric_pipeline.py", - "tests/unit/test_phase3_contracts.py"} -PHASE3_STEP11_ALLOWED = PHASE3_G | PHASE3_STEP11_NEW | STEP11_TEST_EXCEPTIONS | { - "README.md", "CHANGELOG.md", "docs/data_schema.md", "docs/privacy.md", "docs/release_process.md", - "pyproject.toml", "src/recursive_integrity_toolkit/__init__.py", - "tests/integration/test_cli_validation.py", "tests/integration/test_license_notices.py", - "tests/integration/test_prohibited_structure.py", -} -STEP10_RUNTIME_SHA256 = '11b427fa9add50bcd7b2230170ed84d2e4cb4985ade9ee8f9829da0460f12f35' -STEP10_FROZEN_FILES = { - "PHASE_3_PLAN.md": "e1c9a6776e3cd2511d37a6bbbdb4a9d5a33b66ff6b00bc00bf3e08a01e6576f1", - "examples/hero/EXPECTED_OUTPUTS.md": "64d5e696c5e6067b7c34fe85b4d75760588c77fbf19bdd61ee18120385ce8c67", - "examples/hero/config.json": "2e069eb9c0105bf05c145ca15eec47834eb66f4408e8cdde9ff4935f5b920241", - "examples/hero/provenance.csv": "fc4ed3921d087e37996228dd8a37c4c51a0a3c3c91ded3e62a21e1b10f407438", - "examples/hero/records_v1.csv": "b62c9aee057c61f641009b949acb110e003acbea6a399cc607a78ee6e6f0543d", - "examples/hero/records_v2.csv": "d691605fd6bb37e785945ae34c8c18ea8409201f7d78449d39df08df7770599b", - "examples/hero/version_order.json": "3e33ca8b2095f35f55d22265071bce00119c9282b72bb02cee3275c646898387", - "pyproject.toml": "bf3954bc42f3bd9c8275bd7c6858b32266ef014a8c17148b68453a97f3625033", - "schemas/config.schema.json": "cc73411e59d77ded0f9a2a493a507c71901e0a30c7dbca548763a237241927d1", - "schemas/normalized_manifest.schema.json": "13598968bd3ed9ec017759ac4e71a118965ee8895c8c6ba503d0e225fdfad766", - "schemas/report.schema.json": "17130701fc683b3813f649bec45332fe923cd7259a76e6e8e261910207afc23b", - "schemas/schema_mapping.schema.json": "31ef2691dbb11c7861caf15853d7bd0a6dcae25fd10bf26a7b4b1aaa968df85d", - "schemas/version_order.schema.json": "94b10a5842b9f432703789d3388ec3c3d5d07e9224df4266ab29010c0390836d", - "tests/golden/phase3_math_cases.json": "b494d50a1a9bd1009c060c998e4cb3c7433d0e8949f73a8d627e538fa38884b2", - "tests/golden/phase3_math_cases.md": "5f3603ad6ad8f2b1274547f2c239461e7ee8d3b7d552b04d0dd868b862af4044" +ACCEPTED_COMMIT = "15d50c3d992265e716b0b27e69c43107d947d21d" +PROTECTED_PREFIXES = ("src/", "schemas/", "examples/hero/") +CURRENT_IMPLEMENTATION_PATHS = frozenset({ + "src/recursive_integrity_toolkit/__init__.py", + "pyproject.toml", +}) +PARQUET_CASES = { + "test_PR002_parquet_real_roundtrip", + "test_PR002_parquet_real_row_limit", + "test_PR002_parquet_real_invalid_file", + "test_context_table_uses_existing_local_parsers_and_normalizer[parquet]", + "test_context_loaders_and_repeated_inputs_keep_primary_only_denominators[parquet]", } -CUMULATIVE_ALLOWED = STEP1_ALLOWED | STEP2_ALLOWED | STEP3_ALLOWED | STEP4_ALLOWED | STEP5_ALLOWED | STEP6_ALLOWED | STEP7_ALLOWED | STEP8_ALLOWED | STEP9_ALLOWED | PHASE3_STEP10_ALLOWED | PHASE3_STEP11_ALLOWED -CUMULATIVE_NEW = STEP1_NEW | STEP2_NEW | STEP3_NEW | STEP4_NEW | STEP5_NEW | STEP6_NEW | STEP7_NEW | STEP8_NEW | STEP9_NEW | PHASE3_STEP10_NEW | PHASE3_STEP11_NEW -COMPLETION = {"PHASE_2_COMPLETION.md", "PHASE_2_VALIDATION_REPORT.md", "PHASE_2_ARCHITECTURE_COMPLIANCE_REPORT.md"} -PROHIBITED = {"server", "webapp", "cloud", "telemetry", "plugins", "agents", "llm", "auth", "database", "policy_enforcement"} -PARQUET_CASES = {"test_PR002_parquet_real_roundtrip", "test_PR002_parquet_real_row_limit", "test_PR002_parquet_real_invalid_file"} -RESTORATION_PATH = "VALIDATION_PLAN.md" -RESTORATION_BEFORE_SHA256 = "564abf56c91141d0cd8ca09024f1cf6dda63bf4ba12acf6563bb7cbc158a84f7" -RESTORATION_AFTER_SHA256 = "16f5fe539da2b3cff1c3e0a2854208bd7fbc1e4c5b332684265b06a03a90cf2f" +HERO_NAMES = {"records_v1.csv", "records_v2.csv", "provenance.csv", "config.json", "version_order.json", "EXPECTED_OUTPUTS.md"} +RESOURCES = {f"src/recursive_integrity_toolkit/data/hero/{name}": f"examples/hero/{name}" for name in HERO_NAMES} +RESOURCES["src/recursive_integrity_toolkit/data/report.schema.json"] = "schemas/report.schema.json" -def verify_approved_restoration(path: str, status: str, before: bytes, after: bytes) -> dict: - """Historical guard; no new restoration permission is created.""" - if path != RESTORATION_PATH or status != "M": - raise ValueError("Restoration exception does not authorize this path or change kind") - if (type(before) is not bytes or type(after) is not bytes - or hashlib.sha256(before).hexdigest() != RESTORATION_BEFORE_SHA256 - or hashlib.sha256(after).hexdigest() != RESTORATION_AFTER_SHA256): - raise ValueError("Restoration exception requires the exact approved byte transition") - return {"path": path, "before_sha256": RESTORATION_BEFORE_SHA256, - "after_sha256": RESTORATION_AFTER_SHA256, "authorization": "Theory Owner exact restoration"} +def git(*arguments: str) -> bytes: + return subprocess.check_output(["git", "-C", str(ROOT), *arguments]) -def git(*args: str) -> bytes: - return subprocess.check_output(["git", "-C", str(ROOT), *args]) - - -def verify_phase3_control(control: dict, step: int = ACTIVE_STEP) -> None: - expected = {"active_phase": 3, "active_step": ACTIVE_STEP, "baseline_commit": PHASE2_FINAL, - "baseline_tree": PHASE2_TREE, "baseline_test_tree": PHASE2_TEST_TREE, - "approved_plan_sha256": PLAN_SHA256, "baseline_core_tests": 1159, "baseline_parquet_tests": 1162, - "permitted_paths": sorted(PHASE3_STEP11_ALLOWED), "approved_decisions": [f"P3-D{i:02d}" for i in range(1, 11)], - "phase_complete": PHASE3_COMPLETE, "next_step_authorized": False, - "previous_step_commit": STEP10_FINAL, "previous_step_tree": STEP10_TREE, - "new_files_permitted": sorted(PHASE3_STEP11_NEW), "main_merge_authorized": False, - "publication_authorized": False, - "implementation_acceptance_commit": PHASE3_IMPLEMENTATION_ACCEPTANCE, - "implementation_acceptance_tree": PHASE3_IMPLEMENTATION_TREE} - if type(control) is not dict or type(step) is not int or step != ACTIVE_STEP: - raise ValueError("Unsupported active phase or step") - for key, value in expected.items(): - if type(control.get(key)) is not type(value) or control[key] != value: - raise ValueError(f"Frozen Phase 3 control mismatch: {key}") - - -def verify_phase3_changes(changes: list[tuple[str, str]], *, incremental: bool = False) -> None: - allowed = PHASE3_STEP11_ALLOWED if incremental else CUMULATIVE_ALLOWED - new_files = PHASE3_STEP11_NEW if incremental else CUMULATIVE_NEW - for status, path in changes: - if path not in allowed or status not in ("M", "A"): - raise ValueError(f"Unauthorized active-stage change: {status} {path}") - if status == "A" and path not in new_files: - raise ValueError(f"Unapproved new file: {path}") - - -def verify_prior_step_changes(changes: list[tuple[str, str]], *, step: int) -> None: - """Retain historical negative tests against their original permission sets.""" - boundaries = {1: (STEP1_ALLOWED, STEP1_NEW), 2: (STEP2_ALLOWED, STEP2_NEW), - 3: (STEP3_ALLOWED, STEP3_NEW), 4: (STEP4_ALLOWED, STEP4_NEW), 5: (STEP5_ALLOWED, STEP5_NEW), 6: (STEP6_ALLOWED, STEP6_NEW), 7: (STEP7_ALLOWED, STEP7_NEW), 8: (STEP8_ALLOWED, STEP8_NEW), 9: (STEP9_ALLOWED, STEP9_NEW), 10: (PHASE3_STEP10_ALLOWED, PHASE3_STEP10_NEW)} - if type(step) is not int or step not in boundaries: - raise ValueError("Unsupported historical step") - allowed, new_files = boundaries[step] - for status, path in changes: - if path not in allowed or status not in ("M", "A") or (status == "A" and path not in new_files): - raise ValueError("Change was not authorized in the historical step") - - -def verify_step2_contract_test_migration(before: bytes, after: bytes) -> None: - """Only the explicitly authorized forged-stage expression may change.""" - old = b'if key in ("active_phase","active_step"): control[key]=2' - new = b'if key in ("active_phase","active_step"): control[key]+=1' - if (type(before) is not bytes or type(after) is not bytes - or before.count(old) != 1 or after != before.replace(old, new, 1)): - raise ValueError("Step 2 contract-test exception exceeded its exact approved edit") - - -def verify_step10_control(control: dict) -> None: - """Verify the immutable historical control independently of the active stage.""" - if (type(control) is not dict - or hashlib.sha256(json.dumps(control, sort_keys=True, separators=(",", ":")).encode()).hexdigest() - != STEP10_CONTROL_SHA256): - raise ValueError("Historical Step 10 control changed") +@lru_cache(maxsize=1) +def _accepted_files() -> dict[str, bytes]: + """Read the existing accepted Git authority, with no new file registry.""" + raw = git("archive", "--format=zip", ACCEPTED_COMMIT) + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + return {member.filename: archive.read(member) for member in archive.infolist() if not member.is_dir()} -def verify_step11_contract_test_migration(before: bytes, after: bytes) -> None: - """Only the two explicitly approved historical-stage bindings may change.""" - replacements = ( - (b'elif key=="phase_complete": control[key]=True', - b'elif key=="phase_complete": control[key]=not control[key]'), - (b'def test_phase3_final_completion_records_not_created():', - b'def test_phase3_final_completion_records_not_created(phase3_step10_snapshot):'), - (b'assert not (ROOT/name).exists()', - b'assert not (phase3_step10_snapshot/name).exists()'), - ) - if type(before) is not bytes or type(after) is not bytes: - raise ValueError("Test migration requires source bytes") - expected = before - for old, new in replacements: - if expected.count(old) != 1: - raise ValueError("Historical contract test binding missing or duplicated") - expected = expected.replace(old, new, 1) - if after != expected: - raise ValueError("Step 11 contract-test exception exceeded its exact approved edits") +def _regular_file(root: Path, relative: str) -> Path: + path = root / relative + inside = [root.joinpath(*Path(relative).parts[:index]) + for index in range(len(Path(relative).parts) + 1)] + if not path.is_file() or any(part.is_symlink() for part in inside): + raise ValueError(f"Missing or aliased protected file: {relative}") + return path -def verify_step11_pipeline_test_migration(before: bytes, after: bytes) -> None: - """Freeze every inherited test byte except the three approved stage bindings. +def verify_source_scope(root: Path = ROOT) -> dict: + """Reject product changes outside the currently authorized lineage owners. - Fresh Step 11 tests may only be appended. No inherited numerical assertion, - parameter, fixture identity or test name is removed or weakened. + Later authorized implementation updates this single current boundary. It does + not append another historical dispatcher or grant authority through metadata. """ - replacements = ( - (b'gate["verify_phase3_changes"]([("M",path)],incremental=True)', - b'gate["verify_prior_step_changes"]([("M",path)],step=10)'), - (b'def test_phase3_step10_scope_is_fixed_not_self_authorized(repo_root):', - b'def test_phase3_step10_scope_is_fixed_not_self_authorized(repo_root,phase3_step10_snapshot):'), - (b'control = json.loads((repo_root/"PHASE_3_BASELINE.json").read_bytes())', - b'control = json.loads((phase3_step10_snapshot/"PHASE_3_BASELINE.json").read_bytes())'), - (b'gate["verify_phase3_control"](control,10)', b'gate["verify_step10_control"](control)'), - (b'gate["verify_phase3_control"](altered,10)', b'gate["verify_step10_control"](altered)'), - (b'gate["verify_step10_snapshot"]()["runtime_modules_unchanged"]', - b'gate["verify_step10_snapshot"](phase3_step10_snapshot)["runtime_modules_unchanged"]'), - (b'def test_phase3_step10_snapshot_rejects_byte_changes(repo_root,tmp_path,path):', - b'def test_phase3_step10_snapshot_rejects_byte_changes(repo_root,tmp_path,path,phase3_step10_snapshot):'), - (b'shutil.copytree(repo_root/"src/recursive_integrity_toolkit",', - b'shutil.copytree(phase3_step10_snapshot/"src/recursive_integrity_toolkit",'), - (b'target.write_bytes((repo_root/name).read_bytes())', - b'target.write_bytes((phase3_step10_snapshot/name).read_bytes())'), - ) - if type(before) is not bytes or type(after) is not bytes: - raise ValueError("Test migration requires source bytes") - expected = before - for old, new in replacements: - if old not in expected: - raise ValueError("Historical pipeline test binding missing") - expected = expected.replace(old, new) - if not after.startswith(expected): - raise ValueError("Step 11 changed an unapproved inherited pipeline test") - additions = ast.parse(after[len(expected):]) - if not additions.body or any(not isinstance(node, ast.FunctionDef) - or not node.name.startswith("test_phase3_step11_") for node in additions.body): - raise ValueError("Only new Step 11 test functions may be appended") - - -def verify_step11_snapshot(root: Path = ROOT) -> dict: - """Require accepted bytes after undoing exactly the two approved version edits.""" - digest = hashlib.sha256() - paths = sorted((root / "src/recursive_integrity_toolkit").rglob("*.py")) - for path in paths: - raw = path.read_bytes() - if path.relative_to(root).as_posix() == "src/recursive_integrity_toolkit/__init__.py": - current = b'__version__ = "0.1.0.dev2"' - if raw.count(current) != 1: - raise ValueError("Expected the single approved root version literal") - raw = raw.replace(current, b'__version__ = "0.1.0.dev1"', 1) - digest.update(path.relative_to(root).as_posix().encode() + b"\0" + raw + b"\0") - if len(paths) != 40 or digest.hexdigest() != STEP10_RUNTIME_SHA256: - raise ValueError("Step 11 changed runtime beyond the approved version literal") - for name, expected in STEP10_FROZEN_FILES.items(): - raw = (root / name).read_bytes() - if name == "pyproject.toml": - current = b'version = "0.1.0.dev2"' - if raw.count(current) != 1: - raise ValueError("Expected the single approved package version literal") - raw = raw.replace(current, b'version = "0.1.0.dev1"', 1) - if hashlib.sha256(raw).hexdigest() != expected: - raise ValueError(f"Step 11 frozen input, dependency or oracle changed: {name}") - return {"package_modules": 40, "approved_version_literals": 2, - "all_other_runtime_bytes_unchanged": True, "frozen_input_files": len(STEP10_FROZEN_FILES)} - - -def verify_step10_snapshot(root: Path = ROOT) -> dict: - """Require all forty accepted runtime modules and frozen inputs byte-for-byte.""" - digest = hashlib.sha256() - paths = sorted((root / "src/recursive_integrity_toolkit").rglob("*.py")) - for path in paths: - digest.update(path.relative_to(root).as_posix().encode() + b"\0" + path.read_bytes() + b"\0") - if len(paths) != 40 or digest.hexdigest() != STEP10_RUNTIME_SHA256: - raise ValueError("Step 10 cannot change the accepted runtime implementation") - for name, expected in STEP10_FROZEN_FILES.items(): - if hashlib.sha256((root / name).read_bytes()).hexdigest() != expected: - raise ValueError(f"Step 10 frozen input or oracle changed: {name}") - return {"runtime_modules_unchanged": 40, "frozen_input_files": len(STEP10_FROZEN_FILES)} - - -def audit(phase: int = 2, step: int = ACTIVE_STEP) -> dict: - required = {"LICENSE", "NOTICE", "THIRD_PARTY_NOTICES.md", "THEORY_SOURCES.md"} | COMPLETION - if phase == 3: - required |= CUMULATIVE_NEW - elif phase != 2: - raise ValueError("Unsupported audit phase") - for name in required: - if not (ROOT / name).is_file(): - raise ValueError(f"Missing required file: {name}") - project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"] - if (project["name"] != "recursive-integrity-toolkit" or project["requires-python"] != ">=3.11" - or project["license"] != "Apache-2.0" or project["version"] != "0.1.0.dev2" - or project["dependencies"] != ["numpy>=2.0", "pandas>=2.2"]): - raise ValueError("Unapproved package identity, version or dependency change") - tree = ast.parse((ROOT / "src/recursive_integrity_toolkit/__init__.py").read_text(encoding="utf-8")) - versions = [ast.literal_eval(n.value) for n in tree.body if isinstance(n, ast.Assign) - and any(isinstance(t, ast.Name) and t.id == "__version__" for t in n.targets)] - if versions != [project["version"]]: - raise ValueError("Package version mismatch") - entries = re.findall(r"\| `([^`]+\.md)` \| `([0-9a-f]{64})` \|", (ROOT / "PHASE_0_APPROVAL.md").read_text(encoding="utf-8")) - if len(entries) != 16 or any(hashlib.sha256((ROOT / n).read_bytes()).hexdigest() != h for n, h in entries): - raise ValueError("Approved Phase 0 hash mismatch") - if "Apache License" not in (ROOT / "LICENSE").read_text(encoding="utf-8"): - raise ValueError("License text missing") - notices = (ROOT / "THIRD_PARTY_NOTICES.md").read_text(encoding="utf-8") - if any(name not in notices for name in ("NumPy", "pandas", "PyArrow", "pytest", "pip-audit")): - raise ValueError("Dependency notices incomplete") - if len(list((ROOT / "src/recursive_integrity_toolkit").rglob("*.py"))) != 40: - raise ValueError("Expected forty package modules") - if any(p.is_dir() and p.name in PROHIBITED for p in ROOT.rglob("*")): - raise ValueError("Prohibited directory") - if phase == 3: - control = json.loads((ROOT / "PHASE_3_BASELINE.json").read_text(encoding="utf-8")) - verify_phase3_control(control, step) - verify_step11_snapshot() - if hashlib.sha256((ROOT / "PHASE_3_PLAN.md").read_bytes()).hexdigest() != PLAN_SHA256: - raise ValueError("Approved plan bytes changed") - if control["phase0_sha256"] != dict(entries): - raise ValueError("Authority hashes differ from original approval") - result = {"phase": phase, "active_step": step if phase == 3 else None, "package_modules": 40, - "phase0_hashes_verified": 16, "license_notices": "PASS", "runtime_dependencies": "UNCHANGED", - "phase_complete": PHASE3_COMPLETE if phase == 3 else None, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_step10_diff() -> dict: - """Historical gate, restricted to its original final checkout.""" - if git("rev-parse", "HEAD").decode().strip() != PHASE2_FINAL: - raise ValueError("Historical Phase 2 gate requires its frozen checkout; choose --phase 3") - changes, restorations = [], [] - for line in git("diff", "--name-status", "--no-renames", STEP9_BASELINE, "HEAD").decode().splitlines(): - status, path = line.split("\t", 1) - if path == RESTORATION_PATH: - restorations.append(verify_approved_restoration(path, status, git("show", f"{STEP9_BASELINE}:{path}"), git("show", f"HEAD:{path}"))) - elif path not in STEP10_ALLOWED or status not in ("M", "A"): - raise ValueError("Unapproved historical change") - if status == "A" and path not in COMPLETION: - raise ValueError("Unapproved historical new file") - changes.append((status, path)) - if git("diff", "--name-only", STEP9_BASELINE, "HEAD", "--", "src", "schemas", "examples/hero", "pyproject.toml"): - raise ValueError("Historical protected files changed") - return {"changes": changes, "approved_exact_restorations": restorations} + expected = {name: raw for name, raw in _accepted_files().items() + if name.startswith(PROTECTED_PREFIXES) or name == "pyproject.toml"} + for name, raw in expected.items(): + current = _regular_file(root, name).read_bytes() + if name not in CURRENT_IMPLEMENTATION_PATHS and current != raw: + raise ValueError(f"Unauthorized product mutation outside Step 10 scope: {name}") + for prefix in PROTECTED_PREFIXES: + paths = list((root / prefix).rglob("*")) + if (root / prefix).is_symlink() or any(path.is_symlink() for path in paths): + raise ValueError(f"Aliased protected content: {prefix}") + actual = {path.relative_to(root).as_posix() for path in paths + if path.is_file() and "__pycache__" not in path.parts + # setuptools writes this metadata during normal wheel installs. + # It is outside the package; all product paths stay protected. + and not path.relative_to(root).as_posix().startswith( + "src/recursive_integrity_toolkit.egg-info/")} + if actual != {name for name in expected if name.startswith(prefix)}: + raise ValueError(f"Unauthorized product file-set mutation during Step 10: {prefix}") + return {"protected_product_files": len(expected) - len(CURRENT_IMPLEMENTATION_PATHS), + "authorized_implementation_files": len(CURRENT_IMPLEMENTATION_PATHS), + "product_file_inventory": "unchanged"} + + +def verify_frozen_specifications(root: Path = ROOT) -> dict: + """Use the already approved specification hashes from the accepted commit.""" + hashes = json.loads(_accepted_files()["PHASE_4_BASELINE.json"])["phase0_sha256"] + for name, expected in hashes.items(): + if hashlib.sha256(_regular_file(root, name).read_bytes()).hexdigest() != expected: + raise ValueError(f"Frozen specification changed: {name}") + return {"frozen_specifications": len(hashes)} + + +def verify_resources(root: Path = ROOT) -> dict: + for destination, source in RESOURCES.items(): + if _regular_file(root, destination).read_bytes() != _regular_file(root, source).read_bytes(): + raise ValueError(f"Packaged resource differs from canonical source: {destination}") + resource_root = root / "src/recursive_integrity_toolkit/data" + if {path.relative_to(root).as_posix() for path in resource_root.rglob("*") if path.is_file()} != set(RESOURCES): + raise ValueError("Packaged resource inventory differs") + return {"canonical_resource_copies": len(RESOURCES)} -def audit_phase3_diff(step: int = ACTIVE_STEP) -> dict: - audit(3, step) - if git("rev-parse", PHASE3_IMPLEMENTATION_ACCEPTANCE + "^{tree}").decode().strip() != PHASE3_IMPLEMENTATION_TREE: - raise ValueError("Accepted Phase 3 implementation tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE3_IMPLEMENTATION_ACCEPTANCE, "HEAD"], check=True) - if git("rev-parse", PHASE2_FINAL + "^{tree}").decode().strip() != PHASE2_TREE: - raise ValueError("Baseline tree mismatch") - if git("rev-parse", PHASE2_FINAL + ":tests").decode().strip() != PHASE2_TEST_TREE: - raise ValueError("Baseline test tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE2_FINAL, "HEAD"], check=True) - changes = [tuple(line.split("\t", 1)) for line in git("diff", "--name-status", "--no-renames", PHASE2_FINAL, "HEAD").decode().splitlines()] - verify_phase3_changes(changes) - if git("rev-parse", STEP1_FINAL + "^{tree}").decode().strip() != STEP1_TREE: - raise ValueError("Previous accepted Step 1 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP1_FINAL, "HEAD"], check=True) - if git("rev-parse", STEP2_FINAL + "^{tree}").decode().strip() != STEP2_TREE: - raise ValueError("Previous accepted Step 2 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP2_FINAL, "HEAD"], check=True) - if git("rev-parse", STEP3_FINAL + "^{tree}").decode().strip() != STEP3_TREE: - raise ValueError("Previous accepted Step 3 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP3_FINAL, "HEAD"], check=True) - if git("rev-parse", STEP4_FINAL + "^{tree}").decode().strip() != STEP4_TREE: - raise ValueError("Previous accepted Step 4 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP4_FINAL, "HEAD"], check=True) - if git("rev-parse", STEP5_FINAL + "^{tree}").decode().strip() != STEP5_TREE: - raise ValueError("Previous accepted Step 5 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP5_FINAL, "HEAD"], check=True) - if git("rev-parse", STEP6_FINAL + "^{tree}").decode().strip() != STEP6_TREE: - raise ValueError("Previous accepted Step 6 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP6_FINAL, "HEAD"], check=True) - if git("rev-parse", STEP7_FINAL + "^{tree}").decode().strip() != STEP7_TREE: - raise ValueError("Previous accepted Step 7 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP7_FINAL, "HEAD"], check=True) - if git("rev-parse", STEP8_FINAL + "^{tree}").decode().strip() != STEP8_TREE: - raise ValueError("Previous accepted Step 8 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP8_FINAL, "HEAD"], check=True) - if git("rev-parse", STEP9_FINAL + "^{tree}").decode().strip() != STEP9_TREE: - raise ValueError("Accepted Step 9 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP9_FINAL, "HEAD"], check=True) - if git("diff", "--name-only", STEP9_FINAL, STEP10_FINAL, "--", "src", "schemas", "examples/hero", "pyproject.toml"): - raise ValueError("Step 10 changed a protected implementation or input") - if git("rev-parse", STEP10_FINAL + "^{tree}").decode().strip() != STEP10_TREE: - raise ValueError("Accepted Step 10 tree mismatch") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", STEP10_FINAL, "HEAD"], check=True) - historical = [tuple(line.split("\t", 1)) for line in - git("diff", "--name-status", "--no-renames", STEP9_FINAL, STEP10_FINAL).decode().splitlines()] - verify_prior_step_changes(historical, step=10) - verify_step11_contract_test_migration(git("show", f"{STEP10_FINAL}:tests/unit/test_phase3_contracts.py"), - (ROOT / "tests/unit/test_phase3_contracts.py").read_bytes()) - verify_step11_pipeline_test_migration(git("show", f"{STEP10_FINAL}:tests/integration/test_phase3_metric_pipeline.py"), - (ROOT / "tests/integration/test_phase3_metric_pipeline.py").read_bytes()) - # Step 9 may add pair functions but cannot alter any preexisting single-scope formula. - diversity_path = "src/recursive_integrity_toolkit/metrics/diversity.py" - before = ast.parse(git("show", f"{STEP8_FINAL}:{diversity_path}")) - current = ast.parse((ROOT / diversity_path).read_bytes()) - definitions = {n.name: ast.dump(n, include_attributes=False) for n in current.body - if isinstance(n, (ast.ClassDef, ast.FunctionDef))} - for node in before.body: - if isinstance(node, (ast.ClassDef, ast.FunctionDef)) and definitions.get(node.name) != ast.dump(node, include_attributes=False): - raise ValueError("Step 9 altered an inherited single-scope calculation") - incremental = [tuple(line.split("\t", 1)) for line in - git("diff", "--name-status", "--no-renames", STEP10_FINAL, "HEAD").decode().splitlines()] - verify_phase3_changes(incremental, incremental=True) - verify_step2_contract_test_migration(git("show", f"{STEP1_FINAL}:{STEP2_EXCEPTION}"), - git("show", f"{STEP2_FINAL}:{STEP2_EXCEPTION}")) - if git("diff", "--name-only", "HEAD", "--"): - raise ValueError("Tracked checkout differs from tested commit") - for path in ("src/recursive_integrity_toolkit/models.py", "src/recursive_integrity_toolkit/errors.py"): - before, current = ast.parse(git("show", f"{PHASE2_FINAL}:{path}")), ast.parse((ROOT / path).read_bytes()) - definitions = {n.name: ast.dump(n, include_attributes=False) for n in current.body if isinstance(n, (ast.ClassDef, ast.FunctionDef))} - for node in before.body: - if isinstance(node, (ast.ClassDef, ast.FunctionDef)) and definitions.get(node.name) != ast.dump(node, include_attributes=False): - raise ValueError(f"Inherited contract changed: {path}:{node.name}") - result = {"baseline": PHASE2_FINAL, "head": git("rev-parse", "HEAD").decode().strip(), - "changes": changes, "incremental_changes": incremental, "previous_step": STEP10_FINAL, - "old_runtime_definitions": "UNCHANGED", "active_step": step} +def audit(root: Path = ROOT) -> dict: + result = {**verify_source_scope(root), **verify_frozen_specifications(root), **verify_resources(root)} print(json.dumps(result, indent=2)) return result @@ -516,149 +135,8 @@ def verify_junit(path: Path, *, require_parquet: bool = False, minimum: int = 1) return result -def _collect(root: Path) -> tuple[list[str], str]: - env = dict(os.environ, PYTHONPATH=str(root / "src"), PYTHONDONTWRITEBYTECODE="1") - result = subprocess.run([sys.executable, "-m", "pytest", "-p", "no:cacheprovider", "--collect-only", "-q"], - cwd=root, env=env, capture_output=True, text=True, encoding="utf-8", check=True) - nodes = sorted(line.strip() for line in result.stdout.splitlines() if line.startswith("tests/") and "::" in line) - if len(nodes) != len(set(nodes)): - raise ValueError("Duplicate test node identity") - return nodes, result.stdout - - -def baseline_evidence(output: Path) -> dict: - output.mkdir(parents=True, exist_ok=True) - files = {} - with tempfile.TemporaryDirectory(prefix="rit-p3-baseline-") as temp: - root = Path(temp) - for entry in git("ls-tree", "-rz", PHASE2_FINAL).split(b"\0"): - if not entry: - continue - info, path_bytes = entry.split(b"\t", 1) - mode, kind, blob = info.decode().split() - if kind != "blob": - raise ValueError("Unexpected baseline object kind") - path = path_bytes.decode(); raw = git("show", f"{PHASE2_FINAL}:{path}") - target = root / path; target.parent.mkdir(parents=True, exist_ok=True); target.write_bytes(raw) - files[path] = {"mode": mode, "git_blob": blob, "sha256": hashlib.sha256(raw).hexdigest()} - old, old_log = _collect(root) - new, new_log = _collect(ROOT) - import io - with tempfile.TemporaryDirectory(prefix="rit-p3-step1-tests-") as temp: - prior_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP1_FINAL))) as archive: - archive.extractall(prior_root) - prior, prior_log = _collect(prior_root) - expected_prior = 1273 if os.environ.get("RIT_TEST_PARQUET") == "1" else 1270 - if len(prior) != expected_prior or set(prior) - set(new): - raise ValueError("Accepted Step 1 test identities were lost") - (output / "step1-collection.log").write_text(prior_log, encoding="utf-8") - with tempfile.TemporaryDirectory(prefix="rit-p3-step2-tests-") as temp: - prior2_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP2_FINAL))) as archive: - archive.extractall(prior2_root) - prior2, prior2_log = _collect(prior2_root) - expected_prior2 = 1432 if os.environ.get("RIT_TEST_PARQUET") == "1" else 1429 - if len(prior2) != expected_prior2 or set(prior2) - set(new): - raise ValueError("Accepted Step 2 test identities were lost") - (output / "step2-collection.log").write_text(prior2_log, encoding="utf-8") - with tempfile.TemporaryDirectory(prefix="rit-p3-step3-tests-") as temp: - prior3_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP3_FINAL))) as archive: - archive.extractall(prior3_root) - prior3, prior3_log = _collect(prior3_root) - expected_prior3 = 1578 if os.environ.get("RIT_TEST_PARQUET") == "1" else 1575 - if len(prior3) != expected_prior3 or set(prior3) - set(new): - raise ValueError("Accepted Step 3 test identities were lost") - (output / "step3-collection.log").write_text(prior3_log, encoding="utf-8") - with tempfile.TemporaryDirectory(prefix="rit-p3-step4-tests-") as temp: - prior4_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP4_FINAL))) as archive: - archive.extractall(prior4_root) - prior4, prior4_log = _collect(prior4_root) - expected_prior4 = 1717 if os.environ.get("RIT_TEST_PARQUET") == "1" else 1714 - if len(prior4) != expected_prior4 or set(prior4) - set(new): - raise ValueError("Accepted Step 4 test identities were lost") - (output / "step4-collection.log").write_text(prior4_log, encoding="utf-8") - with tempfile.TemporaryDirectory(prefix="rit-p3-step5-tests-") as temp: - prior5_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP5_FINAL))) as archive: - archive.extractall(prior5_root) - prior5, prior5_log = _collect(prior5_root) - expected_prior5 = 1891 if os.environ.get("RIT_TEST_PARQUET") == "1" else 1888 - if len(prior5) != expected_prior5 or set(prior5) - set(new): - raise ValueError("Accepted Step 5 test identities were lost") - (output / "step5-collection.log").write_text(prior5_log, encoding="utf-8") - with tempfile.TemporaryDirectory(prefix="rit-p3-step6-tests-") as temp: - prior6_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP6_FINAL))) as archive: - archive.extractall(prior6_root) - prior6, prior6_log = _collect(prior6_root) - expected_prior6 = 1990 if os.environ.get("RIT_TEST_PARQUET") == "1" else 1987 - if len(prior6) != expected_prior6 or set(prior6) - set(new): - raise ValueError("Accepted Step 6 test identities were lost") - (output / "step6-collection.log").write_text(prior6_log, encoding="utf-8") - with tempfile.TemporaryDirectory(prefix="rit-p3-step7-tests-") as temp: - prior7_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP7_FINAL))) as archive: - archive.extractall(prior7_root) - prior7, prior7_log = _collect(prior7_root) - expected_prior7 = 2120 if os.environ.get("RIT_TEST_PARQUET") == "1" else 2117 - if len(prior7) != expected_prior7 or set(prior7) - set(new): - raise ValueError("Accepted Step 7 test identities were lost") - (output / "step7-collection.log").write_text(prior7_log, encoding="utf-8") - with tempfile.TemporaryDirectory(prefix="rit-p3-step8-tests-") as temp: - prior8_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP8_FINAL))) as archive: - archive.extractall(prior8_root) - prior8, prior8_log = _collect(prior8_root) - expected_prior8 = 2271 if os.environ.get("RIT_TEST_PARQUET") == "1" else 2268 - if len(prior8) != expected_prior8 or set(prior8) - set(new): - raise ValueError("Accepted Step 8 test identities were lost") - (output / "step8-collection.log").write_text(prior8_log, encoding="utf-8") - with tempfile.TemporaryDirectory(prefix="rit-p3-step9-tests-") as temp: - prior9_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP9_FINAL))) as archive: - archive.extractall(prior9_root) - prior9, prior9_log = _collect(prior9_root) - expected_prior9 = 2407 if os.environ.get("RIT_TEST_PARQUET") == "1" else 2404 - if len(prior9) != expected_prior9 or set(prior9) - set(new): - raise ValueError("Accepted Step 9 test identities were lost") - (output / "step9-collection.log").write_text(prior9_log, encoding="utf-8") - with tempfile.TemporaryDirectory(prefix="rit-p3-step10-tests-") as temp: - prior10_root = Path(temp) - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", STEP10_FINAL))) as archive: - archive.extractall(prior10_root) - prior10, prior10_log = _collect(prior10_root) - expected_prior10 = 2475 if os.environ.get("RIT_TEST_PARQUET") == "1" else 2472 - if len(prior10) != expected_prior10 or set(prior10) - set(new): - raise ValueError("Accepted Step 10 test identities were lost") - (output / "step10-collection.log").write_text(prior10_log, encoding="utf-8") - expected = 1162 if os.environ.get("RIT_TEST_PARQUET") == "1" else 1159 - if len(files) != 197 or len(old) != expected or set(old) - set(new): - raise ValueError(f"Frozen file/test identities do not reconcile: {len(files)} files, {len(old)} baseline tests, {len(set(old)-set(new))} missing") - payload = {"baseline_commit": PHASE2_FINAL, "files": files, "baseline_nodeids": old, "current_nodeids": new, - "previous_step_commit": STEP10_FINAL, "previous_step_nodeids": prior10, - "step9_commit": STEP9_FINAL, "step9_nodeids": prior9, - "step8_commit": STEP8_FINAL, "step8_nodeids": prior8, - "step7_commit": STEP7_FINAL, "step7_nodeids": prior7, - "step6_commit": STEP6_FINAL, "step6_nodeids": prior6, - "step5_commit": STEP5_FINAL, "step5_nodeids": prior5, - "step4_commit": STEP4_FINAL, "step4_nodeids": prior4, - "step3_commit": STEP3_FINAL, "step3_nodeids": prior3, - "step2_commit": STEP2_FINAL, "step2_nodeids": prior2, - "step1_commit": STEP1_FINAL, "step1_nodeids": prior, - "missing_nodeids": [], "nodeids_sha256": hashlib.sha256(("\n".join(old)+"\n").encode()).hexdigest()} - (output / "baseline-identities.json").write_text(json.dumps(payload, indent=2)+"\n", encoding="utf-8") - (output / "baseline-collection.log").write_text(old_log, encoding="utf-8") - (output / "current-collection.log").write_text(new_log, encoding="utf-8") - summary = {"baseline_files": len(files), "inherited_tests": len(old), "current_tests": len(new), "missing_tests": 0, "previous_step_tests": len(prior10), "step9_tests": len(prior9), "step8_tests": len(prior8), "step7_tests": len(prior7), "step6_tests": len(prior6), "step5_tests": len(prior5), "step4_tests": len(prior4), "step3_tests": len(prior3), "step2_tests": len(prior2), "step1_tests": len(prior), - "nodeids_sha256": payload["nodeids_sha256"]} - print(json.dumps(summary, indent=2)) - return summary - - -def verify_distributions(directory: Path, expected_version: str = "0.1.0.dev2") -> tuple[Path, Path]: +def verify_distributions(directory: Path, expected_version: str | None = None) -> tuple[Path, Path]: + expected_version = expected_version or tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"]["version"] wheels, sdists = list(directory.glob("*.whl")), list(directory.glob("*.tar.gz")) if len(wheels) != 1 or len(sdists) != 1: raise ValueError("Require one wheel and one sdist") @@ -668,8 +146,10 @@ def verify_distributions(directory: Path, expected_version: str = "0.1.0.dev2") if wheel.read(path) != raw: raise ValueError(f"Wheel code mismatch: {path}") names = wheel.namelist() + if len(names) != len(set(names)): + raise ValueError("Duplicate wheel entries") if {n for n in names if n.endswith(".py")} != set(expected): - raise ValueError("Wheel module set differs from the forty source modules") + raise ValueError("Wheel module set differs from source") metadata = [n for n in names if n.endswith(".dist-info/METADATA")] if len(metadata) != 1 or f"Version: {expected_version}\n" not in wheel.read(metadata[0]).decode(): raise ValueError("Wheel metadata mismatch") @@ -677,14 +157,17 @@ def verify_distributions(directory: Path, expected_version: str = "0.1.0.dev2") if not any(n.endswith("/"+notice) for n in names): raise ValueError(f"Wheel lacks {notice}") with tarfile.open(sdists[0], "r:gz") as archive: - members = {m.name: m for m in archive.getmembers()} + entries = archive.getmembers() + members = {m.name: m for m in entries} + if len(members) != len(entries): + raise ValueError("Duplicate sdist entries") roots = {n.split("/", 1)[0] for n in members} if len(roots) != 1: raise ValueError("Source archive requires one root") prefix = next(iter(roots)) if {n.removeprefix(prefix + "/src/") for n in members if n.startswith(prefix + "/src/recursive_integrity_toolkit/") and n.endswith(".py")} != set(expected): - raise ValueError("Sdist module set differs from the forty source modules") + raise ValueError("Sdist module set differs from source") for notice in ("LICENSE", "NOTICE"): if f"{prefix}/{notice}" not in members: raise ValueError(f"Sdist lacks {notice}") @@ -692,7 +175,25 @@ def verify_distributions(directory: Path, expected_version: str = "0.1.0.dev2") stream = archive.extractfile(members[f"{prefix}/src/{path}"]) if stream is None or stream.read() != raw: raise ValueError(f"Sdist code mismatch: {path}") - print("wheel/sdist: 40 packaged modules match source") + with zipfile.ZipFile(wheels[0]) as archive: + actual = {name for name in archive.namelist() if name.startswith("recursive_integrity_toolkit/data/") and not name.endswith("/")} + if actual != {path.removeprefix("src/") for path in RESOURCES}: + raise ValueError("Wheel resource inventory differs") + for destination, source in RESOURCES.items(): + if archive.read(destination.removeprefix("src/")) != (ROOT / source).read_bytes(): + raise ValueError("Wheel resource bytes differ") + with tarfile.open(sdists[0], "r:gz") as archive: + prefix = next(iter({m.name.split("/")[0] for m in archive.getmembers()})) + actual = {m.name.removeprefix(prefix + "/") for m in archive.getmembers() + if m.name.startswith(prefix + "/src/recursive_integrity_toolkit/data/") and m.isfile()} + if actual != set(RESOURCES): + raise ValueError("Sdist resource inventory differs") + for destination, source in RESOURCES.items(): + member = archive.getmember(prefix + "/" + destination) + stream = archive.extractfile(member) + if not member.isfile() or stream is None or stream.read() != (ROOT / source).read_bytes(): + raise ValueError("Sdist resource bytes differ") + print(f"wheel/sdist: {len(expected)} source modules and canonical resources match") return wheels[0], sdists[0] @@ -719,7 +220,7 @@ def blocked(*args, **kwargs): assert not Path(package.__file__).resolve().is_relative_to(Path(sys.argv[1])/'src') assert package.__version__==importlib.metadata.version('recursive-integrity-toolkit') names=[package.__name__]+[m.name for m in pkgutil.walk_packages(package.__path__,package.__name__+'.')] -assert len(names)==40 +assert names for name in names: importlib.import_module(name) from recursive_integrity_toolkit.models import AuditBundle,InputSource,FileRole,CapabilityKey,CapabilityStatus,NumericalPolicy from recursive_integrity_toolkit.io.validation import validate_bundle @@ -740,7 +241,7 @@ def blocked(*args, **kwargs): assert result.observability.capabilities[CapabilityKey.MODEL_LONGITUDINAL].status is CapabilityStatus.UNAVAILABLE assert before=={n:hashlib.sha256((hero/n).read_bytes()).hexdigest() for n in files} assert not {'numpy','pandas','pyarrow'} & set(sys.modules) -print('installed: 40 imports, Step 2 field assignment, input-only Hero Level 4, no dependencies/network: PASS') +print('installed: all package imports, field assignment, input-only Hero Level 4, no dependencies/network: PASS') ''' subprocess.run([str(python), "-I", "-c", program, str(ROOT)], cwd=work, check=True) for arg in ("--help", "version"): @@ -749,7 +250,6 @@ def blocked(*args, **kwargs): subprocess.run([str(bindir / (name+".exe" if os.name == "nt" else name)), arg], cwd=work, check=True) - def smoke_installed_duplicates(wheel: Path) -> None: """A separate installed calculation check with approved dependencies present.""" with tempfile.TemporaryDirectory(prefix="rit-p3-installed-duplicates-") as temp: @@ -860,1053 +360,7 @@ def blocked(*args, **kwargs): subprocess.run([str(python), "-I", "-c", program, str(work)], cwd=work, check=True) -def verify_step10_evidence(junit: Path, output: Path | None = None) -> dict: - """Require twenty executed frozen cases and seven measured operation records. - - These are test-run observations, not a public analytical report or speed SLA. - General JUnit validation rejects failures and skips before evidence extraction. - """ - from math import isfinite - verify_junit(junit) - root = ET.parse(junit).getroot() - cases = list(root.iter("testcase")) - expected_ids = {"test_phase3_frozen_case[" + case["case_id"] + "]" - for case in json.loads((ROOT / "tests/golden/phase3_math_cases.json").read_bytes())["cases"]} - actual = {case.get("name") for case in cases if case.get("classname", "").endswith("test_phase3_math")} - if len(expected_ids) != 20 or not expected_ids <= actual: - raise ValueError("All twenty frozen mathematical cases must actually execute") - required = {"hero_input_plus_calculations", "metadata_100k_setup", "metadata_100k_field_representation", - "metadata_100k_support_diversity", "metadata_100k_provenance_join", - "metadata_100k_composition", "metadata_100k_exact_duplicates"} - observations = [] - seen = set() - for case in cases: - for prop in case.findall("./properties/property"): - if prop.get("name") != "phase3_performance": - continue - data = json.loads(prop.get("value", "")) - name = data.get("name") - if name not in required or name in seen: - raise ValueError("Missing, duplicated or unexpected performance observation") - for key in ("elapsed_seconds", "traced_current_bytes", "traced_peak_bytes"): - value = data.get(key) - if type(value) not in (int, float) or not isfinite(value) or value < 0: - raise ValueError("Performance evidence requires finite nonnegative measurements") - if data["traced_peak_bytes"] < data["traced_current_bytes"]: - raise ValueError("Invalid traced-memory evidence") - if data.get("whole_product_report_target_certified") is not False: - raise ValueError("A calculation observation cannot certify the final report target") - if data.get("record_count") != (16 if name == "hero_input_plus_calculations" else 100000): - raise ValueError("Performance evidence used an incorrect dataset size") - for key in ("memory_method", "timing_method", "python", "platform"): - if type(data.get(key)) is not str or not data[key]: - raise ValueError("Performance environment and measurement methods must be disclosed") - seen.add(name) - observations.append(data) - if seen != required: - raise ValueError("Required Step 10 performance measurements were not all executed") - result = {"frozen_cases_executed": 20, "performance_observations": sorted(observations, key=lambda d: d["name"]), - "whole_product_report_target_certified": False} - if output is not None: - output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") - return result - - -def candidate(output: Path, step: int = ACTIVE_STEP) -> None: - result = {"audit": audit_phase3_diff(step), "phase_complete": PHASE3_COMPLETE, - "acceptance_requires_all_four_workflow_roles": True, "next_step_authorized": False} - output.mkdir(parents=True, exist_ok=True) - result["core"] = verify_junit(output / "core.xml", minimum=2472) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=2475) - result["core_integration"] = verify_step10_evidence(output / "core.xml", output / "phase3-core-observations.json") - result["parquet_integration"] = verify_step10_evidence(output / "parquet.xml", output / "phase3-parquet-observations.json") - result["identities"] = baseline_evidence(output) - verify_distributions(output / "dist") - archive = output / "recursive-integrity-toolkit-phase3.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {n for n in git("ls-files", "-z").decode().split("\0") if n} - with zipfile.ZipFile(archive) as zipped: - actual = {n.removeprefix("recursive-integrity-toolkit/") for n in zipped.namelist() if not n.endswith("/")} - if actual != tracked: - raise ValueError("Candidate archive file set mismatch") - for path in actual: - if any(p in {".git", "__pycache__", ".pytest_cache", ".venv", "venv", "build", "dist"} for p in Path(path).parts) or path.endswith((".pyc", ".pyo")): - raise ValueError("Generated file in source archive") - if zipped.read("recursive-integrity-toolkit/"+path) != (ROOT / path).read_bytes(): - raise ValueError("Candidate archive byte mismatch") - (output / "phase3_repository_files.sha256").write_text( - "".join(f"{hashlib.sha256((ROOT / name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - result["source_archive"] = archive.name - result["tracked_files"] = len(tracked) - result["python"] = sys.version; result["platform"] = sys.platform; result["versions"] = {} - for name in ("numpy", "pandas", "pyarrow", "pytest", "build", "setuptools", "twine"): - try: - result["versions"][name] = importlib.metadata.version(name) - except importlib.metadata.PackageNotFoundError: - result["versions"][name] = None - (output / "phase3_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - files = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase3_artifacts.sha256") - (output / "phase3_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in files), encoding="utf-8") - print(f"Step 11 source artifact: {len(tracked)} tracked files verified. Final acceptance also requires all workflow roles on this commit.") - - -# Phase 4 gates are additive. Historical Phase 2/3 functions and constants above -# keep their original contracts; the active Phase 4 dispatcher is separate. -from functools import lru_cache -import io -import importlib.util -from types import MappingProxyType - -PHASE4_FINAL = "e3ffb8c0a88bfe31f669f9662d9b5213da628b3a" -PHASE4_TREE = "e2a25f8cfdc66c3317809c479f80fdae162e6ba9" -PHASE4_TEST_TREE = "6ab22cb9197a8f094f455b29a07a851062bb26c9" -PHASE4_PLAN_SHA256 = "5a6d65696720a426630e876d001937b2837d24774fcb6c2bd43b8e09f5ae93f0" -PHASE4_STEP1_NEW = { - "PHASE_4_PLAN.md", "PHASE_4_BASELINE.json", "PHASE_4_DECISIONS.md", - "tests/unit/test_phase4_contracts.py", "tests/integration/test_phase4_gates.py", -} -PHASE4_G = { - "PHASE_4_BASELINE.json", "PHASE_4_DECISIONS.md", "scripts/check_traceability.py", - "scripts/check_spec_consistency.py", "scripts/release_check.py", "tests/conftest.py", - "tests/unit/test_phase4_contracts.py", "tests/integration/test_phase4_gates.py", - "tests/integration/test_no_algorithms.py", "tests/integration/test_ci_workflows.py", - "tests/integration/test_repository_structure.py", "tests/integration/test_owner_ids.py", - "tests/integration/test_package_import.py", "tests/integration/test_package_install.py", - "tests/integration/test_optional_dependency.py", "tests/integration/test_no_network.py", - "tests/integration/test_schema_json.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_prohibited_structure.py", "tests/integration/test_license_notices.py", - ".github/workflows/ci.yml", ".github/workflows/security.yml", - ".github/workflows/golden.yml", ".github/workflows/release.yml", - "docs/architecture.md", "docs/theory_traceability.md", -} -PHASE4_MIGRATION_PATHS = { - "tests/unit/test_PR012_evidence_classes.py", "tests/unit/test_PR013_report_schema.py", - "tests/unit/test_PR014_unavailable.py", "tests/unit/test_PR015_redaction.py", - "tests/unit/test_PR016_determinism.py", "tests/unit/test_PR018_language.py", - "tests/integration/test_no_algorithms.py", "tests/integration/test_phase3_metric_pipeline.py", - "tests/integration/test_cli_validation.py", "tests/integration/test_ci_workflows.py", -} -PHASE4_STEP1_ALLOWED = PHASE4_G | PHASE4_STEP1_NEW | PHASE4_MIGRATION_PATHS -PHASE4_FORBIDDEN_OUTPUTS = { - "PHASE_4_COMPLETION.md", "PHASE_4_VALIDATION_REPORT.md", - "PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md", "report.json", "report.md", -} -PHASE4_APPROVAL = "批准，开始 **Phase 4 Step 1**" -PHASE4_MIGRATIONS = {'tests/integration/test_ci_workflows.py': [{'new': 'def ' - 'test_workflows_preserve_phase_boundary(repo_root: ' - 'Path,phase3_final_snapshot) -> None:\n' - ' repo_root = phase3_final_snapshot\n', - 'node': 'test_workflows_preserve_phase_boundary', - 'old': 'def ' - 'test_workflows_preserve_phase_boundary(repo_root: ' - 'Path) -> None:\n'}, - {'new': 'def ' - 'test_phase2_delivery_builds_both_formats_and_tests_installed_wheel(repo_root,phase3_final_snapshot):\n' - ' repo_root = phase3_final_snapshot\n', - 'node': 'test_phase2_delivery_builds_both_formats_and_tests_installed_wheel', - 'old': 'def ' - 'test_phase2_delivery_builds_both_formats_and_tests_installed_wheel(repo_root):\n'}, - {'new': 'def ' - 'test_phase2_hero_workflow_preserves_scaffold_only_golden_boundary(repo_root,phase3_final_snapshot):\n' - ' repo_root = phase3_final_snapshot\n', - 'node': 'test_phase2_hero_workflow_preserves_scaffold_only_golden_boundary', - 'old': 'def ' - 'test_phase2_hero_workflow_preserves_scaffold_only_golden_boundary(repo_root):\n'}], - 'tests/integration/test_cli_validation.py': [{'new': 'def ' - 'test_cli_help_runs(subprocess_env,phase3_final_subprocess_env) ' - '-> None:\n' - ' subprocess_env = ' - 'phase3_final_subprocess_env\n', - 'node': 'test_cli_help_runs', - 'old': 'def test_cli_help_runs(subprocess_env) -> ' - 'None:\n'}], - 'tests/integration/test_no_algorithms.py': [{'new': 'def ' - 'test_only_step8_authorized_modules_gain_behavior(package_root,phase3_final_package_root) ' - '-> None:\n' - ' package_root = phase3_final_package_root\n', - 'node': 'test_only_step8_authorized_modules_gain_behavior', - 'old': 'def ' - 'test_only_step8_authorized_modules_gain_behavior(package_root) ' - '-> None:\n'}, - {'new': 'def ' - 'test_protected_phase3_plus_modules_remain_placeholders(package_root,phase3_final_package_root) ' - '-> None:\n' - ' package_root = phase3_final_package_root\n', - 'node': 'test_protected_phase3_plus_modules_remain_placeholders', - 'old': 'def ' - 'test_protected_phase3_plus_modules_remain_placeholders(package_root) ' - '-> None:\n'}, - {'new': 'def ' - 'test_PR003_traceability_script_enforces_step8_scope(repo_root,phase3_final_snapshot) ' - '-> None:\n' - ' repo_root = phase3_final_snapshot\n', - 'node': 'test_PR003_traceability_script_enforces_step8_scope', - 'old': 'def ' - 'test_PR003_traceability_script_enforces_step8_scope(repo_root) ' - '-> None:\n'}], - 'tests/integration/test_phase3_metric_pipeline.py': [{'new': 'def ' - 'test_phase3_later_implementations_remain_empty(package_root,module,phase3_final_package_root):\n' - ' package_root = ' - 'phase3_final_package_root\n', - 'node': 'test_phase3_later_implementations_remain_empty', - 'old': 'def ' - 'test_phase3_later_implementations_remain_empty(package_root,module):\n'}, - {'new': 'def ' - 'test_phase3_step11_rejects_changes_beyond_version_literals(repo_root,tmp_path,path,phase3_final_snapshot):\n' - ' repo_root = phase3_final_snapshot\n', - 'node': 'test_phase3_step11_rejects_changes_beyond_version_literals', - 'old': 'def ' - 'test_phase3_step11_rejects_changes_beyond_version_literals(repo_root,tmp_path,path):\n'}, - {'new': 'def ' - 'test_phase3_step11_test_exceptions_are_exact(repo_root,phase3_step10_snapshot,phase3_final_snapshot):\n' - ' repo_root = phase3_final_snapshot\n', - 'node': 'test_phase3_step11_test_exceptions_are_exact', - 'old': 'def ' - 'test_phase3_step11_test_exceptions_are_exact(repo_root,phase3_step10_snapshot):\n'}], - 'tests/unit/test_PR012_evidence_classes.py': [{'new': 'def ' - 'test_PR012_evidence_classes_owner_and_placeholder(owner_checker, ' - 'placeholder_checker,phase3_final_owner_checker,phase3_final_placeholder_checker):\n' - ' owner_checker = phase3_final_owner_checker\n' - ' placeholder_checker = ' - 'phase3_final_placeholder_checker\n', - 'node': 'test_PR012_evidence_classes_owner_and_placeholder', - 'old': 'def ' - 'test_PR012_evidence_classes_owner_and_placeholder(owner_checker, ' - 'placeholder_checker):\n'}], - 'tests/unit/test_PR013_report_schema.py': [{'new': 'def ' - 'test_PR013_report_schema_owner_and_placeholder(owner_checker, ' - 'placeholder_checker,phase3_final_owner_checker,phase3_final_placeholder_checker):\n' - ' owner_checker = phase3_final_owner_checker\n' - ' placeholder_checker = ' - 'phase3_final_placeholder_checker\n', - 'node': 'test_PR013_report_schema_owner_and_placeholder', - 'old': 'def ' - 'test_PR013_report_schema_owner_and_placeholder(owner_checker, ' - 'placeholder_checker):\n'}], - 'tests/unit/test_PR014_unavailable.py': [{'new': 'def ' - 'test_PR014_unavailable_owner_and_placeholder(owner_checker, ' - 'placeholder_checker,phase3_final_owner_checker,phase3_final_placeholder_checker):\n' - ' owner_checker = phase3_final_owner_checker\n' - ' placeholder_checker = ' - 'phase3_final_placeholder_checker\n', - 'node': 'test_PR014_unavailable_owner_and_placeholder', - 'old': 'def ' - 'test_PR014_unavailable_owner_and_placeholder(owner_checker, ' - 'placeholder_checker):\n'}], - 'tests/unit/test_PR015_redaction.py': [{'new': 'def ' - 'test_PR015_redaction_owner_and_placeholder(owner_checker, ' - 'placeholder_checker,phase3_final_owner_checker,phase3_final_placeholder_checker):\n' - ' owner_checker = phase3_final_owner_checker\n' - ' placeholder_checker = ' - 'phase3_final_placeholder_checker\n', - 'node': 'test_PR015_redaction_owner_and_placeholder', - 'old': 'def ' - 'test_PR015_redaction_owner_and_placeholder(owner_checker, ' - 'placeholder_checker):\n'}], - 'tests/unit/test_PR016_determinism.py': [{'new': 'def ' - 'test_PR016_ordering_owner_and_no_later_behavior(owner_checker, ' - 'package_root, ' - 'placeholder_checker,phase3_final_placeholder_checker):\n' - ' placeholder_checker = ' - 'phase3_final_placeholder_checker\n', - 'node': 'test_PR016_ordering_owner_and_no_later_behavior', - 'old': 'def ' - 'test_PR016_ordering_owner_and_no_later_behavior(owner_checker, ' - 'package_root, placeholder_checker):\n'}], - 'tests/unit/test_PR018_language.py': [{'new': 'def ' - 'test_PR018_language_owner_and_placeholder(owner_checker, ' - 'placeholder_checker,phase3_final_owner_checker,phase3_final_placeholder_checker):\n' - ' owner_checker = phase3_final_owner_checker\n' - ' placeholder_checker = ' - 'phase3_final_placeholder_checker\n', - 'node': 'test_PR018_language_owner_and_placeholder', - 'old': 'def ' - 'test_PR018_language_owner_and_placeholder(owner_checker, ' - 'placeholder_checker):\n'}]} - - -@lru_cache(maxsize=1) -def _phase4_baseline_files(): - """Read immutable Git objects once, independently of the editable manifest.""" - for suffix, expected in (("^{commit}", PHASE4_FINAL), ("^{tree}", PHASE4_TREE), - (":tests", PHASE4_TEST_TREE)): - if git("rev-parse", PHASE4_FINAL + suffix).decode().strip() != expected: - raise ValueError("Pinned Phase 3 baseline identity mismatch") - blobs = {} - for entry in git("ls-tree", "-rz", PHASE4_FINAL).split(b"\0"): - if not entry: - continue - info, name = entry.split(b"\t", 1) - mode, kind, oid = info.split() - path = name.decode("utf-8") - if mode not in (b"100644", b"100755") or kind != b"blob": - raise ValueError("Unexpected baseline file type") - if Path(path).is_absolute() or ".." in Path(path).parts or path in blobs: - raise ValueError("Unsafe baseline path") - blobs[path] = oid.decode("ascii") - result = {} - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_FINAL))) as archive: - names = [n for n in archive.namelist() if not n.endswith("/")] - if len(names) != len(set(names)) or set(names) != set(blobs) or len(names) != 222: - raise ValueError("Pinned baseline archive file set mismatch") - for name in names: - raw = archive.read(name) - oid = hashlib.sha1(b"blob " + str(len(raw)).encode() + b"\0" + raw).hexdigest() - if oid != blobs[name]: - raise ValueError(f"Pinned baseline blob mismatch: {name}") - result[name] = raw - return MappingProxyType(result) - - -def phase4_expected_control() -> dict: - """Independent exact authorization, protected-byte and migration registry.""" - files = _phase4_baseline_files() - authority = re.findall(r"\| `([^`]+\.md)` \| `([0-9a-f]{64})` \|", - files["PHASE_0_APPROVAL.md"].decode()) - if len(authority) != 16: - raise ValueError("Expected sixteen original authority hashes") - return { - "control_version": "1.0", "active_phase": 4, "active_step": 1, - "baseline_commit": PHASE4_FINAL, "baseline_tree": PHASE4_TREE, - "baseline_test_tree": PHASE4_TEST_TREE, "baseline_branch": "phase3-metrics", - "work_branch": "phase4-reports-cli", - "main_at_authorization": "cfe1bd0941c1125498ac3d9d9ebf3adafa2c2fcb", - "approved_plan_sha256": PHASE4_PLAN_SHA256, "approval_date": "2026-09-18", - "approval_basis": PHASE4_APPROVAL, - "approved_decisions": [f"P4-D{i:02d}" for i in range(1, 12)], - "baseline_core_tests": 2489, "baseline_parquet_tests": 2492, - "baseline_tracked_files": 222, "baseline_package_modules": 40, - "baseline_placeholders": 16, "phase0_sha256": dict(authority), - "baseline_files_sha256": {p: hashlib.sha256(raw).hexdigest() for p, raw in sorted(files.items())}, - "permitted_paths": sorted(PHASE4_STEP1_ALLOWED), - "new_files_permitted": sorted(PHASE4_STEP1_NEW), - "historical_migration_nodes": {p: [r["node"] for r in rows] - for p, rows in sorted(PHASE4_MIGRATIONS.items())}, - "runtime_changes_authorized": False, "schema_changes_authorized": False, - "phase_complete": False, "next_step_authorized": False, - "main_merge_authorized": False, "publication_authorized": False, - } - - -def verify_phase4_control(control: dict, step: int = 1) -> None: - if type(step) is not int or step != 1 or type(control) is not dict: - raise ValueError("Unsupported Phase 4 stage/control") - try: - actual = json.dumps(control, sort_keys=True, ensure_ascii=True, allow_nan=False) - expected = json.dumps(phase4_expected_control(), sort_keys=True, ensure_ascii=True, allow_nan=False) - except (TypeError, ValueError) as error: - raise ValueError("Invalid Phase 4 control") from error - if actual != expected: - raise ValueError("Phase 4 control differs from the fixed approved Step 1 contract") - - -def verify_phase4_changes(changes: list[tuple[str, str]]) -> None: - seen = set() - for status, path in changes: - if path in seen or path not in PHASE4_STEP1_ALLOWED: - raise ValueError(f"Unapproved or repeated Phase 4 path: {path}") - seen.add(path) - required_status = "A" if path in PHASE4_STEP1_NEW else "M" - if status != required_status: - raise ValueError(f"Unapproved Phase 4 change kind: {status} {path}") - - -def verify_phase4_test_migration(path: str, before: bytes, after: bytes) -> None: - """Permit exact source bindings, retaining original assertions/parameters.""" - if path not in PHASE4_MIGRATIONS or before != _phase4_baseline_files()[path]: - raise ValueError("Migration requires the named pinned Phase 3 source") - expected = before - for row in PHASE4_MIGRATIONS[path]: - old, new = row["old"].encode(), row["new"].encode() - if expected.count(old) != 1: - raise ValueError("Historical binding is not unique") - expected = expected.replace(old, new, 1) - if not after.startswith(expected): - raise ValueError(f"Inherited test changed beyond its approved binding: {path}") - _phase4_append_only(expected, after, path) - - -def _phase4_preserve_function_header(node, path): - """Reject import-time effects in newly permitted function definitions.""" - decorators = [ast.unparse(value) for value in node.decorator_list] - expected = [] - if path == "tests/conftest.py" and node.name.startswith("phase3_final_"): - expected = ["pytest.fixture(scope='session')"] - elif path == "scripts/release_check.py" and node.name == "_phase4_baseline_files": - expected = ["lru_cache(maxsize=1)"] - if decorators != expected: - raise ValueError(f"Unapproved added function decorator: {path}:{node.name}") - for value in [*node.args.defaults, *node.args.kw_defaults]: - if value is None or isinstance(value, ast.Name) and value.id == "ROOT": - continue - try: - ast.literal_eval(value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError(f"Nonliteral added function default: {path}:{node.name}") from error - arguments = [*node.args.posonlyargs, *node.args.args, *node.args.kwonlyargs] - arguments.extend(value for value in (node.args.vararg, node.args.kwarg) if value is not None) - annotations = [value.annotation for value in arguments] + [node.returns] - allowed_nodes = (ast.Name, ast.Constant, ast.Subscript, ast.Tuple, ast.List, - ast.BinOp, ast.BitOr, ast.Load) - allowed_names = {"Path", "list", "str", "bytes", "dict", "int", "bool", "tuple", "set"} - for value in annotations: - if value is not None and any( - not isinstance(child, allowed_nodes) - or isinstance(child, ast.Name) and child.id not in allowed_names - for child in ast.walk(value)): - raise ValueError(f"Unapproved added function annotation: {path}:{node.name}") - - -def _phase4_append_only(before: bytes, after: bytes, path: str) -> None: - if not after.startswith(before): - raise ValueError(f"Inherited source prefix changed: {path}") - # Appended tests/helpers cannot shadow inherited globals or redefine nodes. - old_tree, new_tree = ast.parse(before), ast.parse(after) - def bindings(tree): - names = [] - for node in tree.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - names.append(node.name) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names.extend(n.id for target in targets for n in ast.walk(target) if isinstance(n, ast.Name)) - return names - names = bindings(new_tree) - if len(names) != len(set(names)) or any(names.count(n) != 1 for n in bindings(old_tree)): - raise ValueError(f"Appended source redefines an inherited binding: {path}") - for node in new_tree.body[len(old_tree.body):]: - if not isinstance(node, ast.FunctionDef): - raise ValueError(f"Appended test source executes or rebinds at module scope: {path}") - prefix = "phase3_final_" if path == "tests/conftest.py" else "test_phase4_" - if not node.name.startswith(prefix): - raise ValueError(f"Appended function is outside the Step 1 test/fixture scope: {path}") - _phase4_preserve_function_header(node, path) - - -def _phase4_preserve_tooling(before: bytes, after: bytes, path: str) -> None: - """All inherited maintainer definitions/constants remain exact.""" - old_text, new_text = before.decode(), after.decode() - old, new = ast.parse(old_text), ast.parse(new_text) - def entry_guard(node): - return isinstance(node, ast.If) and isinstance(node.test, ast.Compare) and "__name__" in ast.unparse(node.test) - historical = [n for n in old.body if not entry_guard(n)] - guards = [n for n in new.body if entry_guard(n)] - expected_guard = ast.parse('if __name__ == "__main__":\n raise SystemExit(cli_main())').body[0] - if len(guards) != 1 or ast.dump(guards[0]) != ast.dump(expected_guard): - raise ValueError(f"Unapproved maintainer entrypoint: {path}") - cursor = 0 - allowed_imports = {"from functools import lru_cache", "import io", "import importlib.util", "from types import MappingProxyType"} - for node in new.body: - if entry_guard(node): - continue - segment = ast.get_source_segment(new_text, node) - if cursor < len(historical) and segment == ast.get_source_segment(old_text, historical[cursor]): - cursor += 1 - continue - if isinstance(node, ast.FunctionDef) and (node.name == "cli_main" or node.name.startswith( - ("phase4_", "_phase4_", "verify_phase4_", "audit_phase4"))): - _phase4_preserve_function_header(node, path) - continue - if isinstance(node, ast.Assign) and all(isinstance(t, ast.Name) and t.id.startswith("PHASE4_") for t in node.targets): - approved_union = ast.parse("PHASE4_G | PHASE4_STEP1_NEW | PHASE4_MIGRATION_PATHS", mode="eval").body - if ([target.id for target in node.targets] == ["PHASE4_STEP1_ALLOWED"] - and ast.dump(node.value) == ast.dump(approved_union)): - continue - try: - ast.literal_eval(node.value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError(f"Nonliteral added maintainer constant: {path}") from error - continue - if isinstance(node, (ast.Import, ast.ImportFrom)) and ast.unparse(node) in allowed_imports: - continue - raise ValueError(f"Unapproved executable maintainer addition: {path}") - if cursor != len(historical): - raise ValueError(f"Historical maintainer statement changed: {path}") - def named(tree): - result = {} - for node in tree.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - keys = [node.name] - elif isinstance(node, ast.Assign): - keys = [] - for target in node.targets: - if isinstance(target, ast.Name): - keys.append(target.id) - elif isinstance(target, (ast.Tuple, ast.List)): - keys.extend(n.id for n in ast.walk(target) if isinstance(n, ast.Name)) - else: - continue - for key in keys: - if key in result: - raise ValueError(f"Duplicate maintainer binding: {key}") - result[key] = node - return result - current = named(new) - for key, node in named(old).items(): - if key not in current or ast.get_source_segment(old_text, node) != ast.get_source_segment(new_text, current[key]): - raise ValueError(f"Historical maintainer contract changed: {path}:{key}") - - -def verify_phase4_snapshot(root: Path = ROOT) -> dict: - files = _phase4_baseline_files() - for path, raw in files.items(): - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError(f"Missing or aliased inherited file: {path}") - current = target.read_bytes() - if path not in PHASE4_STEP1_ALLOWED and current != raw: - raise ValueError(f"Protected Phase 3 bytes changed: {path}") - if path in PHASE4_MIGRATIONS: - verify_phase4_test_migration(path, raw, current) - elif path.startswith("tests/") and path in PHASE4_STEP1_ALLOWED: - _phase4_append_only(raw, current, path) - elif path.startswith("scripts/") and path in PHASE4_STEP1_ALLOWED: - _phase4_preserve_tooling(raw, current, path) - actual_modules = {p.relative_to(root).as_posix() for p in (root / "src/recursive_integrity_toolkit").rglob("*.py")} - expected_modules = {p for p in files if p.startswith("src/") and p.endswith(".py")} - if actual_modules != expected_modules or len(actual_modules) != 40: - raise ValueError("Step 1 cannot change the runtime module set") - if {p.name for p in (root / "schemas").iterdir()} != {Path(p).name for p in files if p.startswith("schemas/")}: - raise ValueError("Step 1 cannot change the schema set") - for path in PHASE4_STEP1_NEW: - if not (root / path).is_file() or (root / path).is_symlink(): - raise ValueError(f"Missing or unsafe Step 1 control/test file: {path}") - if hashlib.sha256((root / "PHASE_4_PLAN.md").read_bytes()).hexdigest() != PHASE4_PLAN_SHA256: - raise ValueError("Approved Phase 4 plan bytes changed") - verify_phase4_control(json.loads((root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))) - decisions = (root / "PHASE_4_DECISIONS.md").read_text(encoding="utf-8") - if PHASE4_APPROVAL not in decisions or any(f"P4-D{i:02d}" not in decisions for i in range(1, 12)): - raise ValueError("Actual Phase 4 approval record missing") - if any((root / name).exists() for name in PHASE4_FORBIDDEN_OUTPUTS): - raise ValueError("Step 1 cannot create Phase 4 completion records or audit reports") - return {"package_modules_unchanged": 40, "schemas_unchanged": 5, - "hero_files_unchanged": 6, "historical_migrated_nodes": 16, - "phase_complete": False, "runtime_behavior_added": False} - - -def audit_phase4(step: int = 1) -> dict: - verify_phase4_control(json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")), step) - result = {"phase": 4, "active_step": step, **verify_phase4_snapshot(), - "phase0_hashes_verified": 16, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_phase4_diff(step: int = 1) -> dict: - if type(step) is not int or step != 1: - raise ValueError("Unsupported Phase 4 step") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE4_FINAL, "HEAD"], check=True) - raw = git("diff", "--name-status", "--no-renames", "-z", PHASE4_FINAL, "--").split(b"\0") - raw = [part.decode("utf-8") for part in raw if part] - if len(raw) % 2: - raise ValueError("Malformed Git difference records") - changes = list(zip(raw[::2], raw[1::2])) - changes.extend(("A", p.decode()) for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") if p) - verify_phase4_changes(changes) - result = {"baseline_commit": PHASE4_FINAL, "changed_files": len(changes), - "changes": changes, "step": 1, "scope": "PASS"} - print(json.dumps(result, indent=2)) - return result - - -def phase4_baseline_evidence(output: Path) -> dict: - """Reconcile final Phase 3 identities plus all previously required identities.""" - output.mkdir(parents=True, exist_ok=True) - inherited = baseline_evidence(output / "phase3-inherited") - with tempfile.TemporaryDirectory(prefix="rit-p4-baseline-") as temp: - baseline = Path(temp) - for name, raw in _phase4_baseline_files().items(): - target = baseline / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(raw) - old, old_log = _collect(baseline) - current, current_log = _collect(ROOT) - expected = 2492 if os.environ.get("RIT_TEST_PARQUET") == "1" else 2489 - if len(old) != expected or set(old) - set(current): - raise ValueError("Accepted final Phase 3 test identities were lost") - result = {"baseline_commit": PHASE4_FINAL, "baseline_test_tree": PHASE4_TEST_TREE, - "baseline_nodeids": old, "current_nodeids": current, "missing_nodeids": [], - "baseline_tests": len(old), "current_tests": len(current), "inherited": inherited, - "nodeids_sha256": hashlib.sha256(("\n".join(old)+"\n").encode()).hexdigest()} - (output / "phase4_test_identity_manifest.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase3-final-collection.log").write_text(old_log, encoding="utf-8") - (output / "phase4-current-collection.log").write_text(current_log, encoding="utf-8") - print(json.dumps({k: result[k] for k in ("baseline_tests", "current_tests", "missing_nodeids")}, indent=2)) - return result - - -def phase4_candidate(output: Path, step: int = 1) -> None: - """Archive a tested intermediate Step 1; never certify phase completion.""" - if os.environ.get("RIT_TEST_PARQUET") != "1": - raise ValueError("Step 1 candidate evidence requires RIT_TEST_PARQUET=1 and real PyArrow") - if importlib.util.find_spec("pyarrow") is None: - raise ValueError("Step 1 candidate evidence requires real PyArrow") - output.mkdir(parents=True, exist_ok=True) - result = audit_phase4(step) - result["diff"] = audit_phase4_diff(step) - if git("status", "--porcelain").strip(): - raise ValueError("Candidate archive requires committed, clean source") - result["core"] = verify_junit(output / "core.xml", minimum=2489) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=2492) - result["core_math_measurements"] = verify_step10_evidence(output / "core.xml", output / "phase4-core-observations.json") - result["parquet_math_measurements"] = verify_step10_evidence(output / "parquet.xml", output / "phase4-parquet-observations.json") - identity = phase4_baseline_evidence(output) - result["test_identity"] = {k: identity[k] for k in ("baseline_tests", "current_tests", "nodeids_sha256")} - expected = set() - for node in identity["current_nodeids"]: - base, bracket, parameter = node.partition("[") - owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), - name + bracket + parameter)) - for name, parquet in (("core.xml", False), ("parquet.xml", True)): - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(output / name).getroot().iter("testcase")} - target = expected if parquet else {c for c in expected if c[1] not in PARQUET_CASES} - if cases != target: - raise ValueError(f"Candidate JUnit does not execute the entire current suite: {name}") - verify_distributions(output / "dist") - archive = output / "recursive-integrity-toolkit-phase4-step1-candidate.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {p for p in git("ls-files", "-z").decode().split("\0") if p} - with zipfile.ZipFile(archive) as zipped: - names = {p.removeprefix("recursive-integrity-toolkit/") for p in zipped.namelist() if not p.endswith("/")} - if names != tracked: - raise ValueError("Step 1 source archive file set mismatch") - for name in names: - if zipped.read("recursive-integrity-toolkit/"+name) != (ROOT / name).read_bytes(): - raise ValueError(f"Step 1 source archive byte mismatch: {name}") - result.update({"commit": git("rev-parse", "HEAD").decode().strip(), - "tree": git("rev-parse", "HEAD^{tree}").decode().strip(), - "source_archive": archive.name, "tracked_files": len(tracked), - "python": sys.version, "platform": sys.platform, - "versions": {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "pytest")}}) - (output / "phase4_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4_repository_files.sha256").write_text("".join(f"{hashlib.sha256((ROOT/name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - paths = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase4_artifacts.sha256") - (output / "phase4_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in paths), encoding="utf-8") - print(f"Phase 4 Step 1 candidate: {len(tracked)} source files verified; Phase 4 remains incomplete.") - - -def phase4_cli_main() -> int: - parser = argparse.ArgumentParser(description="Phase 4 Step 1 maintainer gate") - parser.add_argument("--phase", type=int, choices=(4,), required=True) - parser.add_argument("--step", type=int, choices=(1,), required=True) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) - parser.add_argument("--require-parquet", action="store_true") - parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - raise ValueError("Phase 4 Step 1 cannot certify a final phase delivery") - if args.junit is not None: - verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - phase4_baseline_evidence(args.baseline_evidence) - elif args.dist is not None: - audit_phase4(args.step) - wheel, _ = verify_distributions(args.dist) - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - elif args.smoke_wheel is not None: - smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - phase4_candidate(args.candidate, args.step) - else: - audit_phase4(args.step) - if args.diff: - audit_phase4_diff(args.step) - return 0 - - -# Phase 4 Step 2 is additive. Step 1 gates above retain their original semantics. -PHASE4_STEP1_FINAL = "a7f3c46d6ca05de36bfcb60f496ebb2d5ab4a37c" -PHASE4_STEP1_LOCAL = "5aa9cd6d06bfffec9cc280a52277aa9152a8627e" -PHASE4_STEP1_TREE = "7f8ef492568456bf5d46fb1b7e2631cb9b94e3fd" -PHASE4_STEP1_TEST_TREE = "2ce77e331a0fc377387735bb55dffd3be630b59c" -PHASE4_STEP2_APPROVAL = "批准，开始 **Phase 4 Step 2**" -PHASE4_STEP2_APPROVAL_DATE = "2026-09-19" -PHASE4_STEP2_REPORT_SCHEMA_SHA256 = "34b7edb8e8672fca3f3d4f6fe647b4d8967acb6f0435544ba449c1c44f7bc1f6" -PHASE4_STEP2_ALLOWED = { - "PHASE_4_BASELINE.json", "PHASE_4_DECISIONS.md", "scripts/check_traceability.py", - "scripts/check_spec_consistency.py", "scripts/release_check.py", "tests/conftest.py", - "tests/unit/test_phase4_contracts.py", "tests/integration/test_phase4_gates.py", - "tests/integration/test_no_algorithms.py", "tests/integration/test_ci_workflows.py", - "tests/integration/test_repository_structure.py", "tests/integration/test_owner_ids.py", - "tests/integration/test_package_import.py", "tests/integration/test_package_install.py", - "tests/integration/test_optional_dependency.py", "tests/integration/test_no_network.py", - "tests/integration/test_schema_json.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_prohibited_structure.py", "tests/integration/test_license_notices.py", - ".github/workflows/ci.yml", ".github/workflows/security.yml", - ".github/workflows/golden.yml", ".github/workflows/release.yml", - "docs/architecture.md", "docs/theory_traceability.md", - "src/recursive_integrity_toolkit/result.py", "schemas/report.schema.json", - "docs/report_schema.md", "tests/unit/test_PR012_evidence_classes.py", - "tests/unit/test_PR013_report_schema.py", -} -PHASE4_STEP2_NEW = () -PHASE4_STEP2_MIGRATIONS = {'tests/integration/test_phase4_gates.py': [{'new': 'def phase4_mutation_tree(repo_root, ' - 'tmp_path_factory, phase4_gate_tools, ' - 'phase4_step1_snapshot):', - 'node': 'phase4_mutation_tree', - 'old': 'def phase4_mutation_tree(repo_root, ' - 'tmp_path_factory, phase4_gate_tools):'}, - {'new': 'shutil.copyfile(phase4_step1_snapshot / ' - 'relative, destination)', - 'node': 'phase4_mutation_tree', - 'old': 'shutil.copyfile(repo_root / relative, ' - 'destination)'}, - {'new': 'def ' - 'test_phase4_final_snapshot_preserves_all_current_runtime_bytes(repo_root, ' - 'phase3_final_snapshot, ' - 'phase4_step1_snapshot):\n' - ' repo_root = phase4_step1_snapshot', - 'node': 'test_phase4_final_snapshot_preserves_all_current_runtime_bytes', - 'old': 'def ' - 'test_phase4_final_snapshot_preserves_all_current_runtime_bytes(repo_root, ' - 'phase3_final_snapshot):'}, - {'new': 'def ' - 'test_phase4_active_workflows_preserve_full_matrix_and_use_current_dispatch(repo_root, ' - 'phase4_step1_snapshot):\n' - ' repo_root = phase4_step1_snapshot', - 'node': 'test_phase4_active_workflows_preserve_full_matrix_and_use_current_dispatch', - 'old': 'def ' - 'test_phase4_active_workflows_preserve_full_matrix_and_use_current_dispatch(repo_root):'}], - 'tests/unit/test_phase4_contracts.py': [{'new': 'def phase4_control(phase4_step1_snapshot):\n' - ' return json.loads((phase4_step1_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))', - 'node': 'phase4_control', - 'old': 'def phase4_control():\n' - ' return json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))'}, - {'new': 'def ' - 'test_phase4_registered_migrations_accept_only_the_approved_inventory(phase4_tools, ' - 'phase3_final_snapshot, phase4_step1_snapshot):', - 'node': 'test_phase4_registered_migrations_accept_only_the_approved_inventory', - 'old': 'def ' - 'test_phase4_registered_migrations_accept_only_the_approved_inventory(phase4_tools, ' - 'phase3_final_snapshot):'}, - {'new': 'path, (phase3_final_snapshot / ' - 'path).read_bytes(), (phase4_step1_snapshot / ' - 'path).read_bytes(),', - 'node': 'test_phase4_registered_migrations_accept_only_the_approved_inventory', - 'old': 'path, (phase3_final_snapshot / ' - 'path).read_bytes(), (ROOT / ' - 'path).read_bytes(),'}]} - - -def _phase4_step1_files(): - """Read the exact accepted Step 1 Git tree and verify every blob identity.""" - for suffix, expected in (("^{commit}", PHASE4_STEP1_FINAL), ("^{tree}", PHASE4_STEP1_TREE), - (":tests", PHASE4_STEP1_TEST_TREE)): - if git("rev-parse", PHASE4_STEP1_FINAL + suffix).decode().strip() != expected: - raise ValueError("Pinned Phase 4 Step 1 identity mismatch") - objects = {} - for entry in git("ls-tree", "-rz", PHASE4_STEP1_FINAL).split(b"\0"): - if not entry: - continue - metadata, raw_path = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - path = raw_path.decode("utf-8") - if (mode not in (b"100644", b"100755") or kind != b"blob" or path in objects - or path.startswith("/") or ".." in path.split("/") or ".git" in path.split("/")): - raise ValueError("Unsafe pinned Step 1 Git object") - objects[path] = oid.decode("ascii") - if len(objects) != 227: - raise ValueError("Pinned Step 1 must contain exactly 227 files") - files = {} - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_STEP1_FINAL))) as archive: - names = [item.filename for item in archive.infolist() if not item.is_dir()] - if len(names) != len(objects) or set(names) != set(objects): - raise ValueError("Pinned Step 1 archive identity mismatch") - for name in names: - raw = archive.read(name) - oid = hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - if oid != objects[name]: - raise ValueError(f"Pinned Step 1 blob mismatch: {name}") - files[name] = raw - return MappingProxyType(files) - - -def phase4_step2_expected_control() -> dict: - """Actual Step 2 approval is independent of the mutable control document.""" - prior = _phase4_step1_files() - result = phase4_expected_control() - result.update({ - "control_version": "1.1", "active_step": 2, - "approval_date": PHASE4_STEP2_APPROVAL_DATE, "approval_basis": PHASE4_STEP2_APPROVAL, - "previous_step_commit": PHASE4_STEP1_FINAL, - "previous_step_local_commit": PHASE4_STEP1_LOCAL, - "previous_step_tree": PHASE4_STEP1_TREE, - "previous_step_test_tree": PHASE4_STEP1_TEST_TREE, - "previous_step_core_tests": 2584, "previous_step_parquet_tests": 2587, - "previous_step_files_sha256": {p: hashlib.sha256(raw).hexdigest() for p, raw in sorted(prior.items())}, - "permitted_paths": sorted(PHASE4_STEP2_ALLOWED), "new_files_permitted": [], - "runtime_changes_authorized": True, "schema_changes_authorized": True, - "runtime_paths_authorized": ["src/recursive_integrity_toolkit/result.py"], - "schema_paths_authorized": ["schemas/report.schema.json"], - "step1_historical_binding_nodes": {p: sorted({r["node"] for r in rows}) - for p, rows in sorted(PHASE4_STEP2_MIGRATIONS.items())}, - }) - return result - - -def verify_phase4_step2_control(control: dict, step: int = 2) -> None: - if type(step) is not int or step != 2 or type(control) is not dict: - raise ValueError("Unsupported Phase 4 Step 2 stage/control") - try: - actual = json.dumps(control, sort_keys=True, ensure_ascii=True, allow_nan=False) - expected = json.dumps(phase4_step2_expected_control(), sort_keys=True, ensure_ascii=True, allow_nan=False) - except (TypeError, ValueError) as error: - raise ValueError("Invalid Phase 4 Step 2 control") from error - if actual != expected: - raise ValueError("Phase 4 control differs from the independently approved Step 2 contract") - - -def verify_phase4_step2_changes(changes: list[tuple[str, str]]) -> None: - seen = set() - for status, path in changes: - if path in seen or path not in PHASE4_STEP2_ALLOWED or status != "M": - raise ValueError(f"Unapproved Phase 4 Step 2 path/operation: {status} {path}") - seen.add(path) - - -def _phase4_step2_header(node, path): - """Allow explicit test parametrization/fixtures, never definition-time effects.""" - for decorator in node.decorator_list: - if not isinstance(decorator, ast.Call) or ast.unparse(decorator.func) not in { - "pytest.fixture", "pytest.mark.parametrize"}: - raise ValueError(f"Unapproved Step 2 test decorator: {path}:{node.name}") - if ast.unparse(decorator.func) == "pytest.fixture" and not node.name.startswith("phase4_step"): - raise ValueError("A Step 2 fixture must have an explicit scoped name") - if any(keyword.arg is None for keyword in decorator.keywords): - raise ValueError("Decorator expansion is outside the Step 2 contract") - for value in [*decorator.args, *(keyword.value for keyword in decorator.keywords)]: - try: - ast.literal_eval(value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError("Step 2 decorator arguments must be literal") from error - clone = ast.parse(ast.unparse(node)).body[0] - clone.decorator_list = [] - _phase4_preserve_function_header(clone, path) - - -def _phase4_step2_append_only(before: bytes, after: bytes, path: str) -> None: - if not after.startswith(before): - raise ValueError(f"Inherited Step 1 prefix changed: {path}") - old, new = ast.parse(before), ast.parse(after) - bindings = set() - for node in old.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - bindings.add(node.name) - elif isinstance(node, (ast.Import, ast.ImportFrom)): - bindings.update(alias.asname or alias.name.split(".")[0] for alias in node.names) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - bindings.update(child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)) - for node in new.body[len(old.body):]: - if not isinstance(node, ast.FunctionDef) or node.name in bindings: - raise ValueError(f"Step 2 addition executes, rebinds or shadows inherited source: {path}") - if not node.name.startswith(("test_phase4_step2_", "phase4_step2_", "phase4_step1_")): - raise ValueError(f"Step 2 added test/helper is not explicitly scoped: {path}:{node.name}") - if path == "tests/conftest.py" and node.name != "phase4_step1_snapshot": - raise ValueError("Only the pinned Step 1 shared fixture is authorized") - bindings.add(node.name) - _phase4_step2_header(node, path) - - -def verify_phase4_step2_test_migration(path: str, before: bytes, after: bytes) -> None: - prior = _phase4_step1_files() - if path not in prior or path not in PHASE4_STEP2_ALLOWED or before != prior[path] or not path.startswith("tests/"): - raise ValueError("Step 2 migration requires the exact named Step 1 source") - _phase4_step2_check_test_migration(path, before, after) - - -def _phase4_step2_check_test_migration(path: str, before: bytes, after: bytes) -> None: - """Check a source already read from the verified Step 1 immutable mapping.""" - expected = before - for row in PHASE4_STEP2_MIGRATIONS.get(path, []): - old, new = row["old"].encode(), row["new"].encode() - if expected.count(old) != 1: - raise ValueError("Step 1 historical binding is not unique") - expected = expected.replace(old, new, 1) - _phase4_step2_append_only(expected, after, path) - - -def _phase4_step2_preserve_tooling(before: bytes, after: bytes, path: str) -> None: - """Normalize only the exact new dispatch branch, then preserve all old code.""" - if path == "scripts/release_check.py": - old = ' return phase4_cli_main() if explicit_phase4 else main()' - new = (' explicit_step2 = "--step=2" in argv or any(a == "--step" and b == "2" for a, b in zip(argv, argv[1:]))\n' - ' if explicit_phase4 and explicit_step2:\n' - ' return phase4_step2_cli_main()\n' - ' return phase4_cli_main() if explicit_phase4 else main()') - replacements = [(old, new)] - else: - old = ' if args.phase == 4 and args.step == 1:\n return phase4_main(step=args.step)\n' - new = old + ' if args.phase == 4 and args.step == 2:\n return phase4_step2_main(step=args.step)\n' - replacements = [(old, new), - ('Choose explicit --phase 3 --step 11 or --phase 4 --step 1', - 'Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2')] - expected = before - for old, new in replacements: - if expected.count(old.encode()) != 1: - raise ValueError(f"Historical dispatcher identity mismatch: {path}") - expected = expected.replace(old.encode(), new.encode(), 1) - _phase4_preserve_tooling(expected, after, path) - - -def verify_phase4_step2_snapshot(root: Path = ROOT) -> dict: - prior = _phase4_step1_files() - for path, raw in prior.items(): - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError(f"Missing or aliased inherited Step 1 file: {path}") - current = target.read_bytes() - if path not in PHASE4_STEP2_ALLOWED and current != raw: - raise ValueError(f"Protected Step 1 bytes changed: {path}") - if path.startswith("tests/") and path in PHASE4_STEP2_ALLOWED: - _phase4_step2_check_test_migration(path, raw, current) - elif path.startswith("scripts/") and path in PHASE4_STEP2_ALLOWED: - _phase4_step2_preserve_tooling(raw, current, path) - elif path in {"docs/architecture.md", "docs/theory_traceability.md", "PHASE_4_DECISIONS.md"}: - if not current.startswith(raw): - raise ValueError(f"Step 1 historical documentation prefix changed: {path}") - actual_modules = {p.relative_to(root).as_posix() for p in (root / "src/recursive_integrity_toolkit").rglob("*.py")} - expected_modules = {p for p in prior if p.startswith("src/") and p.endswith(".py")} - if actual_modules != expected_modules or len(actual_modules) != 40: - raise ValueError("Step 2 cannot change the runtime module set") - if {p.name for p in (root / "schemas").iterdir()} != {Path(p).name for p in prior if p.startswith("schemas/")}: - raise ValueError("Step 2 cannot change the schema set") - if hashlib.sha256((root / "PHASE_4_PLAN.md").read_bytes()).hexdigest() != PHASE4_PLAN_SHA256: - raise ValueError("Approved Phase 4 plan changed") - verify_phase4_step2_control(json.loads((root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))) - decisions = (root / "PHASE_4_DECISIONS.md").read_text(encoding="utf-8") - if PHASE4_STEP2_APPROVAL not in decisions or PHASE4_STEP1_FINAL not in decisions: - raise ValueError("Actual Step 2 authorization or Step 1 evidence anchor missing") - if any((root / name).exists() for name in PHASE4_FORBIDDEN_OUTPUTS): - raise ValueError("Step 2 cannot create Phase 4 completion records or audit outputs") - import runpy - checker = runpy.run_path(str(root / "scripts/check_traceability.py"), run_name="phase4_step2_contract_boundary") - checker["phase4_step2_result_boundary"](root / "src/recursive_integrity_toolkit/result.py") - schema_bytes = (root / "schemas/report.schema.json").read_bytes() - if hashlib.sha256(schema_bytes).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Step 2 report schema differs from its independently reviewed bytes") - schema = json.loads(schema_bytes) - if (schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema" - or schema.get("type") != "object" or schema.get("additionalProperties") is not False): - raise ValueError("Step 2 report schema dialect/closed root mismatch") - return {"package_modules": 40, "frozen_runtime_modules": 39, "frozen_schemas": 4, - "hero_files_unchanged": 6, "historical_phase3_migrated_nodes": 16, - "phase_complete": False, "result_contracts_enabled": True, - "adapters_enabled": False, "cli_analysis_enabled": False} - - -def audit_phase4_step2(step: int = 2) -> dict: - verify_phase4_step2_control(json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")), step) - result = {"phase": 4, "active_step": step, **verify_phase4_step2_snapshot(), - "phase0_hashes_verified": 16, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_phase4_step2_diff(step: int = 2) -> dict: - if type(step) is not int or step != 2: - raise ValueError("Unsupported Phase 4 Step 2 stage") - _phase4_step1_files() - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE4_STEP1_FINAL, "HEAD"], check=True) - raw = git("diff", "--name-status", "--no-renames", "-z", PHASE4_STEP1_FINAL, "--").split(b"\0") - raw = [part.decode("utf-8") for part in raw if part] - if len(raw) % 2: - raise ValueError("Malformed Step 2 Git difference records") - changes = list(zip(raw[::2], raw[1::2])) - changes.extend(("A", p.decode()) for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") if p) - verify_phase4_step2_changes(changes) - result = {"previous_step_commit": PHASE4_STEP1_FINAL, "changed_files": len(changes), - "changes": changes, "step": 2, "scope": "PASS"} - print(json.dumps(result, indent=2)) - return result - - -def phase4_step2_baseline_evidence(output: Path) -> dict: - """Reconcile all inherited identities, including the accepted Step 1 suite.""" - output.mkdir(parents=True, exist_ok=True) - inherited = phase4_baseline_evidence(output / "phase4-step1-inherited") - with tempfile.TemporaryDirectory(prefix="rit-p4-step1-identities-") as temp: - baseline = Path(temp) - for name, raw in _phase4_step1_files().items(): - target = baseline / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(raw) - old, old_log = _collect(baseline) - current, current_log = _collect(ROOT) - expected = 2587 if os.environ.get("RIT_TEST_PARQUET") == "1" else 2584 - if len(old) != expected or set(old) - set(current): - raise ValueError("Accepted Phase 4 Step 1 test identities were lost") - result = {"baseline_commit": PHASE4_STEP1_FINAL, "baseline_test_tree": PHASE4_STEP1_TEST_TREE, - "baseline_nodeids": old, "current_nodeids": current, "missing_nodeids": [], - "baseline_tests": len(old), "current_tests": len(current), "inherited": inherited, - "nodeids_sha256": hashlib.sha256(("\n".join(old)+"\n").encode()).hexdigest()} - (output / "phase4_step2_test_identity_manifest.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4-step1-collection.log").write_text(old_log, encoding="utf-8") - (output / "phase4-step2-collection.log").write_text(current_log, encoding="utf-8") - print(json.dumps({k: result[k] for k in ("baseline_tests", "current_tests", "missing_nodeids")}, indent=2)) - return result - - -def phase4_step2_candidate(output: Path, step: int = 2) -> None: - """Build tested Step 2 intermediate evidence without declaring Phase 4 complete.""" - if os.environ.get("RIT_TEST_PARQUET") != "1" or importlib.util.find_spec("pyarrow") is None: - raise ValueError("Step 2 candidate evidence requires RIT_TEST_PARQUET=1 and real PyArrow") - output.mkdir(parents=True, exist_ok=True) - result = audit_phase4_step2(step) - result["diff"] = audit_phase4_step2_diff(step) - if git("status", "--porcelain").strip(): - raise ValueError("Step 2 candidate archive requires committed, clean source") - result["core"] = verify_junit(output / "core.xml", minimum=2584) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=2587) - result["core_math_measurements"] = verify_step10_evidence(output / "core.xml", output / "phase4-step2-core-observations.json") - result["parquet_math_measurements"] = verify_step10_evidence(output / "parquet.xml", output / "phase4-step2-parquet-observations.json") - identity = phase4_step2_baseline_evidence(output) - result["test_identity"] = {k: identity[k] for k in ("baseline_tests", "current_tests", "nodeids_sha256")} - expected = set() - for node in identity["current_nodeids"]: - base, bracket, parameter = node.partition("[") - owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) - for name, parquet in (("core.xml", False), ("parquet.xml", True)): - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(output / name).getroot().iter("testcase")} - target = expected if parquet else {c for c in expected if c[1] not in PARQUET_CASES} - if cases != target: - raise ValueError(f"Step 2 JUnit does not execute the entire current suite: {name}") - wheel, _ = verify_distributions(output / "dist") - phase4_step2_installed_contract_smoke(wheel) - archive = output / "recursive-integrity-toolkit-phase4-step2-candidate.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {p for p in git("ls-files", "-z").decode().split("\0") if p} - with zipfile.ZipFile(archive) as zipped: - names = {p.removeprefix("recursive-integrity-toolkit/") for p in zipped.namelist() if not p.endswith("/")} - if names != tracked or len(tracked) != 227: - raise ValueError("Step 2 source archive file set mismatch") - for name in names: - if zipped.read("recursive-integrity-toolkit/"+name) != (ROOT / name).read_bytes(): - raise ValueError(f"Step 2 source archive byte mismatch: {name}") - result.update({"commit": git("rev-parse", "HEAD").decode().strip(), - "tree": git("rev-parse", "HEAD^{tree}").decode().strip(), - "source_archive": archive.name, "tracked_files": len(tracked), - "python": sys.version, "platform": sys.platform, - "versions": {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "pytest")}}) - (output / "phase4_step2_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4_step2_repository_files.sha256").write_text("".join(f"{hashlib.sha256((ROOT/name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - paths = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase4_step2_artifacts.sha256") - (output / "phase4_step2_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in paths), encoding="utf-8") - print(f"Phase 4 Step 2 candidate: {len(tracked)} source files verified; Phase 4 remains incomplete.") - - -def phase4_step2_installed_contract_smoke(wheel: Path) -> None: +def smoke_installed_contract(wheel: Path) -> None: """Exercise installed canonical contracts with no optional or analytical imports.""" with tempfile.TemporaryDirectory(prefix="rit-p4-step2-installed-contract-") as temp: work = Path(temp) @@ -1952,7 +406,7 @@ def reject_schema_files(event, args): payload = { 'run': { 'run_id': 'installed-empty-case', 'toolkit_version': '0.1.0.dev2', - 'report_schema_version': '1.0', 'started_at': '2026-09-19T00:00:00+00:00', + 'report_schema_version': '1.1', 'started_at': '2026-09-19T00:00:00+00:00', 'completed_at': '2026-09-19T00:00:00+00:00', 'duration_seconds': 0, 'python_version': '3.12', 'platform': 'isolated-installed-smoke', 'command': 'rit validate', 'config_hash': '0123456789abcdef' * 4, @@ -2028,570 +482,7 @@ def reject_schema_files(event, args): subprocess.run([str(python), "-I", "-c", program, str(work), str(ROOT)], cwd=work, check=True) -def phase4_step2_cli_main() -> int: - parser = argparse.ArgumentParser(description="Phase 4 Step 2 maintainer gate") - parser.add_argument("--phase", type=int, choices=(4,), required=True) - parser.add_argument("--step", type=int, choices=(2,), required=True) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) - parser.add_argument("--require-parquet", action="store_true") - parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - raise ValueError("Phase 4 Step 2 cannot certify a final phase delivery") - if args.junit is not None: - verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - phase4_step2_baseline_evidence(args.baseline_evidence) - elif args.dist is not None: - audit_phase4_step2(args.step) - wheel, _ = verify_distributions(args.dist) - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - elif args.smoke_wheel is not None: - smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - phase4_step2_candidate(args.candidate, args.step) - else: - audit_phase4_step2(args.step) - if args.diff: - audit_phase4_step2_diff(args.step) - return 0 - - -# Phase 4 Step 3 adds pure adapters. All earlier stage gates remain callable. -PHASE4_STEP2_FINAL = "fcea74e2b1858e83cdbfd1b8212d15343d63ff08" -PHASE4_STEP2_TREE = "c3ce887fd393cfc6c18364546005f2b74a63b746" -PHASE4_STEP2_TEST_TREE = "9d6ac0cbf3827ab54c7852503d99d442a3796259" -PHASE4_STEP3_APPROVAL = "开始 **Phase 4 Step 3**" -PHASE4_STEP3_APPROVAL_DATE = "2026-09-19" -PHASE4_STEP3_ALLOWED = {'.github/workflows/ci.yml', - '.github/workflows/golden.yml', - '.github/workflows/release.yml', - '.github/workflows/security.yml', - 'PHASE_4_BASELINE.json', - 'PHASE_4_DECISIONS.md', - 'docs/architecture.md', - 'docs/report_schema.md', - 'docs/theory_traceability.md', - 'scripts/check_spec_consistency.py', - 'scripts/check_traceability.py', - 'scripts/release_check.py', - 'src/recursive_integrity_toolkit/reports/assembly.py', - 'tests/conftest.py', - 'tests/integration/test_ci_workflows.py', - 'tests/integration/test_hero_structure.py', - 'tests/integration/test_license_notices.py', - 'tests/integration/test_no_algorithms.py', - 'tests/integration/test_no_network.py', - 'tests/integration/test_optional_dependency.py', - 'tests/integration/test_owner_ids.py', - 'tests/integration/test_package_import.py', - 'tests/integration/test_package_install.py', - 'tests/integration/test_partial_lineage_report.py', - 'tests/integration/test_partial_provenance_report.py', - 'tests/integration/test_phase4_gates.py', - 'tests/integration/test_prohibited_structure.py', - 'tests/integration/test_repository_structure.py', - 'tests/integration/test_schema_json.py', - 'tests/unit/test_PR012_evidence_classes.py', - 'tests/unit/test_PR014_unavailable.py', - 'tests/unit/test_PR018_language.py', - 'tests/unit/test_phase4_contracts.py'} -PHASE4_STEP3_NEW = () -PHASE4_STEP3_DOCUMENT_CORRECTIONS = { - "docs/report_schema.md": [{ - "old": "| `derived_metrics.provenance.weighted_source_type_masses` | number | `user_declared_weight_mass` | `derived_metric` | `PR-005` | `PR-005.source_weight_mass` | 2 | Null only with unavailable reasons |", - "new": "| `derived_metrics.provenance.weighted_source_type_masses` | category_mass_map | `user_declared_weight_mass` | `derived_metric` | `PR-005` | `PR-005.source_weight_mass` | 2 | Null only with unavailable reasons |", - }], -} -PHASE4_STEP3_MIGRATIONS = {'tests/integration/test_phase4_gates.py': [{'new': 'def ' - 'test_phase4_step2_current_runtime_opens_only_the_canonical_result(repo_root, ' - 'phase3_final_snapshot, ' - 'phase4_step2_snapshot):\n' - ' repo_root = phase4_step2_snapshot\n', - 'node': 'test_phase4_step2_current_runtime_opens_only_the_canonical_result', - 'old': 'def ' - 'test_phase4_step2_current_runtime_opens_only_the_canonical_result(repo_root, ' - 'phase3_final_snapshot):\n'}, - {'new': 'def ' - 'test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step2_snapshot):\n' - ' repo_root = phase4_step2_snapshot\n', - 'node': 'test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch', - 'old': 'def ' - 'test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root):\n'}, - {'new': 'def ' - 'test_phase4_step2_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step2_snapshot):', - 'node': 'test_phase4_step2_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step2_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation):'}, - {'new': ' shutil.copyfile(phase4_step2_snapshot ' - '/ relative, destination)', - 'node': 'test_phase4_step2_current_snapshot_checks_valid_tree_before_mutations', - 'old': ' shutil.copyfile(repo_root / relative, ' - 'destination)'}], - 'tests/unit/test_phase4_contracts.py': [{'new': 'def ' - 'test_phase4_step2_approved_control_keeps_independent_step1_anchors(phase4_tools, ' - 'phase4_step2_snapshot):\n' - ' control = json.loads((phase4_step2_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))', - 'node': 'test_phase4_step2_approved_control_keeps_independent_step1_anchors', - 'old': 'def ' - 'test_phase4_step2_approved_control_keeps_independent_step1_anchors(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))'}, - {'new': 'def ' - 'test_phase4_step2_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value, phase4_step2_snapshot):\n' - ' control = json.loads((phase4_step2_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))', - 'node': 'test_phase4_step2_control_rejects_forged_scope_and_stage', - 'old': 'def ' - 'test_phase4_step2_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))'}, - {'new': 'def ' - 'test_phase4_step2_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step, phase4_step2_snapshot):\n' - ' control = json.loads((phase4_step2_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))', - 'node': 'test_phase4_step2_control_rejects_unapproved_dispatch', - 'old': 'def ' - 'test_phase4_step2_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))'}, - {'new': 'def ' - 'test_phase4_step2_control_requires_explicit_approval_fields(phase4_tools, ' - 'field, phase4_step2_snapshot):\n' - ' control = json.loads((phase4_step2_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))', - 'node': 'test_phase4_step2_control_requires_explicit_approval_fields', - 'old': 'def ' - 'test_phase4_step2_control_requires_explicit_approval_fields(phase4_tools, ' - 'field):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))'}, - {'new': 'def ' - 'test_phase4_step2_control_cannot_mint_extra_permission(phase4_tools, ' - 'phase4_step2_snapshot):\n' - ' control = json.loads((phase4_step2_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))', - 'node': 'test_phase4_step2_control_cannot_mint_extra_permission', - 'old': 'def ' - 'test_phase4_step2_control_cannot_mint_extra_permission(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))'}, - {'new': 'def ' - 'test_phase4_step2_historical_migrations_are_five_explicit_step1_bindings(phase4_tools, ' - 'phase4_step1_snapshot, phase4_step2_snapshot):', - 'node': 'test_phase4_step2_historical_migrations_are_five_explicit_step1_bindings', - 'old': 'def ' - 'test_phase4_step2_historical_migrations_are_five_explicit_step1_bindings(phase4_tools, ' - 'phase4_step1_snapshot):'}, - {'new': 'def ' - 'test_phase4_step2_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, ' - 'phase4_step1_snapshot, mutation, ' - 'phase4_step2_snapshot):', - 'node': 'test_phase4_step2_historical_guard_rejects_assertion_and_binding_weakening', - 'old': 'def ' - 'test_phase4_step2_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, ' - 'phase4_step1_snapshot, mutation):'}, - {'new': ' path, (phase4_step1_snapshot / ' - 'path).read_bytes(), (phase4_step2_snapshot / ' - 'path).read_bytes(),', - 'node': 'test_phase4_step2_historical_migrations_are_five_explicit_step1_bindings', - 'old': ' path, (phase4_step1_snapshot / ' - 'path).read_bytes(), (ROOT / path).read_bytes(),'}, - {'new': ' before = (phase4_step1_snapshot / ' - 'path).read_bytes()\n' - ' after = (phase4_step2_snapshot / ' - 'path).read_bytes()', - 'node': 'test_phase4_step2_historical_guard_rejects_assertion_and_binding_weakening', - 'old': ' before = (phase4_step1_snapshot / ' - 'path).read_bytes()\n' - ' after = (ROOT / path).read_bytes()'}]} - - -def _phase4_step2_files(): - """Read the exact accepted Step 2 Git tree and verify every blob identity.""" - for suffix, expected in (("^{commit}", PHASE4_STEP2_FINAL), ("^{tree}", PHASE4_STEP2_TREE), - (":tests", PHASE4_STEP2_TEST_TREE)): - if git("rev-parse", PHASE4_STEP2_FINAL + suffix).decode().strip() != expected: - raise ValueError("Pinned Phase 4 Step 2 identity mismatch") - objects = {} - for entry in git("ls-tree", "-rz", PHASE4_STEP2_FINAL).split(b"\0"): - if not entry: - continue - metadata, raw_path = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - path = raw_path.decode("utf-8") - if (mode not in (b"100644", b"100755") or kind != b"blob" or path in objects - or path.startswith("/") or ".." in path.split("/") or ".git" in path.split("/")): - raise ValueError("Unsafe pinned Step 2 Git object") - objects[path] = oid.decode("ascii") - if len(objects) != 227: - raise ValueError("Pinned Step 2 must contain exactly 227 files") - files = {} - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_STEP2_FINAL))) as archive: - names = [item.filename for item in archive.infolist() if not item.is_dir()] - if len(names) != len(objects) or set(names) != set(objects): - raise ValueError("Pinned Step 2 archive identity mismatch") - for name in names: - raw = archive.read(name) - oid = hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - if oid != objects[name]: - raise ValueError(f"Pinned Step 2 blob mismatch: {name}") - files[name] = raw - return MappingProxyType(files) - - -def phase4_step3_expected_control() -> dict: - """Actual Step 3 approval is independent of the mutable control document.""" - prior = _phase4_step2_files() - result = phase4_step2_expected_control() - result.pop("previous_step_local_commit") - result.update({ - "control_version": "1.2", "active_step": 3, - "approval_date": PHASE4_STEP3_APPROVAL_DATE, "approval_basis": PHASE4_STEP3_APPROVAL, - "previous_step_commit": PHASE4_STEP2_FINAL, - "previous_step_tree": PHASE4_STEP2_TREE, - "previous_step_test_tree": PHASE4_STEP2_TEST_TREE, - "previous_step_core_tests": 2728, "previous_step_parquet_tests": 2731, - "previous_step_files_sha256": {p: hashlib.sha256(raw).hexdigest() for p, raw in sorted(prior.items())}, - "permitted_paths": sorted(PHASE4_STEP3_ALLOWED), "new_files_permitted": [], - "runtime_changes_authorized": True, "schema_changes_authorized": False, - "runtime_paths_authorized": ["src/recursive_integrity_toolkit/reports/assembly.py"], - "schema_paths_authorized": [], - "step2_historical_binding_nodes": {p: sorted({r["node"] for r in rows}) - for p, rows in sorted(PHASE4_STEP3_MIGRATIONS.items())}, - }) - return result - - -def verify_phase4_step3_control(control: dict, step: int = 3) -> None: - if type(step) is not int or step != 3 or type(control) is not dict: - raise ValueError("Unsupported Phase 4 Step 3 stage/control") - try: - actual = json.dumps(control, sort_keys=True, ensure_ascii=True, allow_nan=False) - expected = json.dumps(phase4_step3_expected_control(), sort_keys=True, ensure_ascii=True, allow_nan=False) - except (TypeError, ValueError) as error: - raise ValueError("Invalid Phase 4 Step 3 control") from error - if actual != expected: - raise ValueError("Phase 4 control differs from the independently approved Step 3 contract") - - -def verify_phase4_step3_changes(changes: list[tuple[str, str]]) -> None: - seen = set() - for status, path in changes: - if path in seen or path not in PHASE4_STEP3_ALLOWED or status != "M": - raise ValueError(f"Unapproved Phase 4 Step 3 path/operation: {status} {path}") - seen.add(path) - - -def _phase4_step3_header(node, path): - """Allow explicit test parametrization/fixtures, never definition-time effects.""" - for decorator in node.decorator_list: - if not isinstance(decorator, ast.Call) or ast.unparse(decorator.func) not in { - "pytest.fixture", "pytest.mark.parametrize"}: - raise ValueError(f"Unapproved Step 3 test decorator: {path}:{node.name}") - if ast.unparse(decorator.func) == "pytest.fixture" and not node.name.startswith("phase4_step"): - raise ValueError("A Step 3 fixture must have an explicit scoped name") - if any(keyword.arg is None for keyword in decorator.keywords): - raise ValueError("Decorator expansion is outside the Step 3 contract") - for value in [*decorator.args, *(keyword.value for keyword in decorator.keywords)]: - try: - ast.literal_eval(value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError("Step 3 decorator arguments must be literal") from error - clone = ast.parse(ast.unparse(node)).body[0] - clone.decorator_list = [] - _phase4_preserve_function_header(clone, path) - - -def _phase4_step3_append_only(before: bytes, after: bytes, path: str) -> None: - if not after.startswith(before): - raise ValueError(f"Inherited Step 2 prefix changed: {path}") - old, new = ast.parse(before), ast.parse(after) - bindings = set() - for node in old.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - bindings.add(node.name) - elif isinstance(node, (ast.Import, ast.ImportFrom)): - bindings.update(alias.asname or alias.name.split(".")[0] for alias in node.names) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - bindings.update(child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)) - for node in new.body[len(old.body):]: - if not isinstance(node, ast.FunctionDef) or node.name in bindings: - raise ValueError(f"Step 3 addition executes, rebinds or shadows inherited source: {path}") - allowed_snapshot = path == "tests/conftest.py" and node.name == "phase4_step2_snapshot" - if not allowed_snapshot and not node.name.startswith(("test_phase4_step3_", "phase4_step3_")): - raise ValueError(f"Step 3 added test/helper is not explicitly scoped: {path}:{node.name}") - if path == "tests/conftest.py" and node.name != "phase4_step2_snapshot": - raise ValueError("Only the pinned Step 2 shared fixture is authorized") - bindings.add(node.name) - _phase4_step3_header(node, path) - - -def verify_phase4_step3_test_migration(path: str, before: bytes, after: bytes) -> None: - prior = _phase4_step2_files() - if path not in prior or path not in PHASE4_STEP3_ALLOWED or before != prior[path] or not path.startswith("tests/"): - raise ValueError("Step 3 migration requires the exact named Step 2 source") - _phase4_step3_check_test_migration(path, before, after) - - -def _phase4_step3_check_test_migration(path: str, before: bytes, after: bytes) -> None: - """Check a source already read from the verified Step 2 immutable mapping.""" - expected = before - for row in PHASE4_STEP3_MIGRATIONS.get(path, []): - old, new = row["old"].encode(), row["new"].encode() - if expected.count(old) != 1: - raise ValueError("Step 2 historical binding is not unique") - expected = expected.replace(old, new, 1) - _phase4_step3_append_only(expected, after, path) - - -def _phase4_step3_preserve_tooling(before: bytes, after: bytes, path: str) -> None: - """Permit the exact Step 3 dispatch addition and preserve every older statement.""" - if path == "scripts/release_check.py": - old = ('def cli_main() -> int:\n' - ' argv = sys.argv[1:]\n' - ' explicit_phase4 = "--phase=4" in argv or any(a == "--phase" and b == "4" for a, b in zip(argv, argv[1:]))\n' - ' explicit_step2 = "--step=2" in argv or any(a == "--step" and b == "2" for a, b in zip(argv, argv[1:]))\n' - ' if explicit_phase4 and explicit_step2:\n' - ' return phase4_step2_cli_main()\n' - ' return phase4_cli_main() if explicit_phase4 else main()') - new = old.replace(' return phase4_cli_main() if explicit_phase4 else main()', - ' explicit_step3 = "--step=3" in argv or any(a == "--step" and b == "3" for a, b in zip(argv, argv[1:]))\n' - ' if explicit_phase4 and explicit_step3:\n' - ' return phase4_step3_cli_main()\n' - ' return phase4_cli_main() if explicit_phase4 else main()') - replacements = [(old, new)] - else: - old = ' if args.phase == 4 and args.step == 2:\n return phase4_step2_main(step=args.step)\n' - new = old + ' if args.phase == 4 and args.step == 3:\n return phase4_step3_main(step=args.step)\n' - replacements = [(old, new), - ('Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2', - 'Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3')] - expected = before - for old, new in replacements: - if expected.count(old.encode()) != 1: - raise ValueError(f"Historical dispatcher identity mismatch: {path}") - expected = expected.replace(old.encode(), new.encode(), 1) - _phase4_preserve_tooling(expected, after, path) - - -def verify_phase4_step3_snapshot(root: Path = ROOT) -> dict: - prior = _phase4_step2_files() - for path, raw in prior.items(): - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError(f"Missing or aliased inherited Step 2 file: {path}") - current = target.read_bytes() - if path not in PHASE4_STEP3_ALLOWED and current != raw: - raise ValueError(f"Protected Step 2 bytes changed: {path}") - if path.startswith("tests/") and path in PHASE4_STEP3_ALLOWED: - _phase4_step3_check_test_migration(path, raw, current) - elif path.startswith("scripts/") and path in PHASE4_STEP3_ALLOWED: - _phase4_step3_preserve_tooling(raw, current, path) - elif path in {"docs/architecture.md", "docs/theory_traceability.md", "docs/report_schema.md", "PHASE_4_DECISIONS.md"}: - expected = raw - for correction in PHASE4_STEP3_DOCUMENT_CORRECTIONS.get(path, []): - old, new = correction["old"].encode(), correction["new"].encode() - if expected.count(old) != 1: - raise ValueError(f"Historical documentation correction is not unique: {path}") - expected = expected.replace(old, new, 1) - if not current.startswith(expected): - raise ValueError(f"Step 2 historical documentation prefix changed: {path}") - actual_modules = {p.relative_to(root).as_posix() for p in (root / "src/recursive_integrity_toolkit").rglob("*.py")} - expected_modules = {p for p in prior if p.startswith("src/") and p.endswith(".py")} - if actual_modules != expected_modules or len(actual_modules) != 40: - raise ValueError("Step 3 cannot change the runtime module set") - if {p.name for p in (root / "schemas").iterdir()} != {Path(p).name for p in prior if p.startswith("schemas/")}: - raise ValueError("Step 3 cannot change the schema set") - if hashlib.sha256((root / "PHASE_4_PLAN.md").read_bytes()).hexdigest() != PHASE4_PLAN_SHA256: - raise ValueError("Approved Phase 4 plan changed") - verify_phase4_step3_control(json.loads((root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))) - decisions = (root / "PHASE_4_DECISIONS.md").read_text(encoding="utf-8") - if PHASE4_STEP3_APPROVAL not in decisions or PHASE4_STEP2_FINAL not in decisions: - raise ValueError("Actual Step 3 authorization or Step 2 evidence anchor missing") - if any((root / name).exists() for name in PHASE4_FORBIDDEN_OUTPUTS): - raise ValueError("Step 3 cannot create Phase 4 completion records or audit outputs") - import runpy - checker = runpy.run_path(str(root / "scripts/check_traceability.py"), run_name="phase4_step3_assembly_boundary") - checker["phase4_step2_result_boundary"](root / "src/recursive_integrity_toolkit/result.py") - checker["phase4_step3_assembly_boundary"](root / "src/recursive_integrity_toolkit/reports/assembly.py") - schema_bytes = (root / "schemas/report.schema.json").read_bytes() - if hashlib.sha256(schema_bytes).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Step 3 report schema differs from its independently reviewed bytes") - schema = json.loads(schema_bytes) - if (schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema" - or schema.get("type") != "object" or schema.get("additionalProperties") is not False): - raise ValueError("Step 3 report schema dialect/closed root mismatch") - return {"package_modules": 40, "frozen_runtime_modules": 39, "frozen_schemas": 5, - "hero_files_unchanged": 6, "historical_phase3_migrated_nodes": 16, - "phase_complete": False, "result_contracts_enabled": True, - "adapters_enabled": True, "cli_analysis_enabled": False} - - -def audit_phase4_step3(step: int = 3) -> dict: - verify_phase4_step3_control(json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")), step) - result = {"phase": 4, "active_step": step, **verify_phase4_step3_snapshot(), - "phase0_hashes_verified": 16, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_phase4_step3_diff(step: int = 3) -> dict: - if type(step) is not int or step != 3: - raise ValueError("Unsupported Phase 4 Step 3 stage") - _phase4_step2_files() - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE4_STEP2_FINAL, "HEAD"], check=True) - raw = git("diff", "--name-status", "--no-renames", "-z", PHASE4_STEP2_FINAL, "--").split(b"\0") - raw = [part.decode("utf-8") for part in raw if part] - if len(raw) % 2: - raise ValueError("Malformed Step 3 Git difference records") - changes = list(zip(raw[::2], raw[1::2])) - changes.extend(("A", p.decode()) for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") if p) - verify_phase4_step3_changes(changes) - result = {"previous_step_commit": PHASE4_STEP2_FINAL, "changed_files": len(changes), - "changes": changes, "step": 3, "scope": "PASS"} - print(json.dumps(result, indent=2)) - return result - - -def phase4_step3_baseline_evidence(output: Path) -> dict: - """Reconcile all inherited identities, including the accepted Step 2 suite.""" - output.mkdir(parents=True, exist_ok=True) - inherited = phase4_step2_baseline_evidence(output / "phase4-step2-inherited") - with tempfile.TemporaryDirectory(prefix="rit-p4-step2-identities-") as temp: - baseline = Path(temp) - for name, raw in _phase4_step2_files().items(): - target = baseline / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(raw) - old, old_log = _collect(baseline) - current, current_log = _collect(ROOT) - expected = 2731 if os.environ.get("RIT_TEST_PARQUET") == "1" else 2728 - if len(old) != expected or set(old) - set(current): - raise ValueError("Accepted Phase 4 Step 2 test identities were lost") - result = {"baseline_commit": PHASE4_STEP2_FINAL, "baseline_test_tree": PHASE4_STEP2_TEST_TREE, - "baseline_nodeids": old, "current_nodeids": current, "missing_nodeids": [], - "baseline_tests": len(old), "current_tests": len(current), "inherited": inherited, - "nodeids_sha256": hashlib.sha256(("\n".join(old)+"\n").encode()).hexdigest()} - (output / "phase4_step3_test_identity_manifest.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4-step2-collection.log").write_text(old_log, encoding="utf-8") - (output / "phase4-step3-collection.log").write_text(current_log, encoding="utf-8") - print(json.dumps({k: result[k] for k in ("baseline_tests", "current_tests", "missing_nodeids")}, indent=2)) - return result - - -def phase4_step3_candidate(output: Path, step: int = 3) -> None: - """Build tested Step 3 intermediate evidence without declaring Phase 4 complete.""" - if os.environ.get("RIT_TEST_PARQUET") != "1" or importlib.util.find_spec("pyarrow") is None: - raise ValueError("Step 3 candidate evidence requires RIT_TEST_PARQUET=1 and real PyArrow") - output.mkdir(parents=True, exist_ok=True) - result = audit_phase4_step3(step) - result["diff"] = audit_phase4_step3_diff(step) - if git("status", "--porcelain").strip(): - raise ValueError("Step 3 candidate archive requires committed, clean source") - result["core"] = verify_junit(output / "core.xml", minimum=2728) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=2731) - result["core_math_measurements"] = verify_step10_evidence(output / "core.xml", output / "phase4-step3-core-observations.json") - result["parquet_math_measurements"] = verify_step10_evidence(output / "parquet.xml", output / "phase4-step3-parquet-observations.json") - identity = phase4_step3_baseline_evidence(output) - result["test_identity"] = {k: identity[k] for k in ("baseline_tests", "current_tests", "nodeids_sha256")} - expected = set() - for node in identity["current_nodeids"]: - base, bracket, parameter = node.partition("[") - owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) - for name, parquet in (("core.xml", False), ("parquet.xml", True)): - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(output / name).getroot().iter("testcase")} - target = expected if parquet else {c for c in expected if c[1] not in PARQUET_CASES} - if cases != target: - raise ValueError(f"Step 3 JUnit does not execute the entire current suite: {name}") - wheel, _ = verify_distributions(output / "dist") - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - archive = output / "recursive-integrity-toolkit-phase4-step3-candidate.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {p for p in git("ls-files", "-z").decode().split("\0") if p} - with zipfile.ZipFile(archive) as zipped: - names = {p.removeprefix("recursive-integrity-toolkit/") for p in zipped.namelist() if not p.endswith("/")} - if names != tracked or len(tracked) != 227: - raise ValueError("Step 3 source archive file set mismatch") - for name in names: - if zipped.read("recursive-integrity-toolkit/"+name) != (ROOT / name).read_bytes(): - raise ValueError(f"Step 3 source archive byte mismatch: {name}") - result.update({"commit": git("rev-parse", "HEAD").decode().strip(), - "tree": git("rev-parse", "HEAD^{tree}").decode().strip(), - "source_archive": archive.name, "tracked_files": len(tracked), - "python": sys.version, "platform": sys.platform, - "versions": {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "pytest")}}) - (output / "phase4_step3_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4_step3_repository_files.sha256").write_text("".join(f"{hashlib.sha256((ROOT/name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - paths = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase4_step3_artifacts.sha256") - (output / "phase4_step3_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in paths), encoding="utf-8") - print(f"Phase 4 Step 3 candidate: {len(tracked)} source files verified; Phase 4 remains incomplete.") - - -def phase4_step3_cli_main() -> int: - parser = argparse.ArgumentParser(description="Phase 4 Step 3 maintainer gate") - parser.add_argument("--phase", type=int, choices=(4,), required=True) - parser.add_argument("--step", type=int, choices=(3,), required=True) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) - parser.add_argument("--require-parquet", action="store_true") - parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - raise ValueError("Phase 4 Step 3 cannot certify a final phase delivery") - if args.junit is not None: - verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - phase4_step3_baseline_evidence(args.baseline_evidence) - elif args.dist is not None: - audit_phase4_step3(args.step) - wheel, _ = verify_distributions(args.dist) - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - elif args.smoke_wheel is not None: - smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - phase4_step3_candidate(args.candidate, args.step) - else: - audit_phase4_step3(args.step) - if args.diff: - audit_phase4_step3_diff(args.step) - return 0 - - -def phase4_step3_installed_assembly_smoke(wheel: Path) -> None: +def smoke_installed_assembly(wheel: Path) -> None: """Exercise an installed, hand-authored typed handoff with every effect blocked.""" with tempfile.TemporaryDirectory(prefix="rit-p4-step3-installed-assembly-") as temp: work = Path(temp) @@ -2690,7 +581,7 @@ def deny_effects(event, arguments): ) run = { 'run_id': 'installed-typed-empty-case', 'toolkit_version': '0.1.0.dev2', - 'report_schema_version': '1.0', 'started_at': None, 'completed_at': None, + 'report_schema_version': '1.1', 'started_at': None, 'completed_at': None, 'duration_seconds': None, 'python_version': None, 'platform': None, 'command': None, 'config_hash': None, 'random_seed': None, 'strict_mode': False, 'redacted_mode': False, 'network_call_count': 0, @@ -2717,7 +608,7 @@ def deny_effects(event, arguments): assert payload['inputs']['scope']['record_count'] == 0 assert payload['inputs']['artifacts'] == [] and payload['inputs']['file_hashes'] == [] assert payload['simulations'] == {} and payload['errors'] == [] -assert payload['capabilities']['lineage']['execution_status'] == 'deferred' +assert payload['capabilities']['lineage']['execution_status'] == 'not_requested' assert report.sections['observability']['capabilities'] is report.sections['capabilities'] assert {'model_performance_decline', 'causal_ancestor_effect', 'universal_integrity', 'universal_collapse_prediction'} <= {item['conclusion'] for item in payload['unavailable_conclusions']} @@ -2773,607 +664,7 @@ def deny_effects(event, arguments): subprocess.run([str(python), "-I", "-c", program, str(work), str(ROOT)], cwd=work, check=True) -# Phase 4 Step 4 adds explicit privacy views without changing inherited gates. -PHASE4_STEP3_FINAL = "974545c456e535ba1e1c5b6bf4ae0ce25bc04b57" -PHASE4_STEP3_TREE = "15498f3f00d0ce048c3cba2156eb7a44f096ae88" -PHASE4_STEP3_TEST_TREE = "17b87858f77ee4413c98cfd76f16f52170246741" -PHASE4_STEP4_APPROVAL = "继续" -PHASE4_STEP4_APPROVAL_DATE = "2026-09-19" -PHASE4_STEP4_NEW = () -PHASE4_STEP4_MIGRATIONS = {'tests/integration/test_phase4_gates.py': [{'new': 'def ' - 'test_phase4_step3_current_runtime_opens_only_assembly_and_freezes_schema(repo_root, ' - 'phase4_step2_snapshot, ' - 'phase4_step3_snapshot):\n' - ' repo_root = ' - 'phase4_step3_snapshot\n', - 'node': 'test_phase4_step3_current_runtime_opens_only_assembly_and_freezes_schema', - 'old': 'def ' - 'test_phase4_step3_current_runtime_opens_only_assembly_and_freezes_schema(repo_root, ' - 'phase4_step2_snapshot):\n'}, - {'new': 'def ' - 'test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step3_snapshot):\n' - ' repo_root = ' - 'phase4_step3_snapshot\n', - 'node': 'test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch', - 'old': 'def ' - 'test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root):\n'}, - {'new': 'def ' - 'test_phase4_step3_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, ' - 'phase4_gate_tools, ' - 'mutation, ' - 'phase4_step3_snapshot):\n', - 'node': 'test_phase4_step3_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step3_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, ' - 'phase4_gate_tools, ' - 'mutation):\n'}, - {'new': ' ' - 'shutil.copyfile(phase4_step3_snapshot ' - '/ relative, ' - 'destination)\n', - 'node': 'test_phase4_step3_current_snapshot_checks_valid_tree_before_mutations', - 'old': ' ' - 'shutil.copyfile(repo_root ' - '/ relative, ' - 'destination)\n'}], - 'tests/unit/test_phase4_contracts.py': [{'new': 'def ' - 'test_phase4_step3_approved_control_keeps_independent_step2_anchors(phase4_tools, ' - 'phase4_step3_snapshot):\n' - ' control = ' - 'json.loads((phase4_step3_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step3_approved_control_keeps_independent_step2_anchors', - 'old': 'def ' - 'test_phase4_step3_approved_control_keeps_independent_step2_anchors(phase4_tools):\n' - ' control = ' - 'json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step3_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value, ' - 'phase4_step3_snapshot):\n' - ' control = ' - 'json.loads((phase4_step3_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step3_control_rejects_forged_scope_and_stage', - 'old': 'def ' - 'test_phase4_step3_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value):\n' - ' control = ' - 'json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step3_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step, ' - 'phase4_step3_snapshot):\n' - ' control = ' - 'json.loads((phase4_step3_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step3_control_rejects_unapproved_dispatch', - 'old': 'def ' - 'test_phase4_step3_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step):\n' - ' control = ' - 'json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step3_control_requires_explicit_approval_fields(phase4_tools, ' - 'field, ' - 'phase4_step3_snapshot):\n' - ' control = ' - 'json.loads((phase4_step3_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step3_control_requires_explicit_approval_fields', - 'old': 'def ' - 'test_phase4_step3_control_requires_explicit_approval_fields(phase4_tools, ' - 'field):\n' - ' control = ' - 'json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step3_control_cannot_mint_extra_permission(phase4_tools, ' - 'phase4_step3_snapshot):\n' - ' control = ' - 'json.loads((phase4_step3_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step3_control_cannot_mint_extra_permission', - 'old': 'def ' - 'test_phase4_step3_control_cannot_mint_extra_permission(phase4_tools):\n' - ' control = ' - 'json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step3_historical_migrations_preserve_ten_step2_gate_nodes(phase4_tools, ' - 'phase4_step2_snapshot, ' - 'phase4_step3_snapshot):\n', - 'node': 'test_phase4_step3_historical_migrations_preserve_ten_step2_gate_nodes', - 'old': 'def ' - 'test_phase4_step3_historical_migrations_preserve_ten_step2_gate_nodes(phase4_tools, ' - 'phase4_step2_snapshot):\n'}, - {'new': ' path, ' - '(phase4_step2_snapshot / ' - 'path).read_bytes(), ' - '(phase4_step3_snapshot / ' - 'path).read_bytes(),\n', - 'node': 'test_phase4_step3_historical_migrations_preserve_ten_step2_gate_nodes', - 'old': ' path, ' - '(phase4_step2_snapshot / ' - 'path).read_bytes(), (ROOT / ' - 'path).read_bytes(),\n'}, - {'new': 'def ' - 'test_phase4_step3_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, ' - 'phase4_step2_snapshot, ' - 'mutation, ' - 'phase4_step3_snapshot):\n', - 'node': 'test_phase4_step3_historical_guard_rejects_assertion_and_binding_weakening', - 'old': 'def ' - 'test_phase4_step3_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, ' - 'phase4_step2_snapshot, ' - 'mutation):\n'}, - {'new': ' after = ' - '(phase4_step3_snapshot / ' - 'path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step3_test_migration"]\n', - 'node': 'test_phase4_step3_historical_guard_rejects_assertion_and_binding_weakening', - 'old': ' after = (ROOT / ' - 'path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step3_test_migration"]\n'}]} -PHASE4_STEP4_ALLOWED = {'.github/workflows/ci.yml', - '.github/workflows/golden.yml', - '.github/workflows/release.yml', - '.github/workflows/security.yml', - 'PHASE_4_BASELINE.json', - 'PHASE_4_DECISIONS.md', - 'docs/architecture.md', - 'docs/privacy.md', - 'docs/report_schema.md', - 'docs/theory_traceability.md', - 'scripts/check_spec_consistency.py', - 'scripts/check_traceability.py', - 'scripts/release_check.py', - 'src/recursive_integrity_toolkit/config.py', - 'src/recursive_integrity_toolkit/reports/assembly.py', - 'src/recursive_integrity_toolkit/result.py', - 'src/recursive_integrity_toolkit/utils/hashing.py', - 'src/recursive_integrity_toolkit/utils/logging.py', - 'tests/conftest.py', - 'tests/integration/test_ci_workflows.py', - 'tests/integration/test_hero_structure.py', - 'tests/integration/test_license_notices.py', - 'tests/integration/test_no_algorithms.py', - 'tests/integration/test_no_network.py', - 'tests/integration/test_optional_dependency.py', - 'tests/integration/test_owner_ids.py', - 'tests/integration/test_package_import.py', - 'tests/integration/test_package_install.py', - 'tests/integration/test_phase4_gates.py', - 'tests/integration/test_prohibited_structure.py', - 'tests/integration/test_repository_structure.py', - 'tests/integration/test_schema_json.py', - 'tests/unit/test_PR015_redaction.py', - 'tests/unit/test_PR016_determinism.py', - 'tests/unit/test_PR018_language.py', - 'tests/unit/test_phase4_contracts.py'} - - -def _phase4_step3_files(): - """Read the exact accepted Step 3 Git tree and verify every blob identity.""" - for suffix, expected in (("^{commit}", PHASE4_STEP3_FINAL), ("^{tree}", PHASE4_STEP3_TREE), - (":tests", PHASE4_STEP3_TEST_TREE)): - if git("rev-parse", PHASE4_STEP3_FINAL + suffix).decode().strip() != expected: - raise ValueError("Pinned Phase 4 Step 3 identity mismatch") - objects = {} - for entry in git("ls-tree", "-rz", PHASE4_STEP3_FINAL).split(b"\0"): - if not entry: - continue - metadata, raw_path = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - path = raw_path.decode("utf-8") - if (mode not in (b"100644", b"100755") or kind != b"blob" or path in objects - or path.startswith("/") or ".." in path.split("/") or ".git" in path.split("/")): - raise ValueError("Unsafe pinned Step 3 Git object") - objects[path] = oid.decode("ascii") - if len(objects) != 227: - raise ValueError("Pinned Step 3 must contain exactly 227 files") - files = {} - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_STEP3_FINAL))) as archive: - names = [item.filename for item in archive.infolist() if not item.is_dir()] - if len(names) != len(objects) or set(names) != set(objects): - raise ValueError("Pinned Step 3 archive identity mismatch") - for name in names: - raw = archive.read(name) - oid = hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - if oid != objects[name]: - raise ValueError(f"Pinned Step 3 blob mismatch: {name}") - files[name] = raw - return MappingProxyType(files) - - -def phase4_step4_expected_control() -> dict: - """Actual Step 4 approval is independent of the mutable control document.""" - prior = _phase4_step3_files() - result = phase4_step3_expected_control() - result.update({ - "control_version": "1.3", "active_step": 4, - "approval_date": PHASE4_STEP4_APPROVAL_DATE, "approval_basis": PHASE4_STEP4_APPROVAL, - "previous_step_commit": PHASE4_STEP3_FINAL, - "previous_step_tree": PHASE4_STEP3_TREE, - "previous_step_test_tree": PHASE4_STEP3_TEST_TREE, - "previous_step_core_tests": 2876, "previous_step_parquet_tests": 2879, - "previous_step_files_sha256": {p: hashlib.sha256(raw).hexdigest() for p, raw in sorted(prior.items())}, - "permitted_paths": sorted(PHASE4_STEP4_ALLOWED), "new_files_permitted": [], - "runtime_changes_authorized": True, "schema_changes_authorized": False, - "runtime_paths_authorized": sorted(p for p in PHASE4_STEP4_ALLOWED if p.startswith("src/")), - "schema_paths_authorized": [], - "step3_historical_binding_nodes": {p: sorted({r["node"] for r in rows}) - for p, rows in sorted(PHASE4_STEP4_MIGRATIONS.items())}, - }) - return result - - -def verify_phase4_step4_control(control: dict, step: int = 4) -> None: - if type(step) is not int or step != 4 or type(control) is not dict: - raise ValueError("Unsupported Phase 4 Step 4 stage/control") - try: - actual = json.dumps(control, sort_keys=True, ensure_ascii=True, allow_nan=False) - expected = json.dumps(phase4_step4_expected_control(), sort_keys=True, ensure_ascii=True, allow_nan=False) - except (TypeError, ValueError) as error: - raise ValueError("Invalid Phase 4 Step 4 control") from error - if actual != expected: - raise ValueError("Phase 4 control differs from the independently approved Step 4 contract") - - -def verify_phase4_step4_changes(changes: list[tuple[str, str]]) -> None: - seen = set() - for status, path in changes: - if path in seen or path not in PHASE4_STEP4_ALLOWED or status != "M": - raise ValueError(f"Unapproved Phase 4 Step 4 path/operation: {status} {path}") - seen.add(path) - - -def _phase4_step4_header(node, path): - """Allow explicit test parametrization/fixtures, never definition-time effects.""" - for decorator in node.decorator_list: - if not isinstance(decorator, ast.Call) or ast.unparse(decorator.func) not in { - "pytest.fixture", "pytest.mark.parametrize"}: - raise ValueError(f"Unapproved Step 4 test decorator: {path}:{node.name}") - if ast.unparse(decorator.func) == "pytest.fixture" and not node.name.startswith("phase4_step"): - raise ValueError("A Step 4 fixture must have an explicit scoped name") - if any(keyword.arg is None for keyword in decorator.keywords): - raise ValueError("Decorator expansion is outside the Step 4 contract") - for value in [*decorator.args, *(keyword.value for keyword in decorator.keywords)]: - try: - ast.literal_eval(value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError("Step 4 decorator arguments must be literal") from error - clone = ast.parse(ast.unparse(node)).body[0] - clone.decorator_list = [] - _phase4_preserve_function_header(clone, path) - - -def _phase4_step4_append_only(before: bytes, after: bytes, path: str) -> None: - if not after.startswith(before): - raise ValueError(f"Inherited Step 3 prefix changed: {path}") - old, new = ast.parse(before), ast.parse(after) - bindings = set() - for node in old.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - bindings.add(node.name) - elif isinstance(node, (ast.Import, ast.ImportFrom)): - bindings.update(alias.asname or alias.name.split(".")[0] for alias in node.names) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - bindings.update(child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)) - for node in new.body[len(old.body):]: - if not isinstance(node, ast.FunctionDef) or node.name in bindings: - raise ValueError(f"Step 4 addition executes, rebinds or shadows inherited source: {path}") - allowed_snapshot = path == "tests/conftest.py" and node.name == "phase4_step3_snapshot" - if not allowed_snapshot and not node.name.startswith(("test_phase4_step4_", "phase4_step4_")): - raise ValueError(f"Step 4 added test/helper is not explicitly scoped: {path}:{node.name}") - if path == "tests/conftest.py" and node.name != "phase4_step3_snapshot": - raise ValueError("Only the pinned Step 3 shared fixture is authorized") - bindings.add(node.name) - _phase4_step4_header(node, path) - - -def verify_phase4_step4_test_migration(path: str, before: bytes, after: bytes) -> None: - prior = _phase4_step3_files() - if path not in prior or path not in PHASE4_STEP4_ALLOWED or before != prior[path] or not path.startswith("tests/"): - raise ValueError("Step 4 migration requires the exact named Step 3 source") - _phase4_step4_check_test_migration(path, before, after) - - -def _phase4_step4_check_test_migration(path: str, before: bytes, after: bytes) -> None: - """Check a source already read from the verified Step 3 immutable mapping.""" - expected = before - for row in PHASE4_STEP4_MIGRATIONS.get(path, []): - old, new = row["old"].encode(), row["new"].encode() - if expected.count(old) != 1: - raise ValueError("Step 3 historical binding is not unique") - expected = expected.replace(old, new, 1) - _phase4_step4_append_only(expected, after, path) - - -def _phase4_step4_preserve_tooling(before: bytes, after: bytes, path: str) -> None: - """Preserve exact inherited statements using one source-line split per tree.""" - if path == "scripts/release_check.py": - old = ' return phase4_cli_main() if explicit_phase4 else main()' - new = (' explicit_step4 = "--step=4" in argv or any(a == "--step" and b == "4" for a, b in zip(argv, argv[1:]))\n' - ' if explicit_phase4 and explicit_step4:\n' - ' return phase4_step4_cli_main()\n' + old) - # Earlier preserved dispatch functions contain this source too; bind cli_main only. - parsed = ast.parse(before) - entry = next(node for node in parsed.body if isinstance(node, ast.FunctionDef) and node.name == "cli_main") - old_entry = ast.get_source_segment(before.decode(), entry) - if old_entry.count(old) != 1: - raise ValueError("Step 3 release dispatcher identity mismatch") - replacements = [(old_entry, old_entry.replace(old, new, 1))] - else: - old = ' if args.phase == 4 and args.step == 3:\n return phase4_step3_main(step=args.step)\n' - new = old + ' if args.phase == 4 and args.step == 4:\n return phase4_step4_main(step=args.step)\n' - replacements = [(old, new), - ('Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3', - 'Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4')] - expected = before - for old, new in replacements: - if expected.count(old.encode()) != 1: - raise ValueError(f"Historical dispatcher identity mismatch: {path}") - expected = expected.replace(old.encode(), new.encode(), 1) - trees = [ast.parse(expected), ast.parse(after)] - lines = [expected.splitlines(keepends=True), after.splitlines(keepends=True)] - def segment(index, node): - if node.lineno == node.end_lineno: - return lines[index][node.lineno - 1][node.col_offset:node.end_col_offset] - return (lines[index][node.lineno - 1][node.col_offset:] - + b"".join(lines[index][node.lineno:node.end_lineno - 1]) - + lines[index][node.end_lineno - 1][:node.end_col_offset]) - def entry_guard(node): - return isinstance(node, ast.If) and isinstance(node.test, ast.Compare) and "__name__" in ast.unparse(node.test) - guards = [n for n in trees[1].body if entry_guard(n)] - expected_guard = ast.parse('if __name__ == "__main__":\n raise SystemExit(cli_main())').body[0] - if len(guards) != 1 or ast.dump(guards[0]) != ast.dump(expected_guard): - raise ValueError(f"Unapproved Step 4 maintainer entrypoint: {path}") - historical = [n for n in trees[0].body if not entry_guard(n)] - current = [n for n in trees[1].body if not entry_guard(n)] - old_entries = [(segment(0, n), ast.dump(n)) for n in historical] - cursor = 0 - for node in current: - if cursor < len(old_entries) and (segment(1, node), ast.dump(node)) == old_entries[cursor]: - cursor += 1 - continue - if isinstance(node, ast.FunctionDef) and (node.name == "_phase4_step3_files" or node.name.startswith( - ("phase4_step4_", "_phase4_step4_", "verify_phase4_step4_", "audit_phase4_step4"))): - _phase4_preserve_function_header(node, path) - continue - if isinstance(node, ast.Assign) and all(isinstance(target, ast.Name) and target.id.startswith( - ("PHASE4_STEP4_", "PHASE4_STEP3_")) for target in node.targets): - try: - ast.literal_eval(node.value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError(f"Nonliteral added Step 4 maintainer constant: {path}") from error - continue - raise ValueError(f"Unapproved Step 4 maintainer addition: {path}") - if cursor != len(historical): - raise ValueError(f"Inherited Step 3 maintainer statement changed: {path}") - def binding_counts(tree): - counts = {} - for node in tree.body: - names = [] - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - names = [node.name] - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names = [child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)] - elif isinstance(node, (ast.Import, ast.ImportFrom)): - names = [alias.asname or alias.name.split(".")[0] for alias in node.names] - for name in names: - counts[name] = counts.get(name, 0) + 1 - return counts - inherited_counts = binding_counts(trees[0]) - for name, count in binding_counts(trees[1]).items(): - if count > inherited_counts.get(name, 1): - raise ValueError(f"Duplicate Step 4 maintainer binding: {path}:{name}") - - -def verify_phase4_step4_snapshot(root: Path = ROOT) -> dict: - prior = _phase4_step3_files() - for path, raw in prior.items(): - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError(f"Missing or aliased inherited Step 3 file: {path}") - current = target.read_bytes() - if path not in PHASE4_STEP4_ALLOWED and current != raw: - raise ValueError(f"Protected Step 3 bytes changed: {path}") - if path.startswith("tests/") and path in PHASE4_STEP4_ALLOWED: - _phase4_step4_check_test_migration(path, raw, current) - elif path.startswith("scripts/") and path in PHASE4_STEP4_ALLOWED: - _phase4_step4_preserve_tooling(raw, current, path) - elif path in {"docs/architecture.md", "docs/theory_traceability.md", "docs/report_schema.md", "docs/privacy.md", "PHASE_4_DECISIONS.md"}: - if not current.startswith(raw): - raise ValueError(f"Step 3 historical documentation prefix changed: {path}") - actual_modules = {p.relative_to(root).as_posix() for p in (root / "src/recursive_integrity_toolkit").rglob("*.py")} - expected_modules = {p for p in prior if p.startswith("src/") and p.endswith(".py")} - if actual_modules != expected_modules or len(actual_modules) != 40: - raise ValueError("Step 4 cannot change the runtime module set") - if {p.name for p in (root / "schemas").iterdir()} != {Path(p).name for p in prior if p.startswith("schemas/")}: - raise ValueError("Step 4 cannot change the schema set") - if hashlib.sha256((root / "PHASE_4_PLAN.md").read_bytes()).hexdigest() != PHASE4_PLAN_SHA256: - raise ValueError("Approved Phase 4 plan changed") - verify_phase4_step4_control(json.loads((root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))) - decisions = (root / "PHASE_4_DECISIONS.md").read_text(encoding="utf-8") - if PHASE4_STEP4_APPROVAL not in decisions or PHASE4_STEP3_FINAL not in decisions: - raise ValueError("Actual Step 4 authorization or Step 3 evidence anchor missing") - if any((root / name).exists() for name in PHASE4_FORBIDDEN_OUTPUTS): - raise ValueError("Step 4 cannot create Phase 4 completion records or audit outputs") - import runpy - checker = runpy.run_path(str(root / "scripts/check_traceability.py"), run_name="phase4_step4_assembly_boundary") - checker["phase4_step4_privacy_boundary"](root) - schema_bytes = (root / "schemas/report.schema.json").read_bytes() - if hashlib.sha256(schema_bytes).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Step 4 report schema differs from its independently reviewed bytes") - schema = json.loads(schema_bytes) - if (schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema" - or schema.get("type") != "object" or schema.get("additionalProperties") is not False): - raise ValueError("Step 4 report schema dialect/closed root mismatch") - return {"package_modules": 40, "frozen_runtime_modules": 35, "frozen_schemas": 5, - "hero_files_unchanged": 6, "historical_phase3_migrated_nodes": 16, - "phase_complete": False, "result_contracts_enabled": True, - "adapters_enabled": True, "privacy_views_enabled": True, "cli_analysis_enabled": False} - - -def audit_phase4_step4(step: int = 4) -> dict: - verify_phase4_step4_control(json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")), step) - result = {"phase": 4, "active_step": step, **verify_phase4_step4_snapshot(), - "phase0_hashes_verified": 16, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_phase4_step4_diff(step: int = 4) -> dict: - if type(step) is not int or step != 4: - raise ValueError("Unsupported Phase 4 Step 4 stage") - _phase4_step3_files() - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE4_STEP3_FINAL, "HEAD"], check=True) - raw = git("diff", "--name-status", "--no-renames", "-z", PHASE4_STEP3_FINAL, "--").split(b"\0") - raw = [part.decode("utf-8") for part in raw if part] - if len(raw) % 2: - raise ValueError("Malformed Step 4 Git difference records") - changes = list(zip(raw[::2], raw[1::2])) - changes.extend(("A", p.decode()) for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") if p) - verify_phase4_step4_changes(changes) - result = {"previous_step_commit": PHASE4_STEP3_FINAL, "changed_files": len(changes), - "changes": changes, "step": 4, "scope": "PASS"} - print(json.dumps(result, indent=2)) - return result - - -def phase4_step4_baseline_evidence(output: Path) -> dict: - """Reconcile all inherited identities, including the accepted Step 3 suite.""" - output.mkdir(parents=True, exist_ok=True) - inherited = phase4_step3_baseline_evidence(output / "phase4-step3-inherited") - with tempfile.TemporaryDirectory(prefix="rit-p4-step3-identities-") as temp: - baseline = Path(temp) - for name, raw in _phase4_step3_files().items(): - target = baseline / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(raw) - old, old_log = _collect(baseline) - current, current_log = _collect(ROOT) - expected = 2879 if os.environ.get("RIT_TEST_PARQUET") == "1" else 2876 - if len(old) != expected or set(old) - set(current): - raise ValueError("Accepted Phase 4 Step 3 test identities were lost") - result = {"baseline_commit": PHASE4_STEP3_FINAL, "baseline_test_tree": PHASE4_STEP3_TEST_TREE, - "baseline_nodeids": old, "current_nodeids": current, "missing_nodeids": [], - "baseline_tests": len(old), "current_tests": len(current), "inherited": inherited, - "nodeids_sha256": hashlib.sha256(("\n".join(old)+"\n").encode()).hexdigest()} - (output / "phase4_step4_test_identity_manifest.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4-step3-collection.log").write_text(old_log, encoding="utf-8") - (output / "phase4-step4-collection.log").write_text(current_log, encoding="utf-8") - print(json.dumps({k: result[k] for k in ("baseline_tests", "current_tests", "missing_nodeids")}, indent=2)) - return result - - -def phase4_step4_candidate(output: Path, step: int = 4) -> None: - """Build tested Step 4 intermediate evidence without declaring Phase 4 complete.""" - if os.environ.get("RIT_TEST_PARQUET") != "1" or importlib.util.find_spec("pyarrow") is None: - raise ValueError("Step 4 candidate evidence requires RIT_TEST_PARQUET=1 and real PyArrow") - output.mkdir(parents=True, exist_ok=True) - result = audit_phase4_step4(step) - result["diff"] = audit_phase4_step4_diff(step) - if git("status", "--porcelain").strip(): - raise ValueError("Step 4 candidate archive requires committed, clean source") - result["core"] = verify_junit(output / "core.xml", minimum=2876) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=2879) - result["core_math_measurements"] = verify_step10_evidence(output / "core.xml", output / "phase4-step4-core-observations.json") - result["parquet_math_measurements"] = verify_step10_evidence(output / "parquet.xml", output / "phase4-step4-parquet-observations.json") - identity = phase4_step4_baseline_evidence(output) - result["test_identity"] = {k: identity[k] for k in ("baseline_tests", "current_tests", "nodeids_sha256")} - expected = set() - for node in identity["current_nodeids"]: - base, bracket, parameter = node.partition("[") - owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) - for name, parquet in (("core.xml", False), ("parquet.xml", True)): - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(output / name).getroot().iter("testcase")} - target = expected if parquet else {c for c in expected if c[1] not in PARQUET_CASES} - if cases != target: - raise ValueError(f"Step 4 JUnit does not execute the entire current suite: {name}") - wheel, _ = verify_distributions(output / "dist") - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - archive = output / "recursive-integrity-toolkit-phase4-step4-candidate.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {p for p in git("ls-files", "-z").decode().split("\0") if p} - with zipfile.ZipFile(archive) as zipped: - names = {p.removeprefix("recursive-integrity-toolkit/") for p in zipped.namelist() if not p.endswith("/")} - if names != tracked or len(tracked) != 227: - raise ValueError("Step 4 source archive file set mismatch") - for name in names: - if zipped.read("recursive-integrity-toolkit/"+name) != (ROOT / name).read_bytes(): - raise ValueError(f"Step 4 source archive byte mismatch: {name}") - result.update({"commit": git("rev-parse", "HEAD").decode().strip(), - "tree": git("rev-parse", "HEAD^{tree}").decode().strip(), - "source_archive": archive.name, "tracked_files": len(tracked), - "python": sys.version, "platform": sys.platform, - "versions": {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "pytest")}}) - (output / "phase4_step4_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4_step4_repository_files.sha256").write_text("".join(f"{hashlib.sha256((ROOT/name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - paths = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase4_step4_artifacts.sha256") - (output / "phase4_step4_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in paths), encoding="utf-8") - print(f"Phase 4 Step 4 candidate: {len(tracked)} source files verified; Phase 4 remains incomplete.") - - -def phase4_step4_cli_main() -> int: - parser = argparse.ArgumentParser(description="Phase 4 Step 4 maintainer gate") - parser.add_argument("--phase", type=int, choices=(4,), required=True) - parser.add_argument("--step", type=int, choices=(4,), required=True) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) - parser.add_argument("--require-parquet", action="store_true") - parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - raise ValueError("Phase 4 Step 4 cannot certify a final phase delivery") - if args.junit is not None: - verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - phase4_step4_baseline_evidence(args.baseline_evidence) - elif args.dist is not None: - audit_phase4_step4(args.step) - wheel, _ = verify_distributions(args.dist) - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - elif args.smoke_wheel is not None: - smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - phase4_step4_candidate(args.candidate, args.step) - else: - audit_phase4_step4(args.step) - if args.diff: - audit_phase4_step4_diff(args.step) - return 0 - - -def phase4_step4_installed_privacy_smoke(wheel: Path) -> None: +def smoke_installed_privacy(wheel: Path) -> None: """Exercise installed privacy, config and diagnostics after accepted calculation.""" with tempfile.TemporaryDirectory(prefix="rit-p4-step4-installed-privacy-") as temp: work = Path(temp) @@ -3476,7 +767,7 @@ def deny_effects(event, arguments): ) run = { 'run_id': 'installed-typed-empty-case', 'toolkit_version': '0.1.0.dev2', - 'report_schema_version': '1.0', 'started_at': None, 'completed_at': None, + 'report_schema_version': '1.1', 'started_at': None, 'completed_at': None, 'duration_seconds': None, 'python_version': None, 'platform': None, 'command': None, 'config_hash': None, 'random_seed': None, 'strict_mode': False, 'redacted_mode': False, 'network_call_count': 0, @@ -3503,7 +794,7 @@ def deny_effects(event, arguments): assert payload['inputs']['scope']['record_count'] == 0 assert payload['inputs']['artifacts'] == [] and payload['inputs']['file_hashes'] == [] assert payload['simulations'] == {} and payload['errors'] == [] -assert payload['capabilities']['lineage']['execution_status'] == 'deferred' +assert payload['capabilities']['lineage']['execution_status'] == 'not_requested' assert report.sections['observability']['capabilities'] is report.sections['capabilities'] assert {'model_performance_decline', 'causal_ancestor_effect', 'universal_integrity', 'universal_collapse_prediction'} <= {item['conclusion'] for item in payload['unavailable_conclusions']} @@ -3636,591 +927,7 @@ def deny_effects(event, arguments): subprocess.run([str(python), "-I", "-c", program, str(work), str(ROOT)], cwd=work, check=True) -# Phase 4 Step 5 adds pure renderers while retaining every inherited gate. - - -PHASE4_STEP4_FINAL = "49454a9b162cb8d35e38d1cb1ae32cb208d3a01c" - - -PHASE4_STEP4_TREE = "79264d845e08f73ee68e160dea04f7786f18fd0a" - - -PHASE4_STEP4_TEST_TREE = "4aa4ca0b22f8b8c25afe27feeaba713d6f0830bf" - - -PHASE4_STEP5_APPROVAL = "很好，Phase 4 Step 5 继续" - - -PHASE4_STEP5_APPROVAL_DATE = "2026-09-20" - - -PHASE4_STEP5_NEW = () - - -PHASE4_STEP5_MIGRATIONS = {'tests/integration/test_phase4_gates.py': [{'new': 'def ' - 'test_phase4_step4_current_runtime_opens_only_privacy_metadata_modules_and_freezes_schema(repo_root, ' - 'phase4_step3_snapshot, phase4_step4_snapshot):\n' - ' repo_root = phase4_step4_snapshot\n', - 'node': 'test_phase4_step4_current_runtime_opens_only_privacy_metadata_modules_and_freezes_schema', - 'old': 'def ' - 'test_phase4_step4_current_runtime_opens_only_privacy_metadata_modules_and_freezes_schema(repo_root, ' - 'phase4_step3_snapshot):\n'}, - {'new': 'def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\n' - ' repo_root = phase4_step4_snapshot\n', - 'node': 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch', - 'old': 'def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root):\n'}, - {'new': 'def ' - 'test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step4_snapshot):\n', - 'node': 'test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation):\n'}, - {'new': ' shutil.copyfile(phase4_step4_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step4_snapshot"]\n', - 'node': 'test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations', - 'old': ' shutil.copyfile(repo_root / relative, ' - 'destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step4_snapshot"]\n'}], - 'tests/unit/test_phase4_contracts.py': [{'new': 'def ' - 'test_phase4_step4_approved_control_keeps_independent_step3_anchors(phase4_tools, ' - 'phase4_step4_snapshot):\n' - ' control = json.loads((phase4_step4_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step4_approved_control_keeps_independent_step3_anchors', - 'old': 'def ' - 'test_phase4_step4_approved_control_keeps_independent_step3_anchors(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step4_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value, phase4_step4_snapshot):\n' - ' control = json.loads((phase4_step4_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step4_control_rejects_forged_scope_and_stage', - 'old': 'def ' - 'test_phase4_step4_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step4_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step, phase4_step4_snapshot):\n' - ' control = json.loads((phase4_step4_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step4_control_rejects_unapproved_dispatch', - 'old': 'def ' - 'test_phase4_step4_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step4_control_requires_explicit_approval_fields(phase4_tools, ' - 'field, phase4_step4_snapshot):\n' - ' control = json.loads((phase4_step4_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step4_control_requires_explicit_approval_fields', - 'old': 'def ' - 'test_phase4_step4_control_requires_explicit_approval_fields(phase4_tools, ' - 'field):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step4_control_cannot_mint_extra_permission(phase4_tools, ' - 'phase4_step4_snapshot):\n' - ' control = json.loads((phase4_step4_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n', - 'node': 'test_phase4_step4_control_cannot_mint_extra_permission', - 'old': 'def ' - 'test_phase4_step4_control_cannot_mint_extra_permission(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n'}, - {'new': 'def ' - 'test_phase4_step4_historical_migrations_preserve_ten_step3_gate_nodes(phase4_tools, ' - 'phase4_step3_snapshot, phase4_step4_snapshot):\n', - 'node': 'test_phase4_step4_historical_migrations_preserve_ten_step3_gate_nodes', - 'old': 'def ' - 'test_phase4_step4_historical_migrations_preserve_ten_step3_gate_nodes(phase4_tools, ' - 'phase4_step3_snapshot):\n'}, - {'new': ' path, (phase4_step3_snapshot / ' - 'path).read_bytes(), (phase4_step4_snapshot / ' - 'path).read_bytes(),\n', - 'node': 'test_phase4_step4_historical_migrations_preserve_ten_step3_gate_nodes', - 'old': ' path, (phase4_step3_snapshot / ' - 'path).read_bytes(), (ROOT / path).read_bytes(),\n'}, - {'new': 'def ' - 'test_phase4_step4_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, ' - 'phase4_step3_snapshot, mutation, phase4_step4_snapshot):\n', - 'node': 'test_phase4_step4_historical_guard_rejects_assertion_and_binding_weakening', - 'old': 'def ' - 'test_phase4_step4_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, ' - 'phase4_step3_snapshot, mutation):\n'}, - {'new': ' after = (phase4_step4_snapshot / path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step4_test_migration"]\n', - 'node': 'test_phase4_step4_historical_guard_rejects_assertion_and_binding_weakening', - 'old': ' after = (ROOT / path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step4_test_migration"]\n'}]} - - -PHASE4_STEP5_ALLOWED = {'.github/workflows/ci.yml', - '.github/workflows/golden.yml', - '.github/workflows/release.yml', - '.github/workflows/security.yml', - 'PHASE_4_BASELINE.json', - 'PHASE_4_DECISIONS.md', - 'docs/architecture.md', - 'docs/report_schema.md', - 'docs/theory_traceability.md', - 'scripts/check_spec_consistency.py', - 'scripts/check_traceability.py', - 'scripts/release_check.py', - 'src/recursive_integrity_toolkit/reports/json_report.py', - 'src/recursive_integrity_toolkit/reports/markdown_report.py', - 'tests/conftest.py', - 'tests/integration/test_ci_workflows.py', - 'tests/integration/test_hero_structure.py', - 'tests/integration/test_license_notices.py', - 'tests/integration/test_no_algorithms.py', - 'tests/integration/test_no_network.py', - 'tests/integration/test_optional_dependency.py', - 'tests/integration/test_owner_ids.py', - 'tests/integration/test_package_import.py', - 'tests/integration/test_package_install.py', - 'tests/integration/test_phase4_gates.py', - 'tests/integration/test_prohibited_structure.py', - 'tests/integration/test_repository_structure.py', - 'tests/integration/test_schema_json.py', - 'tests/unit/test_PR013_report_schema.py', - 'tests/unit/test_PR016_determinism.py', - 'tests/unit/test_PR018_language.py', - 'tests/unit/test_phase4_contracts.py'} - - -def _phase4_step4_files(): - """Read the exact accepted Step 4 Git tree and verify every blob identity.""" - for suffix, expected in (("^{commit}", PHASE4_STEP4_FINAL), ("^{tree}", PHASE4_STEP4_TREE), - (":tests", PHASE4_STEP4_TEST_TREE)): - if git("rev-parse", PHASE4_STEP4_FINAL + suffix).decode().strip() != expected: - raise ValueError("Pinned Phase 4 Step 4 identity mismatch") - objects = {} - for entry in git("ls-tree", "-rz", PHASE4_STEP4_FINAL).split(b"\0"): - if not entry: - continue - metadata, raw_path = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - path = raw_path.decode("utf-8") - if (mode not in (b"100644", b"100755") or kind != b"blob" or path in objects - or path.startswith("/") or ".." in path.split("/") or ".git" in path.split("/")): - raise ValueError("Unsafe pinned Step 4 Git object") - objects[path] = oid.decode("ascii") - if len(objects) != 227: - raise ValueError("Pinned Step 4 must contain exactly 227 files") - files = {} - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_STEP4_FINAL))) as archive: - names = [item.filename for item in archive.infolist() if not item.is_dir()] - if len(names) != len(objects) or set(names) != set(objects): - raise ValueError("Pinned Step 4 archive identity mismatch") - for name in names: - raw = archive.read(name) - oid = hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - if oid != objects[name]: - raise ValueError(f"Pinned Step 4 blob mismatch: {name}") - files[name] = raw - return MappingProxyType(files) - - -def phase4_step5_expected_control() -> dict: - """Actual Step 5 approval is independent of the mutable control document.""" - prior = _phase4_step4_files() - result = phase4_step4_expected_control() - result.update({ - "control_version": "1.4", "active_step": 5, - "approval_date": PHASE4_STEP5_APPROVAL_DATE, "approval_basis": PHASE4_STEP5_APPROVAL, - "previous_step_commit": PHASE4_STEP4_FINAL, - "previous_step_tree": PHASE4_STEP4_TREE, - "previous_step_test_tree": PHASE4_STEP4_TEST_TREE, - "previous_step_core_tests": 3071, "previous_step_parquet_tests": 3074, - "previous_step_files_sha256": {p: hashlib.sha256(raw).hexdigest() for p, raw in sorted(prior.items())}, - "permitted_paths": sorted(PHASE4_STEP5_ALLOWED), "new_files_permitted": [], - "runtime_changes_authorized": True, "schema_changes_authorized": False, - "runtime_paths_authorized": sorted(p for p in PHASE4_STEP5_ALLOWED if p.startswith("src/")), - "schema_paths_authorized": [], - "step4_historical_binding_nodes": {p: sorted({r["node"] for r in rows}) - for p, rows in sorted(PHASE4_STEP5_MIGRATIONS.items())}, - }) - return result - - -def verify_phase4_step5_control(control: dict, step: int = 5) -> None: - if type(step) is not int or step != 5 or type(control) is not dict: - raise ValueError("Unsupported Phase 4 Step 5 stage/control") - try: - actual = json.dumps(control, sort_keys=True, ensure_ascii=True, allow_nan=False) - expected = json.dumps(phase4_step5_expected_control(), sort_keys=True, ensure_ascii=True, allow_nan=False) - except (TypeError, ValueError) as error: - raise ValueError("Invalid Phase 4 Step 5 control") from error - if actual != expected: - raise ValueError("Phase 4 control differs from the independently approved Step 5 contract") - - -def verify_phase4_step5_changes(changes: list[tuple[str, str]]) -> None: - seen = set() - for status, path in changes: - if path in seen or path not in PHASE4_STEP5_ALLOWED or status != "M": - raise ValueError(f"Unapproved Phase 4 Step 5 path/operation: {status} {path}") - seen.add(path) - - -def _phase4_step5_header(node, path): - """Allow explicit test parametrization/fixtures, never definition-time effects.""" - for decorator in node.decorator_list: - if not isinstance(decorator, ast.Call) or ast.unparse(decorator.func) not in { - "pytest.fixture", "pytest.mark.parametrize"}: - raise ValueError(f"Unapproved Step 5 test decorator: {path}:{node.name}") - if ast.unparse(decorator.func) == "pytest.fixture" and not node.name.startswith("phase4_step"): - raise ValueError("A Step 5 fixture must have an explicit scoped name") - if any(keyword.arg is None for keyword in decorator.keywords): - raise ValueError("Decorator expansion is outside the Step 5 contract") - for value in [*decorator.args, *(keyword.value for keyword in decorator.keywords)]: - try: - ast.literal_eval(value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError("Step 5 decorator arguments must be literal") from error - clone = ast.parse(ast.unparse(node)).body[0] - clone.decorator_list = [] - _phase4_preserve_function_header(clone, path) - - -def _phase4_step5_append_only(before: bytes, after: bytes, path: str) -> None: - if not after.startswith(before): - raise ValueError(f"Inherited Step 4 prefix changed: {path}") - old, new = ast.parse(before), ast.parse(after) - bindings = set() - for node in old.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - bindings.add(node.name) - elif isinstance(node, (ast.Import, ast.ImportFrom)): - bindings.update(alias.asname or alias.name.split(".")[0] for alias in node.names) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - bindings.update(child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)) - for node in new.body[len(old.body):]: - if not isinstance(node, ast.FunctionDef) or node.name in bindings: - raise ValueError(f"Step 5 addition executes, rebinds or shadows inherited source: {path}") - allowed_snapshot = path == "tests/conftest.py" and node.name == "phase4_step4_snapshot" - if not allowed_snapshot and not node.name.startswith(("test_phase4_step5_", "phase4_step5_")): - raise ValueError(f"Step 5 added test/helper is not explicitly scoped: {path}:{node.name}") - if path == "tests/conftest.py" and node.name != "phase4_step4_snapshot": - raise ValueError("Only the pinned Step 4 shared fixture is authorized") - bindings.add(node.name) - _phase4_step5_header(node, path) - - -def verify_phase4_step5_test_migration(path: str, before: bytes, after: bytes) -> None: - prior = _phase4_step4_files() - if path not in prior or path not in PHASE4_STEP5_ALLOWED or before != prior[path] or not path.startswith("tests/"): - raise ValueError("Step 5 migration requires the exact named Step 4 source") - _phase4_step5_check_test_migration(path, before, after) - - -def _phase4_step5_check_test_migration(path: str, before: bytes, after: bytes) -> None: - """Check a source already read from the verified Step 4 immutable mapping.""" - expected = before - for row in PHASE4_STEP5_MIGRATIONS.get(path, []): - old, new = row["old"].encode(), row["new"].encode() - if expected.count(old) != 1: - raise ValueError("Step 4 historical binding is not unique") - expected = expected.replace(old, new, 1) - _phase4_step5_append_only(expected, after, path) - - -def _phase4_step5_preserve_tooling(before: bytes, after: bytes, path: str) -> None: - """Preserve exact inherited statements using one source-line split per tree.""" - if path == "scripts/release_check.py": - old = ' return phase4_cli_main() if explicit_phase4 else main()' - new = (' explicit_step5 = "--step=5" in argv or any(a == "--step" and b == "5" for a, b in zip(argv, argv[1:]))\n' - ' if explicit_phase4 and explicit_step5:\n' - ' return phase4_step5_cli_main()\n' + old) - # Earlier preserved dispatch functions contain this source too; bind cli_main only. - parsed = ast.parse(before) - entry = next(node for node in parsed.body if isinstance(node, ast.FunctionDef) and node.name == "cli_main") - old_entry = ast.get_source_segment(before.decode(), entry) - if old_entry.count(old) != 1: - raise ValueError("Step 4 release dispatcher identity mismatch") - replacements = [(old_entry, old_entry.replace(old, new, 1))] - else: - old = ' if args.phase == 4 and args.step == 4:\n return phase4_step4_main(step=args.step)\n' - new = old + ' if args.phase == 4 and args.step == 5:\n return phase4_step5_main(step=args.step)\n' - replacements = [(old, new), - ('Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4', - 'Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5')] - expected = before - for old, new in replacements: - if expected.count(old.encode()) != 1: - raise ValueError(f"Historical dispatcher identity mismatch: {path}") - expected = expected.replace(old.encode(), new.encode(), 1) - trees = [ast.parse(expected), ast.parse(after)] - lines = [expected.splitlines(keepends=True), after.splitlines(keepends=True)] - def segment(index, node): - if node.lineno == node.end_lineno: - return lines[index][node.lineno - 1][node.col_offset:node.end_col_offset] - return (lines[index][node.lineno - 1][node.col_offset:] - + b"".join(lines[index][node.lineno:node.end_lineno - 1]) - + lines[index][node.end_lineno - 1][:node.end_col_offset]) - def entry_guard(node): - return isinstance(node, ast.If) and isinstance(node.test, ast.Compare) and "__name__" in ast.unparse(node.test) - guards = [n for n in trees[1].body if entry_guard(n)] - expected_guard = ast.parse('if __name__ == "__main__":\n raise SystemExit(cli_main())').body[0] - if len(guards) != 1 or ast.dump(guards[0]) != ast.dump(expected_guard): - raise ValueError(f"Unapproved Step 5 maintainer entrypoint: {path}") - historical = [n for n in trees[0].body if not entry_guard(n)] - current = [n for n in trees[1].body if not entry_guard(n)] - old_entries = [(segment(0, n), ast.dump(n)) for n in historical] - cursor = 0 - for node in current: - if cursor < len(old_entries) and (segment(1, node), ast.dump(node)) == old_entries[cursor]: - cursor += 1 - continue - if isinstance(node, ast.FunctionDef) and (node.name == "_phase4_step4_files" or node.name.startswith( - ("phase4_step5_", "_phase4_step5_", "verify_phase4_step5_", "audit_phase4_step5"))): - _phase4_preserve_function_header(node, path) - continue - if isinstance(node, ast.Assign) and all(isinstance(target, ast.Name) and target.id.startswith( - ("PHASE4_STEP5_", "PHASE4_STEP4_")) for target in node.targets): - try: - ast.literal_eval(node.value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError(f"Nonliteral added Step 5 maintainer constant: {path}") from error - continue - raise ValueError(f"Unapproved Step 5 maintainer addition: {path}") - if cursor != len(historical): - raise ValueError(f"Inherited Step 4 maintainer statement changed: {path}") - def binding_counts(tree): - counts = {} - for node in tree.body: - names = [] - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - names = [node.name] - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names = [child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)] - elif isinstance(node, (ast.Import, ast.ImportFrom)): - names = [alias.asname or alias.name.split(".")[0] for alias in node.names] - for name in names: - counts[name] = counts.get(name, 0) + 1 - return counts - inherited_counts = binding_counts(trees[0]) - for name, count in binding_counts(trees[1]).items(): - if count > inherited_counts.get(name, 1): - raise ValueError(f"Duplicate Step 5 maintainer binding: {path}:{name}") - - -def verify_phase4_step5_snapshot(root: Path = ROOT) -> dict: - prior = _phase4_step4_files() - for path, raw in prior.items(): - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError(f"Missing or aliased inherited Step 4 file: {path}") - current = target.read_bytes() - if path not in PHASE4_STEP5_ALLOWED and current != raw: - raise ValueError(f"Protected Step 4 bytes changed: {path}") - if path.startswith("tests/") and path in PHASE4_STEP5_ALLOWED: - _phase4_step5_check_test_migration(path, raw, current) - elif path.startswith("scripts/") and path in PHASE4_STEP5_ALLOWED: - _phase4_step5_preserve_tooling(raw, current, path) - elif path in {"docs/architecture.md", "docs/theory_traceability.md", "docs/report_schema.md", "PHASE_4_DECISIONS.md"}: - if not current.startswith(raw): - raise ValueError(f"Step 4 historical documentation prefix changed: {path}") - actual_modules = {p.relative_to(root).as_posix() for p in (root / "src/recursive_integrity_toolkit").rglob("*.py")} - expected_modules = {p for p in prior if p.startswith("src/") and p.endswith(".py")} - if actual_modules != expected_modules or len(actual_modules) != 40: - raise ValueError("Step 5 cannot change the runtime module set") - if {p.name for p in (root / "schemas").iterdir()} != {Path(p).name for p in prior if p.startswith("schemas/")}: - raise ValueError("Step 5 cannot change the schema set") - if hashlib.sha256((root / "PHASE_4_PLAN.md").read_bytes()).hexdigest() != PHASE4_PLAN_SHA256: - raise ValueError("Approved Phase 4 plan changed") - verify_phase4_step5_control(json.loads((root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))) - decisions = (root / "PHASE_4_DECISIONS.md").read_text(encoding="utf-8") - if PHASE4_STEP5_APPROVAL not in decisions or PHASE4_STEP4_FINAL not in decisions: - raise ValueError("Actual Step 5 authorization or Step 4 evidence anchor missing") - if any((root / name).exists() for name in PHASE4_FORBIDDEN_OUTPUTS): - raise ValueError("Step 5 cannot create Phase 4 completion records or audit outputs") - import runpy - checker = runpy.run_path(str(root / "scripts/check_traceability.py"), run_name="phase4_step5_renderer_boundary") - checker["phase4_step5_renderers_boundary"](root) - schema_bytes = (root / "schemas/report.schema.json").read_bytes() - if hashlib.sha256(schema_bytes).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Step 5 report schema differs from its independently reviewed bytes") - schema = json.loads(schema_bytes) - if (schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema" - or schema.get("type") != "object" or schema.get("additionalProperties") is not False): - raise ValueError("Step 5 report schema dialect/closed root mismatch") - return {"package_modules": 40, "frozen_runtime_modules": 38, "frozen_schemas": 5, - "hero_files_unchanged": 6, "historical_phase3_migrated_nodes": 16, - "phase_complete": False, "result_contracts_enabled": True, - "adapters_enabled": True, "privacy_views_enabled": True, "renderers_enabled": True, "cli_analysis_enabled": False} - - -def audit_phase4_step5(step: int = 5) -> dict: - verify_phase4_step5_control(json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")), step) - result = {"phase": 4, "active_step": step, **verify_phase4_step5_snapshot(), - "phase0_hashes_verified": 16, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_phase4_step5_diff(step: int = 5) -> dict: - if type(step) is not int or step != 5: - raise ValueError("Unsupported Phase 4 Step 5 stage") - _phase4_step4_files() - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE4_STEP4_FINAL, "HEAD"], check=True) - raw = git("diff", "--name-status", "--no-renames", "-z", PHASE4_STEP4_FINAL, "--").split(b"\0") - raw = [part.decode("utf-8") for part in raw if part] - if len(raw) % 2: - raise ValueError("Malformed Step 5 Git difference records") - changes = list(zip(raw[::2], raw[1::2])) - changes.extend(("A", p.decode()) for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") if p) - verify_phase4_step5_changes(changes) - result = {"previous_step_commit": PHASE4_STEP4_FINAL, "changed_files": len(changes), - "changes": changes, "step": 5, "scope": "PASS"} - print(json.dumps(result, indent=2)) - return result - - -def phase4_step5_baseline_evidence(output: Path) -> dict: - """Reconcile all inherited identities, including the accepted Step 4 suite.""" - output.mkdir(parents=True, exist_ok=True) - inherited = phase4_step4_baseline_evidence(output / "phase4-step4-inherited") - with tempfile.TemporaryDirectory(prefix="rit-p4-step4-identities-") as temp: - baseline = Path(temp) - for name, raw in _phase4_step4_files().items(): - target = baseline / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(raw) - old, old_log = _collect(baseline) - current, current_log = _collect(ROOT) - expected = 3074 if os.environ.get("RIT_TEST_PARQUET") == "1" else 3071 - if len(old) != expected or set(old) - set(current): - raise ValueError("Accepted Phase 4 Step 4 test identities were lost") - result = {"baseline_commit": PHASE4_STEP4_FINAL, "baseline_test_tree": PHASE4_STEP4_TEST_TREE, - "baseline_nodeids": old, "current_nodeids": current, "missing_nodeids": [], - "baseline_tests": len(old), "current_tests": len(current), "inherited": inherited, - "nodeids_sha256": hashlib.sha256(("\n".join(old)+"\n").encode()).hexdigest()} - (output / "phase4_step5_test_identity_manifest.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4-step4-collection.log").write_text(old_log, encoding="utf-8") - (output / "phase4-step5-collection.log").write_text(current_log, encoding="utf-8") - print(json.dumps({k: result[k] for k in ("baseline_tests", "current_tests", "missing_nodeids")}, indent=2)) - return result - - -def phase4_step5_candidate(output: Path, step: int = 5) -> None: - """Build tested Step 5 intermediate evidence without declaring Phase 4 complete.""" - if os.environ.get("RIT_TEST_PARQUET") != "1" or importlib.util.find_spec("pyarrow") is None: - raise ValueError("Step 5 candidate evidence requires RIT_TEST_PARQUET=1 and real PyArrow") - output.mkdir(parents=True, exist_ok=True) - result = audit_phase4_step5(step) - result["diff"] = audit_phase4_step5_diff(step) - if git("status", "--porcelain").strip(): - raise ValueError("Step 5 candidate archive requires committed, clean source") - result["core"] = verify_junit(output / "core.xml", minimum=3071) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=3074) - result["core_math_measurements"] = verify_step10_evidence(output / "core.xml", output / "phase4-step5-core-observations.json") - result["parquet_math_measurements"] = verify_step10_evidence(output / "parquet.xml", output / "phase4-step5-parquet-observations.json") - identity = phase4_step5_baseline_evidence(output) - result["test_identity"] = {k: identity[k] for k in ("baseline_tests", "current_tests", "nodeids_sha256")} - expected = set() - for node in identity["current_nodeids"]: - base, bracket, parameter = node.partition("[") - owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) - for name, parquet in (("core.xml", False), ("parquet.xml", True)): - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(output / name).getroot().iter("testcase")} - target = expected if parquet else {c for c in expected if c[1] not in PARQUET_CASES} - if cases != target: - raise ValueError(f"Step 5 JUnit does not execute the entire current suite: {name}") - wheel, _ = verify_distributions(output / "dist") - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - archive = output / "recursive-integrity-toolkit-phase4-step5-candidate.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {p for p in git("ls-files", "-z").decode().split("\0") if p} - with zipfile.ZipFile(archive) as zipped: - entries = [item.filename for item in zipped.infolist() if not item.is_dir()] - prefix = "recursive-integrity-toolkit/" - if (len(entries) != len(set(entries)) or any(not name.startswith(prefix) for name in entries)): - raise ValueError("Step 5 source archive has duplicate or unprefixed entries") - names = {name.removeprefix(prefix) for name in entries} - if names != tracked or len(tracked) != 227 or len(entries) != 227: - raise ValueError("Step 5 source archive file set mismatch") - for name in names: - if zipped.read("recursive-integrity-toolkit/"+name) != (ROOT / name).read_bytes(): - raise ValueError(f"Step 5 source archive byte mismatch: {name}") - result.update({"commit": git("rev-parse", "HEAD").decode().strip(), - "tree": git("rev-parse", "HEAD^{tree}").decode().strip(), - "source_archive": archive.name, "tracked_files": len(tracked), - "python": sys.version, "platform": sys.platform, - "versions": {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "pytest")}}) - (output / "phase4_step5_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4_step5_repository_files.sha256").write_text("".join(f"{hashlib.sha256((ROOT/name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - paths = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase4_step5_artifacts.sha256") - (output / "phase4_step5_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in paths), encoding="utf-8") - print(f"Phase 4 Step 5 candidate: {len(tracked)} source files verified; Phase 4 remains incomplete.") - - -def phase4_step5_cli_main() -> int: - parser = argparse.ArgumentParser(description="Phase 4 Step 5 maintainer gate") - parser.add_argument("--phase", type=int, choices=(4,), required=True) - parser.add_argument("--step", type=int, choices=(5,), required=True) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) - parser.add_argument("--require-parquet", action="store_true") - parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - raise ValueError("Phase 4 Step 5 cannot certify a final phase delivery") - if args.junit is not None: - verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - phase4_step5_baseline_evidence(args.baseline_evidence) - elif args.dist is not None: - audit_phase4_step5(args.step) - wheel, _ = verify_distributions(args.dist) - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - elif args.smoke_wheel is not None: - smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - phase4_step5_candidate(args.candidate, args.step) - else: - audit_phase4_step5(args.step) - if args.diff: - audit_phase4_step5_diff(args.step) - return 0 - - -def phase4_step5_installed_renderers_smoke(wheel: Path) -> None: +def smoke_installed_renderers(wheel: Path) -> None: """Exercise the installed pure renderers against prepared protected evidence.""" with tempfile.TemporaryDirectory(prefix="rit-p4-step5-installed-renderers-") as temp: work = Path(temp) @@ -4306,7 +1013,7 @@ def find_spec(self, fullname, path=None, target=None): views.append((report, view, copy.deepcopy(view.to_dict()))) assert views[0][2]['run']['run_status'] == 'partial' assert views[0][2]['simulations']['closed_resampling']['status'] == 'experimental' -assert views[0][2]['capabilities']['lineage']['execution_status'] == 'deferred' +assert views[0][2]['capabilities']['lineage']['execution_status'] == 'not_requested' # After this point even an already imported owner cannot execute a function. def denied(*args, **kwargs): raise AssertionError('installed renderer attempted file, network or process access') @@ -4378,2363 +1085,7 @@ def deny_effects(event, arguments): subprocess.run([str(python), "-I", "-c", program, str(work), str(ROOT)], cwd=work, check=True) -# Phase 4 Step 6: safe local publication; no CLI or analytical expansion. - - -PHASE4_STEP5_FINAL = 'c1667179e895a798bba2349960162542efa5ef87' - - -PHASE4_STEP5_TREE = 'd83f9fc8e0b569f2d796bd2a83b87422fe350fba' - - -PHASE4_STEP5_TEST_TREE = '974f8e65904c24ed4e226996eaf571aab6123965' - - -PHASE4_STEP6_APPROVAL = '批准开始Phase 4 **Step 6**' - - -PHASE4_STEP6_APPROVAL_DATE = '2026-09-20' - - -PHASE4_STEP6_NEW = ('tests/unit/test_phase4_output_safety.py',) - - -PHASE4_STEP6_MIGRATIONS = {'tests/integration/test_phase4_gates.py': [{'new': 'def ' - 'test_phase4_step5_current_runtime_opens_only_renderers_and_freezes_schema(repo_root, ' - 'phase4_step4_snapshot, phase4_step5_snapshot):\n' - ' repo_root = phase4_step5_snapshot\n' - ' package = "src/recursive_integrity_toolkit"\n' - ' current = {path.relative_to(repo_root).as_posix(): ' - 'hashlib.sha256(path.read_bytes()).hexdigest()\n' - ' for path in (repo_root / ' - 'package).rglob("*.py")}\n' - ' frozen = ' - '{path.relative_to(phase4_step4_snapshot).as_posix(): ' - 'hashlib.sha256(path.read_bytes()).hexdigest()\n' - ' for path in (phase4_step4_snapshot / ' - 'package).rglob("*.py")}\n' - ' assert len(current) == len(frozen) == 40\n' - ' assert current.keys() == frozen.keys()\n' - ' opened = {package + "/" + relative for relative in ' - '(\n' - ' "reports/json_report.py", ' - '"reports/markdown_report.py",\n' - ' )}\n' - ' assert {path for path in current if current[path] ' - '!= frozen[path]} == opened\n' - ' assert {path: digest for path, digest in ' - 'current.items() if path not in opened} == {\n' - ' path: digest for path, digest in frozen.items() ' - 'if path not in opened\n' - ' }\n' - ' schemas = {path.name: path.read_bytes() for path in ' - '(repo_root / "schemas").glob("*.json")}\n' - ' assert len(schemas) == 5\n' - ' assert schemas == {path.name: path.read_bytes() for ' - 'path in (phase4_step4_snapshot / ' - '"schemas").glob("*.json")}', - 'node': 'test_phase4_step5_current_runtime_opens_only_renderers_and_freezes_schema', - 'old': 'def ' - 'test_phase4_step5_current_runtime_opens_only_renderers_and_freezes_schema(repo_root, ' - 'phase4_step4_snapshot):\n' - ' package = "src/recursive_integrity_toolkit"\n' - ' current = {path.relative_to(repo_root).as_posix(): ' - 'hashlib.sha256(path.read_bytes()).hexdigest()\n' - ' for path in (repo_root / ' - 'package).rglob("*.py")}\n' - ' frozen = ' - '{path.relative_to(phase4_step4_snapshot).as_posix(): ' - 'hashlib.sha256(path.read_bytes()).hexdigest()\n' - ' for path in (phase4_step4_snapshot / ' - 'package).rglob("*.py")}\n' - ' assert len(current) == len(frozen) == 40\n' - ' assert current.keys() == frozen.keys()\n' - ' opened = {package + "/" + relative for relative in ' - '(\n' - ' "reports/json_report.py", ' - '"reports/markdown_report.py",\n' - ' )}\n' - ' assert {path for path in current if current[path] ' - '!= frozen[path]} == opened\n' - ' assert {path: digest for path, digest in ' - 'current.items() if path not in opened} == {\n' - ' path: digest for path, digest in frozen.items() ' - 'if path not in opened\n' - ' }\n' - ' schemas = {path.name: path.read_bytes() for path in ' - '(repo_root / "schemas").glob("*.json")}\n' - ' assert len(schemas) == 5\n' - ' assert schemas == {path.name: path.read_bytes() for ' - 'path in (phase4_step4_snapshot / ' - '"schemas").glob("*.json")}'}, - {'new': 'def ' - 'test_phase4_step5_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step5_snapshot):\n' - ' repo_root = phase4_step5_snapshot\n' - ' names = {"ci.yml", "golden.yml", "security.yml", ' - '"release.yml"}\n' - ' root = repo_root / ".github/workflows"\n' - ' expected_timeouts = {"ci.yml": [60, 20], ' - '"golden.yml": [15], "security.yml": [20], ' - '"release.yml": [25]}\n' - ' assert {path.name for path in root.glob("*.yml")} ' - '== names\n' - ' for name in sorted(names):\n' - ' text = (root / ' - 'name).read_text(encoding="utf-8")\n' - ' assert [int(line.split(":", 1)[1]) for line in ' - 'text.splitlines()\n' - ' if ' - 'line.strip().startswith("timeout-minutes:")] == ' - 'expected_timeouts[name]\n' - ' assert "Phase 4 Step 5" in text, name\n' - ' assert "--phase 4 --step 5" in text, name\n' - ' for older in ("--phase 4 --step 4", "--phase 4 ' - '--step 3", "--phase 4 --step 2", "--phase 4 --step 1", ' - '"--phase 3 --step 11"):\n' - ' assert older not in text, (name, older)\n' - ' assert "permissions:\\n contents: read" in ' - 'text\n' - ' assert "persist-credentials: false" in text\n' - ' assert "timeout-minutes:" in text and "set -euo ' - 'pipefail" in text\n' - ' assert "actions/upload-artifact@v4" in text and ' - '"if-no-files-found: error" in text\n' - ' for line in text.splitlines():\n' - ' if "python scripts/release_check.py" in ' - 'line:\n' - ' assert "--phase 4 --step 5" in line, ' - '(name, line)\n' - ' for forbidden in ("continue-on-error:", "|| ' - 'true", "contents: write", "id-token: write", "twine ' - 'upload", "git push"):\n' - ' assert forbidden not in text, (name, ' - 'forbidden)\n' - ' ci = (root / "ci.yml").read_text(encoding="utf-8")\n' - " for required in ('os: [ubuntu-latest, " - 'windows-latest]\', \'python-version: ["3.11", ' - '"3.12"]\',\n' - " 'dependencies: [current, " - 'minimum]\', \'"numpy==2.0.0" "pandas==2.2.2"\',\n' - ' \'RIT_TEST_PARQUET: "0"\', ' - '\'RIT_TEST_PARQUET: "1"\', \'--require-parquet\',\n' - " '--baseline-evidence', 'python -m " - "pip check',\n" - " '--minimum-tests 3071', " - "'--minimum-tests 3074',\n" - ' "find_spec(\'pyarrow\') is None", ' - "'import pyarrow'):\n" - ' assert required in ci\n' - ' assert ci.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2\n' - ' assert " -k " not in ci\n' - ' golden = (root / ' - '"golden.yml").read_text(encoding="utf-8")\n' - ' for required in ' - '("tests/golden/test_phase3_math.py", ' - '"tests/integration/test_hero_structure.py",\n' - ' ' - '"tests/integration/test_phase3_metric_pipeline.py"):\n' - ' assert required in golden\n' - ' security = (root / ' - '"security.yml").read_text(encoding="utf-8")\n' - ' for required in ' - "('tests/integration/test_no_network.py', " - "'tests/integration/test_optional_dependency.py',\n" - ' ' - "'tests/unit/test_PR012_evidence_classes.py', " - "'tests/unit/test_PR013_report_schema.py',\n" - ' ' - "'tests/unit/test_PR014_unavailable.py', " - "'tests/unit/test_PR015_redaction.py',\n" - ' ' - "'tests/unit/test_PR016_determinism.py', " - "'tests/unit/test_PR018_language.py',\n" - ' ' - "'tests/integration/test_partial_provenance_report.py',\n" - ' ' - "'tests/integration/test_partial_lineage_report.py',\n" - ' ' - "'tests/unit/test_phase4_contracts.py', " - "'tests/integration/test_phase4_gates.py'):\n" - ' assert required in security\n' - ' release = (root / ' - '"release.yml").read_text(encoding="utf-8")\n' - ' assert "python -m build" in release and "python -m ' - 'twine check --strict" in release\n' - ' assert "--dist" in release and "--candidate" in ' - 'release\n' - ' assert "--delivery" not in release\n' - ' assert ' - '"recursive-integrity-toolkit-phase4-step5-candidate" in ' - 'release\n' - ' assert "rit-phase4-step4" not in release\n' - ' assert "--minimum-tests 3071" in release and ' - '"--minimum-tests 3074" in release\n' - ' assert release.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2', - 'node': 'test_phase4_step5_current_workflows_preserve_matrix_and_use_active_dispatch', - 'old': 'def ' - 'test_phase4_step5_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root):\n' - ' names = {"ci.yml", "golden.yml", "security.yml", ' - '"release.yml"}\n' - ' root = repo_root / ".github/workflows"\n' - ' expected_timeouts = {"ci.yml": [60, 20], ' - '"golden.yml": [15], "security.yml": [20], ' - '"release.yml": [25]}\n' - ' assert {path.name for path in root.glob("*.yml")} ' - '== names\n' - ' for name in sorted(names):\n' - ' text = (root / ' - 'name).read_text(encoding="utf-8")\n' - ' assert [int(line.split(":", 1)[1]) for line in ' - 'text.splitlines()\n' - ' if ' - 'line.strip().startswith("timeout-minutes:")] == ' - 'expected_timeouts[name]\n' - ' assert "Phase 4 Step 5" in text, name\n' - ' assert "--phase 4 --step 5" in text, name\n' - ' for older in ("--phase 4 --step 4", "--phase 4 ' - '--step 3", "--phase 4 --step 2", "--phase 4 --step 1", ' - '"--phase 3 --step 11"):\n' - ' assert older not in text, (name, older)\n' - ' assert "permissions:\\n contents: read" in ' - 'text\n' - ' assert "persist-credentials: false" in text\n' - ' assert "timeout-minutes:" in text and "set -euo ' - 'pipefail" in text\n' - ' assert "actions/upload-artifact@v4" in text and ' - '"if-no-files-found: error" in text\n' - ' for line in text.splitlines():\n' - ' if "python scripts/release_check.py" in ' - 'line:\n' - ' assert "--phase 4 --step 5" in line, ' - '(name, line)\n' - ' for forbidden in ("continue-on-error:", "|| ' - 'true", "contents: write", "id-token: write", "twine ' - 'upload", "git push"):\n' - ' assert forbidden not in text, (name, ' - 'forbidden)\n' - ' ci = (root / "ci.yml").read_text(encoding="utf-8")\n' - " for required in ('os: [ubuntu-latest, " - 'windows-latest]\', \'python-version: ["3.11", ' - '"3.12"]\',\n' - " 'dependencies: [current, " - 'minimum]\', \'"numpy==2.0.0" "pandas==2.2.2"\',\n' - ' \'RIT_TEST_PARQUET: "0"\', ' - '\'RIT_TEST_PARQUET: "1"\', \'--require-parquet\',\n' - " '--baseline-evidence', 'python -m " - "pip check',\n" - " '--minimum-tests 3071', " - "'--minimum-tests 3074',\n" - ' "find_spec(\'pyarrow\') is None", ' - "'import pyarrow'):\n" - ' assert required in ci\n' - ' assert ci.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2\n' - ' assert " -k " not in ci\n' - ' golden = (root / ' - '"golden.yml").read_text(encoding="utf-8")\n' - ' for required in ' - '("tests/golden/test_phase3_math.py", ' - '"tests/integration/test_hero_structure.py",\n' - ' ' - '"tests/integration/test_phase3_metric_pipeline.py"):\n' - ' assert required in golden\n' - ' security = (root / ' - '"security.yml").read_text(encoding="utf-8")\n' - ' for required in ' - "('tests/integration/test_no_network.py', " - "'tests/integration/test_optional_dependency.py',\n" - ' ' - "'tests/unit/test_PR012_evidence_classes.py', " - "'tests/unit/test_PR013_report_schema.py',\n" - ' ' - "'tests/unit/test_PR014_unavailable.py', " - "'tests/unit/test_PR015_redaction.py',\n" - ' ' - "'tests/unit/test_PR016_determinism.py', " - "'tests/unit/test_PR018_language.py',\n" - ' ' - "'tests/integration/test_partial_provenance_report.py',\n" - ' ' - "'tests/integration/test_partial_lineage_report.py',\n" - ' ' - "'tests/unit/test_phase4_contracts.py', " - "'tests/integration/test_phase4_gates.py'):\n" - ' assert required in security\n' - ' release = (root / ' - '"release.yml").read_text(encoding="utf-8")\n' - ' assert "python -m build" in release and "python -m ' - 'twine check --strict" in release\n' - ' assert "--dist" in release and "--candidate" in ' - 'release\n' - ' assert "--delivery" not in release\n' - ' assert ' - '"recursive-integrity-toolkit-phase4-step5-candidate" in ' - 'release\n' - ' assert "rit-phase4-step4" not in release\n' - ' assert "--minimum-tests 3071" in release and ' - '"--minimum-tests 3074" in release\n' - ' assert release.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2'}, - {'new': 'def ' - 'test_phase4_step5_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step5_snapshot):\n' - ' import json\n' - '\n' - ' paths = ' - '{path.relative_to(phase4_step5_snapshot).as_posix()\n' - ' for path in ' - 'phase4_step5_snapshot.rglob("*") if path.is_file()}\n' - ' assert len(paths) == 227\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step5_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step5_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 38\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["privacy_views_enabled"] is True\n' - ' assert baseline["renderers_enabled"] is True\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 4\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n repo_root = ' - 'phase4_step4_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n"),\n' - ' "historical_document": ' - '("docs/report_schema.md", b"# Report Schema\\n",\n' - ' b"# Unauthorized ' - 'historical document\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "html": ' - '"src/recursive_integrity_toolkit/reports/html_report.py",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "config": ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' "hashing": ' - '"src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' "logging": ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 5 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 6\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)', - 'node': 'test_phase4_step5_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step5_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation):\n' - ' import json\n' - '\n' - ' tracked = subprocess.check_output(["git", "-C", ' - 'str(repo_root), "ls-files", "-z"])\n' - ' paths = set(tracked.decode().split("\\0")) - {""}\n' - ' assert len(paths) == 227\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(repo_root / relative, ' - 'destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step5_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 38\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["privacy_views_enabled"] is True\n' - ' assert baseline["renderers_enabled"] is True\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 4\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n repo_root = ' - 'phase4_step4_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n"),\n' - ' "historical_document": ' - '("docs/report_schema.md", b"# Report Schema\\n",\n' - ' b"# Unauthorized ' - 'historical document\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "html": ' - '"src/recursive_integrity_toolkit/reports/html_report.py",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "config": ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' "hashing": ' - '"src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' "logging": ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 5 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 6\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)'}, - {'new': 'def phase4_mutation_tree(repo_root, tmp_path_factory, ' - 'phase4_gate_tools, phase4_step1_snapshot):\n' - ' """One source copy; each individual mutation is ' - 'restored in a finally block."""\n' - ' root = ' - 'tmp_path_factory.mktemp("phase4-active-mutations")\n' - ' paths = ' - '{path.relative_to(phase4_step1_snapshot).as_posix()\n' - ' for path in ' - 'phase4_step1_snapshot.rglob("*") if path.is_file()}\n' - ' ' - 'paths.update(phase4_gate_tools["PHASE4_STEP1_NEW"])\n' - ' for relative in sorted(paths):\n' - ' destination = root / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step1_snapshot / ' - 'relative, destination)\n' - ' return root', - 'node': 'phase4_mutation_tree', - 'old': 'def phase4_mutation_tree(repo_root, tmp_path_factory, ' - 'phase4_gate_tools, phase4_step1_snapshot):\n' - ' """One source copy; each individual mutation is ' - 'restored in a finally block."""\n' - ' root = ' - 'tmp_path_factory.mktemp("phase4-active-mutations")\n' - ' tracked = subprocess.check_output(["git", "-C", ' - 'str(repo_root), "ls-files", "-z"])\n' - ' paths = set(tracked.decode().split("\\0")) - {""}\n' - ' ' - 'paths.update(phase4_gate_tools["PHASE4_STEP1_NEW"])\n' - ' for relative in sorted(paths):\n' - ' destination = root / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step1_snapshot / ' - 'relative, destination)\n' - ' return root'}, - {'new': 'def ' - 'test_phase4_step2_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step2_snapshot):\n' - ' import json\n' - '\n' - ' paths = ' - '{path.relative_to(phase4_step2_snapshot).as_posix()\n' - ' for path in ' - 'phase4_step2_snapshot.rglob("*") if path.is_file()}\n' - ' assert len(paths) == 227\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step2_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step2_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["frozen_runtime_modules"] == 39\n' - ' assert baseline["frozen_schemas"] == 4\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is False\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'phase4_control["active_step"] == 1\\n\', b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_control(control: dict, step: int = 1) -> ' - 'None:",\n' - ' b"def ' - 'verify_phase4_control(control: dict, step: int = 1) -> ' - 'None:\\n return"),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/config.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md",\n' - ' "premature_report": "report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 2 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 3\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/cli.py")\n' - ' else:\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/config.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' elif mutation.startswith("schema_"):\n' - ' path = tmp_path / "schemas/report.schema.json"\n' - ' schema = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "schema_open_root":\n' - ' schema["additionalProperties"] = True\n' - ' else:\n' - ' schema["$schema"] = ' - '"http://json-schema.org/draft-07/schema#"\n' - ' path.write_text(json.dumps(schema), ' - 'encoding="utf-8")\n' - ' elif mutation == "model_computation_import":\n' - ' path = tmp_path / ' - '"src/recursive_integrity_toolkit/result.py"\n' - ' path.write_bytes(path.read_bytes() + b"\\nfrom ' - '.metrics.diversity import ' - 'effective_state_diversity\\n")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)', - 'node': 'test_phase4_step2_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step2_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step2_snapshot):\n' - ' import json\n' - '\n' - ' tracked = subprocess.check_output(["git", "-C", ' - 'str(repo_root), "ls-files", "-z"])\n' - ' paths = set(tracked.decode().split("\\0")) - {""}\n' - ' assert len(paths) == 227\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step2_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step2_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["frozen_runtime_modules"] == 39\n' - ' assert baseline["frozen_schemas"] == 4\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is False\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'phase4_control["active_step"] == 1\\n\', b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_control(control: dict, step: int = 1) -> ' - 'None:",\n' - ' b"def ' - 'verify_phase4_control(control: dict, step: int = 1) -> ' - 'None:\\n return"),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/config.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md",\n' - ' "premature_report": "report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 2 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 3\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/cli.py")\n' - ' else:\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/config.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' elif mutation.startswith("schema_"):\n' - ' path = tmp_path / "schemas/report.schema.json"\n' - ' schema = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "schema_open_root":\n' - ' schema["additionalProperties"] = True\n' - ' else:\n' - ' schema["$schema"] = ' - '"http://json-schema.org/draft-07/schema#"\n' - ' path.write_text(json.dumps(schema), ' - 'encoding="utf-8")\n' - ' elif mutation == "model_computation_import":\n' - ' path = tmp_path / ' - '"src/recursive_integrity_toolkit/result.py"\n' - ' path.write_bytes(path.read_bytes() + b"\\nfrom ' - '.metrics.diversity import ' - 'effective_state_diversity\\n")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)'}, - {'new': 'def ' - 'test_phase4_step3_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step3_snapshot):\n' - ' import json\n' - '\n' - ' paths = ' - '{path.relative_to(phase4_step3_snapshot).as_posix()\n' - ' for path in ' - 'phase4_step3_snapshot.rglob("*") if path.is_file()}\n' - ' assert len(paths) == 227\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step3_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step3_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 39\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 2\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step2_control(control: dict, step: int = ' - '2) -> None:",\n' - ' b"def ' - 'verify_phase4_step2_control(control: dict, step: int = ' - '2) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step2_snapshot):\\n repo_root = ' - 'phase4_step2_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step2_snapshot):\\n"),\n' - ' "historical_document": ' - '("docs/report_schema.md", b"# Report Schema\\n", b"# ' - 'Modified Report Schema\\n"),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "frozen_schema": "schemas/config.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "renderer": ' - '"src/recursive_integrity_toolkit/reports/json_report.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md",\n' - ' "premature_report": "report.json",\n' - ' }\n' - ' assembly_injections = {\n' - ' "assembly_metric_import": b"\\nfrom ' - '..metrics.diversity import ' - 'effective_state_diversity\\n",\n' - ' "assembly_calculation_import": b"\\nfrom ' - '..metrics import coverage\\n",\n' - ' "assembly_numeric_import": b"\\nimport ' - 'numpy\\n",\n' - ' "assembly_network_import": b"\\nimport ' - 'socket\\n",\n' - ' "assembly_file_io": ' - 'b"\\nopen(\'unapproved.txt\', ' - '\'w\').write(\'data\')\\n",\n' - ' "assembly_dynamic_execution": ' - 'b"\\nexec(\'forged = True\')\\n",\n' - ' "assembly_new_formula": b"\\ndef ' - 'invented_metric(values):\\n return sum(value * value ' - 'for value in values)\\n",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 3 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 4\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/result.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' elif mutation == "schema_open_root":\n' - ' path = tmp_path / "schemas/report.schema.json"\n' - ' schema = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' schema["additionalProperties"] = True\n' - ' path.write_text(json.dumps(schema), ' - 'encoding="utf-8")\n' - ' elif mutation in assembly_injections:\n' - ' path = tmp_path / ' - '"src/recursive_integrity_toolkit/reports/assembly.py"\n' - ' path.write_bytes(path.read_bytes() + ' - 'assembly_injections[mutation])\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)', - 'node': 'test_phase4_step3_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step3_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step3_snapshot):\n' - ' import json\n' - '\n' - ' tracked = subprocess.check_output(["git", "-C", ' - 'str(repo_root), "ls-files", "-z"])\n' - ' paths = set(tracked.decode().split("\\0")) - {""}\n' - ' assert len(paths) == 227\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step3_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step3_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 39\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 2\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step2_control(control: dict, step: int = ' - '2) -> None:",\n' - ' b"def ' - 'verify_phase4_step2_control(control: dict, step: int = ' - '2) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step2_snapshot):\\n repo_root = ' - 'phase4_step2_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step2_snapshot):\\n"),\n' - ' "historical_document": ' - '("docs/report_schema.md", b"# Report Schema\\n", b"# ' - 'Modified Report Schema\\n"),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "frozen_schema": "schemas/config.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "renderer": ' - '"src/recursive_integrity_toolkit/reports/json_report.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md",\n' - ' "premature_report": "report.json",\n' - ' }\n' - ' assembly_injections = {\n' - ' "assembly_metric_import": b"\\nfrom ' - '..metrics.diversity import ' - 'effective_state_diversity\\n",\n' - ' "assembly_calculation_import": b"\\nfrom ' - '..metrics import coverage\\n",\n' - ' "assembly_numeric_import": b"\\nimport ' - 'numpy\\n",\n' - ' "assembly_network_import": b"\\nimport ' - 'socket\\n",\n' - ' "assembly_file_io": ' - 'b"\\nopen(\'unapproved.txt\', ' - '\'w\').write(\'data\')\\n",\n' - ' "assembly_dynamic_execution": ' - 'b"\\nexec(\'forged = True\')\\n",\n' - ' "assembly_new_formula": b"\\ndef ' - 'invented_metric(values):\\n return sum(value * value ' - 'for value in values)\\n",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 3 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 4\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/result.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' elif mutation == "schema_open_root":\n' - ' path = tmp_path / "schemas/report.schema.json"\n' - ' schema = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' schema["additionalProperties"] = True\n' - ' path.write_text(json.dumps(schema), ' - 'encoding="utf-8")\n' - ' elif mutation in assembly_injections:\n' - ' path = tmp_path / ' - '"src/recursive_integrity_toolkit/reports/assembly.py"\n' - ' path.write_bytes(path.read_bytes() + ' - 'assembly_injections[mutation])\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)'}, - {'new': 'def ' - 'test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step4_snapshot):\n' - ' import json\n' - '\n' - ' paths = ' - '{path.relative_to(phase4_step4_snapshot).as_posix()\n' - ' for path in ' - 'phase4_step4_snapshot.rglob("*") if path.is_file()}\n' - ' assert len(paths) == 227\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step4_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step4_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 35\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["privacy_views_enabled"] is True\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 3\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step3_control(control: dict, step: int = ' - '3) -> None:",\n' - ' b"def ' - 'verify_phase4_step3_control(control: dict, step: int = ' - '3) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step3_snapshot):\\n repo_root = ' - 'phase4_step3_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step3_snapshot):\\n"),\n' - ' "historical_document": ("docs/privacy.md", b"# ' - 'Privacy and Local Input Boundaries\\n",\n' - ' b"# Unauthorized ' - 'Privacy and Local Input Boundaries\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "renderer": ' - '"src/recursive_integrity_toolkit/reports/json_report.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 4 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 5\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/json_report.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)', - 'node': 'test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step4_snapshot):\n' - ' import json\n' - '\n' - ' tracked = subprocess.check_output(["git", "-C", ' - 'str(repo_root), "ls-files", "-z"])\n' - ' paths = set(tracked.decode().split("\\0")) - {""}\n' - ' assert len(paths) == 227\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step4_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step4_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 35\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["privacy_views_enabled"] is True\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 3\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step3_control(control: dict, step: int = ' - '3) -> None:",\n' - ' b"def ' - 'verify_phase4_step3_control(control: dict, step: int = ' - '3) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step3_snapshot):\\n repo_root = ' - 'phase4_step3_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step3_snapshot):\\n"),\n' - ' "historical_document": ("docs/privacy.md", b"# ' - 'Privacy and Local Input Boundaries\\n",\n' - ' b"# Unauthorized ' - 'Privacy and Local Input Boundaries\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "renderer": ' - '"src/recursive_integrity_toolkit/reports/json_report.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 4 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 5\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/json_report.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)'}, - {'new': 'def ' - 'test_phase4_step4_privacy_ast_checks_current_source_before_rejecting_scope_injection(repo_root, ' - 'tmp_path, relative, injection, phase4_step5_snapshot):\n' - ' checker = runpy.run_path(str(repo_root / ' - '"scripts/check_traceability.py"), ' - 'run_name="phase4_step4_ast_tests")\n' - ' verify = checker["phase4_step4_runtime_boundary"]\n' - ' repository_path = ' - '"src/recursive_integrity_toolkit/" + relative\n' - ' current = (phase4_step5_snapshot if relative == ' - '"utils/logging.py" else repo_root) / repository_path\n' - ' verify(current, repository_path)\n' - ' copy = tmp_path / "source.py"\n' - ' copy.write_bytes(current.read_bytes() + injection)\n' - ' with pytest.raises(ValueError):\n' - ' verify(copy, repository_path)', - 'node': 'test_phase4_step4_privacy_ast_checks_current_source_before_rejecting_scope_injection', - 'old': 'def ' - 'test_phase4_step4_privacy_ast_checks_current_source_before_rejecting_scope_injection(repo_root, ' - 'tmp_path, relative, injection):\n' - ' checker = runpy.run_path(str(repo_root / ' - '"scripts/check_traceability.py"), ' - 'run_name="phase4_step4_ast_tests")\n' - ' verify = checker["phase4_step4_runtime_boundary"]\n' - ' repository_path = ' - '"src/recursive_integrity_toolkit/" + relative\n' - ' current = repo_root / repository_path\n' - ' verify(current, repository_path)\n' - ' copy = tmp_path / "source.py"\n' - ' copy.write_bytes(current.read_bytes() + injection)\n' - ' with pytest.raises(ValueError):\n' - ' verify(copy, repository_path)'}], - 'tests/unit/test_phase4_contracts.py': [{'new': 'def ' - 'test_phase4_step5_approved_control_keeps_independent_step4_anchors(phase4_tools, ' - 'phase4_step5_snapshot):\n' - ' control = json.loads((phase4_step5_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' assert phase4_tools["PHASE4_STEP4_FINAL"] == ' - '"49454a9b162cb8d35e38d1cb1ae32cb208d3a01c"\n' - ' assert phase4_tools["PHASE4_STEP4_TREE"] == ' - '"79264d845e08f73ee68e160dea04f7786f18fd0a"\n' - ' assert phase4_tools["PHASE4_STEP4_TEST_TREE"] == ' - '"4aa4ca0b22f8b8c25afe27feeaba713d6f0830bf"\n' - ' assert control["active_phase"] == 4 and ' - 'control["active_step"] == 5\n' - ' assert control["approval_basis"] == "很好，Phase 4 Step 5 ' - '继续"\n' - ' assert control["baseline_commit"] == BASELINE\n' - ' assert control["previous_step_commit"] == ' - '"49454a9b162cb8d35e38d1cb1ae32cb208d3a01c"\n' - ' assert control["previous_step_tree"] == ' - '"79264d845e08f73ee68e160dea04f7786f18fd0a"\n' - ' assert control["previous_step_test_tree"] == ' - '"4aa4ca0b22f8b8c25afe27feeaba713d6f0830bf"\n' - ' assert control["previous_step_core_tests"] == 3071\n' - ' assert control["previous_step_parquet_tests"] == 3074\n' - ' assert len(control["previous_step_files_sha256"]) == ' - '227\n' - ' assert set(control["runtime_paths_authorized"]) == {\n' - ' ' - '"src/recursive_integrity_toolkit/reports/json_report.py",\n' - ' ' - '"src/recursive_integrity_toolkit/reports/markdown_report.py",\n' - ' }\n' - ' assert control["schema_changes_authorized"] is False\n' - ' assert control["schema_paths_authorized"] == []\n' - ' assert control["new_files_permitted"] == []\n' - ' for name in ("phase_complete", "next_step_authorized", ' - '"main_merge_authorized", "publication_authorized"):\n' - ' assert control[name] is False\n' - ' phase4_tools["verify_phase4_step5_control"](control, ' - 'step=5)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step4_control"](control, ' - 'step=4)', - 'node': 'test_phase4_step5_approved_control_keeps_independent_step4_anchors', - 'old': 'def ' - 'test_phase4_step5_approved_control_keeps_independent_step4_anchors(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' assert phase4_tools["PHASE4_STEP4_FINAL"] == ' - '"49454a9b162cb8d35e38d1cb1ae32cb208d3a01c"\n' - ' assert phase4_tools["PHASE4_STEP4_TREE"] == ' - '"79264d845e08f73ee68e160dea04f7786f18fd0a"\n' - ' assert phase4_tools["PHASE4_STEP4_TEST_TREE"] == ' - '"4aa4ca0b22f8b8c25afe27feeaba713d6f0830bf"\n' - ' assert control["active_phase"] == 4 and ' - 'control["active_step"] == 5\n' - ' assert control["approval_basis"] == "很好，Phase 4 Step 5 ' - '继续"\n' - ' assert control["baseline_commit"] == BASELINE\n' - ' assert control["previous_step_commit"] == ' - '"49454a9b162cb8d35e38d1cb1ae32cb208d3a01c"\n' - ' assert control["previous_step_tree"] == ' - '"79264d845e08f73ee68e160dea04f7786f18fd0a"\n' - ' assert control["previous_step_test_tree"] == ' - '"4aa4ca0b22f8b8c25afe27feeaba713d6f0830bf"\n' - ' assert control["previous_step_core_tests"] == 3071\n' - ' assert control["previous_step_parquet_tests"] == 3074\n' - ' assert len(control["previous_step_files_sha256"]) == ' - '227\n' - ' assert set(control["runtime_paths_authorized"]) == {\n' - ' ' - '"src/recursive_integrity_toolkit/reports/json_report.py",\n' - ' ' - '"src/recursive_integrity_toolkit/reports/markdown_report.py",\n' - ' }\n' - ' assert control["schema_changes_authorized"] is False\n' - ' assert control["schema_paths_authorized"] == []\n' - ' assert control["new_files_permitted"] == []\n' - ' for name in ("phase_complete", "next_step_authorized", ' - '"main_merge_authorized", "publication_authorized"):\n' - ' assert control[name] is False\n' - ' phase4_tools["verify_phase4_step5_control"](control, ' - 'step=5)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step4_control"](control, ' - 'step=4)'}, - {'new': 'def ' - 'test_phase4_step5_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value, phase4_step5_snapshot):\n' - ' control = json.loads((phase4_step5_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' verify = phase4_tools["verify_phase4_step5_control"]\n' - ' verify(control, step=5)\n' - ' control[field] = value\n' - ' with pytest.raises(ValueError):\n' - ' verify(control, step=5)', - 'node': 'test_phase4_step5_control_rejects_forged_scope_and_stage', - 'old': 'def ' - 'test_phase4_step5_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' verify = phase4_tools["verify_phase4_step5_control"]\n' - ' verify(control, step=5)\n' - ' control[field] = value\n' - ' with pytest.raises(ValueError):\n' - ' verify(control, step=5)'}, - {'new': 'def ' - 'test_phase4_step5_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step, phase4_step5_snapshot):\n' - ' control = json.loads((phase4_step5_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step5_control"](control, ' - 'step=5)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step5_control"](control, ' - 'step=step)', - 'node': 'test_phase4_step5_control_rejects_unapproved_dispatch', - 'old': 'def ' - 'test_phase4_step5_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step5_control"](control, ' - 'step=5)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step5_control"](control, ' - 'step=step)'}, - {'new': 'def ' - 'test_phase4_step5_control_requires_explicit_approval_fields(phase4_tools, ' - 'field, phase4_step5_snapshot):\n' - ' control = json.loads((phase4_step5_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step5_control"](control)\n' - ' del control[field]\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step5_control"](control)', - 'node': 'test_phase4_step5_control_requires_explicit_approval_fields', - 'old': 'def ' - 'test_phase4_step5_control_requires_explicit_approval_fields(phase4_tools, ' - 'field):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step5_control"](control)\n' - ' del control[field]\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step5_control"](control)'}, - {'new': 'def ' - 'test_phase4_step5_control_cannot_mint_extra_permission(phase4_tools, ' - 'phase4_step5_snapshot):\n' - ' control = json.loads((phase4_step5_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step5_control"](control)\n' - ' control["approved_scope_expansion"] = {"step": 6, ' - '"publication": True}\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step5_control"](control)', - 'node': 'test_phase4_step5_control_cannot_mint_extra_permission', - 'old': 'def ' - 'test_phase4_step5_control_cannot_mint_extra_permission(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step5_control"](control)\n' - ' control["approved_scope_expansion"] = {"step": 6, ' - '"publication": True}\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step5_control"](control)'}, - {'new': 'def ' - 'test_phase4_step5_historical_migrations_preserve_ten_step4_gate_nodes(phase4_tools, ' - 'phase4_step4_snapshot, phase4_step5_snapshot):\n' - ' registry = phase4_tools["PHASE4_STEP5_MIGRATIONS"]\n' - ' expected = {\n' - ' "tests/unit/test_phase4_contracts.py": {\n' - ' ' - '"test_phase4_step4_approved_control_keeps_independent_step3_anchors",\n' - ' ' - '"test_phase4_step4_control_rejects_forged_scope_and_stage",\n' - ' ' - '"test_phase4_step4_control_rejects_unapproved_dispatch",\n' - ' ' - '"test_phase4_step4_control_requires_explicit_approval_fields",\n' - ' ' - '"test_phase4_step4_control_cannot_mint_extra_permission",\n' - ' ' - '"test_phase4_step4_historical_migrations_preserve_ten_step3_gate_nodes",\n' - ' ' - '"test_phase4_step4_historical_guard_rejects_assertion_and_binding_weakening",\n' - ' },\n' - ' "tests/integration/test_phase4_gates.py": {\n' - ' ' - '"test_phase4_step4_current_runtime_opens_only_privacy_metadata_modules_and_freezes_schema",\n' - ' ' - '"test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch",\n' - ' ' - '"test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations",\n' - ' },\n' - ' }\n' - ' assert set(registry) == set(expected)\n' - ' assert sum(len(rows) for rows in registry.values()) == ' - '13\n' - ' for path, nodes in expected.items():\n' - ' assert {row["node"] for row in registry[path]} == ' - 'nodes\n' - ' ' - 'phase4_tools["verify_phase4_step5_test_migration"](\n' - ' path, (phase4_step4_snapshot / ' - 'path).read_bytes(), (phase4_step5_snapshot / ' - 'path).read_bytes(),\n' - ' )', - 'node': 'test_phase4_step5_historical_migrations_preserve_ten_step4_gate_nodes', - 'old': 'def ' - 'test_phase4_step5_historical_migrations_preserve_ten_step4_gate_nodes(phase4_tools, ' - 'phase4_step4_snapshot):\n' - ' registry = phase4_tools["PHASE4_STEP5_MIGRATIONS"]\n' - ' expected = {\n' - ' "tests/unit/test_phase4_contracts.py": {\n' - ' ' - '"test_phase4_step4_approved_control_keeps_independent_step3_anchors",\n' - ' ' - '"test_phase4_step4_control_rejects_forged_scope_and_stage",\n' - ' ' - '"test_phase4_step4_control_rejects_unapproved_dispatch",\n' - ' ' - '"test_phase4_step4_control_requires_explicit_approval_fields",\n' - ' ' - '"test_phase4_step4_control_cannot_mint_extra_permission",\n' - ' ' - '"test_phase4_step4_historical_migrations_preserve_ten_step3_gate_nodes",\n' - ' ' - '"test_phase4_step4_historical_guard_rejects_assertion_and_binding_weakening",\n' - ' },\n' - ' "tests/integration/test_phase4_gates.py": {\n' - ' ' - '"test_phase4_step4_current_runtime_opens_only_privacy_metadata_modules_and_freezes_schema",\n' - ' ' - '"test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch",\n' - ' ' - '"test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations",\n' - ' },\n' - ' }\n' - ' assert set(registry) == set(expected)\n' - ' assert sum(len(rows) for rows in registry.values()) == ' - '13\n' - ' for path, nodes in expected.items():\n' - ' assert {row["node"] for row in registry[path]} == ' - 'nodes\n' - ' ' - 'phase4_tools["verify_phase4_step5_test_migration"](\n' - ' path, (phase4_step4_snapshot / ' - 'path).read_bytes(), (ROOT / path).read_bytes(),\n' - ' )'}, - {'new': 'def ' - 'test_phase4_step5_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, ' - 'phase4_step4_snapshot, mutation, phase4_step5_snapshot):\n' - ' path = "tests/unit/test_phase4_contracts.py"\n' - ' before = (phase4_step4_snapshot / path).read_bytes()\n' - ' after = (phase4_step5_snapshot / path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step5_test_migration"]\n' - ' verify(path, before, after)\n' - ' if mutation == "old_assertion":\n' - ' original = b\' assert control["active_phase"] ' - '== 4 and control["active_step"] == 4\\n\'\n' - ' assert after.count(original) == 1\n' - ' after = after.replace(original, b" assert ' - 'True\\n", 1)\n' - ' elif mutation == "wrong_snapshot":\n' - " original = (b'def " - 'test_phase4_step4_control_cannot_mint_extra_permission(phase4_tools, ' - "phase4_step4_snapshot):\\n'\n" - " b' control = " - 'json.loads((phase4_step4_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\\n\')\n' - ' assert after.count(original) == 1\n' - ' after = after.replace(original, ' - "original.replace(b'(phase4_step4_snapshot /', b'(ROOT /'), " - '1)\n' - ' elif mutation == "rebind_node":\n' - ' after += b"\\ndef ' - 'test_phase4_step4_fixed_boundary_opens_only_approved_existing_paths():\\n ' - 'assert True\\n"\n' - ' else:\n' - ' after += b"\\ndef ' - 'test_phase4_step5_mutator(value=globals().clear()):\\n ' - 'pass\\n"\n' - ' with pytest.raises(ValueError):\n' - ' verify(path, before, after)', - 'node': 'test_phase4_step5_historical_guard_rejects_assertion_and_binding_weakening', - 'old': 'def ' - 'test_phase4_step5_historical_guard_rejects_assertion_and_binding_weakening(phase4_tools, ' - 'phase4_step4_snapshot, mutation):\n' - ' path = "tests/unit/test_phase4_contracts.py"\n' - ' before = (phase4_step4_snapshot / path).read_bytes()\n' - ' after = (ROOT / path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step5_test_migration"]\n' - ' verify(path, before, after)\n' - ' if mutation == "old_assertion":\n' - ' original = b\' assert control["active_phase"] ' - '== 4 and control["active_step"] == 4\\n\'\n' - ' assert after.count(original) == 1\n' - ' after = after.replace(original, b" assert ' - 'True\\n", 1)\n' - ' elif mutation == "wrong_snapshot":\n' - " original = (b'def " - 'test_phase4_step4_control_cannot_mint_extra_permission(phase4_tools, ' - "phase4_step4_snapshot):\\n'\n" - " b' control = " - 'json.loads((phase4_step4_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\\n\')\n' - ' assert after.count(original) == 1\n' - ' after = after.replace(original, ' - "original.replace(b'(phase4_step4_snapshot /', b'(ROOT /'), " - '1)\n' - ' elif mutation == "rebind_node":\n' - ' after += b"\\ndef ' - 'test_phase4_step4_fixed_boundary_opens_only_approved_existing_paths():\\n ' - 'assert True\\n"\n' - ' else:\n' - ' after += b"\\ndef ' - 'test_phase4_step5_mutator(value=globals().clear()):\\n ' - 'pass\\n"\n' - ' with pytest.raises(ValueError):\n' - ' verify(path, before, after)'}]} - - -PHASE4_STEP6_ALLOWED = {'.github/workflows/ci.yml', - '.github/workflows/golden.yml', - '.github/workflows/release.yml', - '.github/workflows/security.yml', - 'PHASE_4_BASELINE.json', - 'PHASE_4_DECISIONS.md', - 'docs/architecture.md', - 'docs/cli.md', - 'docs/privacy.md', - 'docs/theory_traceability.md', - 'scripts/check_spec_consistency.py', - 'scripts/check_traceability.py', - 'scripts/release_check.py', - 'src/recursive_integrity_toolkit/utils/logging.py', - 'src/recursive_integrity_toolkit/utils/paths.py', - 'tests/conftest.py', - 'tests/integration/test_ci_workflows.py', - 'tests/integration/test_hero_structure.py', - 'tests/integration/test_license_notices.py', - 'tests/integration/test_no_algorithms.py', - 'tests/integration/test_no_network.py', - 'tests/integration/test_optional_dependency.py', - 'tests/integration/test_owner_ids.py', - 'tests/integration/test_package_import.py', - 'tests/integration/test_package_install.py', - 'tests/integration/test_phase4_gates.py', - 'tests/integration/test_prohibited_structure.py', - 'tests/integration/test_repository_structure.py', - 'tests/integration/test_schema_json.py', - 'tests/unit/test_phase4_contracts.py', - 'tests/unit/test_phase4_output_safety.py'} - - -def _phase4_step5_files(): - """Read the exact accepted Step 5 Git tree and verify every blob identity.""" - for suffix, expected in (("^{commit}", PHASE4_STEP5_FINAL), ("^{tree}", PHASE4_STEP5_TREE), - (":tests", PHASE4_STEP5_TEST_TREE)): - if git("rev-parse", PHASE4_STEP5_FINAL + suffix).decode().strip() != expected: - raise ValueError("Pinned Phase 4 Step 5 identity mismatch") - objects = {} - for entry in git("ls-tree", "-rz", PHASE4_STEP5_FINAL).split(b"\0"): - if not entry: - continue - metadata, raw_path = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - path = raw_path.decode("utf-8") - if (mode not in (b"100644", b"100755") or kind != b"blob" or path in objects - or path.startswith("/") or ".." in path.split("/") or ".git" in path.split("/")): - raise ValueError("Unsafe pinned Step 5 Git object") - objects[path] = oid.decode("ascii") - if len(objects) != 227: - raise ValueError("Pinned Step 5 must contain exactly 227 files") - files = {} - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_STEP5_FINAL))) as archive: - names = [item.filename for item in archive.infolist() if not item.is_dir()] - if len(names) != len(objects) or set(names) != set(objects): - raise ValueError("Pinned Step 5 archive identity mismatch") - for name in names: - raw = archive.read(name) - oid = hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - if oid != objects[name]: - raise ValueError(f"Pinned Step 5 blob mismatch: {name}") - files[name] = raw - return MappingProxyType(files) - - -def phase4_step6_expected_control() -> dict: - """Actual Step 6 approval is independent of the mutable control document.""" - prior = _phase4_step5_files() - result = phase4_step5_expected_control() - result.update({ - "control_version": "1.5", "active_step": 6, - "approval_date": PHASE4_STEP6_APPROVAL_DATE, "approval_basis": PHASE4_STEP6_APPROVAL, - "previous_step_commit": PHASE4_STEP5_FINAL, - "previous_step_tree": PHASE4_STEP5_TREE, - "previous_step_test_tree": PHASE4_STEP5_TEST_TREE, - "previous_step_core_tests": 3215, "previous_step_parquet_tests": 3218, - "previous_step_files_sha256": {p: hashlib.sha256(raw).hexdigest() for p, raw in sorted(prior.items())}, - "permitted_paths": sorted(PHASE4_STEP6_ALLOWED), "new_files_permitted": list(PHASE4_STEP6_NEW), - "runtime_changes_authorized": True, "schema_changes_authorized": False, - "runtime_paths_authorized": sorted(p for p in PHASE4_STEP6_ALLOWED if p.startswith("src/")), - "schema_paths_authorized": [], - "step5_historical_binding_nodes": {p: sorted({r["node"] for r in rows}) - for p, rows in sorted(PHASE4_STEP6_MIGRATIONS.items())}, - }) - return result - - -def verify_phase4_step6_control(control: dict, step: int = 6) -> None: - if type(step) is not int or step != 6 or type(control) is not dict: - raise ValueError("Unsupported Phase 4 Step 6 stage/control") - try: - actual = json.dumps(control, sort_keys=True, ensure_ascii=True, allow_nan=False) - expected = json.dumps(phase4_step6_expected_control(), sort_keys=True, ensure_ascii=True, allow_nan=False) - except (TypeError, ValueError) as error: - raise ValueError("Invalid Phase 4 Step 6 control") from error - if actual != expected: - raise ValueError("Phase 4 control differs from the independently approved Step 6 contract") - - -def verify_phase4_step6_changes(changes: list[tuple[str, str]]) -> None: - seen = set() - for status, path in changes: - if path in seen or path not in PHASE4_STEP6_ALLOWED or status != ("A" if path in PHASE4_STEP6_NEW else "M"): - raise ValueError(f"Unapproved Phase 4 Step 6 path/operation: {status} {path}") - seen.add(path) - - -def _phase4_step6_header(node, path): - """Allow explicit test parametrization/fixtures, never definition-time effects.""" - for decorator in node.decorator_list: - if not isinstance(decorator, ast.Call) or ast.unparse(decorator.func) not in { - "pytest.fixture", "pytest.mark.parametrize"}: - raise ValueError(f"Unapproved Step 6 test decorator: {path}:{node.name}") - if ast.unparse(decorator.func) == "pytest.fixture" and not node.name.startswith("phase4_step"): - raise ValueError("A Step 6 fixture must have an explicit scoped name") - if any(keyword.arg is None for keyword in decorator.keywords): - raise ValueError("Decorator expansion is outside the Step 6 contract") - for value in [*decorator.args, *(keyword.value for keyword in decorator.keywords)]: - try: - ast.literal_eval(value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError("Step 6 decorator arguments must be literal") from error - clone = ast.parse(ast.unparse(node)).body[0] - clone.decorator_list = [] - _phase4_preserve_function_header(clone, path) - - -def _phase4_step6_append_only(before: bytes, after: bytes, path: str) -> None: - if not after.startswith(before): - raise ValueError(f"Inherited Step 5 prefix changed: {path}") - old, new = ast.parse(before), ast.parse(after) - bindings = set() - for node in old.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - bindings.add(node.name) - elif isinstance(node, (ast.Import, ast.ImportFrom)): - bindings.update(alias.asname or alias.name.split(".")[0] for alias in node.names) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - bindings.update(child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)) - for node in new.body[len(old.body):]: - if not isinstance(node, ast.FunctionDef) or node.name in bindings: - raise ValueError(f"Step 6 addition executes, rebinds or shadows inherited source: {path}") - allowed_snapshot = path == "tests/conftest.py" and node.name == "phase4_step5_snapshot" - if not allowed_snapshot and not node.name.startswith(("test_phase4_step6_", "phase4_step6_")): - raise ValueError(f"Step 6 added test/helper is not explicitly scoped: {path}:{node.name}") - if path == "tests/conftest.py" and node.name != "phase4_step5_snapshot": - raise ValueError("Only the pinned Step 5 shared fixture is authorized") - bindings.add(node.name) - _phase4_step6_header(node, path) - - -def verify_phase4_step6_test_migration(path: str, before: bytes, after: bytes) -> None: - prior = _phase4_step5_files() - if path not in prior or path not in PHASE4_STEP6_ALLOWED or before != prior[path] or not path.startswith("tests/"): - raise ValueError("Step 6 migration requires the exact named Step 5 source") - _phase4_step6_check_test_migration(path, before, after) - - -def _phase4_step6_check_test_migration(path: str, before: bytes, after: bytes) -> None: - """Check a source already read from the verified Step 5 immutable mapping.""" - expected = before - for row in PHASE4_STEP6_MIGRATIONS.get(path, []): - old, new = row["old"].encode(), row["new"].encode() - if expected.count(old) != 1: - raise ValueError("Step 5 historical binding is not unique") - expected = expected.replace(old, new, 1) - _phase4_step6_append_only(expected, after, path) - - -def _phase4_step6_preserve_tooling(before: bytes, after: bytes, path: str) -> None: - """Preserve exact inherited statements using one source-line split per tree.""" - if path == "scripts/release_check.py": - old = ' return phase4_cli_main() if explicit_phase4 else main()' - new = (' explicit_step6 = "--step=6" in argv or any(a == "--step" and b == "6" for a, b in zip(argv, argv[1:]))\n' - ' if explicit_phase4 and explicit_step6:\n' - ' return phase4_step6_cli_main()\n' + old) - # Earlier preserved dispatch functions contain this source too; bind cli_main only. - parsed = ast.parse(before) - entry = next(node for node in parsed.body if isinstance(node, ast.FunctionDef) and node.name == "cli_main") - old_entry = ast.get_source_segment(before.decode(), entry) - if old_entry.count(old) != 1: - raise ValueError("Step 5 release dispatcher identity mismatch") - replacements = [(old_entry, old_entry.replace(old, new, 1))] - else: - old = ' if args.phase == 4 and args.step == 5:\n return phase4_step5_main(step=args.step)\n' - new = old + ' if args.phase == 4 and args.step == 6:\n return phase4_step6_main(step=args.step)\n' - replacements = [(old, new), - ('Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5', - 'Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5 or --phase 4 --step 6')] - expected = before - for old, new in replacements: - if expected.count(old.encode()) != 1: - raise ValueError(f"Historical dispatcher identity mismatch: {path}") - expected = expected.replace(old.encode(), new.encode(), 1) - trees = [ast.parse(expected), ast.parse(after)] - lines = [expected.splitlines(keepends=True), after.splitlines(keepends=True)] - def segment(index, node): - if node.lineno == node.end_lineno: - return lines[index][node.lineno - 1][node.col_offset:node.end_col_offset] - return (lines[index][node.lineno - 1][node.col_offset:] - + b"".join(lines[index][node.lineno:node.end_lineno - 1]) - + lines[index][node.end_lineno - 1][:node.end_col_offset]) - def entry_guard(node): - return isinstance(node, ast.If) and isinstance(node.test, ast.Compare) and "__name__" in ast.unparse(node.test) - guards = [n for n in trees[1].body if entry_guard(n)] - expected_guard = ast.parse('if __name__ == "__main__":\n raise SystemExit(cli_main())').body[0] - if len(guards) != 1 or ast.dump(guards[0]) != ast.dump(expected_guard): - raise ValueError(f"Unapproved Step 6 maintainer entrypoint: {path}") - historical = [n for n in trees[0].body if not entry_guard(n)] - current = [n for n in trees[1].body if not entry_guard(n)] - old_entries = [(segment(0, n), ast.dump(n)) for n in historical] - cursor = 0 - for node in current: - if cursor < len(old_entries) and (segment(1, node), ast.dump(node)) == old_entries[cursor]: - cursor += 1 - continue - if isinstance(node, ast.FunctionDef) and (node.name == "_phase4_step5_files" or node.name.startswith( - ("phase4_step6_", "_phase4_step6_", "verify_phase4_step6_", "audit_phase4_step6"))): - _phase4_preserve_function_header(node, path) - continue - if isinstance(node, ast.Assign) and all(isinstance(target, ast.Name) and target.id.startswith( - ("PHASE4_STEP6_", "PHASE4_STEP5_")) for target in node.targets): - try: - ast.literal_eval(node.value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError(f"Nonliteral added Step 6 maintainer constant: {path}") from error - continue - raise ValueError(f"Unapproved Step 6 maintainer addition: {path}") - if cursor != len(historical): - raise ValueError(f"Inherited Step 5 maintainer statement changed: {path}") - def binding_counts(tree): - counts = {} - for node in tree.body: - names = [] - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - names = [node.name] - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names = [child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)] - elif isinstance(node, (ast.Import, ast.ImportFrom)): - names = [alias.asname or alias.name.split(".")[0] for alias in node.names] - for name in names: - counts[name] = counts.get(name, 0) + 1 - return counts - inherited_counts = binding_counts(trees[0]) - for name, count in binding_counts(trees[1]).items(): - if count > inherited_counts.get(name, 1): - raise ValueError(f"Duplicate Step 6 maintainer binding: {path}:{name}") - - -def verify_phase4_step6_snapshot(root: Path = ROOT) -> dict: - prior = _phase4_step5_files() - for path, raw in prior.items(): - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError(f"Missing or aliased inherited Step 5 file: {path}") - current = target.read_bytes() - if path not in PHASE4_STEP6_ALLOWED and current != raw: - raise ValueError(f"Protected Step 5 bytes changed: {path}") - if path.startswith("tests/") and path in PHASE4_STEP6_ALLOWED: - _phase4_step6_check_test_migration(path, raw, current) - elif path.startswith("scripts/") and path in PHASE4_STEP6_ALLOWED: - _phase4_step6_preserve_tooling(raw, current, path) - elif path in {"docs/architecture.md", "docs/theory_traceability.md", "docs/privacy.md", "docs/cli.md", "PHASE_4_DECISIONS.md"}: - if not current.startswith(raw): - raise ValueError(f"Step 5 historical documentation prefix changed: {path}") - for path in PHASE4_STEP6_NEW: - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError("Missing Step 6 output safety tests") - _phase4_step6_new_test(target.read_bytes(), path) - for path in sorted(p for p in PHASE4_STEP6_ALLOWED if p.startswith("src/")): - _phase4_step6_preserve_runtime(prior[path], (root / path).read_bytes(), path) - actual_modules = {p.relative_to(root).as_posix() for p in (root / "src/recursive_integrity_toolkit").rglob("*.py")} - expected_modules = {p for p in prior if p.startswith("src/") and p.endswith(".py")} - if actual_modules != expected_modules or len(actual_modules) != 40: - raise ValueError("Step 6 cannot change the runtime module set") - if {p.name for p in (root / "schemas").iterdir()} != {Path(p).name for p in prior if p.startswith("schemas/")}: - raise ValueError("Step 6 cannot change the schema set") - if hashlib.sha256((root / "PHASE_4_PLAN.md").read_bytes()).hexdigest() != PHASE4_PLAN_SHA256: - raise ValueError("Approved Phase 4 plan changed") - verify_phase4_step6_control(json.loads((root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))) - decisions = (root / "PHASE_4_DECISIONS.md").read_text(encoding="utf-8") - if PHASE4_STEP6_APPROVAL not in decisions or PHASE4_STEP5_FINAL not in decisions: - raise ValueError("Actual Step 6 authorization or Step 5 evidence anchor missing") - if any((root / name).exists() for name in PHASE4_FORBIDDEN_OUTPUTS): - raise ValueError("Step 6 cannot create Phase 4 completion records or audit outputs") - import runpy - checker = runpy.run_path(str(root / "scripts/check_traceability.py"), run_name="phase4_step6_renderer_boundary") - checker["phase4_step6_output_boundary"](root) - schema_bytes = (root / "schemas/report.schema.json").read_bytes() - if hashlib.sha256(schema_bytes).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Step 6 report schema differs from its independently reviewed bytes") - schema = json.loads(schema_bytes) - if (schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema" - or schema.get("type") != "object" or schema.get("additionalProperties") is not False): - raise ValueError("Step 6 report schema dialect/closed root mismatch") - return {"package_modules": 40, "frozen_runtime_modules": 38, "frozen_schemas": 5, - "hero_files_unchanged": 6, "historical_phase3_migrated_nodes": 16, - "phase_complete": False, "result_contracts_enabled": True, - "adapters_enabled": True, "privacy_views_enabled": True, "renderers_enabled": True, "output_publication_enabled": True, "cli_analysis_enabled": False} - - -def audit_phase4_step6(step: int = 6) -> dict: - verify_phase4_step6_control(json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")), step) - result = {"phase": 4, "active_step": step, **verify_phase4_step6_snapshot(), - "phase0_hashes_verified": 16, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_phase4_step6_diff(step: int = 6) -> dict: - if type(step) is not int or step != 6: - raise ValueError("Unsupported Phase 4 Step 6 stage") - _phase4_step5_files() - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE4_STEP5_FINAL, "HEAD"], check=True) - raw = git("diff", "--name-status", "--no-renames", "-z", PHASE4_STEP5_FINAL, "--").split(b"\0") - raw = [part.decode("utf-8") for part in raw if part] - if len(raw) % 2: - raise ValueError("Malformed Step 6 Git difference records") - changes = list(zip(raw[::2], raw[1::2])) - changes.extend(("A", p.decode()) for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") if p) - verify_phase4_step6_changes(changes) - result = {"previous_step_commit": PHASE4_STEP5_FINAL, "changed_files": len(changes), - "changes": changes, "step": 6, "scope": "PASS"} - print(json.dumps(result, indent=2)) - return result - - -def phase4_step6_baseline_evidence(output: Path) -> dict: - """Reconcile all inherited identities, including the accepted Step 5 suite.""" - output.mkdir(parents=True, exist_ok=True) - inherited = phase4_step5_baseline_evidence(output / "phase4-step5-inherited") - with tempfile.TemporaryDirectory(prefix="rit-p4-step5-identities-") as temp: - baseline = Path(temp) - for name, raw in _phase4_step5_files().items(): - target = baseline / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(raw) - old, old_log = _collect(baseline) - current, current_log = _collect(ROOT) - expected = 3218 if os.environ.get("RIT_TEST_PARQUET") == "1" else 3215 - if len(old) != expected or set(old) - set(current): - raise ValueError("Accepted Phase 4 Step 5 test identities were lost") - result = {"baseline_commit": PHASE4_STEP5_FINAL, "baseline_test_tree": PHASE4_STEP5_TEST_TREE, - "baseline_nodeids": old, "current_nodeids": current, "missing_nodeids": [], - "baseline_tests": len(old), "current_tests": len(current), "inherited": inherited, - "nodeids_sha256": hashlib.sha256(("\n".join(old)+"\n").encode()).hexdigest()} - (output / "phase4_step6_test_identity_manifest.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4-step5-collection.log").write_text(old_log, encoding="utf-8") - (output / "phase4-step6-collection.log").write_text(current_log, encoding="utf-8") - print(json.dumps({k: result[k] for k in ("baseline_tests", "current_tests", "missing_nodeids")}, indent=2)) - return result - - -def phase4_step6_candidate(output: Path, step: int = 6) -> None: - """Build tested Step 6 intermediate evidence without declaring Phase 4 complete.""" - if os.environ.get("RIT_TEST_PARQUET") != "1" or importlib.util.find_spec("pyarrow") is None: - raise ValueError("Step 6 candidate evidence requires RIT_TEST_PARQUET=1 and real PyArrow") - output.mkdir(parents=True, exist_ok=True) - result = audit_phase4_step6(step) - result["diff"] = audit_phase4_step6_diff(step) - if git("status", "--porcelain").strip(): - raise ValueError("Step 6 candidate archive requires committed, clean source") - result["core"] = verify_junit(output / "core.xml", minimum=3215) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=3218) - result["core_math_measurements"] = verify_step10_evidence(output / "core.xml", output / "phase4-step6-core-observations.json") - result["parquet_math_measurements"] = verify_step10_evidence(output / "parquet.xml", output / "phase4-step6-parquet-observations.json") - identity = phase4_step6_baseline_evidence(output) - result["test_identity"] = {k: identity[k] for k in ("baseline_tests", "current_tests", "nodeids_sha256")} - expected = set() - for node in identity["current_nodeids"]: - base, bracket, parameter = node.partition("[") - owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) - for name, parquet in (("core.xml", False), ("parquet.xml", True)): - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(output / name).getroot().iter("testcase")} - target = expected if parquet else {c for c in expected if c[1] not in PARQUET_CASES} - if cases != target: - raise ValueError(f"Step 6 JUnit does not execute the entire current suite: {name}") - wheel, _ = verify_distributions(output / "dist") - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - phase4_step6_installed_publication_smoke(wheel) - archive = output / "recursive-integrity-toolkit-phase4-step6-candidate.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {p for p in git("ls-files", "-z").decode().split("\0") if p} - with zipfile.ZipFile(archive) as zipped: - entries = [item.filename for item in zipped.infolist() if not item.is_dir()] - prefix = "recursive-integrity-toolkit/" - if (len(entries) != len(set(entries)) or any(not name.startswith(prefix) for name in entries)): - raise ValueError("Step 6 source archive has duplicate or unprefixed entries") - names = {name.removeprefix(prefix) for name in entries} - if names != tracked or len(tracked) != 228 or len(entries) != 228: - raise ValueError("Step 6 source archive file set mismatch") - for name in names: - if zipped.read("recursive-integrity-toolkit/"+name) != (ROOT / name).read_bytes(): - raise ValueError(f"Step 6 source archive byte mismatch: {name}") - result.update({"commit": git("rev-parse", "HEAD").decode().strip(), - "tree": git("rev-parse", "HEAD^{tree}").decode().strip(), - "source_archive": archive.name, "tracked_files": len(tracked), - "python": sys.version, "platform": sys.platform, - "versions": {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "pytest")}}) - (output / "phase4_step6_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4_step6_repository_files.sha256").write_text("".join(f"{hashlib.sha256((ROOT/name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - paths = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase4_step6_artifacts.sha256") - (output / "phase4_step6_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in paths), encoding="utf-8") - print(f"Phase 4 Step 6 candidate: {len(tracked)} source files verified; Phase 4 remains incomplete.") - - -def phase4_step6_cli_main() -> int: - parser = argparse.ArgumentParser(description="Phase 4 Step 6 maintainer gate") - parser.add_argument("--phase", type=int, choices=(4,), required=True) - parser.add_argument("--step", type=int, choices=(6,), required=True) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) - parser.add_argument("--require-parquet", action="store_true") - parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - raise ValueError("Phase 4 Step 6 cannot certify a final phase delivery") - if args.junit is not None: - verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - phase4_step6_baseline_evidence(args.baseline_evidence) - elif args.dist is not None: - audit_phase4_step6(args.step) - wheel, _ = verify_distributions(args.dist) - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - phase4_step6_installed_publication_smoke(wheel) - elif args.smoke_wheel is not None: - smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - phase4_step6_candidate(args.candidate, args.step) - else: - audit_phase4_step6(args.step) - if args.diff: - audit_phase4_step6_diff(args.step) - return 0 - - -def _phase4_step6_new_test(raw: bytes, path: str) -> None: - """Only literal test declarations and explicitly listed safe imports.""" - allowed = {"__future__", "errno", "io", "json", "os", "pathlib", "types", "pytest", - "recursive_integrity_toolkit.utils"} - for index, node in enumerate(ast.parse(raw).body): - if index == 0 and isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str): - continue - if isinstance(node, (ast.Import, ast.ImportFrom)): - modules = [node.module] if isinstance(node, ast.ImportFrom) else [alias.name for alias in node.names] - if any(module not in allowed for module in modules): - raise ValueError("Unapproved Step 6 test import") - continue - if not isinstance(node, ast.FunctionDef) or not node.name.startswith(("test_phase4_step6_", "phase4_step6_")): - raise ValueError("Step 6 output tests gained definition-time execution") - _phase4_step6_header(node, path) - - -def _phase4_step6_preserve_runtime(before: bytes, after: bytes, path: str) -> None: - """Allow ownership-header maintenance and byte-exact additive helpers only.""" - try: - old, new = ast.parse(before), ast.parse(after) - except (SyntaxError, UnicodeError, ValueError) as error: - raise ValueError("Output helper source cannot be inspected") from error - if not ast.get_docstring(old) or not ast.get_docstring(new): - raise ValueError("Output helper ownership documentation is required") - old_tail = b"".join(before.splitlines(keepends=True)[old.body[0].end_lineno:]) - new_tail = b"".join(after.splitlines(keepends=True)[new.body[0].end_lineno:]) - if not new_tail.startswith(old_tail): - raise ValueError(f"Inherited input/diagnostic bytes changed: {path}") - - - -def phase4_step6_installed_publication_smoke(wheel: Path) -> None: +def smoke_installed_publication(wheel: Path) -> None: """Exercise installed output safety outside the source checkout.""" with tempfile.TemporaryDirectory(prefix="rit-p4-step6-installed-") as temporary: work = Path(temporary) @@ -6742,1535 +1093,11 @@ def phase4_step6_installed_publication_smoke(wheel: Path) -> None: bindir = work / "venv" / ("Scripts" if os.name == "nt" else "bin") python = bindir / ("python.exe" if os.name == "nt" else "python") subprocess.run([str(python), "-m", "pip", "install", "--no-index", "--no-deps", str(wheel.resolve())], cwd=work, check=True) - program = 'import importlib.abc, json, os, socket, sys\nfrom pathlib import Path\nclass BlockOptional(importlib.abc.MetaPathFinder):\n def find_spec(self, fullname, path=None, target=None):\n if fullname.split(".")[0] in ("numpy", "pandas", "pyarrow"):\n raise AssertionError("optional or analytical dependency imported")\nsys.meta_path.insert(0, BlockOptional())\ndef phase4_step6_view(mode="standard", label="publication-case"):\n from recursive_integrity_toolkit.reports.assembly import privacy_view\n from recursive_integrity_toolkit.result import CanonicalReport\n from recursive_integrity_toolkit.utils.hashing import IdentifierProtection\n\n payload = {\n "run": {\n "run_id": label, "toolkit_version": "0.1.0.dev2", "report_schema_version": "1.0",\n "started_at": "2026-09-20T00:00:00+00:00", "completed_at": "2026-09-20T00:00:00.500000+00:00",\n "duration_seconds": 0.5, "python_version": "3.12.14", "platform": "independent-test-platform",\n "command": "rit validate", "config_hash": "0123456789abcdef" * 4,\n "random_seed": None, "strict_mode": False, "redacted_mode": False,\n "network_call_count": 0, "deterministic": True, "privacy_mode": "standard",\n "run_status": "complete", "null_reasons": {"random_seed": "No stochastic scenario was requested."},\n },\n "inputs": {}, "observability": {}, "capabilities": {}, "observed_facts": {},\n "derived_metrics": {}, "proxy_signals": {}, "simulations": {},\n "unavailable_conclusions": [], "recommended_next_metadata": [], "warnings": [], "errors": [],\n }\n return privacy_view(CanonicalReport.from_dict(payload), mode=mode,\n protection=IdentifierProtection.create(secret=b"publication-test-only-key-32byte!"))\nfrom recursive_integrity_toolkit.utils.paths import publish_reports, PublicationResult\nfrom recursive_integrity_toolkit.utils.logging import format_publication_diagnostic\nfrom recursive_integrity_toolkit.utils import paths\nfrom recursive_integrity_toolkit.reports.json_report import render_json\nfrom recursive_integrity_toolkit.reports.markdown_report import render_markdown\nimport recursive_integrity_toolkit as package\nassert not Path(package.__file__).resolve().is_relative_to(Path(sys.argv[2]).resolve())\nwork=Path(sys.argv[1]); source=work/\'input.txt\'; source.write_bytes(b\'PRIVATE_INSTALLED_SOURCE\')\nview=phase4_step6_view(\'redacted\')\nexpected={\'report.json\':render_json(view).encode(), \'report.md\':render_markdown(view).encode()}\ndef deny(*args, **kwargs):\n raise AssertionError(\'network forbidden\')\nsocket.socket=deny; socket.getaddrinfo=deny\nactual_open=os.open\ndef no_input(path,*args,**kwargs):\n assert Path(path)!=source\n return actual_open(path,*args,**kwargs)\nos.open=no_input\ndef no_analysis(frame,event,arg):\n if event==\'call\':\n name=frame.f_globals.get(\'__name__\',\'\')\n assert not name.startswith((\'recursive_integrity_toolkit.io.\',\'recursive_integrity_toolkit.metrics.\',\'recursive_integrity_toolkit.representations.\'))\nsys.setprofile(no_analysis)\ntry:\n result=publish_reports(view,work/\'out\',input_paths=(source,))\n assert result==PublicationResult(\'complete\',None,0,(\'report.json\',\'report.md\'))\n assert publish_reports(view,work/\'out\',input_paths=(source,)).code==\'E_OUTPUT_EXISTS\'\n actual_publish=paths._output_publish_one\n def fail_second(src,dst):\n if dst.name==\'report.md\':raise OSError(\'PRIVATE_INSTALLED_FAILURE\')\n return actual_publish(src,dst)\n paths._output_publish_one=fail_second\n failed=publish_reports(view,work/\'failure\',input_paths=(source,))\n assert failed.status==\'failed\' and failed.code==\'E_OUTPUT_IO\' and failed.temporary_cleanup_complete\n assert \'PRIVATE_\' not in format_publication_diagnostic(failed)\nfinally:\n sys.setprofile(None)\nassert {p.name:p.read_bytes() for p in (work/\'out\').iterdir()}==expected\nassert list((work/\'failure\').iterdir())==[]\nassert source.read_bytes()==b\'PRIVATE_INSTALLED_SOURCE\'\nprint(\'installed Step 6: exact safe JSON/Markdown bytes, no-overwrite, partial failure cleanup, safe diagnostics, unchanged input, blocked network/analysis: PASS\')\n' + program = 'import importlib.abc, json, os, socket, sys\nfrom pathlib import Path\nclass BlockOptional(importlib.abc.MetaPathFinder):\n def find_spec(self, fullname, path=None, target=None):\n if fullname.split(".")[0] in ("numpy", "pandas", "pyarrow"):\n raise AssertionError("optional or analytical dependency imported")\nsys.meta_path.insert(0, BlockOptional())\ndef phase4_step6_view(mode="standard", label="publication-case"):\n from recursive_integrity_toolkit.reports.assembly import privacy_view\n from recursive_integrity_toolkit.result import CanonicalReport\n from recursive_integrity_toolkit.utils.hashing import IdentifierProtection\n\n payload = {\n "run": {\n "run_id": label, "toolkit_version": "0.1.0.dev2", "report_schema_version": "1.1",\n "started_at": "2026-09-20T00:00:00+00:00", "completed_at": "2026-09-20T00:00:00.500000+00:00",\n "duration_seconds": 0.5, "python_version": "3.12.14", "platform": "independent-test-platform",\n "command": "rit validate", "config_hash": "0123456789abcdef" * 4,\n "random_seed": None, "strict_mode": False, "redacted_mode": False,\n "network_call_count": 0, "deterministic": True, "privacy_mode": "standard",\n "run_status": "complete", "null_reasons": {"random_seed": "No stochastic scenario was requested."},\n },\n "inputs": {}, "observability": {}, "capabilities": {}, "observed_facts": {},\n "derived_metrics": {}, "proxy_signals": {}, "simulations": {},\n "unavailable_conclusions": [], "recommended_next_metadata": [], "warnings": [], "errors": [],\n }\n return privacy_view(CanonicalReport.from_dict(payload), mode=mode,\n protection=IdentifierProtection.create(secret=b"publication-test-only-key-32byte!"))\nfrom recursive_integrity_toolkit.utils.paths import publish_reports, PublicationResult\nfrom recursive_integrity_toolkit.utils.logging import format_publication_diagnostic\nfrom recursive_integrity_toolkit.utils import paths\nfrom recursive_integrity_toolkit.reports.json_report import render_json\nfrom recursive_integrity_toolkit.reports.markdown_report import render_markdown\nimport recursive_integrity_toolkit as package\nassert not Path(package.__file__).resolve().is_relative_to(Path(sys.argv[2]).resolve())\nwork=Path(sys.argv[1]); source=work/\'input.txt\'; source.write_bytes(b\'PRIVATE_INSTALLED_SOURCE\')\nview=phase4_step6_view(\'redacted\')\nexpected={\'report.json\':render_json(view).encode(), \'report.md\':render_markdown(view).encode()}\ndef deny(*args, **kwargs):\n raise AssertionError(\'network forbidden\')\nsocket.socket=deny; socket.getaddrinfo=deny\nactual_open=os.open\ndef no_input(path,*args,**kwargs):\n assert Path(path)!=source\n return actual_open(path,*args,**kwargs)\nos.open=no_input\ndef no_analysis(frame,event,arg):\n if event==\'call\':\n name=frame.f_globals.get(\'__name__\',\'\')\n assert not name.startswith((\'recursive_integrity_toolkit.io.\',\'recursive_integrity_toolkit.metrics.\',\'recursive_integrity_toolkit.representations.\'))\nsys.setprofile(no_analysis)\ntry:\n result=publish_reports(view,work/\'out\',input_paths=(source,))\n assert result==PublicationResult(\'complete\',None,0,(\'report.json\',\'report.md\'))\n assert publish_reports(view,work/\'out\',input_paths=(source,)).code==\'E_OUTPUT_EXISTS\'\n actual_publish=paths._output_publish_one\n def fail_second(src,dst):\n if dst.name==\'report.md\':raise OSError(\'PRIVATE_INSTALLED_FAILURE\')\n return actual_publish(src,dst)\n paths._output_publish_one=fail_second\n failed=publish_reports(view,work/\'failure\',input_paths=(source,))\n assert failed.status==\'failed\' and failed.code==\'E_OUTPUT_IO\' and failed.temporary_cleanup_complete\n assert \'PRIVATE_\' not in format_publication_diagnostic(failed)\nfinally:\n sys.setprofile(None)\nassert {p.name:p.read_bytes() for p in (work/\'out\').iterdir()}==expected\nassert list((work/\'failure\').iterdir())==[]\nassert source.read_bytes()==b\'PRIVATE_INSTALLED_SOURCE\'\nprint(\'installed Step 6: exact safe JSON/Markdown bytes, no-overwrite, partial failure cleanup, safe diagnostics, unchanged input, blocked network/analysis: PASS\')\n' subprocess.run([str(python), "-I", "-c", program, str(work), str(ROOT)], cwd=work, check=True) - -# Phase 4 Step 7: single-version CLI orchestration, no later-stage expansion. - - -PHASE4_STEP6_FINAL = 'aa2355f2359c3af4fc05345c792f9903b0d5858c' - - -PHASE4_STEP6_TREE = 'd9b6910079f49a2c739f86eceec05040918a41a5' - - -PHASE4_STEP6_TEST_TREE = '8d3a03c61da807d1fef8d26fc4af0d6f72e4546e' - - -PHASE4_STEP7_APPROVAL = 'Phase 4 Step 7 继续' - - -PHASE4_STEP7_APPROVAL_DATE = '2026-09-21' - - -PHASE4_STEP7_NEW = ('tests/integration/test_phase4_cli.py',) - - -PHASE4_STEP7_ALLOWED = {'.github/workflows/ci.yml', - '.github/workflows/golden.yml', - '.github/workflows/release.yml', - '.github/workflows/security.yml', - 'PHASE_4_BASELINE.json', - 'PHASE_4_DECISIONS.md', - 'docs/architecture.md', - 'docs/cli.md', - 'docs/theory_traceability.md', - 'scripts/check_spec_consistency.py', - 'scripts/check_traceability.py', - 'scripts/release_check.py', - 'src/recursive_integrity_toolkit/cli.py', - 'src/recursive_integrity_toolkit/config.py', - 'tests/conftest.py', - 'tests/integration/test_ci_workflows.py', - 'tests/integration/test_cli_validation.py', - 'tests/integration/test_hero_structure.py', - 'tests/integration/test_license_notices.py', - 'tests/integration/test_no_algorithms.py', - 'tests/integration/test_no_network.py', - 'tests/integration/test_optional_dependency.py', - 'tests/integration/test_owner_ids.py', - 'tests/integration/test_package_import.py', - 'tests/integration/test_package_install.py', - 'tests/integration/test_phase4_cli.py', - 'tests/integration/test_phase4_gates.py', - 'tests/integration/test_prohibited_structure.py', - 'tests/integration/test_repository_structure.py', - 'tests/integration/test_schema_json.py', - 'tests/unit/test_phase4_contracts.py'} - - -PHASE4_STEP7_MIGRATIONS = {'tests/integration/test_phase4_gates.py': [{'new': 'def ' - 'test_phase4_step6_current_runtime_opens_only_output_helpers_and_freezes_schema(repo_root, ' - 'phase4_step5_snapshot, phase4_step6_snapshot):\n' - ' repo_root = phase4_step6_snapshot\n' - ' package = "src/recursive_integrity_toolkit"\n' - ' current = {path.relative_to(repo_root).as_posix(): ' - 'hashlib.sha256(path.read_bytes()).hexdigest()\n' - ' for path in (repo_root / ' - 'package).rglob("*.py")}\n' - ' frozen = ' - '{path.relative_to(phase4_step5_snapshot).as_posix(): ' - 'hashlib.sha256(path.read_bytes()).hexdigest()\n' - ' for path in (phase4_step5_snapshot / ' - 'package).rglob("*.py")}\n' - ' assert len(current) == len(frozen) == 40\n' - ' assert current.keys() == frozen.keys()\n' - ' opened = {package + "/" + relative for relative in ' - '(\n' - ' "utils/paths.py", "utils/logging.py",\n' - ' )}\n' - ' assert {path for path in current if current[path] ' - '!= frozen[path]} == opened\n' - ' assert {path: digest for path, digest in ' - 'current.items() if path not in opened} == {\n' - ' path: digest for path, digest in frozen.items() ' - 'if path not in opened\n' - ' }\n' - ' schemas = {path.name: path.read_bytes() for path in ' - '(repo_root / "schemas").glob("*.json")}\n' - ' assert len(schemas) == 5\n' - ' assert schemas == {path.name: path.read_bytes() for ' - 'path in (phase4_step5_snapshot / ' - '"schemas").glob("*.json")}', - 'node': 'test_phase4_step6_current_runtime_opens_only_output_helpers_and_freezes_schema', - 'old': 'def ' - 'test_phase4_step6_current_runtime_opens_only_output_helpers_and_freezes_schema(repo_root, ' - 'phase4_step5_snapshot):\n' - ' package = "src/recursive_integrity_toolkit"\n' - ' current = {path.relative_to(repo_root).as_posix(): ' - 'hashlib.sha256(path.read_bytes()).hexdigest()\n' - ' for path in (repo_root / ' - 'package).rglob("*.py")}\n' - ' frozen = ' - '{path.relative_to(phase4_step5_snapshot).as_posix(): ' - 'hashlib.sha256(path.read_bytes()).hexdigest()\n' - ' for path in (phase4_step5_snapshot / ' - 'package).rglob("*.py")}\n' - ' assert len(current) == len(frozen) == 40\n' - ' assert current.keys() == frozen.keys()\n' - ' opened = {package + "/" + relative for relative in ' - '(\n' - ' "utils/paths.py", "utils/logging.py",\n' - ' )}\n' - ' assert {path for path in current if current[path] ' - '!= frozen[path]} == opened\n' - ' assert {path: digest for path, digest in ' - 'current.items() if path not in opened} == {\n' - ' path: digest for path, digest in frozen.items() ' - 'if path not in opened\n' - ' }\n' - ' schemas = {path.name: path.read_bytes() for path in ' - '(repo_root / "schemas").glob("*.json")}\n' - ' assert len(schemas) == 5\n' - ' assert schemas == {path.name: path.read_bytes() for ' - 'path in (phase4_step5_snapshot / ' - '"schemas").glob("*.json")}'}, - {'new': 'def ' - 'test_phase4_step6_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step6_snapshot):\n' - ' repo_root = phase4_step6_snapshot\n' - ' names = {"ci.yml", "golden.yml", "security.yml", ' - '"release.yml"}\n' - ' root = repo_root / ".github/workflows"\n' - ' expected_timeouts = {"ci.yml": [60, 20], ' - '"golden.yml": [15], "security.yml": [20], ' - '"release.yml": [25]}\n' - ' assert {path.name for path in root.glob("*.yml")} ' - '== names\n' - ' for name in sorted(names):\n' - ' text = (root / ' - 'name).read_text(encoding="utf-8")\n' - ' assert [int(line.split(":", 1)[1]) for line in ' - 'text.splitlines()\n' - ' if ' - 'line.strip().startswith("timeout-minutes:")] == ' - 'expected_timeouts[name]\n' - ' assert "Phase 4 Step 6" in text, name\n' - ' assert "--phase 4 --step 6" in text, name\n' - ' for older in ("--phase 4 --step 5", "--phase 4 ' - '--step 4", "--phase 4 --step 3", "--phase 4 --step 2", ' - '"--phase 4 --step 1", "--phase 3 --step 11"):\n' - ' assert older not in text, (name, older)\n' - ' assert "permissions:\\n contents: read" in ' - 'text\n' - ' assert "persist-credentials: false" in text\n' - ' assert "timeout-minutes:" in text and "set -euo ' - 'pipefail" in text\n' - ' assert "actions/upload-artifact@v4" in text and ' - '"if-no-files-found: error" in text\n' - ' for line in text.splitlines():\n' - ' if "python scripts/release_check.py" in ' - 'line:\n' - ' assert "--phase 4 --step 6" in line, ' - '(name, line)\n' - ' for forbidden in ("continue-on-error:", "|| ' - 'true", "contents: write", "id-token: write", "twine ' - 'upload", "git push"):\n' - ' assert forbidden not in text, (name, ' - 'forbidden)\n' - ' ci = (root / "ci.yml").read_text(encoding="utf-8")\n' - " for required in ('os: [ubuntu-latest, " - 'windows-latest]\', \'python-version: ["3.11", ' - '"3.12"]\',\n' - " 'dependencies: [current, " - 'minimum]\', \'"numpy==2.0.0" "pandas==2.2.2"\',\n' - ' \'RIT_TEST_PARQUET: "0"\', ' - '\'RIT_TEST_PARQUET: "1"\', \'--require-parquet\',\n' - " '--baseline-evidence', 'python -m " - "pip check',\n" - " '--minimum-tests 3215', " - "'--minimum-tests 3218',\n" - ' "find_spec(\'pyarrow\') is None", ' - "'import pyarrow'):\n" - ' assert required in ci\n' - ' assert ci.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2\n' - ' assert " -k " not in ci\n' - ' golden = (root / ' - '"golden.yml").read_text(encoding="utf-8")\n' - ' for required in ' - '("tests/golden/test_phase3_math.py", ' - '"tests/integration/test_hero_structure.py",\n' - ' ' - '"tests/integration/test_phase3_metric_pipeline.py"):\n' - ' assert required in golden\n' - ' security = (root / ' - '"security.yml").read_text(encoding="utf-8")\n' - ' for required in ' - "('tests/integration/test_no_network.py', " - "'tests/integration/test_optional_dependency.py',\n" - ' ' - "'tests/unit/test_PR012_evidence_classes.py', " - "'tests/unit/test_PR013_report_schema.py',\n" - ' ' - "'tests/unit/test_PR014_unavailable.py', " - "'tests/unit/test_PR015_redaction.py',\n" - ' ' - "'tests/unit/test_PR016_determinism.py', " - "'tests/unit/test_PR018_language.py',\n" - ' ' - "'tests/integration/test_partial_provenance_report.py',\n" - ' ' - "'tests/integration/test_partial_lineage_report.py',\n" - ' ' - "'tests/unit/test_phase4_output_safety.py', " - "'tests/unit/test_phase4_contracts.py', " - "'tests/integration/test_phase4_gates.py'):\n" - ' assert required in security\n' - ' release = (root / ' - '"release.yml").read_text(encoding="utf-8")\n' - ' assert "python -m build" in release and "python -m ' - 'twine check --strict" in release\n' - ' assert "--dist" in release and "--candidate" in ' - 'release\n' - ' assert "--delivery" not in release\n' - ' assert ' - '"recursive-integrity-toolkit-phase4-step6-candidate" in ' - 'release\n' - ' assert "rit-phase4-step4" not in release\n' - ' assert "--minimum-tests 3215" in release and ' - '"--minimum-tests 3218" in release\n' - ' assert release.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2', - 'node': 'test_phase4_step6_current_workflows_preserve_matrix_and_use_active_dispatch', - 'old': 'def ' - 'test_phase4_step6_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root):\n' - ' names = {"ci.yml", "golden.yml", "security.yml", ' - '"release.yml"}\n' - ' root = repo_root / ".github/workflows"\n' - ' expected_timeouts = {"ci.yml": [60, 20], ' - '"golden.yml": [15], "security.yml": [20], ' - '"release.yml": [25]}\n' - ' assert {path.name for path in root.glob("*.yml")} ' - '== names\n' - ' for name in sorted(names):\n' - ' text = (root / ' - 'name).read_text(encoding="utf-8")\n' - ' assert [int(line.split(":", 1)[1]) for line in ' - 'text.splitlines()\n' - ' if ' - 'line.strip().startswith("timeout-minutes:")] == ' - 'expected_timeouts[name]\n' - ' assert "Phase 4 Step 6" in text, name\n' - ' assert "--phase 4 --step 6" in text, name\n' - ' for older in ("--phase 4 --step 5", "--phase 4 ' - '--step 4", "--phase 4 --step 3", "--phase 4 --step 2", ' - '"--phase 4 --step 1", "--phase 3 --step 11"):\n' - ' assert older not in text, (name, older)\n' - ' assert "permissions:\\n contents: read" in ' - 'text\n' - ' assert "persist-credentials: false" in text\n' - ' assert "timeout-minutes:" in text and "set -euo ' - 'pipefail" in text\n' - ' assert "actions/upload-artifact@v4" in text and ' - '"if-no-files-found: error" in text\n' - ' for line in text.splitlines():\n' - ' if "python scripts/release_check.py" in ' - 'line:\n' - ' assert "--phase 4 --step 6" in line, ' - '(name, line)\n' - ' for forbidden in ("continue-on-error:", "|| ' - 'true", "contents: write", "id-token: write", "twine ' - 'upload", "git push"):\n' - ' assert forbidden not in text, (name, ' - 'forbidden)\n' - ' ci = (root / "ci.yml").read_text(encoding="utf-8")\n' - " for required in ('os: [ubuntu-latest, " - 'windows-latest]\', \'python-version: ["3.11", ' - '"3.12"]\',\n' - " 'dependencies: [current, " - 'minimum]\', \'"numpy==2.0.0" "pandas==2.2.2"\',\n' - ' \'RIT_TEST_PARQUET: "0"\', ' - '\'RIT_TEST_PARQUET: "1"\', \'--require-parquet\',\n' - " '--baseline-evidence', 'python -m " - "pip check',\n" - " '--minimum-tests 3215', " - "'--minimum-tests 3218',\n" - ' "find_spec(\'pyarrow\') is None", ' - "'import pyarrow'):\n" - ' assert required in ci\n' - ' assert ci.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2\n' - ' assert " -k " not in ci\n' - ' golden = (root / ' - '"golden.yml").read_text(encoding="utf-8")\n' - ' for required in ' - '("tests/golden/test_phase3_math.py", ' - '"tests/integration/test_hero_structure.py",\n' - ' ' - '"tests/integration/test_phase3_metric_pipeline.py"):\n' - ' assert required in golden\n' - ' security = (root / ' - '"security.yml").read_text(encoding="utf-8")\n' - ' for required in ' - "('tests/integration/test_no_network.py', " - "'tests/integration/test_optional_dependency.py',\n" - ' ' - "'tests/unit/test_PR012_evidence_classes.py', " - "'tests/unit/test_PR013_report_schema.py',\n" - ' ' - "'tests/unit/test_PR014_unavailable.py', " - "'tests/unit/test_PR015_redaction.py',\n" - ' ' - "'tests/unit/test_PR016_determinism.py', " - "'tests/unit/test_PR018_language.py',\n" - ' ' - "'tests/integration/test_partial_provenance_report.py',\n" - ' ' - "'tests/integration/test_partial_lineage_report.py',\n" - ' ' - "'tests/unit/test_phase4_output_safety.py', " - "'tests/unit/test_phase4_contracts.py', " - "'tests/integration/test_phase4_gates.py'):\n" - ' assert required in security\n' - ' release = (root / ' - '"release.yml").read_text(encoding="utf-8")\n' - ' assert "python -m build" in release and "python -m ' - 'twine check --strict" in release\n' - ' assert "--dist" in release and "--candidate" in ' - 'release\n' - ' assert "--delivery" not in release\n' - ' assert ' - '"recursive-integrity-toolkit-phase4-step6-candidate" in ' - 'release\n' - ' assert "rit-phase4-step4" not in release\n' - ' assert "--minimum-tests 3215" in release and ' - '"--minimum-tests 3218" in release\n' - ' assert release.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2'}, - {'new': 'def ' - 'test_phase4_step6_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step6_snapshot):\n' - ' import json\n' - '\n' - ' paths = ' - '{path.relative_to(phase4_step6_snapshot).as_posix()\n' - ' for path in ' - 'phase4_step6_snapshot.rglob("*") if path.is_file()}\n' - ' assert len(paths) == 228\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step6_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step6_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 38\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["privacy_views_enabled"] is True\n' - ' assert baseline["renderers_enabled"] is True\n' - ' assert baseline["output_publication_enabled"] is ' - 'True\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 4\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n repo_root = ' - 'phase4_step4_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n"),\n' - ' "historical_document": ("docs/cli.md", b"# ' - 'CLI\\n",\n' - ' b"# Unauthorized ' - 'historical document\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "html": ' - '"src/recursive_integrity_toolkit/reports/html_report.py",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "config": ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' "hashing": ' - '"src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' "logging": ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 6 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 7\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)', - 'node': 'test_phase4_step6_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step6_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation):\n' - ' import json\n' - '\n' - ' tracked = subprocess.check_output(["git", "-C", ' - 'str(repo_root), "ls-files", "-z"])\n' - ' paths = set(tracked.decode().split("\\0")) - {""}\n' - ' assert len(paths) == 228\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(repo_root / relative, ' - 'destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step6_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 38\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["privacy_views_enabled"] is True\n' - ' assert baseline["renderers_enabled"] is True\n' - ' assert baseline["output_publication_enabled"] is ' - 'True\n' - ' assert baseline["cli_analysis_enabled"] is False\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 4\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n repo_root = ' - 'phase4_step4_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n"),\n' - ' "historical_document": ("docs/cli.md", b"# ' - 'CLI\\n",\n' - ' b"# Unauthorized ' - 'historical document\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "html": ' - '"src/recursive_integrity_toolkit/reports/html_report.py",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "config": ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' "hashing": ' - '"src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' "logging": ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 6 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 7\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)'}, - {'new': 'def ' - 'test_phase4_step4_privacy_ast_checks_current_source_before_rejecting_scope_injection(repo_root, ' - 'tmp_path, relative, injection, phase4_step5_snapshot, ' - 'phase4_step6_snapshot):\n' - ' checker = runpy.run_path(str(repo_root / ' - '"scripts/check_traceability.py"), ' - 'run_name="phase4_step4_ast_tests")\n' - ' verify = checker["phase4_step4_runtime_boundary"]\n' - ' repository_path = ' - '"src/recursive_integrity_toolkit/" + relative\n' - ' current = (phase4_step5_snapshot if relative == ' - '"utils/logging.py" else phase4_step6_snapshot if ' - 'relative == "config.py" else repo_root) / ' - 'repository_path\n' - ' verify(current, repository_path)\n' - ' copy = tmp_path / "source.py"\n' - ' copy.write_bytes(current.read_bytes() + injection)\n' - ' with pytest.raises(ValueError):\n' - ' verify(copy, repository_path)', - 'node': 'test_phase4_step4_privacy_ast_checks_current_source_before_rejecting_scope_injection', - 'old': 'def ' - 'test_phase4_step4_privacy_ast_checks_current_source_before_rejecting_scope_injection(repo_root, ' - 'tmp_path, relative, injection, phase4_step5_snapshot):\n' - ' checker = runpy.run_path(str(repo_root / ' - '"scripts/check_traceability.py"), ' - 'run_name="phase4_step4_ast_tests")\n' - ' verify = checker["phase4_step4_runtime_boundary"]\n' - ' repository_path = ' - '"src/recursive_integrity_toolkit/" + relative\n' - ' current = (phase4_step5_snapshot if relative == ' - '"utils/logging.py" else repo_root) / repository_path\n' - ' verify(current, repository_path)\n' - ' copy = tmp_path / "source.py"\n' - ' copy.write_bytes(current.read_bytes() + injection)\n' - ' with pytest.raises(ValueError):\n' - ' verify(copy, repository_path)'}, - {'new': 'def ' - 'test_phase4_step1_keeps_current_help_and_future_commands_unopened(subprocess_env, ' - 'tmp_path, phase4_step6_snapshot):\n' - ' subprocess_env = dict(subprocess_env, ' - 'PYTHONPATH=str(phase4_step6_snapshot / "src"))\n' - ' before = set(tmp_path.iterdir())\n' - ' result = subprocess.run(\n' - ' [sys.executable, "-m", ' - '"recursive_integrity_toolkit", "--help"],\n' - ' cwd=tmp_path, env=subprocess_env, text=True, ' - 'capture_output=True, check=False,\n' - ' )\n' - ' assert result.returncode == 0 and "version" in ' - 'result.stdout\n' - ' assert "Analytical audit functionality is not ' - 'implemented" in " ".join(result.stdout.split())\n' - ' rejected = subprocess.run(\n' - ' [sys.executable, "-m", ' - '"recursive_integrity_toolkit", "audit"],\n' - ' cwd=tmp_path, env=subprocess_env, text=True, ' - 'capture_output=True, check=False,\n' - ' )\n' - ' assert rejected.returncode == 2\n' - ' assert set(tmp_path.iterdir()) == before', - 'node': 'test_phase4_step1_keeps_current_help_and_future_commands_unopened', - 'old': 'def ' - 'test_phase4_step1_keeps_current_help_and_future_commands_unopened(subprocess_env, ' - 'tmp_path):\n' - ' before = set(tmp_path.iterdir())\n' - ' result = subprocess.run(\n' - ' [sys.executable, "-m", ' - '"recursive_integrity_toolkit", "--help"],\n' - ' cwd=tmp_path, env=subprocess_env, text=True, ' - 'capture_output=True, check=False,\n' - ' )\n' - ' assert result.returncode == 0 and "version" in ' - 'result.stdout\n' - ' assert "Analytical audit functionality is not ' - 'implemented" in " ".join(result.stdout.split())\n' - ' rejected = subprocess.run(\n' - ' [sys.executable, "-m", ' - '"recursive_integrity_toolkit", "audit"],\n' - ' cwd=tmp_path, env=subprocess_env, text=True, ' - 'capture_output=True, check=False,\n' - ' )\n' - ' assert rejected.returncode == 2\n' - ' assert set(tmp_path.iterdir()) == before'}], - 'tests/unit/test_phase4_contracts.py': [{'new': 'def ' - 'test_phase4_step6_approved_control_keeps_independent_step5_anchors(phase4_tools, ' - 'phase4_step6_snapshot):\n' - ' control = json.loads((phase4_step6_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' assert phase4_tools["PHASE4_STEP5_FINAL"] == ' - '"c1667179e895a798bba2349960162542efa5ef87"\n' - ' assert phase4_tools["PHASE4_STEP5_TREE"] == ' - '"d83f9fc8e0b569f2d796bd2a83b87422fe350fba"\n' - ' assert phase4_tools["PHASE4_STEP5_TEST_TREE"] == ' - '"974f8e65904c24ed4e226996eaf571aab6123965"\n' - ' assert control["active_phase"] == 4 and ' - 'control["active_step"] == 6\n' - ' assert control["approval_basis"] == "批准开始Phase 4 ' - '**Step 6**"\n' - ' assert control["baseline_commit"] == BASELINE\n' - ' assert control["previous_step_commit"] == ' - '"c1667179e895a798bba2349960162542efa5ef87"\n' - ' assert control["previous_step_tree"] == ' - '"d83f9fc8e0b569f2d796bd2a83b87422fe350fba"\n' - ' assert control["previous_step_test_tree"] == ' - '"974f8e65904c24ed4e226996eaf571aab6123965"\n' - ' assert control["previous_step_core_tests"] == 3215\n' - ' assert control["previous_step_parquet_tests"] == 3218\n' - ' assert len(control["previous_step_files_sha256"]) == ' - '227\n' - ' assert set(control["runtime_paths_authorized"]) == {\n' - ' "src/recursive_integrity_toolkit/utils/paths.py",\n' - ' ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' }\n' - ' assert control["schema_changes_authorized"] is False\n' - ' assert control["schema_paths_authorized"] == []\n' - ' assert control["new_files_permitted"] == ' - '["tests/unit/test_phase4_output_safety.py"]\n' - ' for name in ("phase_complete", "next_step_authorized", ' - '"main_merge_authorized", "publication_authorized"):\n' - ' assert control[name] is False\n' - ' phase4_tools["verify_phase4_step6_control"](control, ' - 'step=6)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step4_control"](control, ' - 'step=4)', - 'node': 'test_phase4_step6_approved_control_keeps_independent_step5_anchors', - 'old': 'def ' - 'test_phase4_step6_approved_control_keeps_independent_step5_anchors(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' assert phase4_tools["PHASE4_STEP5_FINAL"] == ' - '"c1667179e895a798bba2349960162542efa5ef87"\n' - ' assert phase4_tools["PHASE4_STEP5_TREE"] == ' - '"d83f9fc8e0b569f2d796bd2a83b87422fe350fba"\n' - ' assert phase4_tools["PHASE4_STEP5_TEST_TREE"] == ' - '"974f8e65904c24ed4e226996eaf571aab6123965"\n' - ' assert control["active_phase"] == 4 and ' - 'control["active_step"] == 6\n' - ' assert control["approval_basis"] == "批准开始Phase 4 ' - '**Step 6**"\n' - ' assert control["baseline_commit"] == BASELINE\n' - ' assert control["previous_step_commit"] == ' - '"c1667179e895a798bba2349960162542efa5ef87"\n' - ' assert control["previous_step_tree"] == ' - '"d83f9fc8e0b569f2d796bd2a83b87422fe350fba"\n' - ' assert control["previous_step_test_tree"] == ' - '"974f8e65904c24ed4e226996eaf571aab6123965"\n' - ' assert control["previous_step_core_tests"] == 3215\n' - ' assert control["previous_step_parquet_tests"] == 3218\n' - ' assert len(control["previous_step_files_sha256"]) == ' - '227\n' - ' assert set(control["runtime_paths_authorized"]) == {\n' - ' "src/recursive_integrity_toolkit/utils/paths.py",\n' - ' ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' }\n' - ' assert control["schema_changes_authorized"] is False\n' - ' assert control["schema_paths_authorized"] == []\n' - ' assert control["new_files_permitted"] == ' - '["tests/unit/test_phase4_output_safety.py"]\n' - ' for name in ("phase_complete", "next_step_authorized", ' - '"main_merge_authorized", "publication_authorized"):\n' - ' assert control[name] is False\n' - ' phase4_tools["verify_phase4_step6_control"](control, ' - 'step=6)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step4_control"](control, ' - 'step=4)'}, - {'new': 'def ' - 'test_phase4_step6_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value, phase4_step6_snapshot):\n' - ' control = json.loads((phase4_step6_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' verify = phase4_tools["verify_phase4_step6_control"]\n' - ' verify(control, step=6)\n' - ' control[field] = value\n' - ' with pytest.raises(ValueError):\n' - ' verify(control, step=6)', - 'node': 'test_phase4_step6_control_rejects_forged_scope_and_stage', - 'old': 'def ' - 'test_phase4_step6_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' verify = phase4_tools["verify_phase4_step6_control"]\n' - ' verify(control, step=6)\n' - ' control[field] = value\n' - ' with pytest.raises(ValueError):\n' - ' verify(control, step=6)'}, - {'new': 'def ' - 'test_phase4_step6_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step, phase4_step6_snapshot):\n' - ' control = json.loads((phase4_step6_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step6_control"](control, ' - 'step=6)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step6_control"](control, ' - 'step=step)', - 'node': 'test_phase4_step6_control_rejects_unapproved_dispatch', - 'old': 'def ' - 'test_phase4_step6_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step6_control"](control, ' - 'step=6)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step6_control"](control, ' - 'step=step)'}, - {'new': 'def ' - 'test_phase4_step6_control_requires_explicit_approval_fields(phase4_tools, ' - 'field, phase4_step6_snapshot):\n' - ' control = json.loads((phase4_step6_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step6_control"](control)\n' - ' del control[field]\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step6_control"](control)', - 'node': 'test_phase4_step6_control_requires_explicit_approval_fields', - 'old': 'def ' - 'test_phase4_step6_control_requires_explicit_approval_fields(phase4_tools, ' - 'field):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step6_control"](control)\n' - ' del control[field]\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step6_control"](control)'}, - {'new': 'def ' - 'test_phase4_step6_control_cannot_mint_extra_permission(phase4_tools, ' - 'phase4_step6_snapshot):\n' - ' control = json.loads((phase4_step6_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step6_control"](control)\n' - ' control["approved_scope_expansion"] = {"step": 6, ' - '"publication": True}\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step6_control"](control)', - 'node': 'test_phase4_step6_control_cannot_mint_extra_permission', - 'old': 'def ' - 'test_phase4_step6_control_cannot_mint_extra_permission(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step6_control"](control)\n' - ' control["approved_scope_expansion"] = {"step": 6, ' - '"publication": True}\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step6_control"](control)'}, - {'new': 'def ' - 'test_phase4_step6_historical_bindings_preserve_assertions_and_headers(phase4_tools, ' - 'phase4_step5_snapshot, phase4_step6_snapshot):\n' - ' import ast\n' - '\n' - ' registry = phase4_tools["PHASE4_STEP6_MIGRATIONS"]\n' - ' assert set(registry) == ' - '{"tests/unit/test_phase4_contracts.py", ' - '"tests/integration/test_phase4_gates.py"}\n' - ' assert sorted(len(rows) for rows in registry.values()) ' - '== [7, 8]\n' - ' expected_inventory = {"phase4_mutation_tree"} | {\n' - ' "test_phase4_step" + str(step) + ' - '"_current_snapshot_checks_valid_tree_before_mutations"\n' - ' for step in (2, 3, 4, 5)\n' - ' }\n' - ' inventory_seen = set()\n' - ' for path, rows in registry.items():\n' - ' ' - 'phase4_tools["verify_phase4_step6_test_migration"](\n' - ' path, (phase4_step5_snapshot / ' - 'path).read_bytes(), (phase4_step6_snapshot / ' - 'path).read_bytes())\n' - ' for row in rows:\n' - ' trees = [ast.parse(row[key]) for key in ' - '("old", "new")]\n' - ' assertions = [[ast.dump(n) for n in ' - 'ast.walk(tree) if isinstance(n, ast.Assert)] for tree in ' - 'trees]\n' - ' assert assertions[0] == assertions[1]\n' - ' assert trees[0].body[0].name == ' - 'trees[1].body[0].name == row["node"]\n' - ' if row["node"] in expected_inventory:\n' - ' inventory_seen.add(row["node"])\n' - ' assert \'"ls-files"\' in row["old"] and ' - '\'rglob("*")\' in row["new"]\n' - ' assert inventory_seen == expected_inventory', - 'node': 'test_phase4_step6_historical_bindings_preserve_assertions_and_headers', - 'old': 'def ' - 'test_phase4_step6_historical_bindings_preserve_assertions_and_headers(phase4_tools, ' - 'phase4_step5_snapshot):\n' - ' import ast\n' - '\n' - ' registry = phase4_tools["PHASE4_STEP6_MIGRATIONS"]\n' - ' assert set(registry) == ' - '{"tests/unit/test_phase4_contracts.py", ' - '"tests/integration/test_phase4_gates.py"}\n' - ' assert sorted(len(rows) for rows in registry.values()) ' - '== [7, 8]\n' - ' expected_inventory = {"phase4_mutation_tree"} | {\n' - ' "test_phase4_step" + str(step) + ' - '"_current_snapshot_checks_valid_tree_before_mutations"\n' - ' for step in (2, 3, 4, 5)\n' - ' }\n' - ' inventory_seen = set()\n' - ' for path, rows in registry.items():\n' - ' ' - 'phase4_tools["verify_phase4_step6_test_migration"](\n' - ' path, (phase4_step5_snapshot / ' - 'path).read_bytes(), (ROOT / path).read_bytes())\n' - ' for row in rows:\n' - ' trees = [ast.parse(row[key]) for key in ' - '("old", "new")]\n' - ' assertions = [[ast.dump(n) for n in ' - 'ast.walk(tree) if isinstance(n, ast.Assert)] for tree in ' - 'trees]\n' - ' assert assertions[0] == assertions[1]\n' - ' assert trees[0].body[0].name == ' - 'trees[1].body[0].name == row["node"]\n' - ' if row["node"] in expected_inventory:\n' - ' inventory_seen.add(row["node"])\n' - ' assert \'"ls-files"\' in row["old"] and ' - '\'rglob("*")\' in row["new"]\n' - ' assert inventory_seen == expected_inventory'}, - {'new': 'def ' - 'test_phase4_step6_historical_guard_rejects_weakening(phase4_tools, ' - 'phase4_step5_snapshot, mutation, phase4_step6_snapshot):\n' - ' path = "tests/integration/test_phase4_gates.py" if ' - 'mutation in ("inventory", "logging") else ' - '"tests/unit/test_phase4_contracts.py"\n' - ' before = (phase4_step5_snapshot / path).read_bytes()\n' - ' after = (phase4_step6_snapshot / path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step6_test_migration"]\n' - ' verify(path, before, after)\n' - ' if mutation == "assertion":\n' - ' needle = b\' assert control["active_phase"] == ' - '4 and control["active_step"] == 5\\n\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b" assert ' - 'True\\n", 1)\n' - ' elif mutation == "binding":\n' - " needle = b'json.loads((phase4_step5_snapshot / " - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\'\n' - ' assert needle in after\n' - ' after = after.replace(needle, ' - 'needle.replace(b"phase4_step5_snapshot", b"ROOT"), 1)\n' - ' elif mutation == "inventory":\n' - " needle = b'for path in " - 'phase4_step1_snapshot.rglob("*") if path.is_file()\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, ' - 'needle.replace(b"phase4_step1_snapshot", b"repo_root"), ' - '1)\n' - ' elif mutation == "logging":\n' - " needle = b'(phase4_step5_snapshot if relative == " - '"utils/logging.py" else repo_root)\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b"repo_root", 1)\n' - ' elif mutation == "shadow":\n' - ' after += b"\\ndef ' - 'test_phase4_step5_fixed_boundary_opens_only_approved_existing_paths():\\n ' - 'assert True\\n"\n' - ' else:\n' - ' after += b"\\ndef ' - 'test_phase4_step6_bad(value=globals().clear()):\\n ' - 'pass\\n"\n' - ' with pytest.raises(ValueError):\n' - ' verify(path, before, after)', - 'node': 'test_phase4_step6_historical_guard_rejects_weakening', - 'old': 'def ' - 'test_phase4_step6_historical_guard_rejects_weakening(phase4_tools, ' - 'phase4_step5_snapshot, mutation):\n' - ' path = "tests/integration/test_phase4_gates.py" if ' - 'mutation in ("inventory", "logging") else ' - '"tests/unit/test_phase4_contracts.py"\n' - ' before = (phase4_step5_snapshot / path).read_bytes()\n' - ' after = (ROOT / path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step6_test_migration"]\n' - ' verify(path, before, after)\n' - ' if mutation == "assertion":\n' - ' needle = b\' assert control["active_phase"] == ' - '4 and control["active_step"] == 5\\n\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b" assert ' - 'True\\n", 1)\n' - ' elif mutation == "binding":\n' - " needle = b'json.loads((phase4_step5_snapshot / " - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\'\n' - ' assert needle in after\n' - ' after = after.replace(needle, ' - 'needle.replace(b"phase4_step5_snapshot", b"ROOT"), 1)\n' - ' elif mutation == "inventory":\n' - " needle = b'for path in " - 'phase4_step1_snapshot.rglob("*") if path.is_file()\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, ' - 'needle.replace(b"phase4_step1_snapshot", b"repo_root"), ' - '1)\n' - ' elif mutation == "logging":\n' - " needle = b'(phase4_step5_snapshot if relative == " - '"utils/logging.py" else repo_root)\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b"repo_root", 1)\n' - ' elif mutation == "shadow":\n' - ' after += b"\\ndef ' - 'test_phase4_step5_fixed_boundary_opens_only_approved_existing_paths():\\n ' - 'assert True\\n"\n' - ' else:\n' - ' after += b"\\ndef ' - 'test_phase4_step6_bad(value=globals().clear()):\\n ' - 'pass\\n"\n' - ' with pytest.raises(ValueError):\n' - ' verify(path, before, after)'}]} - - -def _phase4_step6_files(): - """Read the exact accepted Step 6 Git tree and verify every blob identity.""" - for suffix, expected in (("^{commit}", PHASE4_STEP6_FINAL), ("^{tree}", PHASE4_STEP6_TREE), - (":tests", PHASE4_STEP6_TEST_TREE)): - if git("rev-parse", PHASE4_STEP6_FINAL + suffix).decode().strip() != expected: - raise ValueError("Pinned Phase 4 Step 6 identity mismatch") - objects = {} - for entry in git("ls-tree", "-rz", PHASE4_STEP6_FINAL).split(b"\0"): - if not entry: - continue - metadata, raw_path = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - path = raw_path.decode("utf-8") - if (mode not in (b"100644", b"100755") or kind != b"blob" or path in objects - or path.startswith("/") or ".." in path.split("/") or ".git" in path.split("/")): - raise ValueError("Unsafe pinned Step 6 Git object") - objects[path] = oid.decode("ascii") - if len(objects) != 228: - raise ValueError("Pinned Step 6 must contain exactly 228 files") - files = {} - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_STEP6_FINAL))) as archive: - names = [item.filename for item in archive.infolist() if not item.is_dir()] - if len(names) != len(objects) or set(names) != set(objects): - raise ValueError("Pinned Step 6 archive identity mismatch") - for name in names: - raw = archive.read(name) - oid = hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - if oid != objects[name]: - raise ValueError(f"Pinned Step 6 blob mismatch: {name}") - files[name] = raw - return MappingProxyType(files) - - -def phase4_step7_expected_control() -> dict: - """Actual Step 7 approval is independent of the mutable control document.""" - prior = _phase4_step6_files() - result = phase4_step6_expected_control() - result.update({ - "control_version": "1.6", "active_step": 7, - "approval_date": PHASE4_STEP7_APPROVAL_DATE, "approval_basis": PHASE4_STEP7_APPROVAL, - "previous_step_commit": PHASE4_STEP6_FINAL, - "previous_step_tree": PHASE4_STEP6_TREE, - "previous_step_test_tree": PHASE4_STEP6_TEST_TREE, - "previous_step_core_tests": 3420, "previous_step_parquet_tests": 3423, - "previous_step_files_sha256": {p: hashlib.sha256(raw).hexdigest() for p, raw in sorted(prior.items())}, - "permitted_paths": sorted(PHASE4_STEP7_ALLOWED), "new_files_permitted": list(PHASE4_STEP7_NEW), - "runtime_changes_authorized": True, "schema_changes_authorized": False, - "runtime_paths_authorized": sorted(p for p in PHASE4_STEP7_ALLOWED if p.startswith("src/")), - "schema_paths_authorized": [], - "step6_historical_binding_nodes": {p: sorted({r["node"] for r in rows}) - for p, rows in sorted(PHASE4_STEP7_MIGRATIONS.items())}, - }) - return result - - -def verify_phase4_step7_control(control: dict, step: int = 7) -> None: - if type(step) is not int or step != 7 or type(control) is not dict: - raise ValueError("Unsupported Phase 4 Step 7 stage/control") - try: - actual = json.dumps(control, sort_keys=True, ensure_ascii=True, allow_nan=False) - expected = json.dumps(phase4_step7_expected_control(), sort_keys=True, ensure_ascii=True, allow_nan=False) - except (TypeError, ValueError) as error: - raise ValueError("Invalid Phase 4 Step 7 control") from error - if actual != expected: - raise ValueError("Phase 4 control differs from the independently approved Step 7 contract") - - -def verify_phase4_step7_changes(changes: list[tuple[str, str]]) -> None: - seen = set() - for status, path in changes: - if path in seen or path not in PHASE4_STEP7_ALLOWED or status != ("A" if path in PHASE4_STEP7_NEW else "M"): - raise ValueError(f"Unapproved Phase 4 Step 7 path/operation: {status} {path}") - seen.add(path) - - -def _phase4_step7_header(node, path): - """Allow explicit test parametrization/fixtures, never definition-time effects.""" - for decorator in node.decorator_list: - if not isinstance(decorator, ast.Call) or ast.unparse(decorator.func) not in { - "pytest.fixture", "pytest.mark.parametrize"}: - raise ValueError(f"Unapproved Step 7 test decorator: {path}:{node.name}") - if ast.unparse(decorator.func) == "pytest.fixture" and not node.name.startswith("phase4_step"): - raise ValueError("A Step 7 fixture must have an explicit scoped name") - if any(keyword.arg is None for keyword in decorator.keywords): - raise ValueError("Decorator expansion is outside the Step 7 contract") - for value in [*decorator.args, *(keyword.value for keyword in decorator.keywords)]: - try: - ast.literal_eval(value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError("Step 7 decorator arguments must be literal") from error - clone = ast.parse(ast.unparse(node)).body[0] - clone.decorator_list = [] - _phase4_preserve_function_header(clone, path) - - -def _phase4_step7_append_only(before: bytes, after: bytes, path: str) -> None: - if not after.startswith(before): - raise ValueError(f"Inherited Step 6 prefix changed: {path}") - old, new = ast.parse(before), ast.parse(after) - bindings = set() - for node in old.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - bindings.add(node.name) - elif isinstance(node, (ast.Import, ast.ImportFrom)): - bindings.update(alias.asname or alias.name.split(".")[0] for alias in node.names) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - bindings.update(child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)) - for node in new.body[len(old.body):]: - if not isinstance(node, ast.FunctionDef) or node.name in bindings: - raise ValueError(f"Step 7 addition executes, rebinds or shadows inherited source: {path}") - allowed_snapshot = path == "tests/conftest.py" and node.name == "phase4_step6_snapshot" - if not allowed_snapshot and not node.name.startswith(("test_phase4_step7_", "phase4_step7_")): - raise ValueError(f"Step 7 added test/helper is not explicitly scoped: {path}:{node.name}") - if path == "tests/conftest.py" and node.name != "phase4_step6_snapshot": - raise ValueError("Only the pinned Step 6 shared fixture is authorized") - bindings.add(node.name) - _phase4_step7_header(node, path) - - -def verify_phase4_step7_test_migration(path: str, before: bytes, after: bytes) -> None: - prior = _phase4_step6_files() - if path not in prior or path not in PHASE4_STEP7_ALLOWED or before != prior[path] or not path.startswith("tests/"): - raise ValueError("Step 7 migration requires the exact named Step 6 source") - _phase4_step7_check_test_migration(path, before, after) - - -def _phase4_step7_check_test_migration(path: str, before: bytes, after: bytes) -> None: - """Check a source already read from the verified Step 6 immutable mapping.""" - expected = before - for row in PHASE4_STEP7_MIGRATIONS.get(path, []): - old, new = row["old"].encode(), row["new"].encode() - if expected.count(old) != 1: - raise ValueError("Step 6 historical binding is not unique") - expected = expected.replace(old, new, 1) - _phase4_step7_append_only(expected, after, path) - - -def _phase4_step7_preserve_tooling(before: bytes, after: bytes, path: str) -> None: - """Preserve exact inherited statements using one source-line split per tree.""" - if path == "scripts/release_check.py": - old = ' return phase4_cli_main() if explicit_phase4 else main()' - new = (' explicit_step7 = "--step=7" in argv or any(a == "--step" and b == "7" for a, b in zip(argv, argv[1:]))\n' - ' if explicit_phase4 and explicit_step7:\n' - ' return phase4_step7_cli_main()\n' + old) - # Earlier preserved dispatch functions contain this source too; bind cli_main only. - parsed = ast.parse(before) - entry = next(node for node in parsed.body if isinstance(node, ast.FunctionDef) and node.name == "cli_main") - old_entry = ast.get_source_segment(before.decode(), entry) - if old_entry.count(old) != 1: - raise ValueError("Step 6 release dispatcher identity mismatch") - replacements = [(old_entry, old_entry.replace(old, new, 1))] - else: - old = ' if args.phase == 4 and args.step == 6:\n return phase4_step6_main(step=args.step)\n' - new = old + ' if args.phase == 4 and args.step == 7:\n return phase4_step7_main(step=args.step)\n' - replacements = [(old, new), - ('Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5 or --phase 4 --step 6', - 'Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5 or --phase 4 --step 6 or --phase 4 --step 7')] - expected = before - for old, new in replacements: - if expected.count(old.encode()) != 1: - raise ValueError(f"Historical dispatcher identity mismatch: {path}") - expected = expected.replace(old.encode(), new.encode(), 1) - trees = [ast.parse(expected), ast.parse(after)] - lines = [expected.splitlines(keepends=True), after.splitlines(keepends=True)] - def segment(index, node): - if node.lineno == node.end_lineno: - return lines[index][node.lineno - 1][node.col_offset:node.end_col_offset] - return (lines[index][node.lineno - 1][node.col_offset:] - + b"".join(lines[index][node.lineno:node.end_lineno - 1]) - + lines[index][node.end_lineno - 1][:node.end_col_offset]) - def entry_guard(node): - return isinstance(node, ast.If) and isinstance(node.test, ast.Compare) and "__name__" in ast.unparse(node.test) - guards = [n for n in trees[1].body if entry_guard(n)] - expected_guard = ast.parse('if __name__ == "__main__":\n raise SystemExit(cli_main())').body[0] - if len(guards) != 1 or ast.dump(guards[0]) != ast.dump(expected_guard): - raise ValueError(f"Unapproved Step 7 maintainer entrypoint: {path}") - historical = [n for n in trees[0].body if not entry_guard(n)] - current = [n for n in trees[1].body if not entry_guard(n)] - old_entries = [(segment(0, n), ast.dump(n)) for n in historical] - cursor = 0 - for node in current: - if cursor < len(old_entries) and (segment(1, node), ast.dump(node)) == old_entries[cursor]: - cursor += 1 - continue - if isinstance(node, ast.FunctionDef) and (node.name == "_phase4_step6_files" or node.name.startswith( - ("phase4_step7_", "_phase4_step7_", "verify_phase4_step7_", "audit_phase4_step7"))): - _phase4_preserve_function_header(node, path) - continue - if isinstance(node, ast.Assign) and all(isinstance(target, ast.Name) and target.id.startswith( - ("PHASE4_STEP7_", "PHASE4_STEP6_")) for target in node.targets): - try: - ast.literal_eval(node.value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError(f"Nonliteral added Step 7 maintainer constant: {path}") from error - continue - raise ValueError(f"Unapproved Step 7 maintainer addition: {path}") - if cursor != len(historical): - raise ValueError(f"Inherited Step 6 maintainer statement changed: {path}") - def binding_counts(tree): - counts = {} - for node in tree.body: - names = [] - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - names = [node.name] - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names = [child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)] - elif isinstance(node, (ast.Import, ast.ImportFrom)): - names = [alias.asname or alias.name.split(".")[0] for alias in node.names] - for name in names: - counts[name] = counts.get(name, 0) + 1 - return counts - inherited_counts = binding_counts(trees[0]) - for name, count in binding_counts(trees[1]).items(): - if count > inherited_counts.get(name, 1): - raise ValueError(f"Duplicate Step 7 maintainer binding: {path}:{name}") - - -def verify_phase4_step7_snapshot(root: Path = ROOT) -> dict: - prior = _phase4_step6_files() - for path, raw in prior.items(): - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError(f"Missing or aliased inherited Step 6 file: {path}") - current = target.read_bytes() - if path not in PHASE4_STEP7_ALLOWED and current != raw: - raise ValueError(f"Protected Step 6 bytes changed: {path}") - if path.startswith("tests/") and path in PHASE4_STEP7_ALLOWED: - _phase4_step7_check_test_migration(path, raw, current) - elif path.startswith("scripts/") and path in PHASE4_STEP7_ALLOWED: - _phase4_step7_preserve_tooling(raw, current, path) - elif path in {"docs/architecture.md", "docs/theory_traceability.md", "PHASE_4_DECISIONS.md"}: - if not current.startswith(raw): - raise ValueError(f"Step 6 historical documentation prefix changed: {path}") - if not (root / "docs/cli.md").read_bytes().endswith(prior["docs/cli.md"]): - raise ValueError("Step 7 CLI documentation must retain the historical notes verbatim") - for path in PHASE4_STEP7_NEW: - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError("Missing Step 7 output safety tests") - _phase4_step7_new_test(target.read_bytes(), path) - for path in ("src/recursive_integrity_toolkit/config.py",): - _phase4_step7_preserve_runtime(prior[path], (root / path).read_bytes(), path) - actual_modules = {p.relative_to(root).as_posix() for p in (root / "src/recursive_integrity_toolkit").rglob("*.py")} - expected_modules = {p for p in prior if p.startswith("src/") and p.endswith(".py")} - if actual_modules != expected_modules or len(actual_modules) != 40: - raise ValueError("Step 7 cannot change the runtime module set") - if {p.name for p in (root / "schemas").iterdir()} != {Path(p).name for p in prior if p.startswith("schemas/")}: - raise ValueError("Step 7 cannot change the schema set") - if hashlib.sha256((root / "PHASE_4_PLAN.md").read_bytes()).hexdigest() != PHASE4_PLAN_SHA256: - raise ValueError("Approved Phase 4 plan changed") - verify_phase4_step7_control(json.loads((root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))) - decisions = (root / "PHASE_4_DECISIONS.md").read_text(encoding="utf-8") - if PHASE4_STEP7_APPROVAL not in decisions or PHASE4_STEP6_FINAL not in decisions: - raise ValueError("Actual Step 7 authorization or Step 6 evidence anchor missing") - if any((root / name).exists() for name in PHASE4_FORBIDDEN_OUTPUTS): - raise ValueError("Step 7 cannot create Phase 4 completion records or audit outputs") - import runpy - checker = runpy.run_path(str(root / "scripts/check_traceability.py"), run_name="phase4_step7_renderer_boundary") - checker["phase4_step7_cli_boundary"](root) - schema_bytes = (root / "schemas/report.schema.json").read_bytes() - if hashlib.sha256(schema_bytes).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Step 7 report schema differs from its independently reviewed bytes") - schema = json.loads(schema_bytes) - if (schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema" - or schema.get("type") != "object" or schema.get("additionalProperties") is not False): - raise ValueError("Step 7 report schema dialect/closed root mismatch") - return {"package_modules": 40, "frozen_runtime_modules": 38, "frozen_schemas": 5, - "hero_files_unchanged": 6, "historical_phase3_migrated_nodes": 16, - "phase_complete": False, "result_contracts_enabled": True, - "adapters_enabled": True, "privacy_views_enabled": True, "renderers_enabled": True, "output_publication_enabled": True, "cli_analysis_enabled": True, "comparison_enabled": False, "example_enabled": False} - - -def audit_phase4_step7(step: int = 7) -> dict: - verify_phase4_step7_control(json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")), step) - result = {"phase": 4, "active_step": step, **verify_phase4_step7_snapshot(), - "phase0_hashes_verified": 16, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_phase4_step7_diff(step: int = 7) -> dict: - if type(step) is not int or step != 7: - raise ValueError("Unsupported Phase 4 Step 7 stage") - _phase4_step6_files() - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE4_STEP6_FINAL, "HEAD"], check=True) - raw = git("diff", "--name-status", "--no-renames", "-z", PHASE4_STEP6_FINAL, "--").split(b"\0") - raw = [part.decode("utf-8") for part in raw if part] - if len(raw) % 2: - raise ValueError("Malformed Step 7 Git difference records") - changes = list(zip(raw[::2], raw[1::2])) - changes.extend(("A", p.decode()) for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") if p) - verify_phase4_step7_changes(changes) - result = {"previous_step_commit": PHASE4_STEP6_FINAL, "changed_files": len(changes), - "changes": changes, "step": 7, "scope": "PASS"} - print(json.dumps(result, indent=2)) - return result - - -def phase4_step7_baseline_evidence(output: Path) -> dict: - """Reconcile all inherited identities, including the accepted Step 6 suite.""" - output.mkdir(parents=True, exist_ok=True) - inherited = phase4_step6_baseline_evidence(output / "phase4-step6-inherited") - with tempfile.TemporaryDirectory(prefix="rit-p4-step6-identities-") as temp: - baseline = Path(temp) - for name, raw in _phase4_step6_files().items(): - target = baseline / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(raw) - old, old_log = _collect(baseline) - current, current_log = _collect(ROOT) - expected = 3423 if os.environ.get("RIT_TEST_PARQUET") == "1" else 3420 - if len(old) != expected or set(old) - set(current): - raise ValueError("Accepted Phase 4 Step 6 test identities were lost") - result = {"baseline_commit": PHASE4_STEP6_FINAL, "baseline_test_tree": PHASE4_STEP6_TEST_TREE, - "baseline_nodeids": old, "current_nodeids": current, "missing_nodeids": [], - "baseline_tests": len(old), "current_tests": len(current), "inherited": inherited, - "nodeids_sha256": hashlib.sha256(("\n".join(old)+"\n").encode()).hexdigest()} - (output / "phase4_step7_test_identity_manifest.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4-step6-collection.log").write_text(old_log, encoding="utf-8") - (output / "phase4-step7-collection.log").write_text(current_log, encoding="utf-8") - print(json.dumps({k: result[k] for k in ("baseline_tests", "current_tests", "missing_nodeids")}, indent=2)) - return result - - -def phase4_step7_candidate(output: Path, step: int = 7) -> None: - """Build tested Step 7 intermediate evidence without declaring Phase 4 complete.""" - if os.environ.get("RIT_TEST_PARQUET") != "1" or importlib.util.find_spec("pyarrow") is None: - raise ValueError("Step 7 candidate evidence requires RIT_TEST_PARQUET=1 and real PyArrow") - output.mkdir(parents=True, exist_ok=True) - result = audit_phase4_step7(step) - result["diff"] = audit_phase4_step7_diff(step) - if git("status", "--porcelain").strip(): - raise ValueError("Step 7 candidate archive requires committed, clean source") - result["core"] = verify_junit(output / "core.xml", minimum=3420) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=3423) - result["core_math_measurements"] = verify_step10_evidence(output / "core.xml", output / "phase4-step7-core-observations.json") - result["parquet_math_measurements"] = verify_step10_evidence(output / "parquet.xml", output / "phase4-step7-parquet-observations.json") - identity = phase4_step7_baseline_evidence(output) - result["test_identity"] = {k: identity[k] for k in ("baseline_tests", "current_tests", "nodeids_sha256")} - expected = set() - for node in identity["current_nodeids"]: - base, bracket, parameter = node.partition("[") - owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) - for name, parquet in (("core.xml", False), ("parquet.xml", True)): - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(output / name).getroot().iter("testcase")} - target = expected if parquet else {c for c in expected if c[1] not in PARQUET_CASES} - if cases != target: - raise ValueError(f"Step 7 JUnit does not execute the entire current suite: {name}") - wheel, _ = verify_distributions(output / "dist") - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - phase4_step6_installed_publication_smoke(wheel) - phase4_step7_installed_cli_smoke(wheel) - archive = output / "recursive-integrity-toolkit-phase4-step7-candidate.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {p for p in git("ls-files", "-z").decode().split("\0") if p} - with zipfile.ZipFile(archive) as zipped: - entries = [item.filename for item in zipped.infolist() if not item.is_dir()] - prefix = "recursive-integrity-toolkit/" - if (len(entries) != len(set(entries)) or any(not name.startswith(prefix) for name in entries)): - raise ValueError("Step 7 source archive has duplicate or unprefixed entries") - names = {name.removeprefix(prefix) for name in entries} - if names != tracked or len(tracked) != 229 or len(entries) != 229: - raise ValueError("Step 7 source archive file set mismatch") - for name in names: - if zipped.read("recursive-integrity-toolkit/"+name) != (ROOT / name).read_bytes(): - raise ValueError(f"Step 7 source archive byte mismatch: {name}") - result.update({"commit": git("rev-parse", "HEAD").decode().strip(), - "tree": git("rev-parse", "HEAD^{tree}").decode().strip(), - "source_archive": archive.name, "tracked_files": len(tracked), - "python": sys.version, "platform": sys.platform, - "versions": {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "pytest")}}) - (output / "phase4_step7_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4_step7_repository_files.sha256").write_text("".join(f"{hashlib.sha256((ROOT/name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - paths = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase4_step7_artifacts.sha256") - (output / "phase4_step7_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in paths), encoding="utf-8") - print(f"Phase 4 Step 7 candidate: {len(tracked)} source files verified; Phase 4 remains incomplete.") - - -def phase4_step7_cli_main() -> int: - parser = argparse.ArgumentParser(description="Phase 4 Step 7 maintainer gate") - parser.add_argument("--phase", type=int, choices=(4,), required=True) - parser.add_argument("--step", type=int, choices=(7,), required=True) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) - parser.add_argument("--require-parquet", action="store_true") - parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - raise ValueError("Phase 4 Step 7 cannot certify a final phase delivery") - if args.junit is not None: - verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - phase4_step7_baseline_evidence(args.baseline_evidence) - elif args.dist is not None: - audit_phase4_step7(args.step) - wheel, _ = verify_distributions(args.dist) - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - phase4_step6_installed_publication_smoke(wheel) - phase4_step7_installed_cli_smoke(wheel) - elif args.smoke_wheel is not None: - smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - phase4_step7_candidate(args.candidate, args.step) - else: - audit_phase4_step7(args.step) - if args.diff: - audit_phase4_step7_diff(args.step) - return 0 - - -def _phase4_step7_new_test(raw: bytes, path: str) -> None: - """Only literal test declarations and explicitly listed safe imports.""" - allowed = {"__future__", "json", "pathlib", "pytest"} - for index, node in enumerate(ast.parse(raw).body): - if index == 0 and isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str): - continue - if isinstance(node, (ast.Import, ast.ImportFrom)): - modules = [node.module] if isinstance(node, ast.ImportFrom) else [alias.name for alias in node.names] - if any(module not in allowed for module in modules): - raise ValueError("Unapproved Step 7 test import") - continue - if not isinstance(node, ast.FunctionDef) or not node.name.startswith(("test_phase4_step7_", "phase4_step7_")): - raise ValueError("Step 7 output tests gained definition-time execution") - _phase4_step7_header(node, path) - - -def _phase4_step7_preserve_runtime(before: bytes, after: bytes, path: str) -> None: - """Allow ownership-header maintenance and byte-exact additive helpers only.""" - try: - old, new = ast.parse(before), ast.parse(after) - except (SyntaxError, UnicodeError, ValueError) as error: - raise ValueError("Output helper source cannot be inspected") from error - if not ast.get_docstring(old) or not ast.get_docstring(new): - raise ValueError("Output helper ownership documentation is required") - old_tail = b"".join(before.splitlines(keepends=True)[old.body[0].end_lineno:]) - new_tail = b"".join(after.splitlines(keepends=True)[new.body[0].end_lineno:]) - if not new_tail.startswith(old_tail): - raise ValueError(f"Inherited input/diagnostic bytes changed: {path}") - - -def phase4_step7_installed_cli_smoke(wheel: Path) -> None: +def smoke_installed_cli(wheel: Path) -> None: """Run the installed CLI with caller core dependencies outside the checkout.""" with tempfile.TemporaryDirectory(prefix="rit-p4-step7-installed-") as temporary: work = Path(temporary) @@ -8280,1442 +1107,7 @@ def phase4_step7_installed_cli_smoke(wheel: Path) -> None: subprocess.run([sys.executable, "-I", "-c", program, str(target), str(work), str(ROOT)], cwd=work, check=True) - -# Phase 4 Step 8: explicit pair and exact packaged resources. - - -PHASE4_STEP7_FINAL = 'bacd33ae65e392872776c6d976401fdc3d565f63' - - -PHASE4_STEP7_TREE = '4c16914bb0f0c5281efc36624efb8b1a1d4a174e' - - -PHASE4_STEP7_TEST_TREE = '96317cb7b25236b79efc0f2d93e1e0f5f132fdb2' - - -PHASE4_STEP8_APPROVAL = 'phase 4 step 8 开始' - - -PHASE4_STEP8_APPROVAL_DATE = '2026-09-21' - - -PHASE4_STEP8_NEW = ('src/recursive_integrity_toolkit/data/hero/EXPECTED_OUTPUTS.md', 'src/recursive_integrity_toolkit/data/hero/config.json', 'src/recursive_integrity_toolkit/data/hero/provenance.csv', 'src/recursive_integrity_toolkit/data/hero/records_v1.csv', 'src/recursive_integrity_toolkit/data/hero/records_v2.csv', 'src/recursive_integrity_toolkit/data/hero/version_order.json', 'src/recursive_integrity_toolkit/data/report.schema.json') - - -PHASE4_STEP8_ALLOWED = {'.github/workflows/ci.yml', - '.github/workflows/golden.yml', - '.github/workflows/release.yml', - '.github/workflows/security.yml', - 'PHASE_4_BASELINE.json', - 'PHASE_4_DECISIONS.md', - 'docs/architecture.md', - 'docs/cli.md', - 'docs/theory_traceability.md', - 'pyproject.toml', - 'scripts/check_spec_consistency.py', - 'scripts/check_traceability.py', - 'scripts/release_check.py', - 'src/recursive_integrity_toolkit/cli.py', - 'src/recursive_integrity_toolkit/config.py', - 'src/recursive_integrity_toolkit/data/hero/EXPECTED_OUTPUTS.md', - 'src/recursive_integrity_toolkit/data/hero/config.json', - 'src/recursive_integrity_toolkit/data/hero/provenance.csv', - 'src/recursive_integrity_toolkit/data/hero/records_v1.csv', - 'src/recursive_integrity_toolkit/data/hero/records_v2.csv', - 'src/recursive_integrity_toolkit/data/hero/version_order.json', - 'src/recursive_integrity_toolkit/data/report.schema.json', - 'tests/conftest.py', - 'tests/integration/test_ci_workflows.py', - 'tests/integration/test_hero_end_to_end.py', - 'tests/integration/test_hero_structure.py', - 'tests/integration/test_license_notices.py', - 'tests/integration/test_no_algorithms.py', - 'tests/integration/test_no_network.py', - 'tests/integration/test_optional_dependency.py', - 'tests/integration/test_owner_ids.py', - 'tests/integration/test_package_import.py', - 'tests/integration/test_package_install.py', - 'tests/integration/test_phase4_cli.py', - 'tests/integration/test_phase4_gates.py', - 'tests/integration/test_prohibited_structure.py', - 'tests/integration/test_repository_structure.py', - 'tests/integration/test_schema_json.py', - 'tests/unit/test_phase4_contracts.py'} - - -PHASE4_STEP8_RESOURCES = {'src/recursive_integrity_toolkit/data/hero/EXPECTED_OUTPUTS.md': 'examples/hero/EXPECTED_OUTPUTS.md', - 'src/recursive_integrity_toolkit/data/hero/config.json': 'examples/hero/config.json', - 'src/recursive_integrity_toolkit/data/hero/provenance.csv': 'examples/hero/provenance.csv', - 'src/recursive_integrity_toolkit/data/hero/records_v1.csv': 'examples/hero/records_v1.csv', - 'src/recursive_integrity_toolkit/data/hero/records_v2.csv': 'examples/hero/records_v2.csv', - 'src/recursive_integrity_toolkit/data/hero/version_order.json': 'examples/hero/version_order.json', - 'src/recursive_integrity_toolkit/data/report.schema.json': 'schemas/report.schema.json'} - - -PHASE4_STEP8_PACKAGING = '\n[tool.setuptools.package-data]\nrecursive_integrity_toolkit = ["data/hero/*", "data/report.schema.json"]\n' - - -PHASE4_STEP8_MIGRATIONS = {'tests/integration/test_phase4_gates.py': [{'new': 'def ' - 'test_phase4_step7_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step7_snapshot):\n' - ' repo_root = phase4_step7_snapshot\n' - ' names = {"ci.yml", "golden.yml", "security.yml", ' - '"release.yml"}\n' - ' root = repo_root / ".github/workflows"\n' - ' expected_timeouts = {"ci.yml": [60, 20], ' - '"golden.yml": [15], "security.yml": [20], ' - '"release.yml": [40]}\n' - ' assert {path.name for path in root.glob("*.yml")} ' - '== names\n' - ' for name in sorted(names):\n' - ' text = (root / ' - 'name).read_text(encoding="utf-8")\n' - ' assert [int(line.split(":", 1)[1]) for line in ' - 'text.splitlines()\n' - ' if ' - 'line.strip().startswith("timeout-minutes:")] == ' - 'expected_timeouts[name]\n' - ' assert "Phase 4 Step 7" in text, name\n' - ' assert "--phase 4 --step 7" in text, name\n' - ' for older in ("--phase 4 --step 6", "--phase 4 ' - '--step 5", "--phase 4 --step 4", "--phase 4 --step 3", ' - '"--phase 4 --step 2", "--phase 4 --step 1", "--phase 3 ' - '--step 11"):\n' - ' assert older not in text, (name, older)\n' - ' assert "permissions:\\n contents: read" in ' - 'text\n' - ' assert "persist-credentials: false" in text\n' - ' assert "timeout-minutes:" in text and "set -euo ' - 'pipefail" in text\n' - ' assert "actions/upload-artifact@v4" in text and ' - '"if-no-files-found: error" in text\n' - ' for line in text.splitlines():\n' - ' if "python scripts/release_check.py" in ' - 'line:\n' - ' assert "--phase 4 --step 7" in line, ' - '(name, line)\n' - ' for forbidden in ("continue-on-error:", "|| ' - 'true", "contents: write", "id-token: write", "twine ' - 'upload", "git push"):\n' - ' assert forbidden not in text, (name, ' - 'forbidden)\n' - ' ci = (root / "ci.yml").read_text(encoding="utf-8")\n' - " for required in ('os: [ubuntu-latest, " - 'windows-latest]\', \'python-version: ["3.11", ' - '"3.12"]\',\n' - " 'dependencies: [current, " - 'minimum]\', \'"numpy==2.0.0" "pandas==2.2.2"\',\n' - ' \'RIT_TEST_PARQUET: "0"\', ' - '\'RIT_TEST_PARQUET: "1"\', \'--require-parquet\',\n' - " '--baseline-evidence', 'python -m " - "pip check',\n" - " '--minimum-tests 3420', " - "'--minimum-tests 3423',\n" - ' "find_spec(\'pyarrow\') is None", ' - "'import pyarrow'):\n" - ' assert required in ci\n' - ' assert ci.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2\n' - ' assert " -k " not in ci\n' - ' golden = (root / ' - '"golden.yml").read_text(encoding="utf-8")\n' - ' for required in ' - '("tests/golden/test_phase3_math.py", ' - '"tests/integration/test_hero_structure.py",\n' - ' ' - '"tests/integration/test_phase3_metric_pipeline.py"):\n' - ' assert required in golden\n' - ' security = (root / ' - '"security.yml").read_text(encoding="utf-8")\n' - ' for required in ' - "('tests/integration/test_no_network.py', " - "'tests/integration/test_optional_dependency.py',\n" - ' ' - "'tests/unit/test_PR012_evidence_classes.py', " - "'tests/unit/test_PR013_report_schema.py',\n" - ' ' - "'tests/unit/test_PR014_unavailable.py', " - "'tests/unit/test_PR015_redaction.py',\n" - ' ' - "'tests/unit/test_PR016_determinism.py', " - "'tests/unit/test_PR018_language.py',\n" - ' ' - "'tests/integration/test_partial_provenance_report.py',\n" - ' ' - "'tests/integration/test_partial_lineage_report.py',\n" - ' ' - "'tests/integration/test_phase4_cli.py', " - "'tests/unit/test_phase4_output_safety.py', " - "'tests/unit/test_phase4_contracts.py', " - "'tests/integration/test_phase4_gates.py'):\n" - ' assert required in security\n' - ' release = (root / ' - '"release.yml").read_text(encoding="utf-8")\n' - ' assert "python -m build" in release and "python -m ' - 'twine check --strict" in release\n' - ' assert "--dist" in release and "--candidate" in ' - 'release\n' - ' assert "--delivery" not in release\n' - ' assert ' - '"recursive-integrity-toolkit-phase4-step7-candidate" in ' - 'release\n' - ' assert "rit-phase4-step4" not in release\n' - ' assert "--minimum-tests 3420" in release and ' - '"--minimum-tests 3423" in release\n' - ' assert release.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2', - 'node': 'test_phase4_step7_current_workflows_preserve_matrix_and_use_active_dispatch', - 'old': 'def ' - 'test_phase4_step7_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root):\n' - ' names = {"ci.yml", "golden.yml", "security.yml", ' - '"release.yml"}\n' - ' root = repo_root / ".github/workflows"\n' - ' expected_timeouts = {"ci.yml": [60, 20], ' - '"golden.yml": [15], "security.yml": [20], ' - '"release.yml": [40]}\n' - ' assert {path.name for path in root.glob("*.yml")} ' - '== names\n' - ' for name in sorted(names):\n' - ' text = (root / ' - 'name).read_text(encoding="utf-8")\n' - ' assert [int(line.split(":", 1)[1]) for line in ' - 'text.splitlines()\n' - ' if ' - 'line.strip().startswith("timeout-minutes:")] == ' - 'expected_timeouts[name]\n' - ' assert "Phase 4 Step 7" in text, name\n' - ' assert "--phase 4 --step 7" in text, name\n' - ' for older in ("--phase 4 --step 6", "--phase 4 ' - '--step 5", "--phase 4 --step 4", "--phase 4 --step 3", ' - '"--phase 4 --step 2", "--phase 4 --step 1", "--phase 3 ' - '--step 11"):\n' - ' assert older not in text, (name, older)\n' - ' assert "permissions:\\n contents: read" in ' - 'text\n' - ' assert "persist-credentials: false" in text\n' - ' assert "timeout-minutes:" in text and "set -euo ' - 'pipefail" in text\n' - ' assert "actions/upload-artifact@v4" in text and ' - '"if-no-files-found: error" in text\n' - ' for line in text.splitlines():\n' - ' if "python scripts/release_check.py" in ' - 'line:\n' - ' assert "--phase 4 --step 7" in line, ' - '(name, line)\n' - ' for forbidden in ("continue-on-error:", "|| ' - 'true", "contents: write", "id-token: write", "twine ' - 'upload", "git push"):\n' - ' assert forbidden not in text, (name, ' - 'forbidden)\n' - ' ci = (root / "ci.yml").read_text(encoding="utf-8")\n' - " for required in ('os: [ubuntu-latest, " - 'windows-latest]\', \'python-version: ["3.11", ' - '"3.12"]\',\n' - " 'dependencies: [current, " - 'minimum]\', \'"numpy==2.0.0" "pandas==2.2.2"\',\n' - ' \'RIT_TEST_PARQUET: "0"\', ' - '\'RIT_TEST_PARQUET: "1"\', \'--require-parquet\',\n' - " '--baseline-evidence', 'python -m " - "pip check',\n" - " '--minimum-tests 3420', " - "'--minimum-tests 3423',\n" - ' "find_spec(\'pyarrow\') is None", ' - "'import pyarrow'):\n" - ' assert required in ci\n' - ' assert ci.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2\n' - ' assert " -k " not in ci\n' - ' golden = (root / ' - '"golden.yml").read_text(encoding="utf-8")\n' - ' for required in ' - '("tests/golden/test_phase3_math.py", ' - '"tests/integration/test_hero_structure.py",\n' - ' ' - '"tests/integration/test_phase3_metric_pipeline.py"):\n' - ' assert required in golden\n' - ' security = (root / ' - '"security.yml").read_text(encoding="utf-8")\n' - ' for required in ' - "('tests/integration/test_no_network.py', " - "'tests/integration/test_optional_dependency.py',\n" - ' ' - "'tests/unit/test_PR012_evidence_classes.py', " - "'tests/unit/test_PR013_report_schema.py',\n" - ' ' - "'tests/unit/test_PR014_unavailable.py', " - "'tests/unit/test_PR015_redaction.py',\n" - ' ' - "'tests/unit/test_PR016_determinism.py', " - "'tests/unit/test_PR018_language.py',\n" - ' ' - "'tests/integration/test_partial_provenance_report.py',\n" - ' ' - "'tests/integration/test_partial_lineage_report.py',\n" - ' ' - "'tests/integration/test_phase4_cli.py', " - "'tests/unit/test_phase4_output_safety.py', " - "'tests/unit/test_phase4_contracts.py', " - "'tests/integration/test_phase4_gates.py'):\n" - ' assert required in security\n' - ' release = (root / ' - '"release.yml").read_text(encoding="utf-8")\n' - ' assert "python -m build" in release and "python -m ' - 'twine check --strict" in release\n' - ' assert "--dist" in release and "--candidate" in ' - 'release\n' - ' assert "--delivery" not in release\n' - ' assert ' - '"recursive-integrity-toolkit-phase4-step7-candidate" in ' - 'release\n' - ' assert "rit-phase4-step4" not in release\n' - ' assert "--minimum-tests 3420" in release and ' - '"--minimum-tests 3423" in release\n' - ' assert release.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2'}, - {'new': 'def ' - 'test_phase4_step7_current_cli_ast_rejects_injection(repo_root, ' - 'tmp_path, relative, injection, phase4_step7_snapshot):\n' - ' repo_root = phase4_step7_snapshot\n' - ' checker = runpy.run_path(str(repo_root / ' - '"scripts/check_traceability.py"), ' - 'run_name="phase4_step7_ast_tests")\n' - ' verify = checker["phase4_step7_runtime_boundary"]\n' - ' path = "src/recursive_integrity_toolkit/" + ' - 'relative\n' - ' verify(repo_root / path, path)\n' - ' changed = tmp_path / "changed.py"\n' - ' changed.write_bytes((repo_root / path).read_bytes() ' - '+ injection)\n' - ' with pytest.raises(ValueError):\n' - ' verify(changed, path)', - 'node': 'test_phase4_step7_current_cli_ast_rejects_injection', - 'old': 'def ' - 'test_phase4_step7_current_cli_ast_rejects_injection(repo_root, ' - 'tmp_path, relative, injection):\n' - ' checker = runpy.run_path(str(repo_root / ' - '"scripts/check_traceability.py"), ' - 'run_name="phase4_step7_ast_tests")\n' - ' verify = checker["phase4_step7_runtime_boundary"]\n' - ' path = "src/recursive_integrity_toolkit/" + ' - 'relative\n' - ' verify(repo_root / path, path)\n' - ' changed = tmp_path / "changed.py"\n' - ' changed.write_bytes((repo_root / path).read_bytes() ' - '+ injection)\n' - ' with pytest.raises(ValueError):\n' - ' verify(changed, path)'}, - {'new': 'def ' - 'test_phase4_step7_new_cli_tests_reject_definition_effects(repo_root, ' - 'phase4_gate_tools, injection, phase4_step7_snapshot):\n' - ' repo_root = phase4_step7_snapshot\n' - ' path = "tests/integration/test_phase4_cli.py"\n' - ' raw = (repo_root / path).read_bytes()\n' - ' verify = ' - 'phase4_gate_tools["_phase4_step7_new_test"]\n' - ' verify(raw, path)\n' - ' with pytest.raises(ValueError):\n' - ' verify(raw + injection, path)', - 'node': 'test_phase4_step7_new_cli_tests_reject_definition_effects', - 'old': 'def ' - 'test_phase4_step7_new_cli_tests_reject_definition_effects(repo_root, ' - 'phase4_gate_tools, injection):\n' - ' path = "tests/integration/test_phase4_cli.py"\n' - ' raw = (repo_root / path).read_bytes()\n' - ' verify = ' - 'phase4_gate_tools["_phase4_step7_new_test"]\n' - ' verify(raw, path)\n' - ' with pytest.raises(ValueError):\n' - ' verify(raw + injection, path)'}, - {'new': 'def ' - 'test_phase4_step7_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step7_snapshot):\n' - ' import json\n' - '\n' - ' paths = ' - '{path.relative_to(phase4_step7_snapshot).as_posix()\n' - ' for path in ' - 'phase4_step7_snapshot.rglob("*") if path.is_file()}\n' - ' assert len(paths) == 229\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step7_snapshot / ' - 'relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step7_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 38\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["privacy_views_enabled"] is True\n' - ' assert baseline["renderers_enabled"] is True\n' - ' assert baseline["output_publication_enabled"] is ' - 'True\n' - ' assert baseline["cli_analysis_enabled"] is True\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 4\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n repo_root = ' - 'phase4_step4_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n"),\n' - ' "historical_document": ("docs/cli.md", b"# ' - 'CLI\\n",\n' - ' b"# Unauthorized ' - 'historical document\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "html": ' - '"src/recursive_integrity_toolkit/reports/html_report.py",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "config": ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' "hashing": ' - '"src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' "logging": ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 7 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 8\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)', - 'node': 'test_phase4_step7_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step7_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation):\n' - ' import json\n' - '\n' - ' tracked = subprocess.check_output(["git", "-C", ' - 'str(repo_root), "ls-files", "-z"])\n' - ' paths = set(tracked.decode().split("\\0")) - {""}\n' - ' assert len(paths) == 229\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(repo_root / relative, ' - 'destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step7_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] == 38\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert baseline["result_contracts_enabled"] is ' - 'True\n' - ' assert baseline["adapters_enabled"] is True\n' - ' assert baseline["privacy_views_enabled"] is True\n' - ' assert baseline["renderers_enabled"] is True\n' - ' assert baseline["output_publication_enabled"] is ' - 'True\n' - ' assert baseline["cli_analysis_enabled"] is True\n' - ' assert baseline["phase_complete"] is False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - concentration", ' - 'b"diversity = 0.5 - concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", b"numpy>=2.0", ' - 'b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and control["active_step"] ' - "== 4\\n',\n" - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, step: int = ' - '4) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n repo_root = ' - 'phase4_step4_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n"),\n' - ' "historical_document": ("docs/cli.md", b"# ' - 'CLI\\n",\n' - ' b"# Unauthorized ' - 'historical document\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": "schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "html": ' - '"src/recursive_integrity_toolkit/reports/html_report.py",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "config": ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' "hashing": ' - '"src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' "logging": ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 7 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized later-stage ' - 'file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / "PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 8\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py")\n' - ' else:\n' - ' control["schema_changes_authorized"] = ' - 'True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active mutation ' - 'case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)'}], - 'tests/unit/test_phase4_contracts.py': [{'new': 'def ' - 'test_phase4_step7_approved_control_keeps_independent_step6_anchors(phase4_tools, ' - 'phase4_step7_snapshot):\n' - ' control = json.loads((phase4_step7_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' assert phase4_tools["PHASE4_STEP6_FINAL"] == ' - '"aa2355f2359c3af4fc05345c792f9903b0d5858c"\n' - ' assert phase4_tools["PHASE4_STEP6_TREE"] == ' - '"d9b6910079f49a2c739f86eceec05040918a41a5"\n' - ' assert phase4_tools["PHASE4_STEP6_TEST_TREE"] == ' - '"8d3a03c61da807d1fef8d26fc4af0d6f72e4546e"\n' - ' assert control["active_phase"] == 4 and ' - 'control["active_step"] == 7\n' - ' assert control["approval_basis"] == "Phase 4 Step 7 ' - '继续"\n' - ' assert control["baseline_commit"] == BASELINE\n' - ' assert control["previous_step_commit"] == ' - '"aa2355f2359c3af4fc05345c792f9903b0d5858c"\n' - ' assert control["previous_step_tree"] == ' - '"d9b6910079f49a2c739f86eceec05040918a41a5"\n' - ' assert control["previous_step_test_tree"] == ' - '"8d3a03c61da807d1fef8d26fc4af0d6f72e4546e"\n' - ' assert control["previous_step_core_tests"] == 3420\n' - ' assert control["previous_step_parquet_tests"] == 3423\n' - ' assert len(control["previous_step_files_sha256"]) == ' - '228\n' - ' assert set(control["runtime_paths_authorized"]) == {\n' - ' "src/recursive_integrity_toolkit/cli.py",\n' - ' "src/recursive_integrity_toolkit/config.py",\n' - ' }\n' - ' assert control["schema_changes_authorized"] is False\n' - ' assert control["schema_paths_authorized"] == []\n' - ' assert control["new_files_permitted"] == ' - '["tests/integration/test_phase4_cli.py"]\n' - ' for name in ("phase_complete", "next_step_authorized", ' - '"main_merge_authorized", "publication_authorized"):\n' - ' assert control[name] is False\n' - ' phase4_tools["verify_phase4_step7_control"](control, ' - 'step=7)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step4_control"](control, ' - 'step=4)', - 'node': 'test_phase4_step7_approved_control_keeps_independent_step6_anchors', - 'old': 'def ' - 'test_phase4_step7_approved_control_keeps_independent_step6_anchors(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' assert phase4_tools["PHASE4_STEP6_FINAL"] == ' - '"aa2355f2359c3af4fc05345c792f9903b0d5858c"\n' - ' assert phase4_tools["PHASE4_STEP6_TREE"] == ' - '"d9b6910079f49a2c739f86eceec05040918a41a5"\n' - ' assert phase4_tools["PHASE4_STEP6_TEST_TREE"] == ' - '"8d3a03c61da807d1fef8d26fc4af0d6f72e4546e"\n' - ' assert control["active_phase"] == 4 and ' - 'control["active_step"] == 7\n' - ' assert control["approval_basis"] == "Phase 4 Step 7 ' - '继续"\n' - ' assert control["baseline_commit"] == BASELINE\n' - ' assert control["previous_step_commit"] == ' - '"aa2355f2359c3af4fc05345c792f9903b0d5858c"\n' - ' assert control["previous_step_tree"] == ' - '"d9b6910079f49a2c739f86eceec05040918a41a5"\n' - ' assert control["previous_step_test_tree"] == ' - '"8d3a03c61da807d1fef8d26fc4af0d6f72e4546e"\n' - ' assert control["previous_step_core_tests"] == 3420\n' - ' assert control["previous_step_parquet_tests"] == 3423\n' - ' assert len(control["previous_step_files_sha256"]) == ' - '228\n' - ' assert set(control["runtime_paths_authorized"]) == {\n' - ' "src/recursive_integrity_toolkit/cli.py",\n' - ' "src/recursive_integrity_toolkit/config.py",\n' - ' }\n' - ' assert control["schema_changes_authorized"] is False\n' - ' assert control["schema_paths_authorized"] == []\n' - ' assert control["new_files_permitted"] == ' - '["tests/integration/test_phase4_cli.py"]\n' - ' for name in ("phase_complete", "next_step_authorized", ' - '"main_merge_authorized", "publication_authorized"):\n' - ' assert control[name] is False\n' - ' phase4_tools["verify_phase4_step7_control"](control, ' - 'step=7)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step4_control"](control, ' - 'step=4)'}, - {'new': 'def ' - 'test_phase4_step7_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value, phase4_step7_snapshot):\n' - ' control = json.loads((phase4_step7_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' verify = phase4_tools["verify_phase4_step7_control"]\n' - ' verify(control, step=7)\n' - ' control[field] = value\n' - ' with pytest.raises(ValueError):\n' - ' verify(control, step=7)', - 'node': 'test_phase4_step7_control_rejects_forged_scope_and_stage', - 'old': 'def ' - 'test_phase4_step7_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' verify = phase4_tools["verify_phase4_step7_control"]\n' - ' verify(control, step=7)\n' - ' control[field] = value\n' - ' with pytest.raises(ValueError):\n' - ' verify(control, step=7)'}, - {'new': 'def ' - 'test_phase4_step7_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step, phase4_step7_snapshot):\n' - ' control = json.loads((phase4_step7_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step7_control"](control, ' - 'step=7)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step7_control"](control, ' - 'step=step)', - 'node': 'test_phase4_step7_control_rejects_unapproved_dispatch', - 'old': 'def ' - 'test_phase4_step7_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step7_control"](control, ' - 'step=7)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step7_control"](control, ' - 'step=step)'}, - {'new': 'def ' - 'test_phase4_step7_control_requires_explicit_approval_fields(phase4_tools, ' - 'field, phase4_step7_snapshot):\n' - ' control = json.loads((phase4_step7_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step7_control"](control)\n' - ' del control[field]\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step7_control"](control)', - 'node': 'test_phase4_step7_control_requires_explicit_approval_fields', - 'old': 'def ' - 'test_phase4_step7_control_requires_explicit_approval_fields(phase4_tools, ' - 'field):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step7_control"](control)\n' - ' del control[field]\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step7_control"](control)'}, - {'new': 'def ' - 'test_phase4_step7_control_cannot_mint_extra_permission(phase4_tools, ' - 'phase4_step7_snapshot):\n' - ' control = json.loads((phase4_step7_snapshot / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step7_control"](control)\n' - ' control["approved_scope_expansion"] = {"step": 7, ' - '"publication": True}\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step7_control"](control)', - 'node': 'test_phase4_step7_control_cannot_mint_extra_permission', - 'old': 'def ' - 'test_phase4_step7_control_cannot_mint_extra_permission(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' phase4_tools["verify_phase4_step7_control"](control)\n' - ' control["approved_scope_expansion"] = {"step": 7, ' - '"publication": True}\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step7_control"](control)'}, - {'new': 'def ' - 'test_phase4_step7_historical_bindings_preserve_assertions(phase4_tools, ' - 'phase4_step6_snapshot, phase4_step7_snapshot):\n' - ' import ast\n' - ' registry = phase4_tools["PHASE4_STEP7_MIGRATIONS"]\n' - ' assert set(registry) == ' - '{"tests/unit/test_phase4_contracts.py", ' - '"tests/integration/test_phase4_gates.py"}\n' - ' assert sorted(len(rows) for rows in registry.values()) ' - '== [5, 7]\n' - ' for path, rows in registry.items():\n' - ' ' - 'phase4_tools["verify_phase4_step7_test_migration"](path, ' - '(phase4_step6_snapshot / path).read_bytes(), ' - '(phase4_step7_snapshot / path).read_bytes())\n' - ' for row in rows:\n' - ' trees = [ast.parse(row[key]) for key in ' - '("old", "new")]\n' - ' assert trees[0].body[0].name == ' - 'trees[1].body[0].name == row["node"]\n' - ' assertions = [[ast.dump(n) for n in ' - 'ast.walk(tree) if isinstance(n, ast.Assert)] for tree in ' - 'trees]\n' - ' assert assertions[0] == assertions[1]', - 'node': 'test_phase4_step7_historical_bindings_preserve_assertions', - 'old': 'def ' - 'test_phase4_step7_historical_bindings_preserve_assertions(phase4_tools, ' - 'phase4_step6_snapshot):\n' - ' import ast\n' - ' registry = phase4_tools["PHASE4_STEP7_MIGRATIONS"]\n' - ' assert set(registry) == ' - '{"tests/unit/test_phase4_contracts.py", ' - '"tests/integration/test_phase4_gates.py"}\n' - ' assert sorted(len(rows) for rows in registry.values()) ' - '== [5, 7]\n' - ' for path, rows in registry.items():\n' - ' ' - 'phase4_tools["verify_phase4_step7_test_migration"](path, ' - '(phase4_step6_snapshot / path).read_bytes(), (ROOT / ' - 'path).read_bytes())\n' - ' for row in rows:\n' - ' trees = [ast.parse(row[key]) for key in ' - '("old", "new")]\n' - ' assert trees[0].body[0].name == ' - 'trees[1].body[0].name == row["node"]\n' - ' assertions = [[ast.dump(n) for n in ' - 'ast.walk(tree) if isinstance(n, ast.Assert)] for tree in ' - 'trees]\n' - ' assert assertions[0] == assertions[1]'}, - {'new': 'def ' - 'test_phase4_step7_historical_guard_rejects_weakening(phase4_tools, ' - 'phase4_step6_snapshot, mutation, phase4_step7_snapshot):\n' - ' path = "tests/integration/test_phase4_gates.py" if ' - 'mutation == "config" else ' - '"tests/unit/test_phase4_contracts.py"\n' - ' before, after = (phase4_step6_snapshot / ' - 'path).read_bytes(), (phase4_step7_snapshot / ' - 'path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step7_test_migration"]\n' - ' verify(path, before, after)\n' - ' if mutation == "assertion":\n' - ' needle = b\' assert control["active_phase"] == ' - '4 and control["active_step"] == 6\\n\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b" assert ' - 'True\\n", 1)\n' - ' elif mutation == "binding":\n' - " needle = b'json.loads((phase4_step6_snapshot / " - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\'\n' - ' assert needle in after\n' - ' after = after.replace(needle, ' - 'needle.replace(b"phase4_step6_snapshot", b"ROOT"), 1)\n' - ' elif mutation == "config":\n' - " needle = b'else phase4_step6_snapshot if relative " - '== "config.py" else repo_root\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b"else repo_root", ' - '1)\n' - ' elif mutation == "shadow":\n' - ' after += b"\\ndef ' - 'test_phase4_step6_fixed_boundary_opens_only_approved_existing_paths():\\n ' - 'assert True\\n"\n' - ' else:\n' - ' after += b"\\ndef ' - 'test_phase4_step7_bad(value=globals().clear()):\\n ' - 'pass\\n"\n' - ' with pytest.raises(ValueError):\n' - ' verify(path, before, after)', - 'node': 'test_phase4_step7_historical_guard_rejects_weakening', - 'old': 'def ' - 'test_phase4_step7_historical_guard_rejects_weakening(phase4_tools, ' - 'phase4_step6_snapshot, mutation):\n' - ' path = "tests/integration/test_phase4_gates.py" if ' - 'mutation == "config" else ' - '"tests/unit/test_phase4_contracts.py"\n' - ' before, after = (phase4_step6_snapshot / ' - 'path).read_bytes(), (ROOT / path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step7_test_migration"]\n' - ' verify(path, before, after)\n' - ' if mutation == "assertion":\n' - ' needle = b\' assert control["active_phase"] == ' - '4 and control["active_step"] == 6\\n\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b" assert ' - 'True\\n", 1)\n' - ' elif mutation == "binding":\n' - " needle = b'json.loads((phase4_step6_snapshot / " - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\'\n' - ' assert needle in after\n' - ' after = after.replace(needle, ' - 'needle.replace(b"phase4_step6_snapshot", b"ROOT"), 1)\n' - ' elif mutation == "config":\n' - " needle = b'else phase4_step6_snapshot if relative " - '== "config.py" else repo_root\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b"else repo_root", ' - '1)\n' - ' elif mutation == "shadow":\n' - ' after += b"\\ndef ' - 'test_phase4_step6_fixed_boundary_opens_only_approved_existing_paths():\\n ' - 'assert True\\n"\n' - ' else:\n' - ' after += b"\\ndef ' - 'test_phase4_step7_bad(value=globals().clear()):\\n ' - 'pass\\n"\n' - ' with pytest.raises(ValueError):\n' - ' verify(path, before, after)'}]} - - -def _phase4_step7_files(): - """Read the exact accepted Step 7 Git tree and verify every blob identity.""" - for suffix, expected in (("^{commit}", PHASE4_STEP7_FINAL), ("^{tree}", PHASE4_STEP7_TREE), - (":tests", PHASE4_STEP7_TEST_TREE)): - if git("rev-parse", PHASE4_STEP7_FINAL + suffix).decode().strip() != expected: - raise ValueError("Pinned Phase 4 Step 7 identity mismatch") - objects = {} - for entry in git("ls-tree", "-rz", PHASE4_STEP7_FINAL).split(b"\0"): - if not entry: - continue - metadata, raw_path = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - path = raw_path.decode("utf-8") - if (mode not in (b"100644", b"100755") or kind != b"blob" or path in objects - or path.startswith("/") or ".." in path.split("/") or ".git" in path.split("/")): - raise ValueError("Unsafe pinned Step 7 Git object") - objects[path] = oid.decode("ascii") - if len(objects) != 229: - raise ValueError("Pinned Step 7 must contain exactly 229 files") - files = {} - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_STEP7_FINAL))) as archive: - names = [item.filename for item in archive.infolist() if not item.is_dir()] - if len(names) != len(objects) or set(names) != set(objects): - raise ValueError("Pinned Step 7 archive identity mismatch") - for name in names: - raw = archive.read(name) - oid = hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - if oid != objects[name]: - raise ValueError(f"Pinned Step 7 blob mismatch: {name}") - files[name] = raw - return MappingProxyType(files) - - -def phase4_step8_expected_control() -> dict: - """Actual Step 8 approval is independent of the mutable control document.""" - prior = _phase4_step7_files() - result = phase4_step7_expected_control() - result.update({ - "control_version": "1.7", "active_step": 8, - "approval_date": PHASE4_STEP8_APPROVAL_DATE, "approval_basis": PHASE4_STEP8_APPROVAL, - "previous_step_commit": PHASE4_STEP7_FINAL, - "previous_step_tree": PHASE4_STEP7_TREE, - "previous_step_test_tree": PHASE4_STEP7_TEST_TREE, - "previous_step_core_tests": 3620, "previous_step_parquet_tests": 3623, - "previous_step_files_sha256": {p: hashlib.sha256(raw).hexdigest() for p, raw in sorted(prior.items())}, - "permitted_paths": sorted(PHASE4_STEP8_ALLOWED), "new_files_permitted": list(PHASE4_STEP8_NEW), - "runtime_changes_authorized": True, "schema_changes_authorized": False, - "runtime_paths_authorized": sorted(p for p in PHASE4_STEP8_ALLOWED if p.startswith("src/") and p.endswith(".py")), - "schema_paths_authorized": [], "package_resource_copies": PHASE4_STEP8_RESOURCES, - "package_data_declaration_only": True, - "step7_historical_binding_nodes": {p: sorted({r["node"] for r in rows}) - for p, rows in sorted(PHASE4_STEP8_MIGRATIONS.items())}, - }) - return result - - -def verify_phase4_step8_control(control: dict, step: int = 8) -> None: - if type(step) is not int or step != 8 or type(control) is not dict: - raise ValueError("Unsupported Phase 4 Step 8 stage/control") - try: - actual = json.dumps(control, sort_keys=True, ensure_ascii=True, allow_nan=False) - expected = json.dumps(phase4_step8_expected_control(), sort_keys=True, ensure_ascii=True, allow_nan=False) - except (TypeError, ValueError) as error: - raise ValueError("Invalid Phase 4 Step 8 control") from error - if actual != expected: - raise ValueError("Phase 4 control differs from the independently approved Step 8 contract") - - -def verify_phase4_step8_changes(changes: list[tuple[str, str]]) -> None: - seen = set() - for status, path in changes: - if path in seen or path not in PHASE4_STEP8_ALLOWED or status != ("A" if path in PHASE4_STEP8_NEW else "M"): - raise ValueError(f"Unapproved Phase 4 Step 8 path/operation: {status} {path}") - seen.add(path) - - -def _phase4_step8_header(node, path): - """Allow explicit test parametrization/fixtures, never definition-time effects.""" - for decorator in node.decorator_list: - if not isinstance(decorator, ast.Call) or ast.unparse(decorator.func) not in { - "pytest.fixture", "pytest.mark.parametrize"}: - raise ValueError(f"Unapproved Step 8 test decorator: {path}:{node.name}") - if ast.unparse(decorator.func) == "pytest.fixture" and not node.name.startswith("phase4_step"): - raise ValueError("A Step 8 fixture must have an explicit scoped name") - if any(keyword.arg is None for keyword in decorator.keywords): - raise ValueError("Decorator expansion is outside the Step 8 contract") - for value in [*decorator.args, *(keyword.value for keyword in decorator.keywords)]: - try: - ast.literal_eval(value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError("Step 8 decorator arguments must be literal") from error - clone = ast.parse(ast.unparse(node)).body[0] - clone.decorator_list = [] - _phase4_preserve_function_header(clone, path) - - -def _phase4_step8_append_only(before: bytes, after: bytes, path: str) -> None: - if not after.startswith(before): - raise ValueError(f"Inherited Step 7 prefix changed: {path}") - old, new = ast.parse(before), ast.parse(after) - bindings = set() - for node in old.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - bindings.add(node.name) - elif isinstance(node, (ast.Import, ast.ImportFrom)): - bindings.update(alias.asname or alias.name.split(".")[0] for alias in node.names) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - bindings.update(child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)) - for node in new.body[len(old.body):]: - if not isinstance(node, ast.FunctionDef) or node.name in bindings: - raise ValueError(f"Step 8 addition executes, rebinds or shadows inherited source: {path}") - allowed_snapshot = path == "tests/conftest.py" and node.name == "phase4_step7_snapshot" - if not allowed_snapshot and not node.name.startswith(("test_phase4_step8_", "phase4_step8_")): - raise ValueError(f"Step 8 added test/helper is not explicitly scoped: {path}:{node.name}") - if path == "tests/conftest.py" and node.name != "phase4_step7_snapshot": - raise ValueError("Only the pinned Step 7 shared fixture is authorized") - bindings.add(node.name) - _phase4_step8_header(node, path) - - -def verify_phase4_step8_test_migration(path: str, before: bytes, after: bytes) -> None: - prior = _phase4_step7_files() - if path not in prior or path not in PHASE4_STEP8_ALLOWED or before != prior[path] or not path.startswith("tests/"): - raise ValueError("Step 8 migration requires the exact named Step 7 source") - _phase4_step8_check_test_migration(path, before, after) - - -def _phase4_step8_check_test_migration(path: str, before: bytes, after: bytes) -> None: - """Check a source already read from the verified Step 7 immutable mapping.""" - expected = before - for row in PHASE4_STEP8_MIGRATIONS.get(path, []): - old, new = row["old"].encode(), row["new"].encode() - if expected.count(old) != 1: - raise ValueError("Step 7 historical binding is not unique") - expected = expected.replace(old, new, 1) - _phase4_step8_append_only(expected, after, path) - - -def _phase4_step8_preserve_tooling(before: bytes, after: bytes, path: str) -> None: - """Preserve exact inherited statements using one source-line split per tree.""" - if path == "scripts/release_check.py": - old = ' return phase4_cli_main() if explicit_phase4 else main()' - new = (' explicit_step8 = "--step=8" in argv or any(a == "--step" and b == "8" for a, b in zip(argv, argv[1:]))\n' - ' if explicit_phase4 and explicit_step8:\n' - ' return phase4_step8_cli_main()\n' + old) - # Earlier preserved dispatch functions contain this source too; bind cli_main only. - parsed = ast.parse(before) - entry = next(node for node in parsed.body if isinstance(node, ast.FunctionDef) and node.name == "cli_main") - old_entry = ast.get_source_segment(before.decode(), entry) - if old_entry.count(old) != 1: - raise ValueError("Step 7 release dispatcher identity mismatch") - replacements = [(old_entry, old_entry.replace(old, new, 1))] - else: - old = ' if args.phase == 4 and args.step == 7:\n return phase4_step7_main(step=args.step)\n' - new = old + ' if args.phase == 4 and args.step == 8:\n return phase4_step8_main(step=args.step)\n' - replacements = [(old, new), - ('Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5 or --phase 4 --step 6 or --phase 4 --step 7', - 'Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5 or --phase 4 --step 6 or --phase 4 --step 7 or --phase 4 --step 8')] - expected = before - for old, new in replacements: - if expected.count(old.encode()) != 1: - raise ValueError(f"Historical dispatcher identity mismatch: {path}") - expected = expected.replace(old.encode(), new.encode(), 1) - trees = [ast.parse(expected), ast.parse(after)] - lines = [expected.splitlines(keepends=True), after.splitlines(keepends=True)] - def segment(index, node): - if node.lineno == node.end_lineno: - return lines[index][node.lineno - 1][node.col_offset:node.end_col_offset] - return (lines[index][node.lineno - 1][node.col_offset:] - + b"".join(lines[index][node.lineno:node.end_lineno - 1]) - + lines[index][node.end_lineno - 1][:node.end_col_offset]) - def entry_guard(node): - return isinstance(node, ast.If) and isinstance(node.test, ast.Compare) and "__name__" in ast.unparse(node.test) - guards = [n for n in trees[1].body if entry_guard(n)] - expected_guard = ast.parse('if __name__ == "__main__":\n raise SystemExit(cli_main())').body[0] - if len(guards) != 1 or ast.dump(guards[0]) != ast.dump(expected_guard): - raise ValueError(f"Unapproved Step 8 maintainer entrypoint: {path}") - historical = [n for n in trees[0].body if not entry_guard(n)] - current = [n for n in trees[1].body if not entry_guard(n)] - old_entries = [(segment(0, n), ast.dump(n)) for n in historical] - cursor = 0 - for node in current: - if cursor < len(old_entries) and (segment(1, node), ast.dump(node)) == old_entries[cursor]: - cursor += 1 - continue - if isinstance(node, ast.FunctionDef) and (node.name == "_phase4_step7_files" or node.name.startswith( - ("phase4_step8_", "_phase4_step8_", "verify_phase4_step8_", "audit_phase4_step8"))): - _phase4_preserve_function_header(node, path) - continue - if isinstance(node, ast.Assign) and all(isinstance(target, ast.Name) and target.id.startswith( - ("PHASE4_STEP8_", "PHASE4_STEP7_")) for target in node.targets): - try: - ast.literal_eval(node.value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError(f"Nonliteral added Step 8 maintainer constant: {path}") from error - continue - raise ValueError(f"Unapproved Step 8 maintainer addition: {path}") - if cursor != len(historical): - raise ValueError(f"Inherited Step 7 maintainer statement changed: {path}") - def binding_counts(tree): - counts = {} - for node in tree.body: - names = [] - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - names = [node.name] - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names = [child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)] - elif isinstance(node, (ast.Import, ast.ImportFrom)): - names = [alias.asname or alias.name.split(".")[0] for alias in node.names] - for name in names: - counts[name] = counts.get(name, 0) + 1 - return counts - inherited_counts = binding_counts(trees[0]) - for name, count in binding_counts(trees[1]).items(): - if count > inherited_counts.get(name, 1): - raise ValueError(f"Duplicate Step 8 maintainer binding: {path}:{name}") - - -def verify_phase4_step8_snapshot(root: Path = ROOT) -> dict: - prior = _phase4_step7_files() - for path, raw in prior.items(): - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError(f"Missing or aliased inherited Step 7 file: {path}") - current = target.read_bytes() - if path not in PHASE4_STEP8_ALLOWED and current != raw: - raise ValueError(f"Protected Step 7 bytes changed: {path}") - if path.startswith("tests/") and path in PHASE4_STEP8_ALLOWED: - _phase4_step8_check_test_migration(path, raw, current) - elif path.startswith("scripts/") and path in PHASE4_STEP8_ALLOWED: - _phase4_step8_preserve_tooling(raw, current, path) - elif path in {"docs/architecture.md", "docs/theory_traceability.md", "PHASE_4_DECISIONS.md"}: - if not current.startswith(raw): - raise ValueError(f"Step 7 historical documentation prefix changed: {path}") - if not (root / "docs/cli.md").read_bytes().endswith(prior["docs/cli.md"]): - raise ValueError("Step 8 CLI documentation must retain the historical notes verbatim") - phase4_step8_resources(root, prior) - for path in ("src/recursive_integrity_toolkit/config.py",): - _phase4_step8_preserve_runtime(prior[path], (root / path).read_bytes(), path) - actual_modules = {p.relative_to(root).as_posix() for p in (root / "src/recursive_integrity_toolkit").rglob("*.py")} - expected_modules = {p for p in prior if p.startswith("src/") and p.endswith(".py")} - if actual_modules != expected_modules or len(actual_modules) != 40: - raise ValueError("Step 8 cannot change the runtime module set") - if {p.name for p in (root / "schemas").iterdir()} != {Path(p).name for p in prior if p.startswith("schemas/")}: - raise ValueError("Step 8 cannot change the schema set") - if hashlib.sha256((root / "PHASE_4_PLAN.md").read_bytes()).hexdigest() != PHASE4_PLAN_SHA256: - raise ValueError("Approved Phase 4 plan changed") - verify_phase4_step8_control(json.loads((root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))) - decisions = (root / "PHASE_4_DECISIONS.md").read_text(encoding="utf-8") - if PHASE4_STEP8_APPROVAL not in decisions or PHASE4_STEP7_FINAL not in decisions: - raise ValueError("Actual Step 8 authorization or Step 7 evidence anchor missing") - if any((root / name).exists() for name in PHASE4_FORBIDDEN_OUTPUTS): - raise ValueError("Step 8 cannot create Phase 4 completion records or audit outputs") - import runpy - checker = runpy.run_path(str(root / "scripts/check_traceability.py"), run_name="phase4_step8_renderer_boundary") - checker["phase4_step8_cli_boundary"](root) - schema_bytes = (root / "schemas/report.schema.json").read_bytes() - if hashlib.sha256(schema_bytes).hexdigest() != PHASE4_STEP2_REPORT_SCHEMA_SHA256: - raise ValueError("Step 8 report schema differs from its independently reviewed bytes") - schema = json.loads(schema_bytes) - if (schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema" - or schema.get("type") != "object" or schema.get("additionalProperties") is not False): - raise ValueError("Step 8 report schema dialect/closed root mismatch") - return {"package_modules": 40, "frozen_runtime_modules": 38, "frozen_schemas": 5, - "hero_files_unchanged": 6, "historical_phase3_migrated_nodes": 16, - "phase_complete": False, "result_contracts_enabled": True, - "adapters_enabled": True, "privacy_views_enabled": True, "renderers_enabled": True, "output_publication_enabled": True, "cli_analysis_enabled": True, "comparison_enabled": True, "example_enabled": True, "packaged_resources": 7} - - -def audit_phase4_step8(step: int = 8) -> dict: - verify_phase4_step8_control(json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")), step) - result = {"phase": 4, "active_step": step, **verify_phase4_step8_snapshot(), - "phase0_hashes_verified": 16, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_phase4_step8_diff(step: int = 8) -> dict: - if type(step) is not int or step != 8: - raise ValueError("Unsupported Phase 4 Step 8 stage") - _phase4_step7_files() - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE4_STEP7_FINAL, "HEAD"], check=True) - raw = git("diff", "--name-status", "--no-renames", "-z", PHASE4_STEP7_FINAL, "--").split(b"\0") - raw = [part.decode("utf-8") for part in raw if part] - if len(raw) % 2: - raise ValueError("Malformed Step 8 Git difference records") - changes = list(zip(raw[::2], raw[1::2])) - changes.extend(("A", p.decode()) for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") if p) - verify_phase4_step8_changes(changes) - result = {"previous_step_commit": PHASE4_STEP7_FINAL, "changed_files": len(changes), - "changes": changes, "step": 8, "scope": "PASS"} - print(json.dumps(result, indent=2)) - return result - - -def phase4_step8_baseline_evidence(output: Path) -> dict: - """Reconcile all inherited identities, including the accepted Step 7 suite.""" - output.mkdir(parents=True, exist_ok=True) - inherited = phase4_step7_baseline_evidence(output / "phase4-step7-inherited") - with tempfile.TemporaryDirectory(prefix="rit-p4-step7-identities-") as temp: - baseline = Path(temp) - for name, raw in _phase4_step7_files().items(): - target = baseline / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(raw) - old, old_log = _collect(baseline) - current, current_log = _collect(ROOT) - expected = 3623 if os.environ.get("RIT_TEST_PARQUET") == "1" else 3620 - if len(old) != expected or set(old) - set(current): - raise ValueError("Accepted Phase 4 Step 7 test identities were lost") - result = {"baseline_commit": PHASE4_STEP7_FINAL, "baseline_test_tree": PHASE4_STEP7_TEST_TREE, - "baseline_nodeids": old, "current_nodeids": current, "missing_nodeids": [], - "baseline_tests": len(old), "current_tests": len(current), "inherited": inherited, - "nodeids_sha256": hashlib.sha256(("\n".join(old)+"\n").encode()).hexdigest()} - (output / "phase4_step8_test_identity_manifest.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4-step7-collection.log").write_text(old_log, encoding="utf-8") - (output / "phase4-step8-collection.log").write_text(current_log, encoding="utf-8") - print(json.dumps({k: result[k] for k in ("baseline_tests", "current_tests", "missing_nodeids")}, indent=2)) - return result - - -def phase4_step8_candidate(output: Path, step: int = 8) -> None: - """Build tested Step 8 intermediate evidence without declaring Phase 4 complete.""" - if os.environ.get("RIT_TEST_PARQUET") != "1" or importlib.util.find_spec("pyarrow") is None: - raise ValueError("Step 8 candidate evidence requires RIT_TEST_PARQUET=1 and real PyArrow") - output.mkdir(parents=True, exist_ok=True) - result = audit_phase4_step8(step) - result["diff"] = audit_phase4_step8_diff(step) - if git("status", "--porcelain").strip(): - raise ValueError("Step 8 candidate archive requires committed, clean source") - result["core"] = verify_junit(output / "core.xml", minimum=3620) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=3623) - result["core_math_measurements"] = verify_step10_evidence(output / "core.xml", output / "phase4-step8-core-observations.json") - result["parquet_math_measurements"] = verify_step10_evidence(output / "parquet.xml", output / "phase4-step8-parquet-observations.json") - identity = phase4_step8_baseline_evidence(output) - result["test_identity"] = {k: identity[k] for k in ("baseline_tests", "current_tests", "nodeids_sha256")} - expected = set() - for node in identity["current_nodeids"]: - base, bracket, parameter = node.partition("[") - owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) - for name, parquet in (("core.xml", False), ("parquet.xml", True)): - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(output / name).getroot().iter("testcase")} - target = expected if parquet else {c for c in expected if c[1] not in PARQUET_CASES} - if cases != target: - raise ValueError(f"Step 8 JUnit does not execute the entire current suite: {name}") - wheel, sdist = phase4_step8_distributions(output / "dist") - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - phase4_step6_installed_publication_smoke(wheel) - phase4_step7_installed_cli_smoke(wheel) - phase4_step8_installed_example_smoke(wheel) - phase4_step8_sdist_smoke(sdist) - archive = output / "recursive-integrity-toolkit-phase4-step8-candidate.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {p for p in git("ls-files", "-z").decode().split("\0") if p} - with zipfile.ZipFile(archive) as zipped: - entries = [item.filename for item in zipped.infolist() if not item.is_dir()] - prefix = "recursive-integrity-toolkit/" - if (len(entries) != len(set(entries)) or any(not name.startswith(prefix) for name in entries)): - raise ValueError("Step 8 source archive has duplicate or unprefixed entries") - names = {name.removeprefix(prefix) for name in entries} - if names != tracked or len(tracked) != 236 or len(entries) != 236: - raise ValueError("Step 8 source archive file set mismatch") - for name in names: - if zipped.read("recursive-integrity-toolkit/"+name) != (ROOT / name).read_bytes(): - raise ValueError(f"Step 8 source archive byte mismatch: {name}") - result.update({"commit": git("rev-parse", "HEAD").decode().strip(), - "tree": git("rev-parse", "HEAD^{tree}").decode().strip(), - "source_archive": archive.name, "tracked_files": len(tracked), - "python": sys.version, "platform": sys.platform, - "versions": {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "pytest")}}) - (output / "phase4_step8_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4_step8_repository_files.sha256").write_text("".join(f"{hashlib.sha256((ROOT/name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - paths = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase4_step8_artifacts.sha256") - (output / "phase4_step8_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in paths), encoding="utf-8") - print(f"Phase 4 Step 8 candidate: {len(tracked)} source files verified; Phase 4 remains incomplete.") - - -def phase4_step8_cli_main() -> int: - parser = argparse.ArgumentParser(description="Phase 4 Step 8 maintainer gate") - parser.add_argument("--phase", type=int, choices=(4,), required=True) - parser.add_argument("--step", type=int, choices=(8,), required=True) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) - parser.add_argument("--require-parquet", action="store_true") - parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - raise ValueError("Phase 4 Step 8 cannot certify a final phase delivery") - if args.junit is not None: - verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - phase4_step8_baseline_evidence(args.baseline_evidence) - elif args.dist is not None: - audit_phase4_step8(args.step) - wheel, sdist = phase4_step8_distributions(args.dist) - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - phase4_step6_installed_publication_smoke(wheel) - phase4_step7_installed_cli_smoke(wheel) - phase4_step8_installed_example_smoke(wheel) - phase4_step8_sdist_smoke(sdist) - elif args.smoke_wheel is not None: - smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - phase4_step8_candidate(args.candidate, args.step) - else: - audit_phase4_step8(args.step) - if args.diff: - audit_phase4_step8_diff(args.step) - return 0 - - -def _phase4_step8_preserve_runtime(before: bytes, after: bytes, path: str) -> None: - """Allow ownership-header maintenance and byte-exact additive helpers only.""" - try: - old, new = ast.parse(before), ast.parse(after) - except (SyntaxError, UnicodeError, ValueError) as error: - raise ValueError("Output helper source cannot be inspected") from error - if not ast.get_docstring(old) or not ast.get_docstring(new): - raise ValueError("Output helper ownership documentation is required") - old_tail = b"".join(before.splitlines(keepends=True)[old.body[0].end_lineno:]) - new_tail = b"".join(after.splitlines(keepends=True)[new.body[0].end_lineno:]) - if not new_tail.startswith(old_tail): - raise ValueError(f"Inherited input/diagnostic bytes changed: {path}") - - -def phase4_step8_resources(root: Path, prior: dict) -> None: - """Exact approved copies, with no build/dependency/version change.""" - for destination, source in PHASE4_STEP8_RESOURCES.items(): - path = root / destination - if not path.is_file() or path.is_symlink() or path.read_bytes() != (root / source).read_bytes(): - raise ValueError("Step 8 packaged resource differs from its canonical source") - resource_root = root / "src/recursive_integrity_toolkit/data" - if any(p.is_symlink() for p in resource_root.rglob("*")) or {p.relative_to(root).as_posix() for p in resource_root.rglob("*") if p.is_file()} != set(PHASE4_STEP8_RESOURCES): - raise ValueError("Step 8 packaged resource set differs from its exact allowlist") - before = prior["pyproject.toml"] - marker = b'include = ["recursive_integrity_toolkit*"]\n' - if before.count(marker) != 1 or (root / "pyproject.toml").read_bytes() != before.replace(marker, marker + PHASE4_STEP8_PACKAGING.encode(), 1): - raise ValueError("Step 8 packaging may add only the exact package-data declaration") - - -def phase4_step8_distributions(directory: Path, expected_version: str = "0.1.0.dev2"): - wheel, sdist = verify_distributions(directory, expected_version=expected_version) - with zipfile.ZipFile(wheel) as archive: - actual = {name for name in archive.namelist() if name.startswith("recursive_integrity_toolkit/data/") and not name.endswith("/")} - if actual != {path.removeprefix("src/") for path in PHASE4_STEP8_RESOURCES}: - raise ValueError("Wheel resource inventory differs") - for destination, source in PHASE4_STEP8_RESOURCES.items(): - if archive.read(destination.removeprefix("src/")) != (ROOT / source).read_bytes(): - raise ValueError("Wheel resource bytes differ") - with tarfile.open(sdist, "r:gz") as archive: - root = next(iter({m.name.split("/")[0] for m in archive.getmembers()})) - for destination, source in PHASE4_STEP8_RESOURCES.items(): - member = archive.getmember(root + "/" + destination) - if not member.isfile() or archive.extractfile(member).read() != (ROOT / source).read_bytes(): - raise ValueError("Sdist resource bytes differ") - print("Step 8 wheel/sdist: seven exact canonical resources: PASS") - return wheel, sdist - - - -def phase4_step8_example_program() -> str: +def installed_example_program() -> str: """Independent installed acceptance, using frozen section 8 Hero values.""" return r'''import hashlib, importlib.abc, json, socket, sys, urllib.request from pathlib import Path @@ -9753,7 +1145,7 @@ def deny(*args,**kwargs):raise AssertionError('network forbidden') assert report['observed_facts']['provenance'][name]['value']==1.0 assert report['observed_facts']['provenance'][name]['scope']['dataset_versions']==['v2'] assert [report['derived_metrics']['closure_exposure']['direct'][n]['value'] for n in ('lower_bound','upper_bound','interval_width')]==[0.5,0.5,0] -assert report['capabilities']['lineage']['execution_status']=='deferred' +assert report['capabilities']['lineage']['execution_status']=='not_requested' assert report['capabilities']['dataset_longitudinal']['execution_status']=='partial' assert report['simulations']=={} and report['errors']==[] assert {'model_performance_decline','causal_ancestor_effect','universal_integrity','universal_collapse_prediction'} <= {x['conclusion'] for x in report['unavailable_conclusions']} @@ -9768,1592 +1160,195 @@ def deny(*args,**kwargs):raise AssertionError('network forbidden') assert redacted['derived_metrics']['support']['support_delta']['value']==-3 assert redacted['run']['redacted_mode'] is True and redacted['simulations']=={} assert str(work) not in (work/'redacted/reports/report.json').read_text(encoding='utf-8') +for name,privacy in (('lineage',[]),('lineage-redacted',['--redacted'])): + destination=work/name + assert main(['example','--lineage','--out',str(destination),*privacy])==0 + current=json.loads((destination/'reports/report.json').read_bytes()) + jsonschema.Draft202012Validator(json.loads(resources.joinpath('data','report.schema.json').read_bytes())).validate(current) + assert current['run']['report_schema_version']=='1.1' and current['run']['run_status']=='complete' + assert current['capabilities']['lineage']['execution_status']=='completed' + assert current['capabilities']==current['observability']['capabilities'] + assert current['simulations']=={} and current['errors']==[] + facts=current['observed_facts']['lineage'];metrics=current['derived_metrics']['lineage'] + graph=facts['graph_scope']['value'] + assert (graph['target_record_count'],graph['loaded_record_count'],graph['context_record_count'])==(8,16,8) + assert [metrics[k]['value'] for k in ('grounded_record_count','closed_record_count','unresolved_record_count')]==[8,0,0] + assert metrics['distinct_external_root_count']['value']==5 + assert metrics['ancestry_concentration_hhi']['value']==0.25 + assert metrics['effective_external_root_count']['value']==4.0 + assert [metrics[k]['value'] for k in ('resolved_lineage_coverage','external_ancestry_coverage','resolved_parent_edge_coverage')]==[1.0,1.0,1.0] + assert facts['declared_parent_edge_count']['value']==facts['resolved_parent_edge_count']['value']==8 + assert facts['cycle_status']['value']=='acyclic' + roots=metrics['top_shared_ancestors']['value']['items'] + assert [root['incidence_count'] for root in roots]==[3,2,1,1,1] + assert [root['normalized_weight'] for root in roots]==[3/8,2/8,1/8,1/8,1/8] + assert [current['derived_metrics']['closure_exposure']['lineage'][k]['value'] for k in ('lower_bound','upper_bound','interval_width')]==[0.0,0.0,0.0] + assert [current['derived_metrics']['closure_exposure']['direct'][k]['value'] for k in ('lower_bound','upper_bound','interval_width')]==[0.5,0.5,0.0] + assert current['derived_metrics']['support']['support_delta']['value']==-3 + assert current['derived_metrics']['diversity']['gini_simpson_diversity_delta']['value']==-0.125 + assert current['proxy_signals']['shared_ancestry_dependence']['level']=='present' + assert {'model_performance_decline','causal_ancestor_effect','universal_integrity','universal_collapse_prediction'} <= {x['conclusion'] for x in current['unavailable_conclusions']} + markdown=(destination/'reports/report.md').read_text(encoding='utf-8') + assert '`["derived_metrics"]["lineage"]["ancestry_concentration_hhi"]`' in markdown + if privacy: + assert str(work) not in (destination/'reports/report.json').read_text(encoding='utf-8') + assert str(work) not in markdown + assert all(root['record_key']['record_id'] not in {f'v1_{i:02d}' for i in range(1,9)} for root in roots) + else: + assert graph['target_dataset_version']=='v2' + assert roots[0]['record_key']=={'dataset_version':'v1','record_id':'v1_01'} + for p in (destination/'inputs').iterdir(): + assert p.read_bytes()==resources.joinpath('data','hero',p.name).read_bytes() for relative,digest in expected.items(): assert hashlib.sha256(resources.joinpath(*relative.split('/')).read_bytes()).hexdigest()==digest for p in (target/'inputs').iterdir(): assert p.read_bytes()==resources.joinpath('data','hero',p.name).read_bytes() -print('installed Step 8: hand-counted Hero, local schema, exact resources, core-only, standard/redacted, no overwrite, blocked network: PASS') +print('installed Hero: ordinary and explicit lineage, frozen numerical oracles, local schema, exact resources, core-only, standard/redacted, no overwrite, blocked network: PASS') ''' -def phase4_step8_installed_example_smoke(wheel: Path) -> None: +def smoke_installed_example(wheel: Path) -> None: with tempfile.TemporaryDirectory(prefix="rit-p4-step8-wheel-") as temporary: work = Path(temporary) target = work / "installed" subprocess.run([sys.executable, "-m", "pip", "install", "--no-index", "--no-deps", "--target", str(target), str(wheel.resolve())], cwd=work, check=True) - hashes = {destination.removeprefix("src/recursive_integrity_toolkit/"): hashlib.sha256((ROOT/source).read_bytes()).hexdigest() for destination,source in PHASE4_STEP8_RESOURCES.items()} - subprocess.run([sys.executable, "-I", "-c", phase4_step8_example_program(), str(target), str(work), str(ROOT), json.dumps(hashes)], cwd=work, check=True) + hashes = {destination.removeprefix("src/recursive_integrity_toolkit/"): hashlib.sha256((ROOT/source).read_bytes()).hexdigest() for destination,source in RESOURCES.items()} + subprocess.run([sys.executable, "-I", "-c", installed_example_program(), str(target), str(work), str(ROOT), json.dumps(hashes)], cwd=work, check=True) + + +def extract_sdist(sdist: Path, destination: Path) -> Path: + """Preflight regular archive members before copying any source bytes.""" + with tarfile.open(sdist, "r:gz") as archive: + members = archive.getmembers() + names = [member.name for member in members] + if len(names) != len(set(names)): + raise ValueError("Duplicate sdist entries") + roots = {PurePosixPath(name).parts[0] for name in names if PurePosixPath(name).parts} + if len(roots) != 1: + raise ValueError("Sdist needs exactly one root") + for member in members: + parts = PurePosixPath(member.name).parts + if (not parts or member.name.startswith("/") or ".." in parts + or "\\" in member.name or any(":" in part for part in parts) + or not (member.isdir() or member.isfile())): + raise ValueError("Unsafe sdist member") + target = destination.joinpath(*parts) + if any(path.is_symlink() for path in (target, *target.parents)): + raise ValueError("Aliased sdist extraction destination") + for member in members: + if member.isdir(): + continue + target = destination / member.name + target.parent.mkdir(parents=True, exist_ok=True) + stream = archive.extractfile(member) + if stream is None: + raise ValueError("Missing regular sdist content") + target.write_bytes(stream.read()) + return destination / next(iter(roots)) -def phase4_step8_sdist_smoke(sdist: Path) -> None: +def smoke_sdist(sdist: Path) -> None: """Extract regular sdist members and audit without build/download operations.""" with tempfile.TemporaryDirectory(prefix="rit-p4-step8-sdist-") as temporary: work = Path(temporary) - extracted = work / "source" - with tarfile.open(sdist, "r:gz") as archive: - roots = {m.name.split("/")[0] for m in archive.getmembers()} - if len(roots) != 1: - raise ValueError("Sdist needs exactly one root") - for member in archive.getmembers(): - if member.isdir(): - continue - if (not member.isfile() or member.name.startswith("/") or ".." in member.name.split("/") or "\\" in member.name): - raise ValueError("Unsafe sdist member") - target = extracted / member.name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(archive.extractfile(member).read()) - source = extracted / next(iter(roots)) / "src" - hashes = {destination.removeprefix("src/recursive_integrity_toolkit/"): hashlib.sha256((ROOT/original).read_bytes()).hexdigest() for destination,original in PHASE4_STEP8_RESOURCES.items()} - subprocess.run([sys.executable, "-I", "-c", phase4_step8_example_program(), str(source), str(work), str(ROOT), json.dumps(hashes)], cwd=work, check=True) - - - -# Phase 4 Step 9: independently authored report oracles and adversarial integration. - - -PHASE4_STEP8_FINAL = '63e347a785838b64a9c32e636ea48c664458cee4' - - -PHASE4_STEP8_TREE = '3ac450b5f67cad738849de18601999fefee98f63' - - -PHASE4_STEP8_TEST_TREE = 'ddaaab24d83080b7c1a27229c15e6e4a3f5b3cd8' - - -PHASE4_STEP9_APPROVAL = 'Phase 4 **Step 9** 开始' - - -PHASE4_STEP9_APPROVAL_DATE = '2026-09-22' - - -PHASE4_STEP9_NEW = ('tests/golden/phase4_hero_redacted.json', - 'tests/golden/phase4_hero_redacted.md', - 'tests/golden/phase4_hero_report.json', - 'tests/golden/phase4_hero_report.md', - 'tests/golden/phase4_report_cases.md', - 'tests/golden/phase4_report_expected.json', - 'tests/golden/test_phase4_reports.py') - - -PHASE4_STEP9_ALLOWED = {'.github/workflows/ci.yml', - '.github/workflows/golden.yml', - '.github/workflows/release.yml', - '.github/workflows/security.yml', - 'PHASE_4_BASELINE.json', - 'PHASE_4_DECISIONS.md', - 'docs/architecture.md', - 'docs/theory_traceability.md', - 'scripts/build_golden.py', - 'scripts/check_spec_consistency.py', - 'scripts/check_traceability.py', - 'scripts/normalize_golden.py', - 'scripts/release_check.py', - 'tests/conftest.py', - 'tests/golden/README.md', - 'tests/golden/phase4_hero_redacted.json', - 'tests/golden/phase4_hero_redacted.md', - 'tests/golden/phase4_hero_report.json', - 'tests/golden/phase4_hero_report.md', - 'tests/golden/phase4_report_cases.md', - 'tests/golden/phase4_report_expected.json', - 'tests/golden/test_phase4_reports.py', - 'tests/integration/test_ci_workflows.py', - 'tests/integration/test_hero_end_to_end.py', - 'tests/integration/test_hero_structure.py', - 'tests/integration/test_license_notices.py', - 'tests/integration/test_no_algorithms.py', - 'tests/integration/test_no_network.py', - 'tests/integration/test_optional_dependency.py', - 'tests/integration/test_owner_ids.py', - 'tests/integration/test_package_import.py', - 'tests/integration/test_package_install.py', - 'tests/integration/test_partial_lineage_report.py', - 'tests/integration/test_partial_provenance_report.py', - 'tests/integration/test_phase4_cli.py', - 'tests/integration/test_phase4_gates.py', - 'tests/integration/test_prohibited_structure.py', - 'tests/integration/test_repository_structure.py', - 'tests/integration/test_schema_json.py', - 'tests/unit/test_phase4_contracts.py'} - - -PHASE4_STEP9_MIGRATIONS = {'tests/integration/test_phase4_gates.py': [{'new': 'def ' - 'test_phase4_step8_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step8_snapshot):\n' - ' repo_root = phase4_step8_snapshot\n' - ' names = {"ci.yml", "golden.yml", ' - '"security.yml", "release.yml"}\n' - ' root = repo_root / ".github/workflows"\n' - ' expected_timeouts = {"ci.yml": [60, 20], ' - '"golden.yml": [15], "security.yml": [20], ' - '"release.yml": [40]}\n' - ' assert {path.name for path in ' - 'root.glob("*.yml")} == names\n' - ' for name in sorted(names):\n' - ' text = (root / ' - 'name).read_text(encoding="utf-8")\n' - ' assert [int(line.split(":", 1)[1]) ' - 'for line in text.splitlines()\n' - ' if ' - 'line.strip().startswith("timeout-minutes:")] ' - '== expected_timeouts[name]\n' - ' assert "Phase 4 Step 8" in text, ' - 'name\n' - ' assert "--phase 4 --step 8" in text, ' - 'name\n' - ' for older in ("--phase 4 --step 7", ' - '"--phase 4 --step 6", "--phase 4 --step 5", ' - '"--phase 4 --step 4", "--phase 4 --step 3", ' - '"--phase 4 --step 2", "--phase 4 --step 1", ' - '"--phase 3 --step 11"):\n' - ' assert older not in text, (name, ' - 'older)\n' - ' assert "permissions:\\n contents: ' - 'read" in text\n' - ' assert "persist-credentials: false" ' - 'in text\n' - ' assert "timeout-minutes:" in text and ' - '"set -euo pipefail" in text\n' - ' assert "actions/upload-artifact@v4" ' - 'in text and "if-no-files-found: error" in ' - 'text\n' - ' for line in text.splitlines():\n' - ' if "python ' - 'scripts/release_check.py" in line:\n' - ' assert "--phase 4 --step 8" ' - 'in line, (name, line)\n' - ' for forbidden in ' - '("continue-on-error:", "|| true", "contents: ' - 'write", "id-token: write", "twine upload", ' - '"git push"):\n' - ' assert forbidden not in text, ' - '(name, forbidden)\n' - ' ci = (root / ' - '"ci.yml").read_text(encoding="utf-8")\n' - " for required in ('os: [ubuntu-latest, " - 'windows-latest]\', \'python-version: ["3.11", ' - '"3.12"]\',\n' - " 'dependencies: [current, " - 'minimum]\', \'"numpy==2.0.0" ' - '"pandas==2.2.2"\',\n' - " 'RIT_TEST_PARQUET: " - '"0"\', \'RIT_TEST_PARQUET: "1"\', ' - "'--require-parquet',\n" - " '--baseline-evidence', " - "'python -m pip check',\n" - " '--minimum-tests 3620', " - "'--minimum-tests 3623',\n" - ' "find_spec(\'pyarrow\') ' - 'is None", \'import pyarrow\'):\n' - ' assert required in ci\n' - ' assert ci.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2\n' - ' assert " -k " not in ci\n' - ' golden = (root / ' - '"golden.yml").read_text(encoding="utf-8")\n' - ' for required in ' - '("tests/golden/test_phase3_math.py", ' - '"tests/integration/test_hero_structure.py",\n' - ' ' - '"tests/integration/test_phase3_metric_pipeline.py", ' - '"tests/integration/test_hero_end_to_end.py"):\n' - ' assert required in golden\n' - ' security = (root / ' - '"security.yml").read_text(encoding="utf-8")\n' - ' for required in ' - "('tests/integration/test_no_network.py', " - "'tests/integration/test_optional_dependency.py',\n" - ' ' - "'tests/unit/test_PR012_evidence_classes.py', " - "'tests/unit/test_PR013_report_schema.py',\n" - ' ' - "'tests/unit/test_PR014_unavailable.py', " - "'tests/unit/test_PR015_redaction.py',\n" - ' ' - "'tests/unit/test_PR016_determinism.py', " - "'tests/unit/test_PR018_language.py',\n" - ' ' - "'tests/integration/test_partial_provenance_report.py',\n" - ' ' - "'tests/integration/test_partial_lineage_report.py',\n" - ' ' - "'tests/integration/test_phase4_cli.py', " - "'tests/unit/test_phase4_output_safety.py', " - "'tests/unit/test_phase4_contracts.py', " - "'tests/integration/test_phase4_gates.py'):\n" - ' assert required in security\n' - ' release = (root / ' - '"release.yml").read_text(encoding="utf-8")\n' - ' assert "python -m build" in release and ' - '"python -m twine check --strict" in release\n' - ' assert "--dist" in release and ' - '"--candidate" in release\n' - ' assert "--delivery" not in release\n' - ' assert ' - '"recursive-integrity-toolkit-phase4-step8-candidate" ' - 'in release\n' - ' assert "rit-phase4-step4" not in release\n' - ' assert "--minimum-tests 3620" in release ' - 'and "--minimum-tests 3623" in release\n' - ' assert release.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2', - 'node': 'test_phase4_step8_current_workflows_preserve_matrix_and_use_active_dispatch', - 'old': 'def ' - 'test_phase4_step8_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root):\n' - ' names = {"ci.yml", "golden.yml", ' - '"security.yml", "release.yml"}\n' - ' root = repo_root / ".github/workflows"\n' - ' expected_timeouts = {"ci.yml": [60, 20], ' - '"golden.yml": [15], "security.yml": [20], ' - '"release.yml": [40]}\n' - ' assert {path.name for path in ' - 'root.glob("*.yml")} == names\n' - ' for name in sorted(names):\n' - ' text = (root / ' - 'name).read_text(encoding="utf-8")\n' - ' assert [int(line.split(":", 1)[1]) ' - 'for line in text.splitlines()\n' - ' if ' - 'line.strip().startswith("timeout-minutes:")] ' - '== expected_timeouts[name]\n' - ' assert "Phase 4 Step 8" in text, ' - 'name\n' - ' assert "--phase 4 --step 8" in text, ' - 'name\n' - ' for older in ("--phase 4 --step 7", ' - '"--phase 4 --step 6", "--phase 4 --step 5", ' - '"--phase 4 --step 4", "--phase 4 --step 3", ' - '"--phase 4 --step 2", "--phase 4 --step 1", ' - '"--phase 3 --step 11"):\n' - ' assert older not in text, (name, ' - 'older)\n' - ' assert "permissions:\\n contents: ' - 'read" in text\n' - ' assert "persist-credentials: false" ' - 'in text\n' - ' assert "timeout-minutes:" in text and ' - '"set -euo pipefail" in text\n' - ' assert "actions/upload-artifact@v4" ' - 'in text and "if-no-files-found: error" in ' - 'text\n' - ' for line in text.splitlines():\n' - ' if "python ' - 'scripts/release_check.py" in line:\n' - ' assert "--phase 4 --step 8" ' - 'in line, (name, line)\n' - ' for forbidden in ' - '("continue-on-error:", "|| true", "contents: ' - 'write", "id-token: write", "twine upload", ' - '"git push"):\n' - ' assert forbidden not in text, ' - '(name, forbidden)\n' - ' ci = (root / ' - '"ci.yml").read_text(encoding="utf-8")\n' - " for required in ('os: [ubuntu-latest, " - 'windows-latest]\', \'python-version: ["3.11", ' - '"3.12"]\',\n' - " 'dependencies: [current, " - 'minimum]\', \'"numpy==2.0.0" ' - '"pandas==2.2.2"\',\n' - " 'RIT_TEST_PARQUET: " - '"0"\', \'RIT_TEST_PARQUET: "1"\', ' - "'--require-parquet',\n" - " '--baseline-evidence', " - "'python -m pip check',\n" - " '--minimum-tests 3620', " - "'--minimum-tests 3623',\n" - ' "find_spec(\'pyarrow\') ' - 'is None", \'import pyarrow\'):\n' - ' assert required in ci\n' - ' assert ci.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2\n' - ' assert " -k " not in ci\n' - ' golden = (root / ' - '"golden.yml").read_text(encoding="utf-8")\n' - ' for required in ' - '("tests/golden/test_phase3_math.py", ' - '"tests/integration/test_hero_structure.py",\n' - ' ' - '"tests/integration/test_phase3_metric_pipeline.py", ' - '"tests/integration/test_hero_end_to_end.py"):\n' - ' assert required in golden\n' - ' security = (root / ' - '"security.yml").read_text(encoding="utf-8")\n' - ' for required in ' - "('tests/integration/test_no_network.py', " - "'tests/integration/test_optional_dependency.py',\n" - ' ' - "'tests/unit/test_PR012_evidence_classes.py', " - "'tests/unit/test_PR013_report_schema.py',\n" - ' ' - "'tests/unit/test_PR014_unavailable.py', " - "'tests/unit/test_PR015_redaction.py',\n" - ' ' - "'tests/unit/test_PR016_determinism.py', " - "'tests/unit/test_PR018_language.py',\n" - ' ' - "'tests/integration/test_partial_provenance_report.py',\n" - ' ' - "'tests/integration/test_partial_lineage_report.py',\n" - ' ' - "'tests/integration/test_phase4_cli.py', " - "'tests/unit/test_phase4_output_safety.py', " - "'tests/unit/test_phase4_contracts.py', " - "'tests/integration/test_phase4_gates.py'):\n" - ' assert required in security\n' - ' release = (root / ' - '"release.yml").read_text(encoding="utf-8")\n' - ' assert "python -m build" in release and ' - '"python -m twine check --strict" in release\n' - ' assert "--dist" in release and ' - '"--candidate" in release\n' - ' assert "--delivery" not in release\n' - ' assert ' - '"recursive-integrity-toolkit-phase4-step8-candidate" ' - 'in release\n' - ' assert "rit-phase4-step4" not in release\n' - ' assert "--minimum-tests 3620" in release ' - 'and "--minimum-tests 3623" in release\n' - ' assert release.count("python -m pytest -p ' - 'no:cacheprovider -q --junitxml=") == 2'}, - {'new': 'def ' - 'test_phase4_step8_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation, ' - 'phase4_step8_snapshot):\n' - ' import json\n' - '\n' - ' paths = ' - '{path.relative_to(phase4_step8_snapshot).as_posix()\n' - ' for path in ' - 'phase4_step8_snapshot.rglob("*") if ' - 'path.is_file()}\n' - ' assert len(paths) == 236\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' ' - 'destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(phase4_step8_snapshot ' - '/ relative, destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step8_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] ' - '== 38\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert ' - 'baseline["result_contracts_enabled"] is True\n' - ' assert baseline["adapters_enabled"] is ' - 'True\n' - ' assert baseline["privacy_views_enabled"] ' - 'is True\n' - ' assert baseline["renderers_enabled"] is ' - 'True\n' - ' assert ' - 'baseline["output_publication_enabled"] is ' - 'True\n' - ' assert baseline["cli_analysis_enabled"] ' - 'is True\n' - ' assert baseline["phase_complete"] is ' - 'False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - ' - 'concentration", b"diversity = 0.5 - ' - 'concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ' - '("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", ' - 'b"numpy>=2.0", b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and ' - 'control["active_step"] == 4\\n\',\n' - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, ' - 'step: int = 4) -> None:",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, ' - 'step: int = 4) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n repo_root = ' - 'phase4_step4_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n"),\n' - ' "historical_document": ' - '("docs/cli.md", b"# CLI\\n",\n' - ' b"# ' - 'Unauthorized historical document\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def ' - 'resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": ' - '"schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "html": ' - '"src/recursive_integrity_toolkit/reports/html_report.py",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "config": ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' "hashing": ' - '"src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' "logging": ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' ' - 'path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 8 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized ' - 'later-stage file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / ' - '"PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 9\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py")\n' - ' else:\n' - ' ' - 'control["schema_changes_authorized"] = True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active ' - 'mutation case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)', - 'node': 'test_phase4_step8_current_snapshot_checks_valid_tree_before_mutations', - 'old': 'def ' - 'test_phase4_step8_current_snapshot_checks_valid_tree_before_mutations(repo_root, ' - 'tmp_path, phase4_gate_tools, mutation):\n' - ' import json\n' - '\n' - ' tracked = subprocess.check_output(["git", ' - '"-C", str(repo_root), "ls-files", "-z"])\n' - ' paths = ' - 'set(tracked.decode().split("\\0")) - {""}\n' - ' assert len(paths) == 236\n' - ' for relative in sorted(paths):\n' - ' destination = tmp_path / relative\n' - ' ' - 'destination.parent.mkdir(parents=True, ' - 'exist_ok=True)\n' - ' shutil.copyfile(repo_root / relative, ' - 'destination)\n' - ' verify = ' - 'phase4_gate_tools["verify_phase4_step8_snapshot"]\n' - ' baseline = verify(tmp_path)\n' - ' assert baseline["package_modules"] == 40\n' - ' assert baseline["frozen_runtime_modules"] ' - '== 38\n' - ' assert baseline["frozen_schemas"] == 5\n' - ' assert ' - 'baseline["result_contracts_enabled"] is True\n' - ' assert baseline["adapters_enabled"] is ' - 'True\n' - ' assert baseline["privacy_views_enabled"] ' - 'is True\n' - ' assert baseline["renderers_enabled"] is ' - 'True\n' - ' assert ' - 'baseline["output_publication_enabled"] is ' - 'True\n' - ' assert baseline["cli_analysis_enabled"] ' - 'is True\n' - ' assert baseline["phase_complete"] is ' - 'False\n' - ' if mutation == "unchanged":\n' - ' return\n' - ' replacements = {\n' - ' "formula": ' - '("src/recursive_integrity_toolkit/metrics/diversity.py",\n' - ' b"diversity = 1.0 - ' - 'concentration", b"diversity = 0.5 - ' - 'concentration"),\n' - ' "math_oracle": ' - '("tests/golden/phase3_math_cases.json", ' - 'b\'"v2_support":5\', b\'"v2_support":3\'),\n' - ' "hero": ' - '("examples/hero/records_v2.csv", ' - 'b"v2_08,v2,", b"v2_99,v2,"),\n' - ' "dependency": ("pyproject.toml", ' - 'b"numpy>=2.0", b"numpy>=3.0"),\n' - ' "historical_assertion": ' - '("tests/unit/test_phase4_contracts.py",\n' - " b' assert " - 'control["active_phase"] == 4 and ' - 'control["active_step"] == 4\\n\',\n' - ' b" assert ' - 'True\\n"),\n' - ' "historical_tooling": ' - '("scripts/release_check.py",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, ' - 'step: int = 4) -> None:",\n' - ' b"def ' - 'verify_phase4_step4_control(control: dict, ' - 'step: int = 4) -> None:\\n return"),\n' - ' "historical_binding": ' - '("tests/integration/test_phase4_gates.py",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n repo_root = ' - 'phase4_step4_snapshot\\n",\n' - ' b"def ' - 'test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, ' - 'phase4_step4_snapshot):\\n"),\n' - ' "historical_document": ' - '("docs/cli.md", b"# CLI\\n",\n' - ' b"# ' - 'Unauthorized historical document\\n"),\n' - ' "input_hash_helper": ' - '("src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' b"return ' - 'hashlib.sha256(data).hexdigest()", b"return ' - 'hashlib.sha256(b\'changed\').hexdigest()"),\n' - ' "config_resolver_helper": ' - '("src/recursive_integrity_toolkit/config.py",\n' - ' b"def ' - 'resolve_config(", b"def ' - 'resolve_config_disabled("),\n' - ' }\n' - ' appended = {\n' - ' "frozen_model": ' - '"src/recursive_integrity_toolkit/models.py",\n' - ' "frozen_schema": ' - '"schemas/report.schema.json",\n' - ' "plan": "PHASE_4_PLAN.md",\n' - ' "cli": ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' "html": ' - '"src/recursive_integrity_toolkit/reports/html_report.py",\n' - ' "assembly": ' - '"src/recursive_integrity_toolkit/reports/assembly.py",\n' - ' "result": ' - '"src/recursive_integrity_toolkit/result.py",\n' - ' "config": ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' "hashing": ' - '"src/recursive_integrity_toolkit/utils/hashing.py",\n' - ' "logging": ' - '"src/recursive_integrity_toolkit/utils/logging.py",\n' - ' "output_paths": ' - '"src/recursive_integrity_toolkit/utils/paths.py",\n' - ' }\n' - ' added = {\n' - ' "unknown_runtime": ' - '"src/recursive_integrity_toolkit/reports/unauthorized.py",\n' - ' "premature_completion": ' - '"PHASE_4_COMPLETION.md", "premature_report": ' - '"report.json",\n' - ' }\n' - ' if mutation in replacements:\n' - ' relative, original, replacement = ' - 'replacements[mutation]\n' - ' path = tmp_path / relative\n' - ' source = path.read_bytes()\n' - ' assert source.count(original) == 1\n' - ' ' - 'path.write_bytes(source.replace(original, ' - 'replacement, 1))\n' - ' elif mutation in appended:\n' - ' path = tmp_path / appended[mutation]\n' - ' path.write_bytes(path.read_bytes() + ' - 'b"\\nUnauthorized Step 8 mutation\\n")\n' - ' elif mutation in added:\n' - ' path = tmp_path / added[mutation]\n' - ' assert not path.exists()\n' - ' path.write_text("Unauthorized ' - 'later-stage file\\n", encoding="utf-8")\n' - ' elif mutation == "missing_runtime":\n' - ' (tmp_path / ' - '"src/recursive_integrity_toolkit/metrics/bounds.py").unlink()\n' - ' elif mutation.startswith("control_"):\n' - ' path = tmp_path / ' - '"PHASE_4_BASELINE.json"\n' - ' control = ' - 'json.loads(path.read_text(encoding="utf-8"))\n' - ' if mutation == "control_step":\n' - ' control["active_step"] = 9\n' - ' elif mutation == "control_scope":\n' - ' ' - 'control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py")\n' - ' else:\n' - ' ' - 'control["schema_changes_authorized"] = True\n' - ' ' - 'control["schema_paths_authorized"].append("schemas/report.schema.json")\n' - ' path.write_text(json.dumps(control), ' - 'encoding="utf-8")\n' - ' else:\n' - ' raise AssertionError(f"Unknown active ' - 'mutation case: {mutation}")\n' - ' with pytest.raises(ValueError):\n' - ' verify(tmp_path)'}], - 'tests/unit/test_phase4_contracts.py': [{'new': 'def ' - 'test_phase4_step8_approved_control_keeps_independent_step7_anchors(phase4_tools, ' - 'phase4_step8_snapshot):\n' - ' control = json.loads((phase4_step8_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' assert phase4_tools["PHASE4_STEP7_FINAL"] == ' - '"bacd33ae65e392872776c6d976401fdc3d565f63"\n' - ' assert phase4_tools["PHASE4_STEP7_TREE"] == ' - '"4c16914bb0f0c5281efc36624efb8b1a1d4a174e"\n' - ' assert ' - 'phase4_tools["PHASE4_STEP7_TEST_TREE"] == ' - '"96317cb7b25236b79efc0f2d93e1e0f5f132fdb2"\n' - ' assert control["active_phase"] == 4 and ' - 'control["active_step"] == 8\n' - ' assert control["approval_basis"] == "phase 4 ' - 'step 8 开始"\n' - ' assert control["baseline_commit"] == ' - 'BASELINE\n' - ' assert control["previous_step_commit"] == ' - '"bacd33ae65e392872776c6d976401fdc3d565f63"\n' - ' assert control["previous_step_tree"] == ' - '"4c16914bb0f0c5281efc36624efb8b1a1d4a174e"\n' - ' assert control["previous_step_test_tree"] == ' - '"96317cb7b25236b79efc0f2d93e1e0f5f132fdb2"\n' - ' assert control["previous_step_core_tests"] ' - '== 3620\n' - ' assert ' - 'control["previous_step_parquet_tests"] == 3623\n' - ' assert ' - 'len(control["previous_step_files_sha256"]) == ' - '229\n' - ' assert ' - 'set(control["runtime_paths_authorized"]) == {\n' - ' ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' }\n' - ' assert control["schema_changes_authorized"] ' - 'is False\n' - ' assert control["schema_paths_authorized"] == ' - '[]\n' - ' assert control["new_files_permitted"] == ' - "['src/recursive_integrity_toolkit/data/hero/EXPECTED_OUTPUTS.md', " - "'src/recursive_integrity_toolkit/data/hero/config.json', " - "'src/recursive_integrity_toolkit/data/hero/provenance.csv', " - "'src/recursive_integrity_toolkit/data/hero/records_v1.csv', " - "'src/recursive_integrity_toolkit/data/hero/records_v2.csv', " - "'src/recursive_integrity_toolkit/data/hero/version_order.json', " - "'src/recursive_integrity_toolkit/data/report.schema.json']\n" - ' for name in ("phase_complete", ' - '"next_step_authorized", "main_merge_authorized", ' - '"publication_authorized"):\n' - ' assert control[name] is False\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control, ' - 'step=8)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step4_control"](control, ' - 'step=4)', - 'node': 'test_phase4_step8_approved_control_keeps_independent_step7_anchors', - 'old': 'def ' - 'test_phase4_step8_approved_control_keeps_independent_step7_anchors(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' assert phase4_tools["PHASE4_STEP7_FINAL"] == ' - '"bacd33ae65e392872776c6d976401fdc3d565f63"\n' - ' assert phase4_tools["PHASE4_STEP7_TREE"] == ' - '"4c16914bb0f0c5281efc36624efb8b1a1d4a174e"\n' - ' assert ' - 'phase4_tools["PHASE4_STEP7_TEST_TREE"] == ' - '"96317cb7b25236b79efc0f2d93e1e0f5f132fdb2"\n' - ' assert control["active_phase"] == 4 and ' - 'control["active_step"] == 8\n' - ' assert control["approval_basis"] == "phase 4 ' - 'step 8 开始"\n' - ' assert control["baseline_commit"] == ' - 'BASELINE\n' - ' assert control["previous_step_commit"] == ' - '"bacd33ae65e392872776c6d976401fdc3d565f63"\n' - ' assert control["previous_step_tree"] == ' - '"4c16914bb0f0c5281efc36624efb8b1a1d4a174e"\n' - ' assert control["previous_step_test_tree"] == ' - '"96317cb7b25236b79efc0f2d93e1e0f5f132fdb2"\n' - ' assert control["previous_step_core_tests"] ' - '== 3620\n' - ' assert ' - 'control["previous_step_parquet_tests"] == 3623\n' - ' assert ' - 'len(control["previous_step_files_sha256"]) == ' - '229\n' - ' assert ' - 'set(control["runtime_paths_authorized"]) == {\n' - ' ' - '"src/recursive_integrity_toolkit/cli.py",\n' - ' ' - '"src/recursive_integrity_toolkit/config.py",\n' - ' }\n' - ' assert control["schema_changes_authorized"] ' - 'is False\n' - ' assert control["schema_paths_authorized"] == ' - '[]\n' - ' assert control["new_files_permitted"] == ' - "['src/recursive_integrity_toolkit/data/hero/EXPECTED_OUTPUTS.md', " - "'src/recursive_integrity_toolkit/data/hero/config.json', " - "'src/recursive_integrity_toolkit/data/hero/provenance.csv', " - "'src/recursive_integrity_toolkit/data/hero/records_v1.csv', " - "'src/recursive_integrity_toolkit/data/hero/records_v2.csv', " - "'src/recursive_integrity_toolkit/data/hero/version_order.json', " - "'src/recursive_integrity_toolkit/data/report.schema.json']\n" - ' for name in ("phase_complete", ' - '"next_step_authorized", "main_merge_authorized", ' - '"publication_authorized"):\n' - ' assert control[name] is False\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control, ' - 'step=8)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step4_control"](control, ' - 'step=4)'}, - {'new': 'def ' - 'test_phase4_step8_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value, phase4_step8_snapshot):\n' - ' control = json.loads((phase4_step8_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' verify = ' - 'phase4_tools["verify_phase4_step8_control"]\n' - ' verify(control, step=8)\n' - ' control[field] = value\n' - ' with pytest.raises(ValueError):\n' - ' verify(control, step=8)', - 'node': 'test_phase4_step8_control_rejects_forged_scope_and_stage', - 'old': 'def ' - 'test_phase4_step8_control_rejects_forged_scope_and_stage(phase4_tools, ' - 'field, value):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' verify = ' - 'phase4_tools["verify_phase4_step8_control"]\n' - ' verify(control, step=8)\n' - ' control[field] = value\n' - ' with pytest.raises(ValueError):\n' - ' verify(control, step=8)'}, - {'new': 'def ' - 'test_phase4_step8_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step, phase4_step8_snapshot):\n' - ' control = json.loads((phase4_step8_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control, ' - 'step=8)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control, ' - 'step=step)', - 'node': 'test_phase4_step8_control_rejects_unapproved_dispatch', - 'old': 'def ' - 'test_phase4_step8_control_rejects_unapproved_dispatch(phase4_tools, ' - 'step):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control, ' - 'step=8)\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control, ' - 'step=step)'}, - {'new': 'def ' - 'test_phase4_step8_control_requires_explicit_approval_fields(phase4_tools, ' - 'field, phase4_step8_snapshot):\n' - ' control = json.loads((phase4_step8_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control)\n' - ' del control[field]\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control)', - 'node': 'test_phase4_step8_control_requires_explicit_approval_fields', - 'old': 'def ' - 'test_phase4_step8_control_requires_explicit_approval_fields(phase4_tools, ' - 'field):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control)\n' - ' del control[field]\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control)'}, - {'new': 'def ' - 'test_phase4_step8_control_cannot_mint_extra_permission(phase4_tools, ' - 'phase4_step8_snapshot):\n' - ' control = json.loads((phase4_step8_snapshot ' - '/ ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control)\n' - ' control["approved_scope_expansion"] = ' - '{"step": 8, "publication": True}\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control)', - 'node': 'test_phase4_step8_control_cannot_mint_extra_permission', - 'old': 'def ' - 'test_phase4_step8_control_cannot_mint_extra_permission(phase4_tools):\n' - ' control = json.loads((ROOT / ' - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control)\n' - ' control["approved_scope_expansion"] = ' - '{"step": 8, "publication": True}\n' - ' with pytest.raises(ValueError):\n' - ' ' - 'phase4_tools["verify_phase4_step8_control"](control)'}, - {'new': 'def ' - 'test_phase4_step8_historical_bindings_preserve_assertions(phase4_tools, ' - 'phase4_step7_snapshot, phase4_step8_snapshot):\n' - ' import ast\n' - ' registry = ' - 'phase4_tools["PHASE4_STEP8_MIGRATIONS"]\n' - ' assert sorted(len(rows) for rows in ' - 'registry.values()) == [4, 7]\n' - ' for path, rows in registry.items():\n' - ' ' - 'phase4_tools["verify_phase4_step8_test_migration"](path, ' - '(phase4_step7_snapshot / path).read_bytes(), ' - '(phase4_step8_snapshot / path).read_bytes())\n' - ' for row in rows:\n' - ' trees = [ast.parse(row[key]) for key ' - 'in ("old", "new")]\n' - ' assert trees[0].body[0].name == ' - 'trees[1].body[0].name == row["node"]\n' - ' assert [[ast.dump(n) for n in ' - 'ast.walk(t) if isinstance(n, ast.Assert)] for t ' - 'in trees][0] == [[ast.dump(n) for n in ' - 'ast.walk(t) if isinstance(n, ast.Assert)] for t ' - 'in trees][1]', - 'node': 'test_phase4_step8_historical_bindings_preserve_assertions', - 'old': 'def ' - 'test_phase4_step8_historical_bindings_preserve_assertions(phase4_tools, ' - 'phase4_step7_snapshot):\n' - ' import ast\n' - ' registry = ' - 'phase4_tools["PHASE4_STEP8_MIGRATIONS"]\n' - ' assert sorted(len(rows) for rows in ' - 'registry.values()) == [4, 7]\n' - ' for path, rows in registry.items():\n' - ' ' - 'phase4_tools["verify_phase4_step8_test_migration"](path, ' - '(phase4_step7_snapshot / path).read_bytes(), ' - '(ROOT / path).read_bytes())\n' - ' for row in rows:\n' - ' trees = [ast.parse(row[key]) for key ' - 'in ("old", "new")]\n' - ' assert trees[0].body[0].name == ' - 'trees[1].body[0].name == row["node"]\n' - ' assert [[ast.dump(n) for n in ' - 'ast.walk(t) if isinstance(n, ast.Assert)] for t ' - 'in trees][0] == [[ast.dump(n) for n in ' - 'ast.walk(t) if isinstance(n, ast.Assert)] for t ' - 'in trees][1]'}, - {'new': 'def ' - 'test_phase4_step8_historical_guard_rejects_weakening(phase4_tools, ' - 'phase4_step7_snapshot, mutation, ' - 'phase4_step8_snapshot):\n' - ' path = ' - '"tests/unit/test_phase4_contracts.py"\n' - ' before, after = (phase4_step7_snapshot / ' - 'path).read_bytes(), (phase4_step8_snapshot / ' - 'path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step8_test_migration"]\n' - ' verify(path, before, after)\n' - ' if mutation == "assertion":\n' - " needle = b' assert " - 'control["active_phase"] == 4 and ' - 'control["active_step"] == 7\\n\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b" ' - 'assert True\\n", 1)\n' - ' elif mutation == "binding":\n' - ' needle = ' - "b'json.loads((phase4_step7_snapshot / " - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\'\n' - ' assert needle in after\n' - ' after = after.replace(needle, ' - 'needle.replace(b"phase4_step7_snapshot", ' - 'b"ROOT"), 1)\n' - ' elif mutation == "shadow":\n' - ' after += b"\\ndef ' - 'test_phase4_step7_fixed_boundary_opens_only_approved_existing_paths():\\n ' - 'assert True\\n"\n' - ' else:\n' - ' after += b"\\ndef ' - 'test_phase4_step8_bad(value=globals().clear()):\\n ' - 'pass\\n"\n' - ' with pytest.raises(ValueError): verify(path, ' - 'before, after)', - 'node': 'test_phase4_step8_historical_guard_rejects_weakening', - 'old': 'def ' - 'test_phase4_step8_historical_guard_rejects_weakening(phase4_tools, ' - 'phase4_step7_snapshot, mutation):\n' - ' path = ' - '"tests/unit/test_phase4_contracts.py"\n' - ' before, after = (phase4_step7_snapshot / ' - 'path).read_bytes(), (ROOT / path).read_bytes()\n' - ' verify = ' - 'phase4_tools["verify_phase4_step8_test_migration"]\n' - ' verify(path, before, after)\n' - ' if mutation == "assertion":\n' - " needle = b' assert " - 'control["active_phase"] == 4 and ' - 'control["active_step"] == 7\\n\'\n' - ' assert after.count(needle) == 1\n' - ' after = after.replace(needle, b" ' - 'assert True\\n", 1)\n' - ' elif mutation == "binding":\n' - ' needle = ' - "b'json.loads((phase4_step7_snapshot / " - '"PHASE_4_BASELINE.json").read_text(encoding="utf-8"))\'\n' - ' assert needle in after\n' - ' after = after.replace(needle, ' - 'needle.replace(b"phase4_step7_snapshot", ' - 'b"ROOT"), 1)\n' - ' elif mutation == "shadow":\n' - ' after += b"\\ndef ' - 'test_phase4_step7_fixed_boundary_opens_only_approved_existing_paths():\\n ' - 'assert True\\n"\n' - ' else:\n' - ' after += b"\\ndef ' - 'test_phase4_step8_bad(value=globals().clear()):\\n ' - 'pass\\n"\n' - ' with pytest.raises(ValueError): verify(path, ' - 'before, after)'}]} - - -PHASE4_STEP9_GOLDEN_SHA256 = {'tests/golden/phase4_hero_redacted.json': 'd2138b4f982d09c967cc3268323511ba29add6420ed6b64d91cf13379652e523', - 'tests/golden/phase4_hero_redacted.md': 'df9a0cfabf6eb95f2f494eddf23adb55cc161e2df197b16fbcc36b654eeba3f9', - 'tests/golden/phase4_hero_report.json': 'ce259014260e296e13707693745308c958b829c26373a0ed27c0826690abe633', - 'tests/golden/phase4_hero_report.md': 'bcab1cf1c3008ec86ee1c2a11cac0b130d668911f8e00868209451d74807da80', - 'tests/golden/phase4_report_cases.md': '3ca7005e2dddc990ea1c18fa3408343f9b51a943e0210fe0f611771111ce9709', - 'tests/golden/phase4_report_expected.json': '94fb170bc4095a8f765f23c9b8b87ffb784020734315d0704f36ec4ba3eba10f'} - - -def _phase4_step8_files(): - """Read the exact accepted Step 8 Git tree and verify every blob identity.""" - for suffix, expected in (("^{commit}", PHASE4_STEP8_FINAL), ("^{tree}", PHASE4_STEP8_TREE), - (":tests", PHASE4_STEP8_TEST_TREE)): - if git("rev-parse", PHASE4_STEP8_FINAL + suffix).decode().strip() != expected: - raise ValueError("Pinned Phase 4 Step 8 identity mismatch") - objects = {} - for entry in git("ls-tree", "-rz", PHASE4_STEP8_FINAL).split(b"\0"): - if not entry: - continue - metadata, raw_path = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - path = raw_path.decode("utf-8") - if (mode not in (b"100644", b"100755") or kind != b"blob" or path in objects - or path.startswith("/") or ".." in path.split("/") or ".git" in path.split("/")): - raise ValueError("Unsafe pinned Step 8 Git object") - objects[path] = oid.decode("ascii") - if len(objects) != 236: - raise ValueError("Pinned Step 8 must contain exactly 236 files") - files = {} - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_STEP8_FINAL))) as archive: - names = [item.filename for item in archive.infolist() if not item.is_dir()] - if len(names) != len(objects) or set(names) != set(objects): - raise ValueError("Pinned Step 8 archive identity mismatch") - for name in names: - raw = archive.read(name) - oid = hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - if oid != objects[name]: - raise ValueError(f"Pinned Step 8 blob mismatch: {name}") - files[name] = raw - return MappingProxyType(files) - - -PHASE4_CURRENT_STEP = 11 -PHASE4_CURRENT_APPROVAL = "继续 Phase 4 Step 11" -PHASE4_CURRENT_BASE = "9142b344bfc2dc2b33dbc696159f07bbc81b223a" -PHASE4_CURRENT_TREE = "a485078d3e7cf739e10b087acb604f317ee19854" -PHASE4_CURRENT_VERSION_PATHS = { - "src/recursive_integrity_toolkit/__init__.py", "pyproject.toml", -} -PHASE4_CURRENT_NEW = { - "PHASE_4_COMPLETION.md", "PHASE_4_VALIDATION_REPORT.md", - "PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md", -} -PHASE4_CURRENT_ALLOWED = PHASE4_G | PHASE4_CURRENT_VERSION_PATHS | PHASE4_CURRENT_NEW | { - "README.md", "CHANGELOG.md", "docs/cli.md", "docs/report_schema.md", - "docs/data_schema.md", "docs/privacy.md", "docs/release_process.md", - "tests/integration/test_cli_validation.py", "tests/integration/test_phase4_cli.py", - "tests/golden/test_phase4_reports.py", -} - - -def _phase4_current_files() -> dict[str, bytes]: - """Use accepted Git content directly, without another source-hash registry.""" - if git("rev-parse", PHASE4_CURRENT_BASE + "^{tree}").decode().strip() != PHASE4_CURRENT_TREE: - raise ValueError("Accepted Phase 4 source tree mismatch") - with zipfile.ZipFile(io.BytesIO(git("archive", "--format=zip", PHASE4_CURRENT_BASE))) as archive: - return {item.filename: archive.read(item) for item in archive.infolist() if not item.is_dir()} - - -def phase4_current_expected_control() -> dict: - """Bind current permission to the user's authorization and accepted Git source.""" - result = json.loads(git("show", PHASE4_CURRENT_BASE + ":PHASE_4_BASELINE.json")) - result.update({ - "control_version": "1.10", "active_step": PHASE4_CURRENT_STEP, - "approval_date": "2026-09-22", "approval_basis": PHASE4_CURRENT_APPROVAL, - "previous_step_commit": PHASE4_CURRENT_BASE, "previous_step_tree": PHASE4_CURRENT_TREE, - "previous_step_test_tree": "2fc4e86c4dee9f069f93af59caab138f0ea63224", - "permitted_paths": sorted(PHASE4_CURRENT_ALLOWED), "new_files_permitted": sorted(PHASE4_CURRENT_NEW), - "report_goldens_authorized": False, - "runtime_changes_authorized": True, - "runtime_paths_authorized": ["src/recursive_integrity_toolkit/__init__.py"], - }) - # Git preserves this accepted snapshot; avoid growing parallel hash registries. - for key in ("previous_step_files_sha256", "previous_step_core_tests", - "previous_step_parquet_tests", "runtime_repair_approval_basis"): - result.pop(key, None) - return result - - -def verify_phase4_current_control(control: dict, step: int = PHASE4_CURRENT_STEP) -> None: - if type(step) is not int or step != PHASE4_CURRENT_STEP or type(control) is not dict: - raise ValueError("Unsupported current Phase 4 stage/control") - try: - actual = json.dumps(control, sort_keys=True, ensure_ascii=True, allow_nan=False) - expected = json.dumps(phase4_current_expected_control(), sort_keys=True, ensure_ascii=True, allow_nan=False) - except (TypeError, ValueError) as error: - raise ValueError("Invalid current Phase 4 control") from error - if actual != expected: - raise ValueError("Phase 4 control differs from the independently authorized current scope") - - -def verify_phase4_current_changes(changes: list[tuple[str, str]]) -> None: - seen = set() - for status, path in changes: - if (path in seen or path not in PHASE4_CURRENT_ALLOWED - or status != ("A" if path in PHASE4_CURRENT_NEW else "M")): - raise ValueError(f"Unapproved current Phase 4 path/operation: {status} {path}") - seen.add(path) - - -def _phase4_step9_header(node, path): - """Allow explicit test parametrization/fixtures, never definition-time effects.""" - for decorator in node.decorator_list: - if not isinstance(decorator, ast.Call) or ast.unparse(decorator.func) not in { - "pytest.fixture", "pytest.mark.parametrize"}: - raise ValueError(f"Unapproved Step 9 test decorator: {path}:{node.name}") - if ast.unparse(decorator.func) == "pytest.fixture" and not node.name.startswith("phase4_step"): - raise ValueError("A Step 9 fixture must have an explicit scoped name") - if any(keyword.arg is None for keyword in decorator.keywords): - raise ValueError("Decorator expansion is outside the Step 9 contract") - for value in [*decorator.args, *(keyword.value for keyword in decorator.keywords)]: - try: - ast.literal_eval(value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError("Step 9 decorator arguments must be literal") from error - clone = ast.parse(ast.unparse(node)).body[0] - clone.decorator_list = [] - _phase4_preserve_function_header(clone, path) - - -def _phase4_step9_append_only(before: bytes, after: bytes, path: str) -> None: - if not after.startswith(before): - raise ValueError(f"Inherited Step 8 prefix changed: {path}") - old, new = ast.parse(before), ast.parse(after) - bindings = set() - for node in old.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - bindings.add(node.name) - elif isinstance(node, (ast.Import, ast.ImportFrom)): - bindings.update(alias.asname or alias.name.split(".")[0] for alias in node.names) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - bindings.update(child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)) - for node in new.body[len(old.body):]: - if not isinstance(node, ast.FunctionDef) or node.name in bindings: - raise ValueError(f"Step 9 addition executes, rebinds or shadows inherited source: {path}") - allowed_snapshot = path == "tests/conftest.py" and node.name == "phase4_step8_snapshot" - if not allowed_snapshot and not node.name.startswith(("test_phase4_step9_", "phase4_step9_")): - raise ValueError(f"Step 9 added test/helper is not explicitly scoped: {path}:{node.name}") - if path == "tests/conftest.py" and node.name != "phase4_step8_snapshot": - raise ValueError("Only the pinned Step 8 shared fixture is authorized") - bindings.add(node.name) - _phase4_step9_header(node, path) - - -def verify_phase4_step9_test_migration(path: str, before: bytes, after: bytes) -> None: - prior = _phase4_step8_files() - if path not in prior or path not in PHASE4_STEP9_ALLOWED or before != prior[path] or not path.startswith("tests/"): - raise ValueError("Step 9 migration requires the exact named Step 8 source") - _phase4_step9_check_test_migration(path, before, after) - - -def _phase4_step9_check_test_migration(path: str, before: bytes, after: bytes) -> None: - """Check a source already read from the verified Step 8 immutable mapping.""" - expected = before - for row in PHASE4_STEP9_MIGRATIONS.get(path, []): - old, new = row["old"].encode(), row["new"].encode() - if expected.count(old) != 1: - raise ValueError("Step 8 historical binding is not unique") - expected = expected.replace(old, new, 1) - _phase4_step9_append_only(expected, after, path) - - -def _phase4_step9_preserve_tooling(before: bytes, after: bytes, path: str) -> None: - """Preserve exact inherited statements using one source-line split per tree.""" - if path == "scripts/release_check.py": - old = ' return phase4_cli_main() if explicit_phase4 else main()' - new = (' explicit_step9 = "--step=9" in argv or any(a == "--step" and b == "9" for a, b in zip(argv, argv[1:]))\n' - ' if explicit_phase4 and explicit_step9:\n' - ' return phase4_step9_cli_main()\n' + old) - # Earlier preserved dispatch functions contain this source too; bind cli_main only. - parsed = ast.parse(before) - entry = next(node for node in parsed.body if isinstance(node, ast.FunctionDef) and node.name == "cli_main") - old_entry = ast.get_source_segment(before.decode(), entry) - if old_entry.count(old) != 1: - raise ValueError("Step 8 release dispatcher identity mismatch") - replacements = [(old_entry, old_entry.replace(old, new, 1))] - else: - old = ' if args.phase == 4 and args.step == 8:\n return phase4_step8_main(step=args.step)\n' - new = old + ' if args.phase == 4 and args.step == 9:\n return phase4_step9_main(step=args.step)\n' - replacements = [(old, new), - ('Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5 or --phase 4 --step 6 or --phase 4 --step 7 or --phase 4 --step 8', - 'Choose explicit --phase 3 --step 11 or --phase 4 --step 1 or --phase 4 --step 2 or --phase 4 --step 3 or --phase 4 --step 4 or --phase 4 --step 5 or --phase 4 --step 6 or --phase 4 --step 7 or --phase 4 --step 8 or --phase 4 --step 9')] - expected = before - for old, new in replacements: - if expected.count(old.encode()) != 1: - raise ValueError(f"Historical dispatcher identity mismatch: {path}") - expected = expected.replace(old.encode(), new.encode(), 1) - trees = [ast.parse(expected), ast.parse(after)] - lines = [expected.splitlines(keepends=True), after.splitlines(keepends=True)] - def segment(index, node): - if node.lineno == node.end_lineno: - return lines[index][node.lineno - 1][node.col_offset:node.end_col_offset] - return (lines[index][node.lineno - 1][node.col_offset:] - + b"".join(lines[index][node.lineno:node.end_lineno - 1]) - + lines[index][node.end_lineno - 1][:node.end_col_offset]) - def entry_guard(node): - return isinstance(node, ast.If) and isinstance(node.test, ast.Compare) and "__name__" in ast.unparse(node.test) - guards = [n for n in trees[1].body if entry_guard(n)] - expected_guard = ast.parse('if __name__ == "__main__":\n raise SystemExit(cli_main())').body[0] - if len(guards) != 1 or ast.dump(guards[0]) != ast.dump(expected_guard): - raise ValueError(f"Unapproved Step 9 maintainer entrypoint: {path}") - historical = [n for n in trees[0].body if not entry_guard(n)] - current = [n for n in trees[1].body if not entry_guard(n)] - old_entries = [(segment(0, n), ast.dump(n)) for n in historical] - cursor = 0 - for node in current: - if cursor < len(old_entries) and (segment(1, node), ast.dump(node)) == old_entries[cursor]: - cursor += 1 - continue - if isinstance(node, ast.FunctionDef) and (node.name == "_phase4_step8_files" or node.name.startswith( - ("phase4_step9_", "_phase4_step9_", "verify_phase4_step9_", "audit_phase4_step9"))): - _phase4_preserve_function_header(node, path) - continue - if isinstance(node, ast.Assign) and all(isinstance(target, ast.Name) and target.id.startswith( - ("PHASE4_STEP9_", "PHASE4_STEP8_")) for target in node.targets): - try: - ast.literal_eval(node.value) - except (ValueError, TypeError, SyntaxError) as error: - raise ValueError(f"Nonliteral added Step 9 maintainer constant: {path}") from error - continue - raise ValueError(f"Unapproved Step 9 maintainer addition: {path}") - if cursor != len(historical): - raise ValueError(f"Inherited Step 8 maintainer statement changed: {path}") - def binding_counts(tree): - counts = {} - for node in tree.body: - names = [] - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - names = [node.name] - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names = [child.id for target in targets for child in ast.walk(target) if isinstance(child, ast.Name)] - elif isinstance(node, (ast.Import, ast.ImportFrom)): - names = [alias.asname or alias.name.split(".")[0] for alias in node.names] - for name in names: - counts[name] = counts.get(name, 0) + 1 - return counts - inherited_counts = binding_counts(trees[0]) - for name, count in binding_counts(trees[1]).items(): - if count > inherited_counts.get(name, 1): - raise ValueError(f"Duplicate Step 9 maintainer binding: {path}:{name}") - - -def phase4_step9_golden_contract(root: Path = ROOT) -> dict: - """Check separately frozen rationale and literal expected reports byte for byte.""" - expected = { - "tests/golden/phase4_report_cases.md", "tests/golden/phase4_report_expected.json", - "tests/golden/phase4_hero_report.json", "tests/golden/phase4_hero_report.md", - "tests/golden/phase4_hero_redacted.json", "tests/golden/phase4_hero_redacted.md", - } - if set(PHASE4_STEP9_GOLDEN_SHA256) != expected: - raise ValueError("Step 9 independently authored golden identities are not frozen") - for name, digest in PHASE4_STEP9_GOLDEN_SHA256.items(): - path = root / name - if not path.is_file() or path.is_symlink() or hashlib.sha256(path.read_bytes()).hexdigest() != digest: - raise ValueError(f"Step 9 golden rationale or expected bytes changed: {name}") - return {"frozen_report_oracle_files": len(expected)} - - -def verify_phase4_current_snapshot(root: Path = ROOT) -> dict: - """Protect unchanged product/authority bytes with the accepted Git snapshot. - - Current maintenance is reviewed and tested by behavior. Obsolete source forms - are retained in Git instead of adding another source-binding migration layer. - """ - prior = _phase4_current_files() - for path, raw in prior.items(): - target = root / path - if not target.is_file() or target.is_symlink(): - raise ValueError(f"Missing or aliased accepted file: {path}") - if path in PHASE4_CURRENT_VERSION_PATHS: - if raw.count(b"0.1.0.dev2") != 1 or target.read_bytes() != raw.replace(b"0.1.0.dev2", b"0.1.0.dev3", 1): - raise ValueError(f"Only the approved dev3 version literal may change: {path}") - if path not in PHASE4_CURRENT_ALLOWED and target.read_bytes() != raw: - raise ValueError(f"Protected accepted bytes changed: {path}") - for prefix in ("src/recursive_integrity_toolkit/", "schemas/", "examples/hero/"): - paths = list((root / prefix).rglob("*")) - if any(path.is_symlink() for path in paths): - raise ValueError(f"Aliased protected content: {prefix}") - current = {path.relative_to(root).as_posix() for path in paths - if path.is_file() and "__pycache__" not in path.parts} - if current != {path for path in prior if path.startswith(prefix)}: - raise ValueError(f"Protected accepted file set changed: {prefix}") - verify_phase4_current_control(json.loads((root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8"))) - decisions = (root / "PHASE_4_DECISIONS.md").read_text(encoding="utf-8") - if PHASE4_CURRENT_APPROVAL not in decisions or PHASE4_CURRENT_BASE not in decisions: - raise ValueError("Actual authorization or accepted source anchor missing") - for name in PHASE4_CURRENT_NEW: - path = root / name - if not path.is_file() or path.is_symlink() or not path.read_text(encoding="utf-8").strip(): - raise ValueError(f"Step 11 requires a nonempty regular completion record: {name}") - if any((root / name).exists() for name in PHASE4_FORBIDDEN_OUTPUTS - PHASE4_CURRENT_NEW): - raise ValueError("Audit outputs cannot be committed as completion records") - return {"package_modules": 40, "frozen_runtime_modules": 39, "version_literal_changes": 2, "frozen_schemas": 5, - "hero_files_unchanged": 6, "packaged_resources": 7, "phase_complete": False, - **phase4_step9_golden_contract(root)} - - -def phase4_step9_golden_evidence(junit: Path) -> dict: - """Require all collected report goldens and all twenty unchanged math cases.""" - verify_junit(junit) - selected = ["tests/golden/test_phase4_reports.py", "tests/golden/test_phase3_math.py"] - process = subprocess.run([sys.executable, "-m", "pytest", "-p", "no:cacheprovider", "--collect-only", "-q", *selected], cwd=ROOT, capture_output=True, text=True, check=True) - nodes = [line.strip() for line in process.stdout.splitlines() if line.startswith("tests/") and "::" in line] - oracle_path = "tests/golden/phase3_math_cases.json" - oracle_bytes = (ROOT / oracle_path).read_bytes() - if hashlib.sha256(oracle_bytes).hexdigest() != STEP10_FROZEN_FILES[oracle_path]: - raise ValueError("Step 9 mathematical oracle bytes differ from the frozen authority") - expected_math = {selected[1] + "::test_phase3_frozen_case[" + case["case_id"] + "]" - for case in json.loads(oracle_bytes)["cases"]} - registry = selected[1] + "::test_phase3_all_twenty_oracles_have_unique_traceable_identity" - actual_math = {n for n in nodes if n.startswith(selected[1] + "::")} - if len(nodes) != len(set(nodes)) or len(expected_math) != 20 or actual_math != expected_math | {registry}: - raise ValueError("Step 9 golden collection lost mathematical cases or their registry identity") - if not any(n.startswith(selected[0] + "::") for n in nodes): - raise ValueError("Step 9 report golden suite is empty") - expected = set() + source = extract_sdist(sdist, work / "source") / "src" + hashes = {destination.removeprefix("src/recursive_integrity_toolkit/"): hashlib.sha256((ROOT/original).read_bytes()).hexdigest() for destination,original in RESOURCES.items()} + subprocess.run([sys.executable, "-I", "-c", installed_example_program(), str(source), str(work), str(ROOT), json.dumps(hashes)], cwd=work, check=True) + + +def smoke_all(wheel: Path, sdist: Path) -> None: + for check in (smoke_installed, smoke_installed_duplicates, smoke_installed_contract, + smoke_installed_assembly, smoke_installed_privacy, smoke_installed_renderers, + smoke_installed_publication, smoke_installed_cli, smoke_installed_example): + check(wheel) + smoke_sdist(sdist) + + +def _canonical_identities() -> set[tuple[str, str]]: + env = dict(os.environ, PYTHONPATH=str(ROOT / "src"), PYTHONDONTWRITEBYTECODE="1", RIT_TEST_PARQUET="1") + result = subprocess.run([sys.executable, "-m", "pytest", "-p", "no:cacheprovider", + "--collect-only", "-q", "--ignore=tests/performance"], + cwd=ROOT, env=env, capture_output=True, text=True, check=True) + nodes = [line.strip() for line in result.stdout.splitlines() if line.startswith("tests/") and "::" in line] + if not nodes or len(nodes) != len(set(nodes)): + raise ValueError("Canonical collection is empty or has duplicate identities") + identities = set() for node in nodes: base, bracket, parameter = node.partition("[") owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(junit).getroot().iter("testcase")} - selected_owners = {p.removesuffix(".py").replace("/", ".") for p in selected} - actual = {case for case in cases if case[0] in selected_owners} - if actual != expected: - raise ValueError("Step 9 JUnit does not execute every collected golden and mathematical case") - return {"mathematical_cases": 20, "report_golden_tests": sum(n.startswith(selected[0] + "::") for n in nodes), - "report_golden_nodeids": [n for n in nodes if n.startswith(selected[0] + "::")], - "mathematical_nodeids": [n for n in nodes if n in expected_math], - "mathematical_registry_nodeid": registry, - "missing_nodeids": [], "golden_nodeids_sha256": hashlib.sha256(("\n".join(nodes) + "\n").encode()).hexdigest()} - - -def audit_phase4_current(step: int = PHASE4_CURRENT_STEP) -> dict: - verify_phase4_current_control(json.loads((ROOT / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")), step) - result = {"phase": 4, "active_step": step, **verify_phase4_current_snapshot(), - "phase0_hashes_verified": 16, "publication_authorized": False} - print(json.dumps(result, indent=2)) - return result - - -def audit_phase4_current_diff(step: int = PHASE4_CURRENT_STEP) -> dict: - if type(step) is not int or step != PHASE4_CURRENT_STEP: - raise ValueError("Unsupported current Phase 4 stage") - subprocess.run(["git", "-C", str(ROOT), "merge-base", "--is-ancestor", PHASE4_CURRENT_BASE, "HEAD"], check=True) - raw = [part.decode("utf-8") for part in git("diff", "--name-status", "--no-renames", "-z", PHASE4_CURRENT_BASE, "--").split(b"\0") if part] - if len(raw) % 2: - raise ValueError("Malformed current Git difference records") - changes = list(zip(raw[::2], raw[1::2])) - changes.extend(("A", p.decode()) for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") if p) - verify_phase4_current_changes(changes) - result = {"previous_step_commit": PHASE4_CURRENT_BASE, "changed_files": len(changes), - "changes": changes, "step": step, "scope": "PASS"} - print(json.dumps(result, indent=2)) - return result + identities.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) + return identities -def phase4_current_suite_evidence(output: Path) -> dict: - """Collect canonical current identities once; historical receipts remain archived.""" - output.mkdir(parents=True, exist_ok=True) - current, log = _collect(ROOT) - result = {"current_nodeids": current, "current_tests": len(current), - "nodeids_sha256": hashlib.sha256(("\n".join(current)+"\n").encode()).hexdigest()} - (output / "phase4_current_test_identity_manifest.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4-current-collection.log").write_text(log, encoding="utf-8") - print(json.dumps({"current_tests": len(current)}, indent=2)) - return result - +def candidate(output: Path) -> None: + """Verify a committed candidate; workflow separately requires its scale job. -def phase4_current_candidate(output: Path, step: int = PHASE4_CURRENT_STEP) -> None: - """Build tested current Phase 4 candidate evidence without declaring Phase 4 complete.""" + Core/Parquet JUnit must belong to this source revision, checked by the existing + workflow before invocation. This entry point does not prove their origin. + """ if os.environ.get("RIT_TEST_PARQUET") != "1" or importlib.util.find_spec("pyarrow") is None: - raise ValueError("current Phase 4 candidate evidence requires RIT_TEST_PARQUET=1 and real PyArrow") - output.mkdir(parents=True, exist_ok=True) - result = audit_phase4_current(step) - result["diff"] = audit_phase4_current_diff(step) + raise ValueError("Candidate verification requires RIT_TEST_PARQUET=1 and real PyArrow") if git("status", "--porcelain").strip(): - raise ValueError("current Phase 4 candidate archive requires committed, clean source") - result["core"] = verify_junit(output / "core.xml", minimum=3768) - result["parquet"] = verify_junit(output / "parquet.xml", require_parquet=True, minimum=3771) - result["core_math_measurements"] = verify_step10_evidence(output / "core.xml", output / "phase4-current-core-observations.json") - result["parquet_math_measurements"] = verify_step10_evidence(output / "parquet.xml", output / "phase4-current-parquet-observations.json") - result["core_report_goldens"] = phase4_step9_golden_evidence(output / "core.xml") - result["parquet_report_goldens"] = phase4_step9_golden_evidence(output / "parquet.xml") - print(json.dumps({name: result[name] for name in ("core_report_goldens", "parquet_report_goldens")}, indent=2)) - identity = phase4_current_suite_evidence(output) - result["test_identity"] = {k: identity[k] for k in ("current_tests", "nodeids_sha256")} - expected = set() - for node in identity["current_nodeids"]: - base, bracket, parameter = node.partition("[") - owner, name = base.rsplit("::", 1) - expected.add((owner.removesuffix(".py").replace("/", ".").replace("::", "."), name + bracket + parameter)) + raise ValueError("Candidate archive requires committed, clean source") + result = audit() + expected = _canonical_identities() for name, parquet in (("core.xml", False), ("parquet.xml", True)): - cases = {(c.get("classname", ""), c.get("name", "")) for c in ET.parse(output / name).getroot().iter("testcase")} - target = expected if parquet else {c for c in expected if c[1] not in PARQUET_CASES} - if cases != target: - raise ValueError(f"current Phase 4 JUnit does not execute the entire current suite: {name}") - wheel, sdist = phase4_step8_distributions(output / "dist", expected_version="0.1.0.dev3") - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - phase4_step6_installed_publication_smoke(wheel) - phase4_step7_installed_cli_smoke(wheel) - phase4_step8_installed_example_smoke(wheel) - phase4_step8_sdist_smoke(sdist) - archive = output / "recursive-integrity-toolkit-phase4-current-candidate.zip" - subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) - tracked = {p for p in git("ls-files", "-z").decode().split("\0") if p} + result[name] = verify_junit(output / name, require_parquet=parquet) + actual = {(case.get("classname", ""), case.get("name", "")) + for case in ET.parse(output / name).getroot().iter("testcase")} + required = expected if parquet else {item for item in expected if item[1] not in PARQUET_CASES} + if actual != required: + raise ValueError(f"Candidate JUnit does not execute the complete canonical regression suite: {name}") + wheel, sdist = verify_distributions(output / "dist") + smoke_all(wheel, sdist) + archive = output / "recursive-integrity-toolkit-source.zip" + subprocess.run(["git", "-C", str(ROOT), "archive", "--format=zip", + "--prefix=recursive-integrity-toolkit/", "HEAD", "-o", str(archive.resolve())], check=True) + tracked = {name for name in git("ls-files", "-z").decode().split("\0") if name} with zipfile.ZipFile(archive) as zipped: - entries = [item.filename for item in zipped.infolist() if not item.is_dir()] + entries = [member.filename for member in zipped.infolist() if not member.is_dir()] prefix = "recursive-integrity-toolkit/" - if (len(entries) != len(set(entries)) or any(not name.startswith(prefix) for name in entries)): - raise ValueError("current Phase 4 source archive has duplicate or unprefixed entries") - names = {name.removeprefix(prefix) for name in entries} - if names != tracked or len(tracked) != 246 or len(entries) != 246: - raise ValueError("current Phase 4 source archive file set mismatch") - for name in names: - if zipped.read("recursive-integrity-toolkit/"+name) != (ROOT / name).read_bytes(): - raise ValueError(f"current Phase 4 source archive byte mismatch: {name}") - result.update({"commit": git("rev-parse", "HEAD").decode().strip(), - "tree": git("rev-parse", "HEAD^{tree}").decode().strip(), - "source_archive": archive.name, "tracked_files": len(tracked), - "python": sys.version, "platform": sys.platform, - "versions": {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "pytest")}}) - (output / "phase4_current_execution_metadata.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8") - (output / "phase4_current_repository_files.sha256").write_text("".join(f"{hashlib.sha256((ROOT/name).read_bytes()).hexdigest()} {name}\n" for name in sorted(tracked)), encoding="utf-8") - paths = sorted(p for p in output.rglob("*") if p.is_file() and p.name != "phase4_current_artifacts.sha256") - (output / "phase4_current_artifacts.sha256").write_text("".join(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {p.relative_to(output).as_posix()}\n" for p in paths), encoding="utf-8") - print(f"Phase 4 current candidate: {len(tracked)} source files verified; Phase 4 remains incomplete.") - - -def phase4_current_cli_main() -> int: - parser = argparse.ArgumentParser(description="Current Phase 4 maintainer gate") - parser.add_argument("--phase", type=int, choices=(4,), required=True) - parser.add_argument("--step", type=int, choices=(PHASE4_CURRENT_STEP,), required=True) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) - parser.add_argument("--golden-junit", type=Path) - parser.add_argument("--require-parquet", action="store_true") - parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - raise ValueError("Current Phase 4 cannot certify a final phase delivery") - if args.golden_junit is not None: - print(json.dumps(phase4_step9_golden_evidence(args.golden_junit), indent=2)) - elif args.junit is not None: - verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - phase4_current_suite_evidence(args.baseline_evidence) - elif args.dist is not None: - audit_phase4_current(args.step) - wheel, sdist = phase4_step8_distributions(args.dist, expected_version="0.1.0.dev3") - smoke_installed(wheel) - smoke_installed_duplicates(wheel) - phase4_step2_installed_contract_smoke(wheel) - phase4_step3_installed_assembly_smoke(wheel) - phase4_step4_installed_privacy_smoke(wheel) - phase4_step5_installed_renderers_smoke(wheel) - phase4_step6_installed_publication_smoke(wheel) - phase4_step7_installed_cli_smoke(wheel) - phase4_step8_installed_example_smoke(wheel) - phase4_step8_sdist_smoke(sdist) - elif args.smoke_wheel is not None: - smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - phase4_current_candidate(args.candidate, args.step) - else: - audit_phase4_current(args.step) - if args.diff: - audit_phase4_current_diff(args.step) - return 0 - - -def cli_main() -> int: - argv = sys.argv[1:] - explicit_phase4 = "--phase=4" in argv or any(a == "--phase" and b == "4" for a, b in zip(argv, argv[1:])) - explicit_step2 = "--step=2" in argv or any(a == "--step" and b == "2" for a, b in zip(argv, argv[1:])) - if explicit_phase4 and explicit_step2: - return phase4_step2_cli_main() - explicit_step3 = "--step=3" in argv or any(a == "--step" and b == "3" for a, b in zip(argv, argv[1:])) - if explicit_phase4 and explicit_step3: - return phase4_step3_cli_main() - explicit_step4 = "--step=4" in argv or any(a == "--step" and b == "4" for a, b in zip(argv, argv[1:])) - if explicit_phase4 and explicit_step4: - return phase4_step4_cli_main() - explicit_step5 = "--step=5" in argv or any(a == "--step" and b == "5" for a, b in zip(argv, argv[1:])) - if explicit_phase4 and explicit_step5: - return phase4_step5_cli_main() - explicit_step6 = "--step=6" in argv or any(a == "--step" and b == "6" for a, b in zip(argv, argv[1:])) - if explicit_phase4 and explicit_step6: - return phase4_step6_cli_main() - explicit_step7 = "--step=7" in argv or any(a == "--step" and b == "7" for a, b in zip(argv, argv[1:])) - if explicit_phase4 and explicit_step7: - return phase4_step7_cli_main() - explicit_step8 = "--step=8" in argv or any(a == "--step" and b == "8" for a, b in zip(argv, argv[1:])) - if explicit_phase4 and explicit_step8: - return phase4_step8_cli_main() - explicit_current = "--step=11" in argv or any(a == "--step" and b == "11" for a, b in zip(argv, argv[1:])) - if explicit_phase4 and explicit_current: - return phase4_current_cli_main() - return phase4_cli_main() if explicit_phase4 else main() - - -def main() -> int: + if len(entries) != len(set(entries)) or {name.removeprefix(prefix) for name in entries} != tracked: + raise ValueError("Candidate source archive file set mismatch") + for name in tracked: + if zipped.read(prefix + name) != (ROOT / name).read_bytes(): + raise ValueError(f"Candidate source archive byte mismatch: {name}") + result.update(commit=git("rev-parse", "HEAD").decode().strip(), + source_archive=archive.name, tracked_files=len(tracked), + canonical_scope="tests excluding performance; designated scale job required separately", + python=sys.version, platform=sys.platform) + (output / "candidate.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print("Current candidate package and installation checks: PASS; no publication performed") + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + if any(arg in {"--phase", "--step"} or arg.startswith(("--phase=", "--step=")) for arg in argv): + raise SystemExit("Historical phase dispatch has been retired. Run the current command without --phase/--step; use Git history at the accepted commit to inspect prior gates.") parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--phase", type=int, choices=(2, 3), default=2) - parser.add_argument("--step", type=int, default=ACTIVE_STEP) - parser.add_argument("--diff", action="store_true") - parser.add_argument("--junit", type=Path) + modes = parser.add_mutually_exclusive_group() + modes.add_argument("--junit", type=Path) + modes.add_argument("--dist", type=Path) + modes.add_argument("--smoke-wheel", type=Path) + modes.add_argument("--candidate", type=Path) parser.add_argument("--require-parquet", action="store_true") parser.add_argument("--minimum-tests", type=int, default=1) - parser.add_argument("--baseline-evidence", type=Path) - parser.add_argument("--dist", type=Path) - parser.add_argument("--smoke-wheel", type=Path) - parser.add_argument("--candidate", type=Path) - parser.add_argument("--delivery", type=Path) - args = parser.parse_args() - if args.delivery is not None: - if args.phase != 3 or args.step != 11: - raise ValueError("Phase 3 delivery requires explicit Step 11") - candidate(args.delivery, args.step) - elif args.junit is not None: + args = parser.parse_args(argv) + if args.minimum_tests < 1: + parser.error("--minimum-tests must be positive") + if args.junit: verify_junit(args.junit, require_parquet=args.require_parquet, minimum=args.minimum_tests) - elif args.baseline_evidence is not None: - baseline_evidence(args.baseline_evidence) - elif args.dist is not None: - wheel, _ = verify_distributions(args.dist); smoke_installed(wheel); smoke_installed_duplicates(wheel) - elif args.smoke_wheel is not None: + elif args.dist: + audit() + smoke_all(*verify_distributions(args.dist)) + elif args.smoke_wheel: smoke_installed(args.smoke_wheel) - elif args.candidate is not None: - if args.phase != 3: - raise ValueError("Candidate requires explicit --phase 3") - candidate(args.candidate, args.step) + elif args.candidate: + candidate(args.candidate) else: - audit(args.phase, args.step) - if args.diff: - audit_phase3_diff(args.step) if args.phase == 3 else audit_step10_diff() + audit() return 0 if __name__ == "__main__": - raise SystemExit(cli_main()) + raise SystemExit(main()) diff --git a/src/recursive_integrity_toolkit/__init__.py b/src/recursive_integrity_toolkit/__init__.py index e87c36b..eeb10da 100644 --- a/src/recursive_integrity_toolkit/__init__.py +++ b/src/recursive_integrity_toolkit/__init__.py @@ -3,10 +3,10 @@ Owner IDs: Technical Maintainer, PR-016 -Future inputs: +Inputs: Installed package metadata only. -Future outputs: +Outputs: Stable package version and intentionally approved public imports. Assumptions: @@ -16,9 +16,9 @@ No data loading, configuration, metric, lineage, report, or optional dependency is imported here. Current phase status: - Phase 1 package scaffold with version exposure only. + Phase 5 development package; root import exposes the version only. """ -__version__ = "0.1.0.dev3" +__version__ = "0.1.0.dev4" __all__ = ["__version__"] diff --git a/src/recursive_integrity_toolkit/cli.py b/src/recursive_integrity_toolkit/cli.py index f2e9230..d00d3b5 100644 --- a/src/recursive_integrity_toolkit/cli.py +++ b/src/recursive_integrity_toolkit/cli.py @@ -10,10 +10,10 @@ Each calculated side uses one version and an explicitly declared representation. Limits: No formulas, implicit representation, weighting, content-reference resolution, - automatic pairs, trajectories, simulation or Phase 5 graph execution. Help and + automatic pairs, trajectories or simulation. Lineage requires explicit opt-in. Help and version import no analytical dependencies or input/report implementation. Current phase status: - Phase 4 Step 8. Accepted kernels own every numerical result. + Phase 5 Step 8. Accepted kernels own every numerical result. """ from __future__ import annotations @@ -49,10 +49,14 @@ def build_parser() -> argparse.ArgumentParser: example = commands.add_parser("example", help="Extract and audit the packaged local Hero.", allow_abbrev=False) example.add_argument("--out", required=True, action=_Once, help="New example workspace; its parent must exist.") example.add_argument("--redacted", action=_Once, nargs=0, help="Protect paths and identifiers in reports.") + example.add_argument("--lineage", action=_Once, nargs=0, help="Calculate ancestry for the packaged Hero target.") for command, description in (("audit", "Audit one version or one explicitly declared earlier/later pair."), ("validate", "Validate inputs without running calculations.")): child = commands.add_parser(command, help=description, description=description, allow_abbrev=False) child.add_argument("--records", required=True, action=_Once, help="Local records file (CSV, JSONL or Parquet).") + child.add_argument("--lineage-records", action="append", help="Local ancestor/context records; repeat for multiple files. Audit requires --lineage or config lineage=true.") + if command == "audit": + child.add_argument("--lineage", action=_Once, nargs=0, help="Calculate lineage for the primary records using all explicitly loaded records.") for flag, help_text in ( ("provenance", "Local provenance manifest."), ("config", "Local JSON or TOML configuration."), ("schema-mapping", "Local declarative schema mapping."), ("version-order", "Local explicit version-order document."), @@ -133,6 +137,9 @@ def _options(namespace): options, inventory = load_phase4_invocation(cli=declarations, config_path=namespace.config, base_directory=Path.cwd()) phase4_pair_requested(options, operation=namespace.command) + if (namespace.command != "validate" and not options.lineage + and any(source.role is FileRole.LINEAGE_CONTEXT for source in options.inputs)): + raise ConfigurationError(ErrorCode.CONFIG_INVALID, "Audit context inputs require explicit lineage execution") return options, inventory @@ -264,6 +271,39 @@ def _pair_calculations(bundle, options): return results, exits +def _primary_versions(bundle): + """Select the declared primary population without adopting context versions.""" + from .models import FileRole + return tuple(sorted({row.record_key.dataset_version for row in bundle.records + if row.location.file_role is FileRole.RECORDS_PRIMARY})) + + +def _lineage_calculations(bundle, options): + """Request graph/root APIs explicitly and retain a bounded family failure.""" + from .errors import CanonicalValidationError, ErrorCode, LineageResourceLimitError + from .lineage.ancestry import analyze_lineage, SharedAncestryDependence + from .lineage.graph import LineageLimits + from .metrics.bounds import lineage_closure_exposure + from .models import CapabilityKey + from .reports.assembly import FamilyFailure + try: + versions = _primary_versions(bundle) + if len(versions) > 1: + raise CanonicalValidationError(ErrorCode.SCHEMA_TYPE, + "Lineage requires one primary dataset version", field="dataset_version") + limits = options.configuration.resource_limits + lineage = analyze_lineage(bundle, target_dataset_version=versions[0] if versions else None, + limits=LineageLimits(max_nodes=limits.max_lineage_nodes, max_edges=limits.max_lineage_edges, + max_root_memberships=limits.max_lineage_root_memberships, + max_root_union_visits=limits.max_lineage_root_union_visits)) + return {"lineage": lineage, "lineage_bounds": lineage_closure_exposure(lineage), + "shared_ancestry": SharedAncestryDependence(lineage)}, _message_exits(lineage.messages) + except Exception as error: + usage = error.resource_usage if isinstance(error, LineageResourceLimitError) else None + failure = FamilyFailure(CapabilityKey.LINEAGE, (_message(error),), lineage_resource_usage=usage) + return {"family_errors": (failure,)}, (_error_exit(error),) + + def _run_metadata(options, operation, started_at, started_clock, *, status="complete"): from datetime import datetime, timezone import platform @@ -332,6 +372,7 @@ def _execute(namespace): input_paths = [getattr(namespace, key) for key in ( "records", "provenance", "compare", "config", "schema_mapping", "version_order", "id_salt_file") if getattr(namespace, key, None) is not None] + input_paths.extend(getattr(namespace, "lineage_records", None) or ()) try: options, config_inventory = _options(namespace) input_paths.extend(source.path for source in options.inputs) @@ -357,7 +398,8 @@ def order_state(path): file_role=FileRole.VERSION_ORDER.value) from None return info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns order_snapshots = [] - if namespace.command != "validate" and any(source.role is FileRole.RECORDS_COMPARE for source in options.inputs): + pair_input = any(source.role is FileRole.RECORDS_COMPARE for source in options.inputs) + if namespace.command != "validate" and (pair_input or options.lineage): for source in options.inputs: if source.role is FileRole.VERSION_ORDER: state = order_state(source.path) @@ -376,7 +418,7 @@ def order_state(path): from .reports.assembly import FamilyFailure if (not isinstance(order_error, ToolkitError) or order_error.code is not ErrorCode.VERSION_ORDER_CONFLICT or namespace.command == "validate" - or not any(source.role is FileRole.RECORDS_COMPARE for source in options.inputs)): + or not (pair_input or options.lineage)): raise # Invalid chronology cannot erase independently valid input evidence. # Revalidate without an ordering claim; retain the original failure. @@ -390,7 +432,9 @@ def order_state(path): if source.role is not FileRole.VERSION_ORDER)), configuration=unordered, base_directory=Path.cwd()) bundle = replace(bundle, inventory=tuple(sorted(bundle.inventory + tuple(entry for entry, _ in order_snapshots), key=lambda entry: (entry.role.value, str(entry.path))))) - order_failure = (FamilyFailure(CapabilityKey.DATASET_LONGITUDINAL, (_message(order_error),)),) + order_failure = tuple(FamilyFailure(family, (_message(order_error),)) for family in ( + *((CapabilityKey.DATASET_LONGITUDINAL,) if pair_input else ()), + *((CapabilityKey.LINEAGE,) if options.lineage else ()))) if config_inventory: bundle = replace(bundle, inventory=tuple(sorted(bundle.inventory + config_inventory, key=lambda entry: (entry.role.value, str(entry.path))))) @@ -399,12 +443,19 @@ def order_state(path): if namespace.command != "validate": from .config import phase4_pair_requested pair = phase4_pair_requested(options, operation=namespace.command) - results, calculation_exits = _pair_calculations(bundle, options) if pair else _calculations(bundle, options) + results, calculation_exits = _pair_calculations(bundle, options) if pair else _calculations( + bundle, options, dataset_versions=_primary_versions(bundle), record_role=FileRole.RECORDS_PRIMARY) exits += calculation_exits + if options.lineage and not order_failure: + lineage_results, lineage_exits = _lineage_calculations(bundle, options) + failures = results.get("family_errors", ()) + lineage_results.pop("family_errors", ()) + results.update(lineage_results) + results["family_errors"] = failures + exits += lineage_exits if order_failure: results["family_errors"] = results.get("family_errors", ()) + order_failure exits += (1,) - if not pair and len(bundle.version_order.loaded_versions) > 1: + if not pair and len(_primary_versions(bundle)) > 1: sys.stderr.write("Audit requires one dataset_version; use rit validate to inspect multiple versions.\n") report = assemble_report(bundle, run=_run_metadata(options, namespace.command, started_at, started_clock), **results) if namespace.command == "validate": @@ -470,7 +521,8 @@ def _example(namespace): "--compare", str(inputs / "records_v1.csv"), "--config", str(inputs / "config.json"), "--provenance", str(inputs / "provenance.csv"), "--version-order", str(inputs / "version_order.json"), "--state-semantics", "Hero topic labels retain their literal meaning across v1 and v2.", - "--out", str(workspace / "reports"), *(["--redacted"] if namespace.redacted else [])]) + "--out", str(workspace / "reports"), *(["--redacted"] if namespace.redacted else []), + *(["--lineage"] if namespace.lineage else [])]) invocation.command = "example" except Exception as error: cleaned = True @@ -487,7 +539,8 @@ def _example(namespace): if not cleaned: sys.stderr.write("E_OUTPUT_IO: Incomplete example files remain in the selected workspace; inspect it before retrying.\n") return _exit_code((_OUTPUT_CODES.get(code, 4), 1 if not cleaned else 0)) - sys.stderr.write("Phase 4 example: lineage execution is deferred. inputs/EXPECTED_OUTPUTS.md is the full-product reference, not a claim of calculated lineage results.\n") + if not namespace.lineage: + sys.stderr.write("Lineage was not requested. Use rit example --lineage to calculate the Hero ancestry results.\n") return _execute(invocation) diff --git a/src/recursive_integrity_toolkit/config.py b/src/recursive_integrity_toolkit/config.py index edd76ef..d7d607f 100644 --- a/src/recursive_integrity_toolkit/config.py +++ b/src/recursive_integrity_toolkit/config.py @@ -1,4 +1,4 @@ -"""Preserve Phase 2 configuration and add explicit inert Phase 4 report options. +"""Resolve explicit inert input, report and opt-in lineage declarations. Owner IDs: PR-007, PR-010, PR-011, PR-015, PR-016 supporting infrastructure, PR-017 @@ -8,18 +8,18 @@ Outputs: Immutable ``ResolvedConfig`` objects that preserve user declarations without - inference; restricted Phase 4 options, safe summaries and normalized hashes. + inference; report and lineage options, safe summaries and normalized hashes. Assumptions: Version order, representation choice, privacy mode, strict-mode promotion, and scenario declarations are explicit. Configuration never activates analysis at import time. Limits: - Phase 4 resolution performs no file ingestion, secret read, version-order + Declaration resolution performs no file ingestion, secret read, version-order inference, metric, lineage, simulation, report assembly or remote access. Current phase status: - Phase 4 Step 7 invocation adapter. Inherited parsers remain unchanged. + Phase 5 Step 8 explicit lineage and repeatable context invocation adapter. """ from __future__ import annotations @@ -27,6 +27,7 @@ import json import math import tomllib +from collections import Counter from dataclasses import dataclass, field from pathlib import Path from typing import Any, Mapping @@ -50,12 +51,21 @@ "resource_limits", "output", "simulation", + "lineage", } _MULTI_ROLES = { FileRole.RECORDS_COMPARE, FileRole.EMBEDDING_DATA, FileRole.EXTERNAL_REFERENCE, + FileRole.LINEAGE_CONTEXT, +} + +_LINEAGE_LIMIT_DEFAULTS = { + "max_lineage_nodes": 200000, + "max_lineage_edges": 1000000, + "max_lineage_root_memberships": 1000000, + "max_lineage_root_union_visits": 10000000, } @@ -76,6 +86,10 @@ class ResourceLimits: max_content_bytes: int | None = None max_parent_list_length: int | None = None max_json_depth: int | None = None + max_lineage_nodes: int = _LINEAGE_LIMIT_DEFAULTS["max_lineage_nodes"] + max_lineage_edges: int = _LINEAGE_LIMIT_DEFAULTS["max_lineage_edges"] + max_lineage_root_memberships: int = _LINEAGE_LIMIT_DEFAULTS["max_lineage_root_memberships"] + max_lineage_root_union_visits: int = _LINEAGE_LIMIT_DEFAULTS["max_lineage_root_union_visits"] @dataclass(frozen=True, slots=True) @@ -98,6 +112,7 @@ class ResolvedConfig: resource_limits: ResourceLimits = ResourceLimits() output: tuple[tuple[str, Any], ...] = () simulation: ScenarioConfig = ScenarioConfig() + lineage: bool = False def load_config(path: str | Path) -> ResolvedConfig: @@ -130,7 +145,7 @@ def load_config(path: str | Path) -> ResolvedConfig: def resolve_config(data: Mapping[str, Any]) -> ResolvedConfig: - """Resolve approved Phase 2 configuration fields without inferring missing evidence.""" + """Resolve approved configuration fields without inferring missing evidence.""" unknown = sorted(set(data) - _ALLOWED_TOP_LEVEL) if unknown: raise ConfigurationError( @@ -157,6 +172,7 @@ def resolve_config(data: Mapping[str, Any]) -> ResolvedConfig: resource_limits = _parse_resource_limits(data.get("resource_limits")) output = _frozen_mapping(data.get("output"), "output") simulation = _parse_simulation(data.get("simulation")) + lineage = _bool_value(data.get("lineage", False), "lineage") return ResolvedConfig( config_version=config_version, @@ -171,6 +187,7 @@ def resolve_config(data: Mapping[str, Any]) -> ResolvedConfig: resource_limits=resource_limits, output=output, simulation=simulation, + lineage=lineage, ) @@ -261,13 +278,18 @@ def _parse_resource_limits(value: Any) -> ResourceLimits: "max_parent_list_length", "max_json_depth", } - unknown = sorted(set(value) - allowed) + unknown = sorted(set(value) - allowed - _LINEAGE_LIMIT_DEFAULTS.keys()) if unknown: _invalid(f"resource_limits has unknown fields: {', '.join(unknown)}") kwargs = { key: _optional_positive_int(value.get(key), f"resource_limits.{key}") for key in allowed } + for key, default in _LINEAGE_LIMIT_DEFAULTS.items(): + item = value.get(key, default) + if type(item) is not int or item <= 0: + _invalid(f"resource_limits.{key} must be a positive integer") + kwargs[key] = item return ResourceLimits(**kwargs) @@ -361,7 +383,7 @@ class Phase4Options: """Explicit inert report options; raw declarations are hidden from repr. Resolution does not read inputs, load a secret, perform calculations or - activate a command. ``configuration`` retains the accepted Phase 2 contract. + activate a command. ``configuration`` retains explicit input declarations. """ configuration: ResolvedConfig = field(repr=False) @@ -376,12 +398,14 @@ class Phase4Options: missing_state_id: str | None = field(repr=False) tail_rule: str | None tail_threshold: int | float | None + lineage: bool = False def __post_init__(self) -> None: if (type(self.configuration) is not ResolvedConfig or type(self.inputs) is not tuple or any(type(source) is not InputSource or type(source.role) is not FileRole or not isinstance(source.path, Path) for source in self.inputs) - or type(self.strict_mode) is not bool or type(self.privacy_mode) is not str + or type(self.strict_mode) is not bool or type(self.lineage) is not bool + or type(self.privacy_mode) is not str or self.privacy_mode not in {"standard", "redacted"} or type(self.record_id_mode) is not str or self.record_id_mode not in {"preserve", "hash", "omit"} or not isinstance(self.directory, Path) @@ -447,12 +471,14 @@ def _phase4_config_data(config: ResolvedConfig) -> dict[str, Any]: "state_mapping": dict(config.state_mapping), "version_order": list(config.version_order), "privacy_mode": config.privacy_mode.value, "strict_mode": config.strict_mode, "strict_warning_codes": list(config.strict_warning_codes), + "lineage": config.lineage, "resource_limits": { "max_file_bytes": config.resource_limits.max_file_bytes, "max_rows": config.resource_limits.max_rows, "max_content_bytes": config.resource_limits.max_content_bytes, "max_parent_list_length": config.resource_limits.max_parent_list_length, "max_json_depth": config.resource_limits.max_json_depth, + **{key: getattr(config.resource_limits, key) for key in _LINEAGE_LIMIT_DEFAULTS}, }, "output": dict(config.output), "simulation": {"enabled": config.simulation.enabled, "seed": config.simulation.seed}, @@ -488,11 +514,11 @@ def resolve_phase4_options(config: ResolvedConfig | Mapping[str, Any] | None = N CLI keys use underscore spellings of the approved option names. Omitted options are absent or ``None``. Raw config mappings retain explicit-field provenance. With a ``ResolvedConfig``, its privacy and strictness values are - authoritative because Phase 2 does not retain omission provenance. + authoritative because resolved declarations do not retain omission provenance. """ allowed_cli = {"records", "provenance", "compare", "schema_mapping", "version_order", "state_semantics", "missing_state_id", "out", "redacted", "record_ids", - "id_salt_file", "strict", "tail_rule", "tail_threshold"} + "id_salt_file", "strict", "tail_rule", "tail_threshold", "lineage", "lineage_records"} if cli is None: cli = {} if not isinstance(cli, Mapping) or any(type(key) is not str or key not in allowed_cli for key in cli): @@ -540,6 +566,13 @@ def resolve_phase4_options(config: ResolvedConfig | Mapping[str, Any] | None = N if "strict_mode" in declared: _invalid("Phase 4 strict declarations compete") strict_mode = options["strict"] + lineage = resolved.lineage + if "lineage" in options: + if type(options["lineage"]) is not bool: + _invalid("lineage selection must be boolean") + if "lineage" in declared: + _invalid("lineage declarations compete") + lineage = options["lineage"] record_id_mode = options.get("record_ids", output.get("record_id_mode", "hash" if privacy_mode == "redacted" else "preserve")) if type(record_id_mode) is not str or record_id_mode not in {"preserve", "hash", "omit"}: _invalid("Phase 4 record ID mode must be preserve, hash or omit") @@ -560,6 +593,11 @@ def resolve_phase4_options(config: ResolvedConfig | Mapping[str, Any] | None = N _invalid("Phase 4 singleton input declarations compete") if key in options: sources.append(InputSource(role, _phase4_option_path(options[key]))) + if "lineage_records" in options: + paths = options["lineage_records"] + if type(paths) not in (list, tuple): + _invalid("lineage_records must be a sequence of local paths") + sources.extend(InputSource(FileRole.LINEAGE_CONTEXT, _phase4_option_path(path)) for path in paths) version_order = resolved.version_order if "version_order" in options: if version_order or any(source.role is FileRole.VERSION_ORDER for source in sources): @@ -595,7 +633,7 @@ def resolve_phase4_options(config: ResolvedConfig | Mapping[str, Any] | None = N _invalid("frequency_at_or_below requires a finite threshold within zero and one") return Phase4Options(resolved, tuple(sorted(sources, key=lambda source: (source.role.value, str(source.path)))), directory, privacy_mode, record_id_mode, salt_file, strict_mode, version_order, - state_semantics, missing_state_id, tail_rule, threshold) + state_semantics, missing_state_id, tail_rule, threshold, lineage) def phase4_config_summary(options: Phase4Options) -> dict[str, Any]: @@ -617,7 +655,7 @@ def phase4_config_summary(options: Phase4Options) -> dict[str, Any]: return {"strict_mode": options.strict_mode, "privacy_mode": options.privacy_mode, "record_id_mode": options.record_id_mode, "tail_selection": tail, "weighted": False, "comparison_requested": any(source.role is FileRole.RECORDS_COMPARE for source in options.inputs), - "scenario_requested": False} + "scenario_requested": False, **({"lineage_requested": True} if options.lineage else {})} def phase4_config_hash(options: Phase4Options) -> str: @@ -645,6 +683,15 @@ def phase4_config_hash(options: Phase4Options) -> str: normalized["missing_state_id"] = options.missing_state_id normalized["tail_rule"] = options.tail_rule normalized["tail_threshold"] = options.tail_threshold + # Disabled lineage with unchanged graph defaults retains the ordinary hash. + # Explicit graph overrides still identify distinct input settings. + if options.lineage: + normalized["lineage"] = True + else: + normalized.pop("lineage") + for key, default in _LINEAGE_LIMIT_DEFAULTS.items(): + if normalized["resource_limits"][key] == default: + del normalized["resource_limits"][key] return sha256_canonical({"encoding": "rit.phase4.config.v1", "options": normalized}) @@ -689,17 +736,22 @@ def load_phase4_invocation(*, cli: Mapping[str, Any], config_path: str | Path | _phase4_literal(str(path)) if options.id_salt_file is not None: _phase4_literal(str(options.id_salt_file)) - configured_roles = {source.role for source in options.configuration.inputs} - sources = tuple(InputSource(source.role, source.path if source.path.is_absolute() else - (config_base if source.role in configured_roles else base) / source.path, - source.declared_format) for source in options.inputs) + configured_sources = Counter(options.configuration.inputs) + sources = [] + for item in options.inputs: + from_config = configured_sources[item] > 0 + if from_config: + configured_sources[item] -= 1 + source_base = config_base if from_config else base + sources.append(InputSource(item.role, item.path if item.path.is_absolute() else source_base / item.path, + item.declared_format)) output = dict(options.configuration.output) directory_base = config_base if "directory" in output else base directory = options.directory if options.directory.is_absolute() else directory_base / options.directory salt = options.id_salt_file if salt is not None and not salt.is_absolute(): salt = (config_base if "id_salt_file" in output else base) / salt - return replace(options, inputs=sources, directory=directory, id_salt_file=salt), inventory + return replace(options, inputs=tuple(sources), directory=directory, id_salt_file=salt), inventory def phase4_validation_configuration(options: Phase4Options) -> dict[str, Any]: @@ -711,6 +763,7 @@ def phase4_validation_configuration(options: Phase4Options) -> dict[str, Any]: data["strict_mode"] = options.strict_mode data["privacy_mode"] = options.privacy_mode data["version_order"] = list(options.version_order) + data["lineage"] = options.lineage return data @@ -724,7 +777,7 @@ def phase4_pair_requested(options: Phase4Options, *, operation: str) -> bool: _invalid("invalid Phase 4 operation") requested = any(source.role is FileRole.RECORDS_COMPARE for source in options.inputs) if operation == "validate": - if options.state_semantics is not None or options.tail_rule is not None: + if options.state_semantics is not None or options.tail_rule is not None or options.lineage: _invalid("input-only validation cannot request calculations") return False if requested != (options.state_semantics is not None): diff --git a/src/recursive_integrity_toolkit/data/report.schema.json b/src/recursive_integrity_toolkit/data/report.schema.json index c3fce78..df4b687 100644 --- a/src/recursive_integrity_toolkit/data/report.schema.json +++ b/src/recursive_integrity_toolkit/data/report.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://recursive-integrity-toolkit.example/schemas/report.schema.json", - "title": "Recursive Integrity Toolkit canonical report 1.0", - "description": "Phase 4 Step 2 public contract. Product metadata and five analytical evidence classes remain separate.", + "title": "Recursive Integrity Toolkit canonical report 1.1", + "description": "Phase 5 public contract with bounded lineage evidence. Product metadata and five analytical evidence classes remain separate.", "type": "object", "properties": { "run": { @@ -19,7 +19,7 @@ "pattern": "^[^\\u0000]*$" }, "report_schema_version": { - "const": "1.0" + "const": "1.1" }, "started_at": { "anyOf": [ @@ -294,6 +294,7 @@ "enum": [ "records_primary", "records_compare", + "lineage_context", "provenance_manifest", "schema_mapping", "config", @@ -6752,7 +6753,12 @@ "const": "observed_fact" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "T6.graph_cycle_check" @@ -7011,7 +7017,7 @@ } ] }, - "ordering_certificate": { + "graph_scope": { "type": "object", "properties": { "value": { @@ -7019,10 +7025,34 @@ { "type": "object", "properties": { - "method": { - "const": "declared_earlier_version_order" + "target_dataset_version": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] }, - "version_order": { + "target_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "loaded_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "context_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "loaded_dataset_versions": { "type": "array", "items": { "type": "string", @@ -7031,15 +7061,14 @@ }, "minItems": 0, "uniqueItems": true - }, - "all_resolved_edges_follow_order": { - "type": "boolean" } }, "required": [ - "method", - "version_order", - "all_resolved_edges_follow_order" + "target_dataset_version", + "target_record_count", + "loaded_record_count", + "context_record_count", + "loaded_dataset_versions" ], "additionalProperties": false }, @@ -7049,7 +7078,7 @@ ] }, "unit": { - "const": "certificate" + "const": "scope" }, "evidence_class": { "const": "observed_fact" @@ -7063,11 +7092,11 @@ ] }, "method_id": { - "const": "PR-008.earlier_version_certificate" + "const": "T4.graph_scope" }, "owner_ids": { "const": [ - "PR-008" + "T4" ] }, "theory_map_ids": { @@ -7310,10 +7339,34 @@ "value": { "type": "object", "properties": { - "method": { - "const": "declared_earlier_version_order" + "target_dataset_version": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] }, - "version_order": { + "target_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "loaded_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "context_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "loaded_dataset_versions": { "type": "array", "items": { "type": "string", @@ -7322,15 +7375,14 @@ }, "minItems": 0, "uniqueItems": true - }, - "all_resolved_edges_follow_order": { - "type": "boolean" } }, "required": [ - "method", - "version_order", - "all_resolved_edges_follow_order" + "target_dataset_version", + "target_record_count", + "loaded_record_count", + "context_record_count", + "loaded_dataset_versions" ], "additionalProperties": false } @@ -7338,71 +7390,3357 @@ } } ] - } - }, - "required": [], - "additionalProperties": false - }, - "state_counts": { - "type": "object", - "properties": { - "by_version": { + }, + "cycle_analysis": { "type": "object", - "propertyNames": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "additionalProperties": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "array", - "items": { + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "detected": { + "type": "boolean" + }, + "cycle_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "counting_method": { + "const": "cyclic_strongly_connected_components" + }, + "cycle_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "target_cycle_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "affected_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "target_affected_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "components": { "type": "object", "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_index": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "member_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "target_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "witness_record_keys": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 2, + "maxItems": 65 + }, + { + "type": "null" + } + ] + }, + "witness_edge_count": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 64 + }, + { + "type": "null" + } + ] + }, + "witness_reason": { + "anyOf": [ + { + "const": "diagnostic_limit" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "component_index", + "member_count", + "target_member_count", + "witness_record_keys", + "witness_edge_count", + "witness_reason" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] }, - "state_count": { + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { "type": "integer", "minimum": 0, "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true } }, "required": [ - "state_id", - "state_count" + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" ], - "additionalProperties": false + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_index": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "member_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "target_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "witness_record_keys": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 2, + "maxItems": 65 + }, + { + "type": "null" + } + ] + }, + "witness_edge_count": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 64 + }, + { + "type": "null" + } + ] + }, + "witness_reason": { + "anyOf": [ + { + "const": "diagnostic_limit" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "component_index", + "member_count", + "target_member_count", + "witness_record_keys", + "witness_edge_count", + "witness_reason" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] }, - "minItems": 0 - }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "records" - }, - "evidence_class": { - "const": "observed_fact" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "T1.state_counts" + "cycle_member_record_keys": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + }, + "affected_record_keys": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + } + }, + "required": [ + "detected", + "cycle_count", + "counting_method", + "cycle_member_count", + "target_cycle_member_count", + "affected_record_count", + "target_affected_record_count", + "components", + "cycle_member_record_keys", + "affected_record_keys" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "cycles" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T6.cyclic_strongly_connected_components" + }, + "owner_ids": { + "const": [ + "T6" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "detected": { + "type": "boolean" + }, + "cycle_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "counting_method": { + "const": "cyclic_strongly_connected_components" + }, + "cycle_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "target_cycle_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "affected_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "target_affected_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "components": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_index": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "member_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "target_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "witness_record_keys": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 2, + "maxItems": 65 + }, + { + "type": "null" + } + ] + }, + "witness_edge_count": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 64 + }, + { + "type": "null" + } + ] + }, + "witness_reason": { + "anyOf": [ + { + "const": "diagnostic_limit" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "component_index", + "member_count", + "target_member_count", + "witness_record_keys", + "witness_edge_count", + "witness_reason" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "component_index": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "member_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "target_member_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "witness_record_keys": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 2, + "maxItems": 65 + }, + { + "type": "null" + } + ] + }, + "witness_edge_count": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 64 + }, + { + "type": "null" + } + ] + }, + "witness_reason": { + "anyOf": [ + { + "const": "diagnostic_limit" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "component_index", + "member_count", + "target_member_count", + "witness_record_keys", + "witness_edge_count", + "witness_reason" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + }, + "cycle_member_record_keys": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + }, + "affected_record_keys": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/record_key" + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + } + }, + "required": [ + "detected", + "cycle_count", + "counting_method", + "cycle_member_count", + "target_cycle_member_count", + "affected_record_count", + "target_affected_record_count", + "components", + "cycle_member_record_keys", + "affected_record_keys" + ], + "additionalProperties": false + } + } + } + } + ] + }, + "depth_summary": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "depth_resolved_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "maximum_resolved_target_depth": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "target_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "depth_resolved_record_count", + "maximum_resolved_target_depth", + "target_record_count" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "edges" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "PR-009.resolved_depth_summary" + }, + "owner_ids": { + "const": [ + "PR-009" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "depth_resolved_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "maximum_resolved_target_depth": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "target_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "depth_resolved_record_count", + "maximum_resolved_target_depth", + "target_record_count" + ], + "additionalProperties": false + } + } + } + } + ] + }, + "unresolved_record_details": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "record_key", + "reason_codes" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "record_key", + "reason_codes" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "records" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T4.unresolved_record_details" + }, + "owner_ids": { + "const": [ + "T4" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "record_key", + "reason_codes" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "record_key", + "reason_codes" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } + } + ] + } + } + } + } + ] + }, + "resource_usage": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "admitted_node_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "admitted_edge_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "stored_root_membership_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "root_union_visit_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limits": { + "type": "object", + "properties": { + "max_nodes": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_edges": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_root_memberships": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_root_union_visits": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "max_nodes", + "max_edges", + "max_root_memberships", + "max_root_union_visits" + ], + "additionalProperties": false + }, + "exhausted_limit": { + "anyOf": [ + { + "type": "string", + "enum": [ + "max_nodes", + "max_edges", + "max_root_memberships", + "max_root_union_visits" + ] + }, + { + "type": "null" + } + ] + }, + "attempted_value": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "admitted_node_count", + "admitted_edge_count", + "stored_root_membership_count", + "root_union_visit_count", + "limits", + "exhausted_limit", + "attempted_value" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "work_units" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "PR-015.lineage_resource_usage" + }, + "owner_ids": { + "const": [ + "PR-015" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "admitted_node_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "admitted_edge_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "stored_root_membership_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "root_union_visit_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limits": { + "type": "object", + "properties": { + "max_nodes": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_edges": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_root_memberships": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "max_root_union_visits": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "max_nodes", + "max_edges", + "max_root_memberships", + "max_root_union_visits" + ], + "additionalProperties": false + }, + "exhausted_limit": { + "anyOf": [ + { + "type": "string", + "enum": [ + "max_nodes", + "max_edges", + "max_root_memberships", + "max_root_union_visits" + ] + }, + { + "type": "null" + } + ] + }, + "attempted_value": { + "anyOf": [ + { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "admitted_node_count", + "admitted_edge_count", + "stored_root_membership_count", + "root_union_visit_count", + "limits", + "exhausted_limit", + "attempted_value" + ], + "additionalProperties": false + } + } + } + } + ] + }, + "ordering_certificate": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "method": { + "const": "declared_earlier_version_order" + }, + "version_order": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "all_resolved_edges_follow_order": { + "type": "boolean" + } + }, + "required": [ + "method", + "version_order", + "all_resolved_edges_follow_order" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "certificate" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "PR-008.earlier_version_certificate" + }, + "owner_ids": { + "const": [ + "PR-008" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "object", + "properties": { + "method": { + "const": "declared_earlier_version_order" + }, + "version_order": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "all_resolved_edges_follow_order": { + "type": "boolean" + } + }, + "required": [ + "method", + "version_order", + "all_resolved_edges_follow_order" + ], + "additionalProperties": false + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + }, + "state_counts": { + "type": "object", + "properties": { + "by_version": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "additionalProperties": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "state_id", + "state_count" + ], + "additionalProperties": false + }, + "minItems": 0 + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "records" + }, + "evidence_class": { + "const": "observed_fact" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T1.state_counts" }, "owner_ids": { "const": [ @@ -8533,134 +11871,445 @@ "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + } + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + } + }, + "support_delta": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "integer", + "minimum": -1.7976931348623157e+308, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "states" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "F-005" + }, + "owner_ids": { + "const": [ + "T1" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true + "minItems": 1 }, "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minItems": 1 } - }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } - }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } + } + }, + "else": { + "properties": { + "value": { + "type": "integer", + "minimum": -1.7976931348623157e+308, + "maximum": 1.7976931348623157e+308 } - ] + } } }, - "required": [], - "additionalProperties": false - } + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] }, - "support_delta": { + "support_retention_ratio": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "integer", - "minimum": -1.7976931348623157e+308, - "maximum": 1.7976931348623157e+308 + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, { "type": "null" @@ -8668,7 +12317,7 @@ ] }, "unit": { - "const": "states" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -8682,7 +12331,7 @@ ] }, "method_id": { - "const": "F-005" + "const": "F-006" }, "owner_ids": { "const": [ @@ -8927,9 +12576,17 @@ "else": { "properties": { "value": { - "type": "integer", - "minimum": -1.7976931348623157e+308, - "maximum": 1.7976931348623157e+308 + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] } } } @@ -8955,23 +12612,15 @@ } ] }, - "support_retention_ratio": { + "support_loss_count": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 }, { "type": "null" @@ -8979,7 +12628,7 @@ ] }, "unit": { - "const": "ratio" + "const": "states" }, "evidence_class": { "const": "derived_metric" @@ -8993,7 +12642,7 @@ ] }, "method_id": { - "const": "F-006" + "const": "T1.support_set_difference" }, "owner_ids": { "const": [ @@ -9238,17 +12887,9 @@ "else": { "properties": { "value": { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 } } } @@ -9274,7 +12915,7 @@ } ] }, - "support_loss_count": { + "support_added_count": { "type": "object", "properties": { "value": { @@ -9577,15 +13218,20 @@ } ] }, - "support_added_count": { + "extinct_states": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true }, { "type": "null" @@ -9593,7 +13239,7 @@ ] }, "unit": { - "const": "states" + "const": "set_of_states" }, "evidence_class": { "const": "derived_metric" @@ -9607,7 +13253,7 @@ ] }, "method_id": { - "const": "T1.support_set_difference" + "const": "T1.earlier_minus_later" }, "owner_ids": { "const": [ @@ -9852,9 +13498,14 @@ "else": { "properties": { "value": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true } } } @@ -9880,7 +13531,7 @@ } ] }, - "extinct_states": { + "added_states": { "type": "object", "properties": { "value": { @@ -9915,7 +13566,7 @@ ] }, "method_id": { - "const": "T1.earlier_minus_later" + "const": "T1.later_minus_earlier" }, "owner_ids": { "const": [ @@ -10193,7 +13844,7 @@ } ] }, - "added_states": { + "retained_states": { "type": "object", "properties": { "value": { @@ -10228,7 +13879,7 @@ ] }, "method_id": { - "const": "T1.later_minus_earlier" + "const": "T1.support_intersection" }, "owner_ids": { "const": [ @@ -10476,50 +14127,276 @@ "type": "array", "items": { "type": "string", - "minLength": 0, + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + } + } + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] + }, + "comparison_details": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "earlier_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "later_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "version_order": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 2, + "uniqueItems": true + }, + "version_order_source": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "earlier_state_semantics": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "later_state_semantics": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "harmonized_state_semantics": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "compatibility_method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "earlier_representation": { + "$ref": "#/$defs/representation" + }, + "later_representation": { + "$ref": "#/$defs/representation" + }, + "harmonized_representation": { + "$ref": "#/$defs/representation" + }, + "original_earlier_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "original_later_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "harmonized_earlier_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "harmonized_later_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "retention_denominator": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "retention_denominator_basis": { + "type": "string", + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } - } - ] - }, - "retained_states": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "state_mapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "target_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "source_state", + "target_state" + ], + "additionalProperties": false + }, + "minItems": 0 + }, + "mapping_effect": { + "type": "array", + "items": { + "type": "object", + "properties": { + "dataset_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "original_support_size": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "harmonized_support_size": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "dataset_version", + "original_support_size", + "harmonized_support_size" + ], + "additionalProperties": false + }, + "minItems": 0 + }, + "collision_groups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "target_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "source_states": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 2, + "uniqueItems": true + } + }, + "required": [ + "target_state", + "source_states" + ], + "additionalProperties": false + }, + "minItems": 0 + } }, - "minItems": 0, - "uniqueItems": true + "required": [ + "earlier_version", + "later_version", + "version_order", + "version_order_source", + "earlier_state_semantics", + "later_state_semantics", + "harmonized_state_semantics", + "compatibility_method", + "earlier_representation", + "later_representation", + "harmonized_representation", + "original_earlier_support", + "original_later_support", + "harmonized_earlier_support", + "harmonized_later_support", + "retention_denominator", + "retention_denominator_basis", + "state_mapping", + "mapping_effect", + "collision_groups" + ], + "additionalProperties": false }, { "type": "null" @@ -10527,7 +14404,7 @@ ] }, "unit": { - "const": "set_of_states" + "const": "comparison" }, "evidence_class": { "const": "derived_metric" @@ -10541,7 +14418,7 @@ ] }, "method_id": { - "const": "T1.support_intersection" + "const": "T1.explicit_pair_basis" }, "owner_ids": { "const": [ @@ -10786,822 +14663,602 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } - } - ] - }, - "comparison_details": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "object", - "properties": { - "earlier_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "later_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "version_order": { - "type": "array", - "items": { + "type": "object", + "properties": { + "earlier_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "later_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "version_order": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 2, + "uniqueItems": true + }, + "version_order_source": { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 2, - "uniqueItems": true - }, - "version_order_source": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "earlier_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "later_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "harmonized_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "compatibility_method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "earlier_representation": { - "$ref": "#/$defs/representation" - }, - "later_representation": { - "$ref": "#/$defs/representation" - }, - "harmonized_representation": { - "$ref": "#/$defs/representation" - }, - "original_earlier_support": { - "type": "array", - "items": { + "earlier_state_semantics": { "type": "string", - "minLength": 0, + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - }, - "original_later_support": { - "type": "array", - "items": { + "later_state_semantics": { "type": "string", - "minLength": 0, + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - }, - "harmonized_earlier_support": { - "type": "array", - "items": { + "harmonized_state_semantics": { "type": "string", - "minLength": 0, + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - }, - "harmonized_later_support": { - "type": "array", - "items": { + "compatibility_method": { "type": "string", - "minLength": 0, + "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 0, - "uniqueItems": true - }, - "retention_denominator": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "earlier_representation": { + "$ref": "#/$defs/representation" + }, + "later_representation": { + "$ref": "#/$defs/representation" + }, + "harmonized_representation": { + "$ref": "#/$defs/representation" + }, + "original_earlier_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" }, - { - "type": "null" - } - ] - }, - "retention_denominator_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "state_mapping": { - "type": "array", - "items": { - "type": "object", - "properties": { - "source_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "target_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - } + "minItems": 0, + "uniqueItems": true + }, + "original_later_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" }, - "required": [ - "source_state", - "target_state" - ], - "additionalProperties": false + "minItems": 0, + "uniqueItems": true }, - "minItems": 0 - }, - "mapping_effect": { - "type": "array", - "items": { - "type": "object", - "properties": { - "dataset_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "original_support_size": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - { - "type": "null" - } - ] + "harmonized_earlier_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "harmonized_later_support": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "retention_denominator": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 }, - "harmonized_support_size": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - { - "type": "null" - } - ] + { + "type": "null" } + ] + }, + "retention_denominator_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "state_mapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "target_state": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "source_state", + "target_state" + ], + "additionalProperties": false + }, + "minItems": 0 + }, + "mapping_effect": { + "type": "array", + "items": { + "type": "object", + "properties": { + "dataset_version": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "original_support_size": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + }, + "harmonized_support_size": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "dataset_version", + "original_support_size", + "harmonized_support_size" + ], + "additionalProperties": false }, - "required": [ - "dataset_version", - "original_support_size", - "harmonized_support_size" - ], - "additionalProperties": false + "minItems": 0 }, - "minItems": 0 - }, - "collision_groups": { - "type": "array", - "items": { - "type": "object", - "properties": { - "target_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "source_states": { - "type": "array", - "items": { + "collision_groups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "target_state": { "type": "string", "minLength": 0, "pattern": "^[^\\u0000]*$" }, - "minItems": 2, - "uniqueItems": true - } + "source_states": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 2, + "uniqueItems": true + } + }, + "required": [ + "target_state", + "source_states" + ], + "additionalProperties": false }, - "required": [ - "target_state", - "source_states" - ], - "additionalProperties": false - }, - "minItems": 0 - } - }, - "required": [ - "earlier_version", - "later_version", - "version_order", - "version_order_source", - "earlier_state_semantics", - "later_state_semantics", - "harmonized_state_semantics", - "compatibility_method", - "earlier_representation", - "later_representation", - "harmonized_representation", - "original_earlier_support", - "original_later_support", - "harmonized_earlier_support", - "harmonized_later_support", - "retention_denominator", - "retention_denominator_basis", - "state_mapping", - "mapping_effect", - "collision_groups" - ], - "additionalProperties": false - }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "comparison" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "T1.explicit_pair_basis" - }, - "owner_ids": { - "const": [ - "T1" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" - } - ] - }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "denominator_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" + "minItems": 0 + } + }, + "required": [ + "earlier_version", + "later_version", + "version_order", + "version_order_source", + "earlier_state_semantics", + "later_state_semantics", + "harmonized_state_semantics", + "compatibility_method", + "earlier_representation", + "later_representation", + "harmonized_representation", + "original_earlier_support", + "original_later_support", + "harmonized_earlier_support", + "harmonized_later_support", + "retention_denominator", + "retention_denominator_basis", + "state_mapping", + "mapping_effect", + "collision_groups" + ], + "additionalProperties": false + } } - ] - }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" + } }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - } - }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ { "if": { "properties": { "status": { - "const": "unavailable" + "enum": [ + "available", + "partial" + ] } } }, "then": { "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 + "representation": { + "$ref": "#/$defs/representation" } } - }, - "else": { + } + } + ] + } + }, + "required": [], + "additionalProperties": false + }, + "diversity": { + "type": "object", + "properties": { + "by_version": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "additionalProperties": { + "type": "object", + "properties": { + "gini_simpson_diversity": { + "type": "object", "properties": { "value": { - "type": "object", - "properties": { - "earlier_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "later_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "version_order": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 2, - "uniqueItems": true - }, - "version_order_source": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "earlier_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "later_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + { + "type": "null" + } + ] + }, + "unit": { + "const": "dimensionless" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "F-003" + }, + "owner_ids": { + "const": [ + "T1" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" }, - "harmonized_state_semantics": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "compatibility_method": { + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "earlier_representation": { - "$ref": "#/$defs/representation" - }, - "later_representation": { - "$ref": "#/$defs/representation" - }, - "harmonized_representation": { - "$ref": "#/$defs/representation" - }, - "original_earlier_support": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "original_later_support": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "harmonized_earlier_support": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "harmonized_later_support": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "retention_denominator": { - "anyOf": [ + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "retention_denominator_basis": { + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "state_mapping": { - "type": "array", - "items": { - "type": "object", - "properties": { - "source_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "target_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - } - }, - "required": [ - "source_state", - "target_state" - ], - "additionalProperties": false + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, - "minItems": 0 - }, - "mapping_effect": { - "type": "array", - "items": { - "type": "object", - "properties": { - "dataset_version": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "original_support_size": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - { - "type": "null" - } - ] - }, - "harmonized_support_size": { - "anyOf": [ - { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - { - "type": "null" - } - ] - } - }, - "required": [ - "dataset_version", - "original_support_size", - "harmonized_support_size" - ], - "additionalProperties": false + "reason_codes": { + "minItems": 1 }, - "minItems": 0 - }, - "collision_groups": { - "type": "array", - "items": { - "type": "object", - "properties": { - "target_state": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 }, - "source_states": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 2, - "uniqueItems": true + { + "exclusiveMaximum": 0 } - }, - "required": [ - "target_state", - "source_states" - ], - "additionalProperties": false - }, - "minItems": 0 + ] + } + } + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } } }, - "required": [ - "earlier_version", - "later_version", - "version_order", - "version_order_source", - "earlier_state_semantics", - "later_state_semantics", - "harmonized_state_semantics", - "compatibility_method", - "earlier_representation", - "later_representation", - "harmonized_representation", - "original_earlier_support", - "original_later_support", - "harmonized_earlier_support", - "harmonized_later_support", - "retention_denominator", - "retention_denominator_basis", - "state_mapping", - "mapping_effect", - "collision_groups" - ], - "additionalProperties": false - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } } - } + ] }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } - } - ] - } - }, - "required": [], - "additionalProperties": false - }, - "diversity": { - "type": "object", - "properties": { - "by_version": { - "type": "object", - "propertyNames": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "additionalProperties": { - "type": "object", - "properties": { - "gini_simpson_diversity": { + "simpson_concentration": { "type": "object", "properties": { "value": { @@ -11639,7 +15296,7 @@ ] }, "method_id": { - "const": "F-003" + "const": "F-004" }, "owner_ids": { "const": [ @@ -11920,7 +15577,7 @@ } ] }, - "simpson_concentration": { + "weighted_gini_simpson_diversity": { "type": "object", "properties": { "value": { @@ -11958,7 +15615,7 @@ ] }, "method_id": { - "const": "F-004" + "const": "F-003" }, "owner_ids": { "const": [ @@ -12239,7 +15896,7 @@ } ] }, - "weighted_gini_simpson_diversity": { + "weighted_simpson_concentration": { "type": "object", "properties": { "value": { @@ -12277,7 +15934,7 @@ ] }, "method_id": { - "const": "F-003" + "const": "F-004" }, "owner_ids": { "const": [ @@ -12558,23 +16215,42 @@ } ] }, - "weighted_simpson_concentration": { + "state_frequencies": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_frequency": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - { - "exclusiveMaximum": 0 - } - ] + "required": [ + "state_id", + "state_frequency" + ], + "additionalProperties": false + }, + "minItems": 0 }, { "type": "null" @@ -12582,7 +16258,7 @@ ] }, "unit": { - "const": "dimensionless" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -12596,7 +16272,7 @@ ] }, "method_id": { - "const": "F-004" + "const": "F-001" }, "owner_ids": { "const": [ @@ -12841,17 +16517,36 @@ "else": { "properties": { "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_frequency": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - { - "exclusiveMaximum": 0 - } - ] + "required": [ + "state_id", + "state_frequency" + ], + "additionalProperties": false + }, + "minItems": 0 } } } @@ -12877,7 +16572,7 @@ } ] }, - "state_frequencies": { + "weighted_state_frequencies": { "type": "object", "properties": { "value": { @@ -13234,7 +16929,7 @@ } ] }, - "weighted_state_frequencies": { + "weighted_state_masses": { "type": "object", "properties": { "value": { @@ -13249,10 +16944,10 @@ "minLength": 0, "pattern": "^[^\\u0000]*$" }, - "state_frequency": { + "state_mass": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -13265,7 +16960,7 @@ }, "required": [ "state_id", - "state_frequency" + "state_mass" ], "additionalProperties": false }, @@ -13277,7 +16972,7 @@ ] }, "unit": { - "const": "ratio" + "const": "user_declared_weight_mass" }, "evidence_class": { "const": "derived_metric" @@ -13291,7 +16986,7 @@ ] }, "method_id": { - "const": "F-001" + "const": "T1.state_weight_mass" }, "owner_ids": { "const": [ @@ -13545,10 +17240,10 @@ "minLength": 0, "pattern": "^[^\\u0000]*$" }, - "state_frequency": { + "state_mass": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -13561,7 +17256,7 @@ }, "required": [ "state_id", - "state_frequency" + "state_mass" ], "additionalProperties": false }, @@ -13591,42 +17286,107 @@ } ] }, - "weighted_state_masses": { + "distribution_basis": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "array", - "items": { - "type": "object", - "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "state_mass": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - } + "type": "object", + "properties": { + "input_basis": { + "type": "string", + "enum": [ + "empirical_assignments", + "explicit_counts_divided_by_included_records", + "weighted_record_mass", + "explicit_probability_vector" + ] }, - "required": [ - "state_id", - "state_mass" - ], - "additionalProperties": false + "analyzed_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "frequency_denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "supplied_probability_total": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "probability_residual": { + "anyOf": [ + { + "type": "number", + "minimum": -1.7976931348623157e+308, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "numerical_policy": { + "$ref": "#/$defs/numerical_policy" + } }, - "minItems": 0 + "required": [ + "input_basis", + "analyzed_record_count", + "frequency_denominator", + "denominator_basis", + "supplied_probability_total", + "probability_residual", + "numerical_policy" + ], + "additionalProperties": false }, { "type": "null" @@ -13634,7 +17394,7 @@ ] }, "unit": { - "const": "user_declared_weight_mass" + "const": "basis" }, "evidence_class": { "const": "derived_metric" @@ -13648,7 +17408,7 @@ ] }, "method_id": { - "const": "T1.state_weight_mass" + "const": "T1.validated_distribution_basis" }, "owner_ids": { "const": [ @@ -13893,36 +17653,101 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "object", - "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "state_mass": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - } + "type": "object", + "properties": { + "input_basis": { + "type": "string", + "enum": [ + "empirical_assignments", + "explicit_counts_divided_by_included_records", + "weighted_record_mass", + "explicit_probability_vector" + ] + }, + "analyzed_record_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "frequency_denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "supplied_probability_total": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "probability_residual": { + "anyOf": [ + { + "type": "number", + "minimum": -1.7976931348623157e+308, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] }, - "required": [ - "state_id", - "state_mass" - ], - "additionalProperties": false + "numerical_policy": { + "$ref": "#/$defs/numerical_policy" + } }, - "minItems": 0 + "required": [ + "input_basis", + "analyzed_record_count", + "frequency_denominator", + "denominator_basis", + "supplied_probability_total", + "probability_residual", + "numerical_policy" + ], + "additionalProperties": false } } } @@ -13947,507 +17772,649 @@ } } ] - }, - "distribution_basis": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "object", - "properties": { - "input_basis": { - "type": "string", - "enum": [ - "empirical_assignments", - "explicit_counts_divided_by_included_records", - "weighted_record_mass", - "explicit_probability_vector" - ] - }, - "analyzed_record_count": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - "frequency_denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "denominator_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "supplied_probability_total": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "probability_residual": { - "anyOf": [ - { - "type": "number", - "minimum": -1.7976931348623157e+308, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "numerical_policy": { - "$ref": "#/$defs/numerical_policy" - } - }, - "required": [ - "input_basis", - "analyzed_record_count", - "frequency_denominator", - "denominator_basis", - "supplied_probability_total", - "probability_residual", - "numerical_policy" - ], - "additionalProperties": false - }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "basis" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "T1.validated_distribution_basis" - }, - "owner_ids": { - "const": [ - "T1" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] + } + }, + "required": [], + "additionalProperties": false + } + }, + "gini_simpson_diversity_delta": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": -1, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "dimensionless" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "F-018" + }, + "owner_ids": { + "const": [ + "T1" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, - "coverage_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" - } - ] + "reason_codes": { + "minItems": 1 }, - "denominator": { - "anyOf": [ + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": -1, + "maximum": 1, + "oneOf": [ { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] - }, - "denominator_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" - } + } + } + } + }, + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" ] - }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + }, + "tail": { + "type": "object", + "properties": { + "tail_support_size": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } + { + "type": "null" + } + ] + }, + "unit": { + "const": "states" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T2.declared_tail_rule" + }, + "owner_ids": { + "const": [ + "T2" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 - } - } + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "else": { - "properties": { - "value": { - "type": "object", - "properties": { - "input_basis": { - "type": "string", - "enum": [ - "empirical_assignments", - "explicit_counts_divided_by_included_records", - "weighted_record_mass", - "explicit_probability_vector" - ] - }, - "analyzed_record_count": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - }, - "frequency_denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "denominator_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "supplied_probability_total": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "probability_residual": { - "anyOf": [ - { - "type": "number", - "minimum": -1.7976931348623157e+308, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "numerical_policy": { - "$ref": "#/$defs/numerical_policy" - } - }, - "required": [ - "input_basis", - "analyzed_record_count", - "frequency_denominator", - "denominator_basis", - "supplied_probability_total", - "probability_residual", - "numerical_policy" - ], - "additionalProperties": false - } - } + { + "exclusiveMaximum": 0 } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" - } - } - } + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 } - ] + } + }, + "else": { + "properties": { + "value": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + } } }, - "required": [], - "additionalProperties": false - } + { + "if": { + "properties": { + "status": { + "enum": [ + "available", + "partial" + ] + } + } + }, + "then": { + "properties": { + "representation": { + "$ref": "#/$defs/representation" + } + } + } + } + ] }, - "gini_simpson_diversity_delta": { + "tail_record_share": { "type": "object", "properties": { "value": { "anyOf": [ { "type": "number", - "minimum": -1, + "minimum": 0, "maximum": 1, "oneOf": [ { @@ -14464,7 +18431,7 @@ ] }, "unit": { - "const": "dimensionless" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -14478,11 +18445,11 @@ ] }, "method_id": { - "const": "F-018" + "const": "T2.tail_record_share" }, "owner_ids": { "const": [ - "T1" + "T2" ] }, "theory_map_ids": { @@ -14724,7 +18691,7 @@ "properties": { "value": { "type": "number", - "minimum": -1, + "minimum": 0, "maximum": 1, "oneOf": [ { @@ -14758,23 +18725,59 @@ } } ] - } - }, - "required": [], - "additionalProperties": false - }, - "tail": { - "type": "object", - "properties": { - "tail_support_size": { + }, + "rarity_ranking": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "state_frequency": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "rarity_rank": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "in_tail": { + "type": "boolean" + } + }, + "required": [ + "state_id", + "state_count", + "state_frequency", + "rarity_rank", + "in_tail" + ], + "additionalProperties": false + }, + "minItems": 0 }, { "type": "null" @@ -14782,7 +18785,7 @@ ] }, "unit": { - "const": "states" + "const": "ordinal_rank" }, "evidence_class": { "const": "derived_metric" @@ -14796,7 +18799,7 @@ ] }, "method_id": { - "const": "T2.declared_tail_rule" + "const": "T2.frequency_count_unicode_order" }, "owner_ids": { "const": [ @@ -15041,9 +19044,52 @@ "else": { "properties": { "value": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 + "type": "array", + "items": { + "type": "object", + "properties": { + "state_id": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "state_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "state_frequency": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "rarity_rank": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "in_tail": { + "type": "boolean" + } + }, + "required": [ + "state_id", + "state_count", + "state_frequency", + "rarity_rank", + "in_tail" + ], + "additionalProperties": false + }, + "minItems": 0 } } } @@ -15069,23 +19115,20 @@ } ] }, - "tail_record_share": { + "tail_states": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true }, { "type": "null" @@ -15093,7 +19136,7 @@ ] }, "unit": { - "const": "ratio" + "const": "set_of_states" }, "evidence_class": { "const": "derived_metric" @@ -15107,7 +19150,7 @@ ] }, "method_id": { - "const": "T2.tail_record_share" + "const": "T2.declared_tail_rule" }, "owner_ids": { "const": [ @@ -15349,20 +19392,17 @@ } } }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "else": { + "properties": { + "value": { + "type": "array", + "items": { + "type": "string", + "minLength": 0, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true } } } @@ -15388,58 +19428,107 @@ } ] }, - "rarity_ranking": { + "tail_rule": { + "type": "string", + "enum": [ + "singleton_count", + "count_at_or_below", + "frequency_at_or_below", + "state_list" + ] + }, + "selection": { + "$ref": "#/$defs/tail_selection" + } + }, + "required": [], + "additionalProperties": false + }, + "provenance": { + "type": "object", + "properties": { + "source_type_shares": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "array", - "items": { - "type": "object", - "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "state_count": { - "type": "integer", - "minimum": 1, - "maximum": 1.7976931348623157e+308 - }, - "state_frequency": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "rarity_rank": { - "type": "integer", - "minimum": 1, - "maximum": 1.7976931348623157e+308 - }, - "in_tail": { - "type": "boolean" - } + "type": "object", + "properties": { + "human": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "required": [ - "state_id", - "state_count", - "state_frequency", - "rarity_rank", - "in_tail" - ], - "additionalProperties": false + "synthetic": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "mixed": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "sensor": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "unknown": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - "minItems": 0 + "required": [ + "human", + "synthetic", + "mixed", + "sensor", + "unknown" + ], + "additionalProperties": false }, { "type": "null" @@ -15447,7 +19536,7 @@ ] }, "unit": { - "const": "ordinal_rank" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -15461,11 +19550,11 @@ ] }, "method_id": { - "const": "T2.frequency_count_unicode_order" + "const": "F-007" }, "owner_ids": { "const": [ - "T2" + "PR-005" ] }, "theory_map_ids": { @@ -15706,91 +19795,170 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "object", - "properties": { - "state_id": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" - }, - "state_count": { - "type": "integer", - "minimum": 1, - "maximum": 1.7976931348623157e+308 - }, - "state_frequency": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "rarity_rank": { - "type": "integer", - "minimum": 1, - "maximum": 1.7976931348623157e+308 - }, - "in_tail": { - "type": "boolean" - } + "type": "object", + "properties": { + "human": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "required": [ - "state_id", - "state_count", - "state_frequency", - "rarity_rank", - "in_tail" - ], - "additionalProperties": false + "synthetic": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "mixed": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "sensor": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "unknown": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - "minItems": 0 - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" + "required": [ + "human", + "synthetic", + "mixed", + "sensor", + "unknown" + ], + "additionalProperties": false } } } } ] }, - "tail_states": { + "weighted_source_type_shares": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "type": "object", + "properties": { + "human": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "synthetic": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "mixed": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "sensor": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "unknown": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - "minItems": 0, - "uniqueItems": true + "required": [ + "human", + "synthetic", + "mixed", + "sensor", + "unknown" + ], + "additionalProperties": false }, { "type": "null" @@ -15798,7 +19966,7 @@ ] }, "unit": { - "const": "set_of_states" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -15812,11 +19980,11 @@ ] }, "method_id": { - "const": "T2.declared_tail_rule" + "const": "F-007" }, "owner_ids": { "const": [ - "T2" + "PR-005" ] }, "theory_map_ids": { @@ -16057,59 +20225,89 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "string", - "minLength": 0, - "pattern": "^[^\\u0000]*$" + "type": "object", + "properties": { + "human": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "synthetic": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "mixed": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "sensor": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "unknown": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } }, - "minItems": 0, - "uniqueItems": true - } - } - } - }, - { - "if": { - "properties": { - "status": { - "enum": [ - "available", - "partial" - ] - } - } - }, - "then": { - "properties": { - "representation": { - "$ref": "#/$defs/representation" + "required": [ + "human", + "synthetic", + "mixed", + "sensor", + "unknown" + ], + "additionalProperties": false } } } } ] }, - "tail_rule": { - "type": "string", - "enum": [ - "singleton_count", - "count_at_or_below", - "frequency_at_or_below", - "state_list" - ] - }, - "selection": { - "$ref": "#/$defs/tail_selection" - } - }, - "required": [], - "additionalProperties": false - }, - "provenance": { - "type": "object", - "properties": { - "source_type_shares": { + "weighted_source_type_masses": { "type": "object", "properties": { "value": { @@ -16120,7 +20318,7 @@ "human": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16133,7 +20331,7 @@ "synthetic": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16146,7 +20344,7 @@ "mixed": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16159,7 +20357,7 @@ "sensor": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16172,7 +20370,7 @@ "unknown": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16198,7 +20396,7 @@ ] }, "unit": { - "const": "ratio" + "const": "user_declared_weight_mass" }, "evidence_class": { "const": "derived_metric" @@ -16212,7 +20410,7 @@ ] }, "method_id": { - "const": "F-007" + "const": "PR-005.source_weight_mass" }, "owner_ids": { "const": [ @@ -16462,7 +20660,7 @@ "human": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16475,7 +20673,7 @@ "synthetic": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16488,7 +20686,7 @@ "mixed": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16501,7 +20699,7 @@ "sensor": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16514,7 +20712,7 @@ "unknown": { "type": "number", "minimum": 0, - "maximum": 1, + "maximum": 1.7976931348623157e+308, "oneOf": [ { "minimum": 0 @@ -16539,88 +20737,23 @@ } ] }, - "weighted_source_type_shares": { + "total_weight": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "object", - "properties": { - "human": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "synthetic": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "mixed": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "sensor": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "unknown": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + { + "exclusiveMaximum": 0 } - }, - "required": [ - "human", - "synthetic", - "mixed", - "sensor", - "unknown" - ], - "additionalProperties": false + ] }, { "type": "null" @@ -16628,7 +20761,7 @@ ] }, "unit": { - "const": "ratio" + "const": "user_declared_weight_mass" }, "evidence_class": { "const": "derived_metric" @@ -16642,7 +20775,7 @@ ] }, "method_id": { - "const": "F-007" + "const": "PR-005.total_weight" }, "owner_ids": { "const": [ @@ -16878,179 +21011,49 @@ }, "reason_codes": { "minItems": 1 - }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "object", - "properties": { - "human": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "synthetic": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "mixed": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "sensor": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "unknown": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - } - }, - "required": [ - "human", - "synthetic", - "mixed", - "sensor", - "unknown" - ], - "additionalProperties": false - } - } - } - } - ] - }, - "weighted_source_type_masses": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "object", - "properties": { - "human": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "synthetic": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "mixed": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "sensor": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } + } + ] + }, + "missing_provenance_weight": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "unknown": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + { + "exclusiveMaximum": 0 } - }, - "required": [ - "human", - "synthetic", - "mixed", - "sensor", - "unknown" - ], - "additionalProperties": false + ] }, { "type": "null" @@ -17072,7 +21075,7 @@ ] }, "method_id": { - "const": "PR-005.source_weight_mass" + "const": "PR-005.missing_provenance_weight" }, "owner_ids": { "const": [ @@ -17317,89 +21320,24 @@ "else": { "properties": { "value": { - "type": "object", - "properties": { - "human": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "synthetic": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "mixed": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - "sensor": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 }, - "unknown": { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + { + "exclusiveMaximum": 0 } - }, - "required": [ - "human", - "synthetic", - "mixed", - "sensor", - "unknown" - ], - "additionalProperties": false + ] } } } } ] }, - "total_weight": { + "missing_provenance_share": { "type": "object", "properties": { "value": { @@ -17407,7 +21345,7 @@ { "type": "number", "minimum": 0, - "maximum": 1.7976931348623157e+308, + "maximum": 1, "oneOf": [ { "minimum": 0 @@ -17423,7 +21361,7 @@ ] }, "unit": { - "const": "user_declared_weight_mass" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -17437,11 +21375,11 @@ ] }, "method_id": { - "const": "PR-005.total_weight" + "const": "PR-004.one_minus_row_coverage" }, "owner_ids": { "const": [ - "PR-005" + "PR-004" ] }, "theory_map_ids": { @@ -17684,7 +21622,7 @@ "value": { "type": "number", "minimum": 0, - "maximum": 1.7976931348623157e+308, + "maximum": 1, "oneOf": [ { "minimum": 0 @@ -17699,7 +21637,7 @@ } ] }, - "missing_provenance_weight": { + "weighted_missing_provenance_share": { "type": "object", "properties": { "value": { @@ -17707,7 +21645,7 @@ { "type": "number", "minimum": 0, - "maximum": 1.7976931348623157e+308, + "maximum": 1, "oneOf": [ { "minimum": 0 @@ -17723,7 +21661,7 @@ ] }, "unit": { - "const": "user_declared_weight_mass" + "const": "ratio" }, "evidence_class": { "const": "derived_metric" @@ -17737,7 +21675,7 @@ ] }, "method_id": { - "const": "PR-005.missing_provenance_weight" + "const": "PR-005.missing_provenance_weight_share" }, "owner_ids": { "const": [ @@ -17984,7 +21922,7 @@ "value": { "type": "number", "minimum": 0, - "maximum": 1.7976931348623157e+308, + "maximum": 1, "oneOf": [ { "minimum": 0 @@ -17993,623 +21931,623 @@ "exclusiveMaximum": 0 } ] - } - } - } - } - ] - }, - "missing_provenance_share": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + }, + "closure_exposure": { + "type": "object", + "properties": { + "direct": { + "type": "object", + "properties": { + "lower_bound": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "F-009" + }, + "owner_ids": { + "const": [ + "T3" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ { - "minimum": 0 + "$ref": "#/$defs/representation" }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "ratio" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "PR-004.one_minus_row_coverage" - }, - "owner_ids": { - "const": [ - "PR-004" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + "coverage_reason": { + "anyOf": [ { - "minimum": 0 + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, { - "exclusiveMaximum": 0 + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ - { + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ { - "type": "null" + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } } ] }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ + "upper_bound": { + "type": "object", + "properties": { + "value": { + "anyOf": [ { - "minimum": 0 + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "denominator_reason": { - "anyOf": [ - { + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "enum": [ + "available", + "partial", + "unavailable" + ] }, - { - "type": "null" - } - ] - }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - } - }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } - }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 + "method_id": { + "const": "F-010" + }, + "owner_ids": { + "const": [ + "T3" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" ] - } - } - } - } - ] - }, - "weighted_missing_provenance_share": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ { - "minimum": 0 + "$ref": "#/$defs/representation" }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "ratio" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "type": "string", - "enum": [ - "available", - "partial", - "unavailable" - ] - }, - "method_id": { - "const": "PR-005.missing_provenance_weight_share" - }, - "owner_ids": { - "const": [ - "PR-005" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + "coverage_reason": { + "anyOf": [ { - "minimum": 0 + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] }, - { - "type": "null" - } - ] - }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ + "denominator_reason": { + "anyOf": [ { - "minimum": 0 + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, { - "exclusiveMaximum": 0 + "type": "null" } ] }, - { - "type": "null" - } - ] - }, - "denominator_reason": { - "anyOf": [ - { + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - { - "type": "null" - } - ] - }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - } - }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" } }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 + "then": { + "properties": { + "value": { + "type": "null" }, - { - "exclusiveMaximum": 0 + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 } - ] + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } } } - } - } - ] - } - }, - "required": [], - "additionalProperties": false - }, - "closure_exposure": { - "type": "object", - "properties": { - "direct": { - "type": "object", - "properties": { - "lower_bound": { + ] + }, + "interval_width": { "type": "object", "properties": { "value": { @@ -18647,7 +22585,7 @@ ] }, "method_id": { - "const": "F-009" + "const": "T3.upper_minus_lower" }, "owner_ids": { "const": [ @@ -18909,7 +22847,20 @@ } ] }, - "upper_bound": { + "classification_basis": { + "const": "toolkit_operationalization" + }, + "confidence_disclosure": { + "const": "provenance_confidence_is_separate_and_does_not_discount_grounding" + } + }, + "required": [], + "additionalProperties": false + }, + "lineage": { + "type": "object", + "properties": { + "lower_bound": { "type": "object", "properties": { "value": { @@ -18947,7 +22898,7 @@ ] }, "method_id": { - "const": "F-010" + "const": "T3.lineage_closure" }, "owner_ids": { "const": [ @@ -19209,7 +23160,7 @@ } ] }, - "interval_width": { + "upper_bound": { "type": "object", "properties": { "value": { @@ -19247,7 +23198,7 @@ ] }, "method_id": { - "const": "T3.upper_minus_lower" + "const": "T3.lineage_closure" }, "owner_ids": { "const": [ @@ -19509,20 +23460,7 @@ } ] }, - "classification_basis": { - "const": "toolkit_operationalization" - }, - "confidence_disclosure": { - "const": "provenance_confidence_is_separate_and_does_not_discount_grounding" - } - }, - "required": [], - "additionalProperties": false - }, - "lineage": { - "type": "object", - "properties": { - "lower_bound": { + "interval_width": { "type": "object", "properties": { "value": { @@ -19552,7 +23490,12 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "T3.lineage_closure" @@ -19626,813 +23569,1682 @@ "representation": { "anyOf": [ { - "$ref": "#/$defs/representation" + "$ref": "#/$defs/representation" + }, + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } + } + ] + } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [], + "additionalProperties": false + }, + "lineage": { + "type": "object", + "properties": { + "resolved_parent_edge_coverage": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] + { + "type": "null" + } + ] + }, + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "PR-008.resolved_edges_over_declared" + }, + "owner_ids": { + "const": [ + "PR-008" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" }, - "coverage_reason": { - "anyOf": [ + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "denominator": { - "anyOf": [ + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "denominator_reason": { - "anyOf": [ + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "no_declared_parents": { + "type": "boolean" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } + } + ] + }, + "resolved_lineage_coverage": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + { + "type": "null" } + ] + }, + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T4.resolved_records_over_scope" + }, + "owner_ids": { + "const": [ + "T4" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } - }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - } - } - } + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" + }, + { + "type": "null" } ] }, - "upper_bound": { - "type": "object", - "properties": { - "value": { - "anyOf": [ + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "unit": { - "const": "ratio" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "const": "unavailable" - }, - "method_id": { - "const": "T3.lineage_closure" + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "owner_ids": { - "const": [ - "T3" + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } ] }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ - { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - { - "type": "null" - } - ] - }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "denominator_reason": { - "anyOf": [ + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } + } + ] + }, + "external_ancestry_coverage": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true + { + "type": "null" + } + ] + }, + "unit": { + "const": "ratio" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T4.external_roots_over_scope" + }, + "owner_ids": { + "const": [ + "T4" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ + { + "$ref": "#/$defs/representation" }, - "weighting": { - "$ref": "#/$defs/weighting" + { + "type": "null" + } + ] + }, + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "input_basis": { + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "redaction": { - "$ref": "#/$defs/redaction" + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] }, - "method": { + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" + }, + { + "type": "null" + } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + } + } + } + } + ] + }, + "distinct_external_root_count": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + { + "type": "null" } + ] + }, + "unit": { + "const": "roots" + }, + "evidence_class": { + "const": "derived_metric" + }, + "status": { + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] + }, + "method_id": { + "const": "T4.root_set_union" + }, + "owner_ids": { + "const": [ + "T4" + ] + }, + "theory_map_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "TM-M01", + "TM-M02", + "TM-M03", + "TM-M04", + "TM-M05", + "TM-M06", + "TM-M07", + "TM-C01", + "TM-C02", + "TM-C03", + "TM-C04", + "TM-C05", + "TM-C06", + "TM-C07", + "TM-C08", + "TM-P01", + "TM-P02", + "TM-P03", + "TM-P04", + "TM-P05", + "TM-S01", + "TM-S02", + "TM-S03", + "TM-E01", + "TM-E02", + "TM-E03", + "TM-E04", + "TM-E05", + "TM-A01", + "TM-A02", + "TM-A03", + "TM-A04" + ] + }, + "minItems": 0, + "uniqueItems": true + }, + "trace_ids": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "T1", + "T2", + "T3", + "T4", + "T5", + "T6" + ] }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ + "minItems": 0, + "uniqueItems": true + }, + "scope": { + "$ref": "#/$defs/scope" + }, + "representation": { + "anyOf": [ { - "if": { - "properties": { - "status": { - "const": "unavailable" - } - } - }, - "then": { - "properties": { - "value": { - "type": "null" - }, - "reason_codes": { - "minItems": 1 - }, - "required_evidence": { - "minItems": 1 - } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - } - } - } + "$ref": "#/$defs/representation" + }, + { + "type": "null" } ] }, - "interval_width": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "unit": { - "const": "ratio" - }, - "evidence_class": { - "const": "derived_metric" - }, - "status": { - "const": "unavailable" - }, - "method_id": { - "const": "T3.lineage_closure" - }, - "owner_ids": { - "const": [ - "T3" - ] - }, - "theory_map_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "TM-M01", - "TM-M02", - "TM-M03", - "TM-M04", - "TM-M05", - "TM-M06", - "TM-M07", - "TM-C01", - "TM-C02", - "TM-C03", - "TM-C04", - "TM-C05", - "TM-C06", - "TM-C07", - "TM-C08", - "TM-P01", - "TM-P02", - "TM-P03", - "TM-P04", - "TM-P05", - "TM-S01", - "TM-S02", - "TM-S03", - "TM-E01", - "TM-E02", - "TM-E03", - "TM-E04", - "TM-E05", - "TM-A01", - "TM-A02", - "TM-A03", - "TM-A04" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "trace_ids": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "T1", - "T2", - "T3", - "T4", - "T5", - "T6" - ] - }, - "minItems": 0, - "uniqueItems": true - }, - "scope": { - "$ref": "#/$defs/scope" - }, - "representation": { - "anyOf": [ - { - "$ref": "#/$defs/representation" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] - }, - "coverage_reason": { - "anyOf": [ + "coverage": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "denominator": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1.7976931348623157e+308, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] - }, - { - "type": "null" - } - ] + { + "type": "null" + } + ] + }, + "coverage_reason": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "denominator_reason": { - "anyOf": [ + { + "type": "null" + } + ] + }, + "denominator": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + "minimum": 0 }, { - "type": "null" + "exclusiveMaximum": 0 } ] }, - "assumptions": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "limitations": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "reason_codes": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "required_evidence": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" - }, - "minItems": 0, - "uniqueItems": true - }, - "weighting": { - "$ref": "#/$defs/weighting" - }, - "input_basis": { + { + "type": "null" + } + ] + }, + "denominator_reason": { + "anyOf": [ + { "type": "string", "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "redaction": { - "$ref": "#/$defs/redaction" - }, - "method": { - "type": "string", - "minLength": 1, - "pattern": "^[^\\u0000]*$" + { + "type": "null" } + ] + }, + "assumptions": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" }, - "required": [ - "value", - "unit", - "evidence_class", - "status", - "method_id", - "owner_ids", - "theory_map_ids", - "trace_ids", - "scope", - "representation", - "coverage", - "coverage_reason", - "denominator", - "denominator_reason", - "assumptions", - "limitations", - "reason_codes", - "required_evidence" - ], - "additionalProperties": false, - "allOf": [ + "minItems": 0, + "uniqueItems": true + }, + "limitations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "reason_codes": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "required_evidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "minItems": 0, + "uniqueItems": true + }, + "weighting": { + "$ref": "#/$defs/weighting" + }, + "input_basis": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + }, + "redaction": { + "$ref": "#/$defs/redaction" + }, + "method": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]*$" + } + }, + "required": [ + "value", + "unit", + "evidence_class", + "status", + "method_id", + "owner_ids", + "theory_map_ids", + "trace_ids", + "scope", + "representation", + "coverage", + "coverage_reason", + "denominator", + "denominator_reason", + "assumptions", + "limitations", + "reason_codes", + "required_evidence" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "unavailable" + } + } + }, + "then": { + "properties": { + "value": { + "type": "null" + }, + "reason_codes": { + "minItems": 1 + }, + "required_evidence": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "value": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + } + } + } + } + ] + }, + "top_shared_ancestors": { + "type": "object", + "properties": { + "value": { + "anyOf": [ { - "if": { - "properties": { - "status": { - "const": "unavailable" - } + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "incidence_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "incidence_share": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "incidence_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "fractional_mass": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "normalized_weight": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "weight_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "record_key", + "incidence_count", + "incidence_share", + "incidence_denominator", + "fractional_mass", + "normalized_weight", + "weight_denominator" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true } }, - "then": { - "properties": { - "value": { - "type": "null" + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } }, - "reason_codes": { - "minItems": 1 + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } }, - "required_evidence": { - "minItems": 1 + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "incidence_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "incidence_share": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "incidence_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "fractional_mass": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "normalized_weight": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "weight_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "record_key", + "incidence_count", + "incidence_share", + "incidence_denominator", + "fractional_mass", + "normalized_weight", + "weight_denominator" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } } - } - }, - "else": { - "properties": { - "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 + }, + { + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 }, - { - "exclusiveMaximum": 0 + "omission_reasons": { + "maxItems": 0 } - ] + } } - } - } - } - ] - } - }, - "required": [], - "additionalProperties": false - } - }, - "required": [], - "additionalProperties": false - }, - "lineage": { - "type": "object", - "properties": { - "resolved_parent_edge_coverage": { - "type": "object", - "properties": { - "value": { - "anyOf": [ - { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 }, { - "exclusiveMaximum": 0 + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } } ] }, @@ -20442,7 +25254,7 @@ ] }, "unit": { - "const": "ratio" + "const": "records" }, "evidence_class": { "const": "derived_metric" @@ -20456,11 +25268,11 @@ ] }, "method_id": { - "const": "PR-008.resolved_edges_over_declared" + "const": "T4.incidence_ranking" }, "owner_ids": { "const": [ - "PR-008" + "T4" ] }, "theory_map_ids": { @@ -20701,15 +25513,287 @@ "else": { "properties": { "value": { - "type": "number", - "minimum": 0, - "maximum": 1, - "oneOf": [ + "type": "object", + "properties": { + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "incidence_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "incidence_share": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "incidence_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "fractional_mass": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "normalized_weight": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "weight_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "record_key", + "incidence_count", + "incidence_share", + "incidence_denominator", + "fractional_mass", + "normalized_weight", + "weight_denominator" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + }, + { + "type": "null" + } + ] + }, + "total_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "returned_count": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "omitted_count": { + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 + }, + "limit": { + "const": 100 + }, + "detail_status": { + "type": "string", + "enum": [ + "complete", + "truncated", + "omitted" + ] + }, + "omission_reasons": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "diagnostic_limit", + "redacted_identity_details" + ] + }, + "minItems": 0, + "uniqueItems": true + } + }, + "required": [ + "items", + "total_count", + "returned_count", + "omitted_count", + "limit", + "detail_status", + "omission_reasons" + ], + "additionalProperties": false, + "allOf": [ { - "minimum": 0 + "if": { + "properties": { + "detail_status": { + "const": "omitted" + } + } + }, + "then": { + "properties": { + "items": { + "type": "null" + }, + "returned_count": { + "const": 0 + }, + "omission_reasons": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "record_key": { + "$ref": "#/$defs/record_key" + }, + "incidence_count": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "incidence_share": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "incidence_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + }, + "fractional_mass": { + "type": "number", + "minimum": 0, + "maximum": 1.7976931348623157e+308, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "normalized_weight": { + "type": "number", + "minimum": 0, + "maximum": 1, + "oneOf": [ + { + "minimum": 0 + }, + { + "exclusiveMaximum": 0 + } + ] + }, + "weight_denominator": { + "type": "integer", + "minimum": 1, + "maximum": 1.7976931348623157e+308 + } + }, + "required": [ + "record_key", + "incidence_count", + "incidence_share", + "incidence_denominator", + "fractional_mass", + "normalized_weight", + "weight_denominator" + ], + "additionalProperties": false + }, + "minItems": 0, + "maxItems": 100 + } + } + } }, { - "exclusiveMaximum": 0 + "if": { + "properties": { + "detail_status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "const": 0 + }, + "omission_reasons": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "detail_status": { + "const": "truncated" + } + } + }, + "then": { + "properties": { + "omitted_count": { + "minimum": 1 + }, + "returned_count": { + "const": 100 + }, + "omission_reasons": { + "const": [ + "diagnostic_limit" + ] + } + } + } } ] } @@ -20718,23 +25802,15 @@ } ] }, - "resolved_lineage_coverage": { + "grounded_record_count": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 }, { "type": "null" @@ -20742,16 +25818,21 @@ ] }, "unit": { - "const": "ratio" + "const": "records" }, "evidence_class": { "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { - "const": "T4.resolved_records_over_scope" + "const": "T4.grounded_record_count" }, "owner_ids": { "const": [ @@ -20996,40 +26077,24 @@ "else": { "properties": { "value": { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 } } } } ] }, - "external_ancestry_coverage": { + "closed_record_count": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 }, { "type": "null" @@ -21037,16 +26102,21 @@ ] }, "unit": { - "const": "ratio" + "const": "records" }, "evidence_class": { "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { - "const": "T4.external_roots_over_scope" + "const": "T4.closed_record_count" }, "owner_ids": { "const": [ @@ -21291,24 +26361,16 @@ "else": { "properties": { "value": { - "type": "number", + "type": "integer", "minimum": 0, - "maximum": 1, - "oneOf": [ - { - "minimum": 0 - }, - { - "exclusiveMaximum": 0 - } - ] + "maximum": 1.7976931348623157e+308 } } } } ] }, - "distinct_external_root_count": { + "unresolved_record_count": { "type": "object", "properties": { "value": { @@ -21324,16 +26386,21 @@ ] }, "unit": { - "const": "roots" + "const": "records" }, "evidence_class": { "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { - "const": "T4.root_set_union" + "const": "T4.unresolved_record_count" }, "owner_ids": { "const": [ @@ -21587,32 +26654,15 @@ } ] }, - "top_shared_ancestors": { + "records_with_resolved_external_ancestry": { "type": "object", "properties": { "value": { "anyOf": [ { - "type": "array", - "items": { - "type": "object", - "properties": { - "record_key": { - "$ref": "#/$defs/record_key" - }, - "incidence_count": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - } - }, - "required": [ - "record_key", - "incidence_count" - ], - "additionalProperties": false - }, - "minItems": 0 + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 }, { "type": "null" @@ -21626,10 +26676,15 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { - "const": "T4.incidence_ranking" + "const": "T4.records_with_resolved_external_ancestry" }, "owner_ids": { "const": [ @@ -21874,26 +26929,9 @@ "else": { "properties": { "value": { - "type": "array", - "items": { - "type": "object", - "properties": { - "record_key": { - "$ref": "#/$defs/record_key" - }, - "incidence_count": { - "type": "integer", - "minimum": 0, - "maximum": 1.7976931348623157e+308 - } - }, - "required": [ - "record_key", - "incidence_count" - ], - "additionalProperties": false - }, - "minItems": 0 + "type": "integer", + "minimum": 0, + "maximum": 1.7976931348623157e+308 } } } @@ -21930,7 +26968,12 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "F-012" @@ -22225,7 +27268,12 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "F-013" @@ -22512,7 +27560,12 @@ "const": "derived_metric" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "PR-009.maximum_resolved_parent_depth" @@ -23342,7 +28395,12 @@ "const": "proxy_signal" }, "status": { - "const": "unavailable" + "type": "string", + "enum": [ + "available", + "partial", + "unavailable" + ] }, "method_id": { "const": "T4.shared_ancestry_dependence" @@ -23514,7 +28572,7 @@ "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 1, + "minItems": 0, "uniqueItems": true }, "required_evidence": { @@ -23524,14 +28582,19 @@ "minLength": 1, "pattern": "^[^\\u0000]*$" }, - "minItems": 1, + "minItems": 0, "uniqueItems": true }, "signal": { "const": "shared_ancestry_dependence" }, "level": { - "const": "indeterminate" + "type": "string", + "enum": [ + "present", + "not_present", + "indeterminate" + ] }, "basis_fields": { "type": "array", @@ -29503,6 +34566,9 @@ }, "scenario_requested": { "type": "boolean" + }, + "lineage_requested": { + "type": "boolean" } }, "required": [], diff --git a/src/recursive_integrity_toolkit/errors.py b/src/recursive_integrity_toolkit/errors.py index 125b4b2..6578b30 100644 --- a/src/recursive_integrity_toolkit/errors.py +++ b/src/recursive_integrity_toolkit/errors.py @@ -24,6 +24,10 @@ from __future__ import annotations from enum import StrEnum +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .lineage.graph import LineageResourceUsage class ErrorCode(StrEnum): @@ -42,6 +46,7 @@ class ErrorCode(StrEnum): PARENT_AMBIGUOUS = "E_PARENT_AMBIGUOUS" PARENT_FUTURE_VERSION = "E_PARENT_FUTURE_VERSION" LINEAGE_CYCLE = "E_LINEAGE_CYCLE" + LINEAGE_RESOURCE_LIMIT_EXCEEDED = "E_LINEAGE_RESOURCE_LIMIT_EXCEEDED" VERSION_ORDER_CONFLICT = "E_VERSION_ORDER_CONFLICT" REPRESENTATION_INCOMPATIBLE = "E_REPRESENTATION_INCOMPATIBLE" MAPPING_SOURCE_FIELD_MISSING = "E_MAPPING_SOURCE_FIELD_MISSING" @@ -96,6 +101,21 @@ class SecurityError(ToolkitError): """Local security-boundary violation.""" +class LineageResourceLimitError(ToolkitError): + """A bounded lineage stage stopped before admitting its next work unit. + + ``resource_usage`` is the immutable ``LineageResourceUsage`` supplied by the + computation owner. It contains counts and limits only, with no input data. + """ + + reason_code = "LINEAGE_RESOURCE_LIMIT_EXCEEDED" + + def __init__(self, resource_usage: LineageResourceUsage) -> None: + self.resource_usage = resource_usage + super().__init__(ErrorCode.LINEAGE_RESOURCE_LIMIT_EXCEEDED, + "lineage computation exceeded an explicit resource limit") + + class IngestionError(InputError): """Physical input failure with a safe location and no source-value echo.""" diff --git a/src/recursive_integrity_toolkit/io/loaders.py b/src/recursive_integrity_toolkit/io/loaders.py index c476935..8333fb1 100644 --- a/src/recursive_integrity_toolkit/io/loaders.py +++ b/src/recursive_integrity_toolkit/io/loaders.py @@ -43,7 +43,8 @@ from ..utils.paths import local_input_path -_TABLE_ROLES = {FileRole.RECORDS_PRIMARY, FileRole.RECORDS_COMPARE, FileRole.PROVENANCE_MANIFEST} +_TABLE_ROLES = {FileRole.RECORDS_PRIMARY, FileRole.RECORDS_COMPARE, + FileRole.LINEAGE_CONTEXT, FileRole.PROVENANCE_MANIFEST} _TABLE_FORMATS = {FileFormat.CSV, FileFormat.JSONL, FileFormat.PARQUET} _EXTENSIONS = {".csv": FileFormat.CSV, ".jsonl": FileFormat.JSONL, ".parquet": FileFormat.PARQUET, ".json": FileFormat.JSON, ".toml": FileFormat.TOML, ".npy": FileFormat.NPY} @@ -78,6 +79,7 @@ def _select_format(source: InputSource) -> FileFormat: allowed = { FileRole.RECORDS_PRIMARY: _TABLE_FORMATS, FileRole.RECORDS_COMPARE: _TABLE_FORMATS, + FileRole.LINEAGE_CONTEXT: _TABLE_FORMATS, FileRole.PROVENANCE_MANIFEST: _TABLE_FORMATS, FileRole.SCHEMA_MAPPING: {FileFormat.JSON}, FileRole.CONFIG: {FileFormat.JSON, FileFormat.TOML}, @@ -319,7 +321,7 @@ def _parse_parquet(source: InputSource, raw: bytes, limits: ResourceLimits) -> t def load_table(source: InputSource, *, limits: ResourceLimits | None = None) -> LoadedTable: - """Parse a records/compare/provenance table; canonical validation is later. + """Parse a records, lineage-context or provenance table before validation. A declared format selects exactly one parser. No fallback sniffing occurs. CSV values stay strings. JSONL and Parquet retain their native values. @@ -336,7 +338,8 @@ def load_table(source: InputSource, *, limits: ResourceLimits | None = None) -> fields, rows = _parse_jsonl(source, _text(source, raw), checked) else: fields, rows = _parse_parquet(source, raw, checked) - if not rows and source.role in {FileRole.RECORDS_PRIMARY, FileRole.RECORDS_COMPARE}: + if not rows and source.role in {FileRole.RECORDS_PRIMARY, FileRole.RECORDS_COMPARE, + FileRole.LINEAGE_CONTEXT}: raise _failure(source, ErrorCode.EMPTY_DATASET, "records table contains no data rows") inventory = FileInventoryEntry(source.role, path, chosen, len(raw), sha256_bytes(raw), len(rows), fields) return LoadedTable(inventory, rows) diff --git a/src/recursive_integrity_toolkit/io/normalization.py b/src/recursive_integrity_toolkit/io/normalization.py index 465bb4d..d9c537f 100644 --- a/src/recursive_integrity_toolkit/io/normalization.py +++ b/src/recursive_integrity_toolkit/io/normalization.py @@ -329,7 +329,7 @@ def normalize_table(table: LoadedTable, *, options: NormalizationOptions | None if type(table) is not LoadedTable: raise CanonicalValidationError(ErrorCode.SCHEMA_TYPE, "normalize_table requires a LoadedTable") role = table.inventory.role - if role in (FileRole.RECORDS_PRIMARY, FileRole.RECORDS_COMPARE): + if role in (FileRole.RECORDS_PRIMARY, FileRole.RECORDS_COMPARE, FileRole.LINEAGE_CONTEXT): kind = "records" elif role is FileRole.PROVENANCE_MANIFEST: kind = "provenance" diff --git a/src/recursive_integrity_toolkit/io/validation.py b/src/recursive_integrity_toolkit/io/validation.py index 9c39f58..98023e9 100644 --- a/src/recursive_integrity_toolkit/io/validation.py +++ b/src/recursive_integrity_toolkit/io/validation.py @@ -31,6 +31,7 @@ import json import math +from collections import deque from pathlib import Path from ..config import ResourceLimits from datetime import datetime, timezone @@ -43,6 +44,7 @@ ProvenanceAssessment, ProvenanceMatch, ProvenanceJoinResult, ValidationCoverage, ValidationMessage, ValidationSeverity, ParentResolutionStatus, VersionOrderResult, ParentReference, ParentValidationResult, + ParentRecordValidation, ParentBatchValidationResult, GenerationAssessment, GenerationValidationResult, AuditBundle, BundleValidationResult, NormalizationOptions, ScenarioParameters, ) @@ -601,33 +603,53 @@ def _parent_lookup(keys: tuple[RecordKey, ...], order: VersionOrderResult return set(keys), by_id, {version: i for i, version in enumerate(order.order)} +def _interpret_parent_reference(text: str, child: RecordKey, + lookup: tuple[set[RecordKey], dict[str, list[RecordKey]], dict[str, int]], + location: RowLocation) -> tuple[RecordKey | None, bool, bool, str]: + """Single interpretation shared by fail-fast and retained-batch adapters.""" + loaded, by_id, positions = lookup + parsed = _reference_key(text, location, child) + bare = parsed is None + if bare: + candidates = by_id.get(text, []) + if len(candidates) > 1: + raise _fail(ErrorCode.PARENT_AMBIGUOUS, "bare parent matches multiple loaded records", "parent_ids", location, child) + parsed = candidates[0] if candidates else None + resolved = parsed in loaded if parsed is not None else False + if parsed == child: + raise _fail(ErrorCode.LINEAGE_CYCLE, "direct self-parent reference is invalid input", "parent_ids", location, child) + temporal = "unavailable" + if parsed is not None: + if parsed.dataset_version == child.dataset_version: + temporal = "same_version" + elif parsed.dataset_version in positions and child.dataset_version in positions: + if positions[parsed.dataset_version] > positions[child.dataset_version]: + raise _fail(ErrorCode.PARENT_FUTURE_VERSION, "parent refers to a later declared version", "parent_ids", location, child) + temporal = "earlier_version" + return parsed, bare, resolved, temporal + + def _resolve_parent_list(child: RecordKey, values: tuple[str, ...] | None, lookup: tuple[set[RecordKey], dict[str, list[RecordKey]], dict[str, int]], - location: RowLocation, promoted: tuple[str, ...]) -> ParentValidationResult: - loaded, by_id, positions = lookup + location: RowLocation, promoted: tuple[str, ...], *, + retain_errors: bool = False) -> ParentValidationResult: messages: list[ValidationMessage] = [] grouped: dict[tuple[str, str], tuple[RecordKey | None, list[str], bool, str]] = {} graph_deferred = False - for text in sorted(values or ()): - parsed = _reference_key(text, location, child) - bare = parsed is None - if bare: - candidates = by_id.get(text, []) - if len(candidates) > 1: - raise _fail(ErrorCode.PARENT_AMBIGUOUS, "bare parent matches multiple loaded records", "parent_ids", location, child) - parsed = candidates[0] if candidates else None - resolved = parsed in loaded if parsed is not None else False - if parsed == child: - raise _fail(ErrorCode.LINEAGE_CYCLE, "direct self-parent reference is invalid input", "parent_ids", location, child) - temporal = "unavailable" - if parsed is not None: - if parsed.dataset_version == child.dataset_version: - temporal = "same_version" - graph_deferred = True - elif parsed.dataset_version in positions and child.dataset_version in positions: - if positions[parsed.dataset_version] > positions[child.dataset_version]: - raise _fail(ErrorCode.PARENT_FUTURE_VERSION, "parent refers to a later declared version", "parent_ids", location, child) - temporal = "earlier_version" + # Invalid native entries are retained by count with static diagnostics; + # legacy callers still pass a parsed all-string tuple and fail immediately. + for text in sorted(values or (), key=lambda item: item if type(item) is str else ""): + try: + if type(text) is not str: + raise _fail(ErrorCode.PARENT_FORMAT, "parent reference must be a string", "parent_ids", location, child) + parsed, bare, resolved, temporal = _interpret_parent_reference(text, child, lookup, location) + except CanonicalValidationError as error: + if not retain_errors: + raise + messages.append(_join_message(error.code.value, ValidationSeverity.ERROR, + error.safe_message, child, location, "parent_ids")) + continue + graph_deferred = graph_deferred or temporal == "same_version" token = ("key", str(parsed)) if parsed is not None else ("bare", text) if token in grouped: grouped[token][1].append(text) @@ -672,6 +694,99 @@ def resolve_parent_references( return _resolve_parent_list(child_key, parents, _parent_lookup(keys, order), loc, promoted) +def resolve_parent_batch( + loaded_keys: tuple[RecordKey, ...], + provenance: tuple[CanonicalRow | ProvenanceAssessment, ...] | None, *, + version_order: VersionOrderResult | None = None, strict_mode: bool = False, + strict_warning_codes: tuple[str, ...] = (), +) -> ParentBatchValidationResult: + """Retain every record's immediate parent evidence with one lookup index. + + Invalid references are diagnosed individually, preserving valid siblings. + This function performs no graph traversal, cycle detection, or ancestry + classification. Missing chronology does not erase a resolved identity. + """ + from dataclasses import replace + + keys = () if type(loaded_keys) is tuple and not loaded_keys else _parent_keys(loaded_keys) + promoted = _join_options(None, strict_mode, strict_warning_codes) + if keys: + order = _checked_order(version_order, tuple(sorted({key.dataset_version for key in keys}))) + else: + order = VersionOrderResult((), (), "empty_scope", MappingProxyType({}), MappingProxyType({})) + if type(version_order) is VersionOrderResult and (version_order.declarations or + version_order.invocation_order is not None): + declared = _version_labels(version_order.order, field="version_order") + checked = resolve_version_order(declared, document=version_order.declarations or None, + invocation_order=version_order.invocation_order) + order = replace(checked, loaded_versions=()) + if version_order is not None and (type(version_order) is not VersionOrderResult or version_order != order): + raise _version_failure("parent batch ordering evidence disagrees with its declarations") + if provenance is not None and type(provenance) is not tuple: + raise CanonicalValidationError(ErrorCode.SCHEMA_TYPE, "parent batch requires explicit provenance rows") + lookup = _parent_lookup(keys, order) + rows: dict[RecordKey, tuple[ProvenanceAssessment, object, str]] = {} + for row in provenance or (): + if type(row) not in (CanonicalRow, ProvenanceAssessment): + raise CanonicalValidationError(ErrorCode.SCHEMA_TYPE, "parent batch requires canonical or assessed provenance") + loc = _join_location(row.location) + values = _join_fields(row.values, loc) + # Parent failures belong to the retained per-record result. Identity and + # other provenance fields keep their existing fail-closed validation. + raw_parents = values.pop("parent_ids", None) + assessed = assess_provenance_row(replace(row, values=MappingProxyType(values))) + key = assessed.record_key + if key in rows: + raise _fail(ErrorCode.PROVENANCE_DUPLICATE_ROW, "duplicate parent batch provenance identity", "record_id", loc, key) + if key not in lookup[0]: + raise _fail(ErrorCode.PROVENANCE_UNMATCHED_ROW, "parent batch provenance has no loaded record", "record_id", loc, key) + absent = "parent_ids" not in row.values + if type(row) is CanonicalRow: + states = _join_fields(row.field_states, loc) + state = states.get("parent_ids") + if state == "absent": + if raw_parents is not None: + raise _fail(ErrorCode.SCHEMA_TYPE, "absent parent state conflicts with its value", "parent_ids", loc, key) + absent = True + elif state == "null" and raw_parents is not None: + raise _fail(ErrorCode.SCHEMA_TYPE, "null parent state conflicts with its value", "parent_ids", loc, key) + declaration = "absent" if absent else ("null" if raw_parents is None else + "empty" if type(raw_parents) in (list, tuple) and not raw_parents else "declared") + rows[key] = (assessed, raw_parents, declaration) + assessments: list[ParentRecordValidation] = [] + messages = [replace(message, severity=ValidationSeverity.ERROR if message.code in promoted + else message.severity) for message in order.messages] + for key in keys: + item = rows.get(key) + if item is None: + missing = _join_message(WarningCode.PROVENANCE_MISSING_ROW.value, + ValidationSeverity.ERROR if WarningCode.PROVENANCE_MISSING_ROW.value in promoted + else ValidationSeverity.WARNING, "record has no provenance for parent validation", key, RowLocation()) + result = ParentValidationResult(key, "absent", (), False, (missing,)) + assessment = ParentRecordValidation(key, False, result, 0, 0, 0, (missing,)) + else: + row, raw_parents, declaration = item + notices = tuple(_join_message(ErrorCode.SCHEMA_REQUIRED_FIELD.value, ValidationSeverity.ERROR, + "parent input lacks a required provenance field", key, row.location, name) + for name in row.missing_required_fields) + if raw_parents is not None and type(raw_parents) not in (list, tuple): + malformed = _join_message(ErrorCode.PARENT_FORMAT.value, ValidationSeverity.ERROR, + "parent declaration must be an array of strings", key, row.location, "parent_ids") + assessment = ParentRecordValidation(key, True, None, None, 0, 0, notices + (malformed,)) + else: + parents = None if raw_parents is None else tuple(raw_parents) + result = _resolve_parent_list(key, parents, lookup, row.location, promoted, retain_errors=True) + result = replace(result, declaration_state=declaration) + resolved = sum(len(ref.source_references) for ref in result.references + if ref.resolution_status is ParentResolutionStatus.RESOLVED) + self_count = sum(message.code == ErrorCode.LINEAGE_CYCLE.value for message in result.messages) + assessment = ParentRecordValidation(key, True, result, len(parents or ()), + resolved, self_count, notices + result.messages) + assessments.append(assessment) + messages.extend(assessment.messages) + return ParentBatchValidationResult(keys, tuple(assessments), order, tuple(messages), promoted) + + def validate_generation_declarations( loaded_keys: tuple[RecordKey, ...], provenance: tuple[CanonicalRow | ProvenanceAssessment, ...], *, version_order: VersionOrderResult | None = None, strict_mode: bool = False, @@ -679,13 +794,15 @@ def validate_generation_declarations( ) -> GenerationValidationResult: """Validate non-grounding counts using only safely established dependencies. - This internal input check does not construct or traverse an ancestry graph. - Monotone bounded scans propagate established expected counts; declarations - never seed expected values. Stalled dependencies remain unavailable, without + This internal input check propagates counts through immediate dependencies; + it does not calculate ancestry or lineage depth. Each established dependency + is consumed once; declarations never seed expected values. Stalled + dependencies remain unavailable, without inferring a cycle. Grounding yes establishes zero directly, independently of lineage depth. Parent validity and incomplete-provenance errors stay visible. """ keys = _parent_keys(loaded_keys) + key_set = set(keys) promoted = _join_options(None, strict_mode, strict_warning_codes) order = _checked_order(version_order, tuple(sorted({key.dataset_version for key in keys}))) if type(provenance) is not tuple: @@ -703,7 +820,7 @@ def validate_generation_declarations( key = assessed.record_key if key in rows: raise _fail(ErrorCode.PROVENANCE_DUPLICATE_ROW, "duplicate generation input identity", "record_id", assessed.location, key) - if key not in keys: + if key not in key_set: raise _fail(ErrorCode.PROVENANCE_UNMATCHED_ROW, "generation input has no matching loaded record", "record_id", assessed.location, key) rows[key] = assessed lookup = _parent_lookup(keys, order) @@ -742,27 +859,31 @@ def validate_generation_declarations( reasons[key] = ("PARENT_UNRESOLVED",) elif any(ref.temporal_status == "unavailable" for ref in parents[key].references): reasons[key] = ("PARENT_CHRONOLOGY_UNAVAILABLE",) - # At most one new count per record. No queue of graph nodes or topological sort. pending = set(keys) - set(expected) - set(reasons) - # An existing declared version order can schedule scans, without deriving a - # topological order from parent edges. Same-version dependencies may need - # repeated scans. No stalled dependency is ever relabelled as a cycle. - positions = lookup[2] - scan_order = tuple(item[3] for item in sorted( - (positions.get(key.dataset_version, 0), key.dataset_version, key.record_id, key) - for key in pending)) - for _ in range(len(pending)): - newly_established: set[RecordKey] = set() - for key in scan_order: - if key not in pending: - continue - dependencies = parents[key].references - if all(ref.parent_key in expected for ref in dependencies): - expected[key] = 1 + max(expected[ref.parent_key] for ref in dependencies) - newly_established.add(key) - if not newly_established: - break - pending.difference_update(newly_established) + # Only validated grounding seeds the queue. Unavailable dependencies never + # enter it, and stalled dependencies are never relabelled as cycles. Grounded + # records still reset to zero independently of their own parent declarations. + dependents: dict[RecordKey, list[RecordKey]] = {} + remaining: dict[RecordKey, int] = {} + largest_parent: dict[RecordKey, int] = {} + for key in keys: + if key not in pending: + continue + dependencies = parents[key].references + remaining[key] = len(dependencies) + largest_parent[key] = 0 + for ref in dependencies: + dependents.setdefault(ref.parent_key, []).append(key) + ready = deque(key for key in keys if key in expected) + while ready: + parent = ready.popleft() + for child in dependents.get(parent, ()): + largest_parent[child] = max(largest_parent[child], expected[parent]) + remaining[child] -= 1 + if remaining[child] == 0: + expected[child] = 1 + largest_parent[child] + pending.remove(child) + ready.append(child) for key in pending: reasons[key] = ("PARENT_GENERATION_UNAVAILABLE",) assessments: list[GenerationAssessment] = [] @@ -1066,7 +1187,8 @@ def validate_bundle(bundle: AuditBundle, *, configuration: dict[str, object] | N source_by_key = {} provenance_supplied = any(source.role is FileRole.PROVENANCE_MANIFEST for source in sources) for source in sources: - if source.role in (FileRole.RECORDS_PRIMARY, FileRole.RECORDS_COMPARE, FileRole.PROVENANCE_MANIFEST): + if source.role in (FileRole.RECORDS_PRIMARY, FileRole.RECORDS_COMPARE, + FileRole.LINEAGE_CONTEXT, FileRole.PROVENANCE_MANIFEST): entry, rows, row_traces, notices = _bundle_table(source, effective, options, mapping) inventory.append(entry) traces.extend(row_traces) @@ -1081,6 +1203,14 @@ def validate_bundle(bundle: AuditBundle, *, configuration: dict[str, object] | N inventory.append(inventory_source(source, limits=effective)) canonical_records = tuple(sorted(records, key=_bundle_row_key)) validate_unique_keys(canonical_records, kind="records") + context_versions = {row.record_key.dataset_version for row in canonical_records + if row.location.file_role is FileRole.LINEAGE_CONTEXT} + selected_versions = {row.record_key.dataset_version for row in canonical_records + if row.location.file_role in (FileRole.RECORDS_PRIMARY, FileRole.RECORDS_COMPARE)} + if context_versions & selected_versions: + raise _fail(ErrorCode.CONFIG_INVALID, + "lineage context versions must be disjoint from primary and comparison versions", + "dataset_version", RowLocation(FileRole.LINEAGE_CONTEXT)) canonical_provenance = tuple(sorted(provenance, key=_bundle_row_key)) if provenance_supplied else None joined = join_provenance(canonical_records, canonical_provenance, strict_mode=config.strict_mode, strict_warning_codes=config.strict_warning_codes) @@ -1088,6 +1218,9 @@ def validate_bundle(bundle: AuditBundle, *, configuration: dict[str, object] | N order = _bundle_order(versions, order_document, config.version_order, invocation_order) messages.extend(joined.messages) messages.extend(order.messages) + parent_validation = resolve_parent_batch(tuple(row.record_key for row in canonical_records), + canonical_provenance, version_order=order, + strict_mode=config.strict_mode, strict_warning_codes=config.strict_warning_codes) generation = None try: generation = validate_generation_declarations(tuple(row.record_key for row in canonical_records), @@ -1100,6 +1233,7 @@ def validate_bundle(bundle: AuditBundle, *, configuration: dict[str, object] | N raise key = RecordKey.parse(error.record_key) if error.record_key is not None else None messages.append(_bundle_error_message(error, RowLocation(FileRole.PROVENANCE_MANIFEST), key)) + messages.extend(parent_validation.messages) content = {} if options.content_mode is ContentMode.LOCAL_REF else None if resolve_local_content: for row in canonical_records: @@ -1125,7 +1259,7 @@ def validate_bundle(bundle: AuditBundle, *, configuration: dict[str, object] | N classification = replace(classification, validation_messages=final_messages) return BundleValidationResult(tuple(sorted(inventory, key=_bundle_inventory_key)), canonical_records, canonical_provenance, joined, order, generation, classification, tuple(traces), - tuple(sorted(content or {})), final_messages) + tuple(sorted(content or {})), final_messages, parent_validation) def _bundle_row_key(row): diff --git a/src/recursive_integrity_toolkit/lineage/ancestry.py b/src/recursive_integrity_toolkit/lineage/ancestry.py index f423e84..ddd48f1 100644 --- a/src/recursive_integrity_toolkit/lineage/ancestry.py +++ b/src/recursive_integrity_toolkit/lineage/ancestry.py @@ -1,20 +1,644 @@ -"""Own future external-root tracing, ancestry incidence, fractional root mass, and concentration. +"""Resolve external roots, coverage, concentration and shared-root evidence. -Owner IDs: - T4 +Owner IDs: T4; Phase 5 Step 7. -Future inputs: - Future validated acyclic lineage graph and explicit external-root classifications. +Validated explicit grounding supplies anchors. Iterative dependency scheduling +unions complete root sets; unknown branches never supply exact roots. Logical +membership and candidate-visit budgets bound propagation before each work unit. +The allocation is topological and makes no causal or scientific quality claim. -Future outputs: - Future root sets, incidence, ancestry HHI, effective roots, and lineage coverage. +Current phase status: + Phase 5 Step 7 binds explicit report handoffs to relevant input evidence. + No file or network I/O, generation calculation or implicit invocation. +""" +from __future__ import annotations -Assumptions: - Ancestry concentration is topological and does not establish causal contribution. +from dataclasses import dataclass, field, replace +from heapq import heapify, heappop, heappush +from math import fsum, isfinite +from types import MappingProxyType +from typing import Literal -Limits: - No root tracing, allocation, concentration, coverage, or graph traversal is implemented here. +from ..errors import ErrorCode, LineageResourceLimitError +from ..io.validation import join_provenance +from ..models import BundleValidationResult, RecordKey, ValidationMessage, ValidationSeverity +from ..result import ExecutionStatus, ReportStatus +from ..utils.hashing import canonical_json_bytes, sha256_canonical +from .cycles import CycleAnalysis, DepthAssessment, analyze_cycles, _local_depth_reasons +from .graph import ( + LineageGraph, LineageLimits, LineageResourceUsage, LineageScope, + _integer, _invalid, _key, _messages, build_lineage_graph, +) -Current phase status: - Phase 1 package scaffold only. Import-safe. No analytical or data-processing logic. -""" + +_ROOT_REASONS = frozenset({ + "MISSING_PROVENANCE", "MISSING_REQUIRED_PROVENANCE", "UNKNOWN_GROUNDING", + "PARENT_DECLARATION_UNAVAILABLE", "INVALID_PARENT_REFERENCE", + "UNRESOLVED_PARENT_REFERENCE", "VERSION_ORDER_UNAVAILABLE", "CYCLE_AFFECTED", + "INCOMPLETE_PARENT_ANCESTRY", "GROUNDED_PARENT_RULE_UNSUPPORTED", +}) +_EXECUTION_REASONS = frozenset({ + "INVALID_LINEAGE_INPUT", "UNRESOLVED_ANCESTRY", "LINEAGE_RESOURCE_LIMIT_EXCEEDED", +}) + + +def _reasons(values: object, allowed: frozenset[str]) -> tuple[str, ...]: + if (type(values) is not tuple + or any(type(value) is not str or value not in allowed for value in values) + or values != tuple(sorted(set(values)))): + raise _invalid("ancestry reasons require canonical known codes") + return values + + +def _coverage(value: object, expected: float | None) -> bool: + return value is None if expected is None else type(value) is float and value == expected + + +def _execution(messages: tuple[ValidationMessage, ...], complete: int, unresolved: int): + if any(message.severity is ValidationSeverity.FATAL for message in messages): + return ExecutionStatus.FAILED, ("INVALID_LINEAGE_INPUT",) + if any(message.severity is ValidationSeverity.ERROR for message in messages): + return (ExecutionStatus.PARTIAL if complete else ExecutionStatus.FAILED), ("INVALID_LINEAGE_INPUT",) + if unresolved: + return ExecutionStatus.PARTIAL, ("UNRESOLVED_ANCESTRY",) + return ExecutionStatus.COMPLETED, () + + +def _lineage_input_signature(validation: BundleValidationResult) -> str: + """Bind report handoffs to relevant declarations without retaining raw input. + + This internal digest detects stale same-key evidence, not authenticity. + Content, URI, path, notes and extras are excluded. No graph or metric runs. + """ + if type(validation) is not BundleValidationResult or validation.parent_validation is None: + raise _invalid("lineage input binding requires retained validation evidence") + names = ("dataset_version", "record_id", "parent_ids", "external_grounding", + "source_type", "provenance_confidence", "transformation", "batch_id", "timestamp") + + def identity(key): + return None if key is None else (key.dataset_version, key.record_id) + + def diagnostics(messages): + return sorted([(message.code, message.severity.value, identity(message.record_key), + message.field, None if message.file_role is None else message.file_role.value) + for message in messages], key=canonical_json_bytes) + + def rows(items): + return tuple((identity(row.record_key), + getattr(row, "kind", None), + None if row.location.file_role is None else row.location.file_role.value, + tuple((name, name in row.values, row.values.get(name)) for name in names), + tuple((name, row.field_states.get(name)) for name in names) + if hasattr(row, "field_states") else ()) + for row in sorted(items, key=lambda row: row.record_key)) + + def chronology(order): + return (order.loaded_versions, order.order, order.order_source, dict(order.source_orders), + dict(order.declarations), order.invocation_order, diagnostics(order.messages)) + + batch = validation.parent_validation + parents = [] + for item in batch.assessments: + parent = item.result + refs = None if parent is None else tuple(sorted([( + tuple(sorted(ref.source_references)), ref.canonical_reference, + identity(ref.parent_key), ref.resolution_status.value, ref.temporal_status) + for ref in parent.references], key=canonical_json_bytes)) + parents.append((identity(item.child_key), item.provenance_available, + item.declared_reference_count, item.resolved_reference_count, + item.invalid_self_reference_count, + None if parent is None else parent.declaration_state, refs, + None if parent is None else parent.graph_validation_deferred, + None if parent is None else diagnostics(parent.messages), + diagnostics(item.messages))) + try: + return sha256_canonical(( + rows(validation.records), None if validation.provenance is None else rows(validation.provenance), + chronology(validation.version_order), chronology(batch.version_order), + tuple(parents), batch.promoted_warning_codes, diagnostics(batch.messages), + diagnostics(validation.validation_messages))) + except (ValueError, TypeError): + raise _invalid("lineage input binding requires canonical declarations") from None + + +@dataclass(frozen=True, slots=True) +class RecordAncestry: + """Exact complete roots, including known-empty, or explicit uncertainty.""" + record_key: RecordKey + classification: Literal["grounded", "closed", "unresolved"] + external_root_keys: frozenset[RecordKey] | None + reason_codes: tuple[str, ...] + lineage_depth: int | None + depth_reason_codes: tuple[str, ...] + + def __post_init__(self) -> None: + _key(self.record_key) + if type(self.classification) is not str: + raise _invalid("ancestry classification requires a literal status") + reasons = _reasons(self.reason_codes, _ROOT_REASONS) + roots = self.external_root_keys + if roots is None: + if self.classification != "unresolved" or not reasons: + raise _invalid("unresolved ancestry requires null roots and explicit reasons") + else: + if type(roots) is not frozenset: + raise _invalid("complete ancestry requires an immutable root set") + for root in roots: + _key(root) + if reasons or self.classification != ("grounded" if roots else "closed"): + raise _invalid("ancestry classification disagrees with its complete root set") + if self.lineage_depth is None: + raise _invalid("complete ancestry requires complete structural paths") + DepthAssessment(self.lineage_depth, self.depth_reason_codes) + + +@dataclass(frozen=True, slots=True) +class RootContribution: + """One root's target incidence and equal-per-record fractional allocation.""" + record_key: RecordKey + incidence_count: int + incidence_share: float + incidence_denominator: int + fractional_mass: float + normalized_weight: float + weight_denominator: int + + def __post_init__(self) -> None: + _key(self.record_key) + for value in (self.incidence_count, self.incidence_denominator, self.weight_denominator): + _integer(value) + if not 1 <= self.incidence_count <= self.weight_denominator <= self.incidence_denominator: + raise _invalid("root incidence requires positive target and grounded denominators") + for value in (self.incidence_share, self.fractional_mass, self.normalized_weight): + if type(value) is not float or not isfinite(value) or value <= 0: + raise _invalid("root contributions require finite positive floating-point values") + try: + incidence_share = self.incidence_count / self.incidence_denominator + normalized_weight = self.fractional_mass / self.weight_denominator + except OverflowError: + raise _invalid("root contribution denominators exceed the supported numeric range") from None + if (self.fractional_mass > self.incidence_count + or self.incidence_share != incidence_share + or self.normalized_weight != normalized_weight): + raise _invalid("root contribution values disagree with their declared denominators") + + +def _root_metrics(records: tuple[RecordAncestry, ...], total: int, grounded: int): + """Aggregate complete target sets before any future display truncation. + + Input targets and each root visit use canonical identity order. fsum retains + small fractional contributions without order-sensitive running totals. The + temporary terms are bounded by the already admitted target memberships. + """ + terms: dict[RecordKey, list[float]] = {} + for record in records: + roots = record.external_root_keys + if roots: + mass = 1.0 / len(roots) + for root in sorted(roots): + terms.setdefault(root, []).append(mass) + contributions = [] + for root in sorted(terms): + incidence = len(terms[root]) + mass = fsum(terms[root]) + contributions.append(RootContribution( + root, incidence, incidence / total, total, mass, mass / grounded, grounded)) + hhi = fsum(row.normalized_weight ** 2 for row in contributions) if contributions else None + effective = 1.0 / hhi if hhi is not None else None + contributions.sort(key=lambda row: (-row.incidence_count, row.record_key)) + return tuple(contributions), len(contributions), hhi, effective + + +@dataclass(frozen=True, slots=True) +class LineageAnalysisResult: + """Targets, complete root metrics and completed structural observations. + + Resource-aborted root propagation supplies no partition or record rows. + Earlier graph/depth/reference observations survive. Node/edge admission + failures precede this result and retain the graph typed exception contract. + """ + scope: LineageScope + execution_status: ExecutionStatus + execution_reason_codes: tuple[str, ...] + records: tuple[RecordAncestry, ...] | None + cycles: CycleAnalysis + grounded_record_count: int | None + closed_record_count: int | None + unresolved_record_count: int | None + records_with_resolved_external_ancestry: int | None + resolved_lineage_coverage: float | None + external_ancestry_coverage: float | None + ancestry_coverage_reason_codes: tuple[str, ...] + declared_parent_reference_count: int | None + resolved_parent_reference_count: int | None + unresolved_parent_reference_count: int | None + resolved_parent_edge_coverage: float | None + no_declared_parents: bool + reference_coverage_reason_codes: tuple[str, ...] + root_contributions: tuple[RootContribution, ...] | None + distinct_external_root_count: int | None + ancestry_concentration_hhi: float | None + effective_external_root_count: float | None + resource_usage: LineageResourceUsage + messages: tuple[ValidationMessage, ...] + input_signature: str = field(repr=False) + + def __post_init__(self) -> None: + if (type(self.input_signature) is not str or len(self.input_signature) != 64 + or any(character not in "0123456789abcdef" for character in self.input_signature)): + raise _invalid("lineage result requires its internal declaration binding") + if type(self.scope) is not LineageScope or type(self.cycles) is not CycleAnalysis: + raise _invalid("ancestry analysis requires typed scope and cycle observations") + scope = replace(self.scope) + cycles = replace(self.cycles) + messages = _messages(self.messages) + if cycles.scope != scope: + raise _invalid("ancestry and structural scopes disagree") + if not set(cycles.messages) <= set(messages): + raise _invalid("ancestry must preserve completed structural diagnostics") + if type(self.resource_usage) is not LineageResourceUsage: + raise _invalid("ancestry analysis requires typed resource usage") + usage = replace(self.resource_usage) + if usage.admitted_node_count != scope.loaded_record_count: + raise _invalid("ancestry resource usage disagrees with the loaded scope") + if type(self.execution_status) is not ExecutionStatus: + raise _invalid("ancestry analysis requires a typed execution status") + _reasons(self.execution_reason_codes, _EXECUTION_REASONS) + counts = (self.grounded_record_count, self.closed_record_count, + self.unresolved_record_count, self.records_with_resolved_external_ancestry) + root_metrics = (self.root_contributions, self.distinct_external_root_count, + self.ancestry_concentration_hhi, self.effective_external_root_count) + if self.records is None: + if (self.execution_status is not ExecutionStatus.FAILED + or self.execution_reason_codes != ("LINEAGE_RESOURCE_LIMIT_EXCEEDED",) + or usage.exhausted_limit not in ("max_root_memberships", "max_root_union_visits") + or any(value is not None for value in counts) + or self.resolved_lineage_coverage is not None + or self.external_ancestry_coverage is not None + or self.ancestry_coverage_reason_codes != ("LINEAGE_RESOURCE_LIMIT_EXCEEDED",) + or any(value is not None for value in root_metrics) + or not any(message.code == ErrorCode.LINEAGE_RESOURCE_LIMIT_EXCEEDED.value + and message.severity in (ValidationSeverity.ERROR, ValidationSeverity.FATAL) + for message in messages)): + raise _invalid("aborted ancestry cannot expose a partial partition as exact") + else: + if type(self.records) is not tuple or usage.exhausted_limit is not None: + raise _invalid("complete ancestry rows require an immutable completed partition") + for record in self.records: + if type(record) is not RecordAncestry: + raise _invalid("ancestry records require typed assessments") + replace(record) + depth = cycles.depths_by_record.get(record.record_key) + if depth != DepthAssessment(record.lineage_depth, record.depth_reason_codes): + raise _invalid("ancestry depth disagrees with structural observations") + if record.external_root_keys is not None and any( + root not in cycles.depths_by_record for root in record.external_root_keys): + raise _invalid("external roots require loaded canonical identities") + if tuple(record.record_key for record in self.records) != scope.target_record_keys: + raise _invalid("ancestry records must cover the entire canonical target scope") + for count in counts: + _integer(count) + ground = sum(record.classification == "grounded" for record in self.records) + closed = sum(record.classification == "closed" for record in self.records) + unresolved = len(self.records) - ground - closed + if counts != (ground, closed, unresolved, ground + closed): + raise _invalid("ancestry counts disagree with the complete target partition") + total = scope.target_record_count + if (not _coverage(self.resolved_lineage_coverage, (ground + closed) / total if total else None) + or not _coverage(self.external_ancestry_coverage, ground / total if total else None) + or self.ancestry_coverage_reason_codes != (() if total else ("EMPTY_TARGET_SCOPE",))): + raise _invalid("ancestry coverage must use the full explicit target denominator") + if (self.execution_status, self.execution_reason_codes) != _execution(messages, ground + closed, unresolved): + raise _invalid("ancestry execution status disagrees with its target partition and diagnostics") + if type(self.root_contributions) is not tuple: + raise _invalid("completed root metrics require immutable contribution rows") + for contribution in self.root_contributions: + if type(contribution) is not RootContribution: + raise _invalid("root contributions require typed values") + replace(contribution) + _integer(self.distinct_external_root_count) + expected_roots, expected_count, expected_hhi, expected_effective = _root_metrics( + self.records, total, ground) + if (self.root_contributions != expected_roots + or self.distinct_external_root_count != expected_count + or not _coverage(self.ancestry_concentration_hhi, expected_hhi) + or not _coverage(self.effective_external_root_count, expected_effective)): + raise _invalid("root metrics disagree with complete target root sets") + references = (self.declared_parent_reference_count, self.resolved_parent_reference_count, + self.unresolved_parent_reference_count) + if type(self.no_declared_parents) is not bool: + raise _invalid("ancestry declaration status requires an explicit boolean") + if self.declared_parent_reference_count is None: + if (any(value is not None for value in references) + or self.resolved_parent_edge_coverage is not None or self.no_declared_parents + or self.reference_coverage_reason_codes != ("PARENT_REFERENCE_COUNT_UNAVAILABLE",)): + raise _invalid("unknown reference cardinality cannot imply zero declared parents") + else: + for count in references: + _integer(count) + declared, resolved, unresolved = references + if (declared != resolved + unresolved or self.no_declared_parents != (declared == 0) + or not _coverage(self.resolved_parent_edge_coverage, resolved / declared if declared else 1.0) + or self.reference_coverage_reason_codes != ()): + raise _invalid("ancestry reference coverage disagrees with original declarations") + object.__setattr__(self, "scope", scope) + object.__setattr__(self, "cycles", cycles) + object.__setattr__(self, "resource_usage", usage) + object.__setattr__(self, "messages", messages) + + @property + def root_metrics_status(self) -> ReportStatus: + if self.records is None: + return ReportStatus.UNAVAILABLE + return ReportStatus.PARTIAL if self.unresolved_record_count else ReportStatus.AVAILABLE + + @property + def concentration_status(self) -> ReportStatus: + return self.root_metrics_status if self.grounded_record_count else ReportStatus.UNAVAILABLE + + @property + def concentration_reason_codes(self) -> tuple[str, ...]: + if self.records is None: + return ("LINEAGE_RESOURCE_LIMIT_EXCEEDED",) + return () if self.grounded_record_count else ("NO_RESOLVED_EXTERNAL_ROOTS",) + + @property + def lineage_depth(self) -> int | None: + return self.cycles.lineage_depth + + @property + def maximum_resolved_target_depth(self) -> int | None: + return self.cycles.maximum_resolved_target_depth + + @property + def depth_resolved_record_count(self) -> int: + return self.cycles.depth_resolved_record_count + + +@dataclass(frozen=True, slots=True) +class SharedAncestryDependence: + """Descriptive exact-root witness with its original coverage and errors. + + A single ranked contribution is sufficient to witness shared support. The + immutable source retains all evidence without copying the contribution list. + Unresolved targets can preserve a witnessed presence but cannot prove absence. + """ + source: LineageAnalysisResult = field(repr=False) + + def __post_init__(self) -> None: + if type(self.source) is not LineageAnalysisResult: + raise _invalid("shared ancestry requires a typed lineage analysis") + object.__setattr__(self, "source", replace(self.source)) + + @property + def witness(self) -> RootContribution | None: + roots = self.source.root_contributions + return roots[0] if roots and roots[0].incidence_count >= 2 else None + + @property + def status(self) -> ReportStatus: + if self.witness is not None: + return ReportStatus.PARTIAL if self.unresolved_record_count else ReportStatus.AVAILABLE + if self.source.records is not None and self.scope.target_record_count and not self.unresolved_record_count: + return ReportStatus.AVAILABLE + return ReportStatus.UNAVAILABLE + + @property + def level(self) -> Literal["present", "not_present", "indeterminate"]: + if self.witness is not None: + return "present" + return "not_present" if self.status is ReportStatus.AVAILABLE else "indeterminate" + + @property + def reason_codes(self) -> tuple[str, ...]: + if self.source.records is None: + return ("LINEAGE_RESOURCE_LIMIT_EXCEEDED",) + if not self.scope.target_record_count: + return ("EMPTY_TARGET_SCOPE",) + return ("UNRESOLVED_ANCESTRY",) if self.unresolved_record_count else () + + @property + def scope(self) -> LineageScope: + return self.source.scope + + @property + def grounded_record_count(self) -> int | None: + return self.source.grounded_record_count + + @property + def closed_record_count(self) -> int | None: + return self.source.closed_record_count + + @property + def unresolved_record_count(self) -> int | None: + return self.source.unresolved_record_count + + @property + def resolved_lineage_coverage(self) -> float | None: + return self.source.resolved_lineage_coverage + + @property + def external_ancestry_coverage(self) -> float | None: + return self.source.external_ancestry_coverage + + @property + def ancestry_coverage_reason_codes(self) -> tuple[str, ...]: + return self.source.ancestry_coverage_reason_codes + + @property + def unresolved_reason_codes(self) -> tuple[str, ...]: + return tuple(sorted({reason for record in self.source.records or () for reason in record.reason_codes})) + + @property + def input_execution_status(self) -> ExecutionStatus: + return self.source.execution_status + + @property + def input_execution_reason_codes(self) -> tuple[str, ...]: + return self.source.execution_reason_codes + + @property + def input_has_errors(self) -> bool: + return any(message.severity in (ValidationSeverity.ERROR, ValidationSeverity.FATAL) + for message in self.source.messages) + + @property + def validation_messages(self) -> tuple[ValidationMessage, ...]: + return self.source.messages + + @property + def evidence_fields(self) -> tuple[str, ...]: + return ("root_contributions.incidence_count", "resolved_lineage_coverage", "external_ancestry_coverage") + + @property + def operationalization_label(self) -> str: + return "theory_guided_operationalization" + + @property + def limitations(self) -> tuple[str, ...]: + return ("Descriptive shared-root topology under supplied metadata; no calibrated risk level.", + "Does not establish causal contribution, semantic error, independent information, " + "biological relatedness, universal integrity or collapse.") + + +def shared_ancestry_dependence(result: LineageAnalysisResult) -> SharedAncestryDependence: + """Explicit pure proxy request; never reload data or rerun graph analysis.""" + return SharedAncestryDependence(result) + + +class _RootBudget: + """Charge each logical unit before performing it; never expose partial roots.""" + def __init__(self, usage: LineageResourceUsage) -> None: + self.initial = usage + self.memberships = 0 + self.visits = 0 + + def usage(self, exhausted: str | None = None) -> LineageResourceUsage: + attempted = None if exhausted is None else getattr(self.initial.limits, exhausted) + 1 + return replace(self.initial, stored_root_membership_count=self.memberships, + root_union_visit_count=self.visits, exhausted_limit=exhausted, + attempted_value=attempted) + + def membership(self) -> None: + if self.memberships == self.initial.limits.max_root_memberships: + raise LineageResourceLimitError(self.usage("max_root_memberships")) + self.memberships += 1 + + def visit(self) -> None: + if self.visits == self.initial.limits.max_root_union_visits: + raise LineageResourceLimitError(self.usage("max_root_union_visits")) + self.visits += 1 + + +def _metadata(validation: BundleValidationResult, graph: LineageGraph): + """Reassess original metadata while the shared graph owns parent syntax. + + Excluding parent_ids from this metadata-only join preserves the graph's + per-record malformed declaration failures. No declaration is reinterpreted. + """ + if not graph.node_keys: + return {}, () + provenance = None if validation.provenance is None else tuple( + replace(row, values=MappingProxyType({name: value for name, value in row.values.items() + if name != "parent_ids"})) + for row in validation.provenance) + promoted = validation.parent_validation.promoted_warning_codes + joined = join_provenance(validation.records, provenance, strict_mode=bool(promoted), + strict_warning_codes=promoted) + return {match.record_key: match for match in joined.matches}, joined.messages + + +def _resolve_roots(graph: LineageGraph, cycles: CycleAnalysis, metadata, budget: _RootBudget): + affected = set(cycles.affected_record_keys) + roots: dict[RecordKey, frozenset[RecordKey] | None] = {} + reasons: dict[RecordKey, tuple[str, ...]] = {} + for key in cycles.affected_record_keys: + roots[key] = None + reasons[key] = tuple(sorted(_local_depth_reasons(graph, key) | {"CYCLE_AFFECTED"})) + indegree = {key: len(graph.parents_by_child[key]) for key in graph.node_keys if key not in affected} + pending = [key for key, count in indegree.items() if count == 0] + heapify(pending) + while pending: + key = heappop(pending) + parents = graph.parents_by_child[key] + failures = _local_depth_reasons(graph, key) + row = metadata[key].provenance + if row is None: + failures.add("MISSING_PROVENANCE") + else: + if not row.required_fields_valid: + failures.add("MISSING_REQUIRED_PROVENANCE") + grounding = row.values.get("external_grounding") + if grounding == "unknown": + failures.add("UNKNOWN_GROUNDING") + if (grounding == "yes" and parents + and (len(parents) != 1 or row.values.get("transformation") != "carryover")): + failures.add("GROUNDED_PARENT_RULE_UNSUPPORTED") + for parent in parents: + if roots[parent] is None: + failures.add("INCOMPLETE_PARENT_ANCESTRY") + failures.update(reasons[parent]) + if failures: + roots[key] = None + reasons[key] = tuple(sorted(failures)) + elif not parents: + if row.values["external_grounding"] == "yes": + budget.membership() + roots[key] = frozenset((key,)) + else: + roots[key] = frozenset() + reasons[key] = () + else: + combined = set() + for parent in parents: + for root in sorted(roots[parent]): + budget.visit() + if root not in combined: + budget.membership() + combined.add(root) + roots[key] = frozenset(combined) + reasons[key] = () + for child in graph.children_by_parent[key]: + if child in indegree: + indegree[child] -= 1 + if indegree[child] == 0: + heappush(pending, child) + return roots, reasons + + +def analyze_lineage( + validation: BundleValidationResult, *, target_dataset_version: str | None, + limits: LineageLimits = LineageLimits(), +) -> LineageAnalysisResult: + """Compute explicit ancestry and root metrics, stopping on exhaustion. + + All loaded context nodes participate in resolution and resource accounting. + Only selected targets enter returned rows, G/C/U, coverage and root metrics. + Canonical dependency-ready nodes precede canonical parent and root visits. + """ + graph = build_lineage_graph(validation, target_dataset_version=target_dataset_version, limits=limits) + cycles = analyze_cycles(graph) + metadata, metadata_messages = _metadata(validation, graph) + input_signature = _lineage_input_signature(validation) + messages = _messages(cycles.messages + metadata_messages) + target_evidence = tuple(graph.parent_evidence_by_record[key] for key in graph.scope.target_record_keys) + count_known = all(item.declared_reference_count is not None for item in target_evidence) + declared = sum(item.declared_reference_count for item in target_evidence) if count_known else None + resolved = sum(item.resolved_reference_count for item in target_evidence) if count_known else None + reference_fields = dict( + declared_parent_reference_count=declared, resolved_parent_reference_count=resolved, + unresolved_parent_reference_count=None if declared is None else declared - resolved, + resolved_parent_edge_coverage=None if declared is None else (resolved / declared if declared else 1.0), + no_declared_parents=declared == 0, + reference_coverage_reason_codes=() if count_known else ("PARENT_REFERENCE_COUNT_UNAVAILABLE",), + ) + budget = _RootBudget(graph.resource_usage) + try: + roots, reasons = _resolve_roots(graph, cycles, metadata, budget) + except LineageResourceLimitError as error: + messages += (ValidationMessage(ErrorCode.LINEAGE_RESOURCE_LIMIT_EXCEEDED.value, + ValidationSeverity.ERROR, "Lineage root propagation exhausted its resource budget."),) + return LineageAnalysisResult( + graph.scope, ExecutionStatus.FAILED, (error.reason_code,), None, cycles, + None, None, None, None, None, None, (error.reason_code,), + **reference_fields, root_contributions=None, distinct_external_root_count=None, + ancestry_concentration_hhi=None, effective_external_root_count=None, + resource_usage=error.resource_usage, messages=messages, input_signature=input_signature) + records = tuple(RecordAncestry( + key, "unresolved" if roots[key] is None else "grounded" if roots[key] else "closed", + roots[key], reasons[key], cycles.depths_by_record[key].lineage_depth, + cycles.depths_by_record[key].reason_codes) for key in graph.scope.target_record_keys) + ground = sum(record.classification == "grounded" for record in records) + closed = sum(record.classification == "closed" for record in records) + unresolved = len(records) - ground - closed + status, execution_reasons = _execution(messages, ground + closed, unresolved) + total = graph.scope.target_record_count + contributions, distinct, hhi, effective = _root_metrics(records, total, ground) + return LineageAnalysisResult( + graph.scope, status, execution_reasons, records, cycles, + ground, closed, unresolved, ground + closed, + (ground + closed) / total if total else None, ground / total if total else None, + () if total else ("EMPTY_TARGET_SCOPE",), + **reference_fields, root_contributions=contributions, distinct_external_root_count=distinct, + ancestry_concentration_hhi=hhi, effective_external_root_count=effective, + resource_usage=budget.usage(), messages=messages, input_signature=input_signature) diff --git a/src/recursive_integrity_toolkit/lineage/cycles.py b/src/recursive_integrity_toolkit/lineage/cycles.py index 1f383ca..2346c20 100644 --- a/src/recursive_integrity_toolkit/lineage/cycles.py +++ b/src/recursive_integrity_toolkit/lineage/cycles.py @@ -1,20 +1,405 @@ -"""Own future cycle detection and affected-node reporting for record-ancestry graphs. +"""Analyze structural cycles, unaffected topology and complete structural depth. -Owner IDs: - T6 +Owner IDs: T6, PR-009 structural depth; Phase 5 Step 3. -Future inputs: - Future resolved record-ancestry graph. +Iterative traversals consume an immutable, revalidated parent graph. Cyclic +strongly connected components include separately retained invalid self-parents. +Complete internal identity sets support later ancestry work; presentation rows +are capped independently. No roots, scientific metrics, rendering or I/O occur. -Future outputs: - Future graph-validity status, cycle records, and affected-node sets. +Current phase status: + Phase 5 Step 3 cycle detection, descendant propagation, topology and depth. + External-root analysis and report integration remain later steps. +""" +from __future__ import annotations -Assumptions: - Detected cycles block ancestry interpretation and are not silently repaired. +from collections import deque +from dataclasses import dataclass, replace +from types import MappingProxyType +from typing import Mapping -Limits: - No traversal, topological ordering, cycle detection, or repair logic is implemented here. +from ..errors import ErrorCode +from ..models import ParentResolutionStatus, RecordKey, ValidationMessage, ValidationSeverity +from .graph import LineageGraph, LineageScope, _integer, _invalid, _key, _keys, _messages -Current phase status: - Phase 1 package scaffold only. Import-safe. No analytical or data-processing logic. -""" + +_DETAIL_LIMIT = 100 +_WITNESS_EDGE_LIMIT = 64 +_DEPTH_REASONS = frozenset({ + "MISSING_PROVENANCE", "PARENT_DECLARATION_UNAVAILABLE", "INVALID_PARENT_REFERENCE", + "UNRESOLVED_PARENT_REFERENCE", "VERSION_ORDER_UNAVAILABLE", "CYCLE_AFFECTED", + "INCOMPLETE_PARENT_DEPTH", +}) + + +@dataclass(frozen=True, slots=True) +class DepthAssessment: + """Complete structural depth, or explicit structural failure reasons.""" + lineage_depth: int | None + reason_codes: tuple[str, ...] = () + + def __post_init__(self) -> None: + if (type(self.reason_codes) is not tuple + or any(type(code) is not str or code not in _DEPTH_REASONS for code in self.reason_codes) + or self.reason_codes != tuple(sorted(set(self.reason_codes)))): + raise _invalid("structural depth requires canonical known reason codes") + if self.lineage_depth is None: + if not self.reason_codes: + raise _invalid("unavailable structural depth requires a reason") + else: + _integer(self.lineage_depth) + if self.reason_codes: + raise _invalid("complete structural depth cannot carry failure reasons") + + +@dataclass(frozen=True, slots=True) +class CycleComponent: + """Bounded component detail, independent of its complete internal membership.""" + component_index: int + member_count: int + target_member_count: int + witness_record_keys: tuple[RecordKey, ...] | None + witness_edge_count: int | None + witness_reason: str | None + + def __post_init__(self) -> None: + _integer(self.component_index, positive=True) + _integer(self.member_count, positive=True) + _integer(self.target_member_count) + if self.target_member_count > self.member_count: + raise _invalid("target cycle members exceed their component") + witness = self.witness_record_keys + if witness is None: + if self.witness_edge_count is not None or self.witness_reason != "diagnostic_limit": + raise _invalid("an omitted cycle witness requires its diagnostic limit reason") + else: + if type(witness) is not tuple or not 2 <= len(witness) <= _WITNESS_EDGE_LIMIT + 1: + raise _invalid("cycle witnesses require bounded immutable key sequences") + for key in witness: + _key(key) + _integer(self.witness_edge_count, positive=True) + if (witness[0] != witness[-1] or self.witness_edge_count != len(witness) - 1 + or self.witness_reason is not None): + raise _invalid("cycle witness must be closed with its exact edge count") + + +@dataclass(frozen=True, slots=True) +class CycleAnalysis: + """Full internal observations with separately bounded component diagnostics. + + Complete components/member/affected keys support later ancestry. Step 7 will + wrap identity-bearing report collections in the approved bounded/privacy + detail shape. Witness work already covers only the first 100 components. + """ + scope: LineageScope + cyclic_components: tuple[tuple[RecordKey, ...], ...] + cycle_member_record_keys: tuple[RecordKey, ...] + affected_record_keys: tuple[RecordKey, ...] + topological_order: tuple[RecordKey, ...] + depths_by_record: Mapping[RecordKey, DepthAssessment] + component_details: tuple[CycleComponent, ...] + lineage_depth: int | None + maximum_resolved_target_depth: int | None + depth_resolved_record_count: int + messages: tuple[ValidationMessage, ...] + + def __post_init__(self) -> None: + if type(self.scope) is not LineageScope: + raise _invalid("cycle analysis requires a typed scope") + scope = replace(self.scope) + if type(self.depths_by_record) not in (dict, MappingProxyType): + raise _invalid("cycle depth assessments require a mapping snapshot") + nodes = _keys(tuple(sorted(self.depths_by_record))) + if (len(nodes) != scope.loaded_record_count + or tuple(key for key in nodes if key.dataset_version == scope.target_dataset_version) != scope.target_record_keys + or tuple(sorted({key.dataset_version for key in nodes})) != scope.loaded_dataset_versions): + raise _invalid("cycle depth coverage disagrees with the loaded scope") + depths = {} + for key in nodes: + assessment = self.depths_by_record[key] + if type(assessment) is not DepthAssessment: + raise _invalid("cycle depths require typed assessments") + depths[key] = replace(assessment) + if type(self.cyclic_components) is not tuple: + raise _invalid("cyclic components require immutable tuples") + components = tuple(_keys(component) for component in self.cyclic_components) + if any(not component for component in components) or components != tuple(sorted(components)): + raise _invalid("cyclic components require canonical nonempty membership") + members = _keys(self.cycle_member_record_keys) + flattened = tuple(sorted(key for component in components for key in component)) + if flattened != members: + raise _invalid("cyclic components must have disjoint, complete member coverage") + affected = _keys(self.affected_record_keys) + affected_set = set(affected) + if not set(members) <= affected_set <= set(nodes): + raise _invalid("cycle affected records must include all members within loaded scope") + if (type(self.topological_order) is not tuple + or any(type(key) is not RecordKey for key in self.topological_order) + or len(set(self.topological_order)) != len(self.topological_order) + or set(self.topological_order) != set(nodes) - affected_set): + raise _invalid("cycle topology must cover exactly the unaffected records") + if any(depths[key].lineage_depth is not None or "CYCLE_AFFECTED" not in depths[key].reason_codes for key in affected): + raise _invalid("cycle affected records cannot have complete structural depth") + if any("CYCLE_AFFECTED" in depths[key].reason_codes for key in nodes if key not in affected_set): + raise _invalid("cycle depth reasons disagree with the affected-record set") + if (type(self.component_details) is not tuple + or len(self.component_details) != min(len(components), _DETAIL_LIMIT)): + raise _invalid("cycle component details must observe the diagnostic cap") + target_set = set(scope.target_record_keys) + details = [] + for index, detail in enumerate(self.component_details, 1): + if type(detail) is not CycleComponent: + raise _invalid("cycle component details require typed rows") + detail = replace(detail) + component = components[index - 1] + if (detail.component_index != index or detail.member_count != len(component) + or detail.target_member_count != sum(key in target_set for key in component) + or (detail.witness_record_keys is not None and not set(detail.witness_record_keys) <= set(component))): + raise _invalid("cycle component detail disagrees with its complete membership") + details.append(detail) + complete = [depths[key].lineage_depth for key in scope.target_record_keys + if depths[key].lineage_depth is not None] + maximum = max(complete, default=None) + whole = maximum if len(complete) == scope.target_record_count else None + _integer(self.depth_resolved_record_count) + for value in (self.lineage_depth, self.maximum_resolved_target_depth): + if value is not None: + _integer(value) + if (self.depth_resolved_record_count != len(complete) + or self.maximum_resolved_target_depth != maximum or self.lineage_depth != whole): + raise _invalid("cycle depth summaries disagree with the complete target subset") + messages = _messages(self.messages) + if components and not any(message.code == ErrorCode.LINEAGE_CYCLE.value + and message.severity is ValidationSeverity.ERROR for message in messages): + raise _invalid("detected cycles require an explicit lineage error") + object.__setattr__(self, "scope", scope) + object.__setattr__(self, "depths_by_record", MappingProxyType(depths)) + object.__setattr__(self, "component_details", tuple(details)) + object.__setattr__(self, "messages", messages) + + @property + def detected(self) -> bool: + return bool(self.cyclic_components) + + @property + def cycle_count(self) -> int: + return len(self.cyclic_components) + + @property + def counting_method(self) -> str: + return "cyclic_strongly_connected_components" + + @property + def cycle_status(self) -> str: + return "cyclic" if self.detected else "acyclic" + + @property + def cycle_status_scope(self) -> str: + return "accepted_edges_and_explicit_self_references" + + @property + def cycle_member_count(self) -> int: + return len(self.cycle_member_record_keys) + + @property + def target_cycle_member_count(self) -> int: + targets = set(self.scope.target_record_keys) + return sum(key in targets for key in self.cycle_member_record_keys) + + @property + def affected_record_count(self) -> int: + return len(self.affected_record_keys) + + @property + def target_affected_record_count(self) -> int: + targets = set(self.scope.target_record_keys) + return sum(key in targets for key in self.affected_record_keys) + + @property + def omitted_component_count(self) -> int: + return self.cycle_count - len(self.component_details) + + @property + def depth_reason_codes(self) -> tuple[str, ...]: + if not self.scope.target_record_count: + return ("EMPTY_TARGET_SCOPE",) + return () if self.lineage_depth is not None else ("INCOMPLETE_TARGET_DEPTH",) + + @property + def input_reason_codes(self) -> tuple[str, ...]: + return tuple(sorted({code for assessment in self.depths_by_record.values() + for code in assessment.reason_codes + if code not in ("CYCLE_AFFECTED", "INCOMPLETE_PARENT_DEPTH")})) + + @property + def has_errors(self) -> bool: + return any(message.severity in (ValidationSeverity.ERROR, ValidationSeverity.FATAL) + for message in self.messages) + + +def _cyclic_components(graph: LineageGraph) -> tuple[tuple[RecordKey, ...], ...]: + """Iterative Kosaraju traversal, retaining only cyclic components.""" + visited, finished = set(), [] + for start in graph.node_keys: + if start in visited: + continue + visited.add(start) + stack = [(start, iter(graph.children_by_parent[start]))] + while stack: + node, neighbors = stack[-1] + neighbor = next(neighbors, None) + if neighbor is None: + finished.append(node) + stack.pop() + elif neighbor not in visited: + visited.add(neighbor) + stack.append((neighbor, iter(graph.children_by_parent[neighbor]))) + visited.clear() + self_parents = set(graph.self_parent_record_keys) + components = [] + for start in reversed(finished): + if start in visited: + continue + component, pending = [], [start] + visited.add(start) + while pending: + node = pending.pop() + component.append(node) + for parent in graph.parents_by_child[node]: + if parent not in visited: + visited.add(parent) + pending.append(parent) + if len(component) > 1 or start in self_parents: + components.append(tuple(sorted(component))) + return tuple(sorted(components)) + + +def _witness(graph: LineageGraph, component: tuple[RecordKey, ...], + self_parents: set[RecordKey]) -> tuple[RecordKey, ...] | None: + """Canonical DFS with at most 64 active nodes; a missed witness stays omitted.""" + membership = set(component) + + def neighbors(node: RecordKey): + needs_self = node in self_parents + for child in graph.children_by_parent[node]: + if needs_self and child > node: + yield node + needs_self = False + if child in membership: + yield child + if needs_self: + yield node + + start = component[0] + path, positions, visited = [start], {start: 0}, {start} + stack = [neighbors(start)] + while stack: + child = next(stack[-1], None) + if child is None: + stack.pop() + positions.pop(path.pop()) + elif child in positions: + return tuple(path[positions[child]:] + [child]) + elif child not in visited: + if len(path) == _WITNESS_EDGE_LIMIT: + return None + visited.add(child) + positions[child] = len(path) + path.append(child) + stack.append(neighbors(child)) + return None + + +def _local_depth_reasons(graph: LineageGraph, key: RecordKey) -> set[str]: + """Inspect structural evidence only; grounding and unrelated fields are inert.""" + evidence = graph.parent_evidence_by_record[key] + reasons = set() + if not evidence.provenance_available: + reasons.add("MISSING_PROVENANCE") + result = evidence.result + if result is None: + reasons.add("INVALID_PARENT_REFERENCE") + return reasons + if result.declaration_state in ("absent", "null"): + reasons.add("PARENT_DECLARATION_UNAVAILABLE") + represented = sum(len(ref.source_references) for ref in result.references) + if (evidence.invalid_self_reference_count + or evidence.declared_reference_count != represented + or (result.declaration_state == "declared" and not represented)): + reasons.add("INVALID_PARENT_REFERENCE") + for reference in result.references: + if reference.resolution_status is ParentResolutionStatus.RESOLVED and reference.parent_key is None: + raise _invalid("resolved lineage references require a canonical parent identity") + parent = reference.parent_key + if parent == key: + raise _invalid("self-parent references require separately retained invalid evidence") + if parent is not None: + same_version = parent.dataset_version == key.dataset_version + if (reference.temporal_status == "same_version") != same_version: + raise _invalid("lineage reference chronology disagrees with its canonical identities") + if reference.resolution_status is not ParentResolutionStatus.RESOLVED: + reasons.add("UNRESOLVED_PARENT_REFERENCE") + if reference.temporal_status == "unavailable": + reasons.add("VERSION_ORDER_UNAVAILABLE") + return reasons + + +def analyze_cycles(graph: LineageGraph) -> CycleAnalysis: + """Compute exact loaded-graph SCCs and target depth without recursive calls. + + Traversals are O(V + E), apart from canonical sorting and constructor + validation. Witness search is bounded and cannot weaken an SCC finding. + ``cycle_status`` describes accepted edges plus explicit self-reference + evidence; ``input_reason_codes`` discloses structural input incompleteness. + """ + if type(graph) is not LineageGraph: + raise _invalid("cycle analysis requires a typed lineage graph") + graph = replace(graph) + components = _cyclic_components(graph) + members = tuple(sorted(key for component in components for key in component)) + affected, pending = set(members), deque(members) + while pending: + for child in graph.children_by_parent[pending.popleft()]: + if child not in affected: + affected.add(child) + pending.append(child) + unaffected = tuple(key for key in graph.node_keys if key not in affected) + indegree = {key: len(graph.parents_by_child[key]) for key in unaffected} + pending = deque(key for key in unaffected if not indegree[key]) + topology = [] + local_reasons = {key: _local_depth_reasons(graph, key) for key in graph.node_keys} + depths = {key: DepthAssessment(None, tuple(sorted(local_reasons[key] | {"CYCLE_AFFECTED"}))) + for key in graph.node_keys if key in affected} + while pending: + key = pending.popleft() + topology.append(key) + reasons = local_reasons[key] + parent_depths = [depths[parent].lineage_depth for parent in graph.parents_by_child[key]] + if any(value is None for value in parent_depths): + reasons.add("INCOMPLETE_PARENT_DEPTH") + value = None if reasons else 1 + max(parent_depths, default=-1) + depths[key] = DepthAssessment(value, tuple(sorted(reasons))) + for child in graph.children_by_parent[key]: + if child not in affected: + indegree[child] -= 1 + if not indegree[child]: + pending.append(child) + if len(topology) != len(unaffected): + raise _invalid("cycle topology did not cover every unaffected record") + targets, self_parents = set(graph.scope.target_record_keys), set(graph.self_parent_record_keys) + details = [] + for index, component in enumerate(components[:_DETAIL_LIMIT], 1): + witness = _witness(graph, component, self_parents) + details.append(CycleComponent(index, len(component), sum(key in targets for key in component), + witness, None if witness is None else len(witness) - 1, + "diagnostic_limit" if witness is None else None)) + complete = [depths[key].lineage_depth for key in graph.scope.target_record_keys + if depths[key].lineage_depth is not None] + maximum = max(complete, default=None) + whole = maximum if len(complete) == graph.scope.target_record_count else None + messages = graph.messages + if components: + messages += (ValidationMessage(ErrorCode.LINEAGE_CYCLE.value, ValidationSeverity.ERROR, + "A cyclic component was detected in the loaded lineage graph."),) + return CycleAnalysis(graph.scope, components, members, tuple(sorted(affected)), tuple(topology), + depths, tuple(details), whole, maximum, len(complete), messages) diff --git a/src/recursive_integrity_toolkit/lineage/graph.py b/src/recursive_integrity_toolkit/lineage/graph.py index 4063639..05f8367 100644 --- a/src/recursive_integrity_toolkit/lineage/graph.py +++ b/src/recursive_integrity_toolkit/lineage/graph.py @@ -1,20 +1,386 @@ -"""Own future record-ancestry graph construction and parent-edge resolution. +"""Build a bounded immutable parent graph from retained input validation. -Owner IDs: - PR-008 +Owner IDs: PR-008; Phase 5 Step 2. The shared input resolver owns reference +identity and chronology. This module selects an explicit target population and +admits accepted edges. No cycle/ancestry traversal, metrics, file or network I/O. +Raw record content, provenance URI fields and locations never enter graph results. +Reference spellings remain identity evidence, including duplicate aliases. -Future inputs: - Future canonical record keys and parent-reference rows. +Current phase status: + Phase 5 Step 2 parent graph construction. General cycle and ancestry + analysis remain deferred to the following implementation steps. +""" +from __future__ import annotations -Future outputs: - Future node, edge, resolution, and graph-scope records. +from dataclasses import dataclass +from types import MappingProxyType +from typing import Mapping -Assumptions: - Parent identity uses explicit composite references. +from ..errors import CanonicalValidationError, ErrorCode, LineageResourceLimitError, WarningCode +from ..models import ( + BundleValidationResult, CanonicalRow, FileRole, ParentBatchValidationResult, + ParentRecordValidation, ParentReference, ParentResolutionStatus, + ParentValidationResult, RecordKey, RowLocation, ValidationMessage, ValidationSeverity, +) -Limits: - No adjacency structure, edge resolution, graph construction, or traversal is implemented here. -Current phase status: - Phase 1 package scaffold only. Import-safe. No analytical or data-processing logic. -""" +def _invalid(message: str, code: ErrorCode = ErrorCode.SCHEMA_TYPE) -> CanonicalValidationError: + return CanonicalValidationError(code, message) + + +def _integer(value: object, *, positive: bool = False) -> None: + if type(value) is not int or value < (1 if positive else 0): + raise _invalid("lineage counts and limits require exact integers in range") + + +def _key(value: object) -> RecordKey: + if type(value) is not RecordKey: + raise _invalid("lineage identities require canonical record keys") + try: + return RecordKey(value.dataset_version, value.record_id) + except (TypeError, ValueError): + raise _invalid("lineage identity violates the canonical contract") from None + + +def _keys(value: object) -> tuple[RecordKey, ...]: + if type(value) is not tuple: + raise _invalid("lineage identity collections require immutable tuples") + checked = tuple(_key(item) for item in value) + if checked != tuple(sorted(set(checked))): + raise _invalid("lineage identities must be unique and canonically ordered") + return checked + + +def _version(value: object) -> str: + try: + return RecordKey(value, "lineage-scope").dataset_version + except (TypeError, ValueError): + raise _invalid("lineage target requires a valid explicit version", ErrorCode.CONFIG_INVALID) from None + + +@dataclass(frozen=True, slots=True) +class LineageLimits: + """Positive work limits, independent of loader limits and performance goals.""" + max_nodes: int = 200000 + max_edges: int = 1000000 + max_root_memberships: int = 1000000 + max_root_union_visits: int = 10000000 + + def __post_init__(self) -> None: + for value in (self.max_nodes, self.max_edges, self.max_root_memberships, self.max_root_union_visits): + _integer(value, positive=True) + + +def _limits(value: object) -> LineageLimits: + if type(value) is not LineageLimits: + raise _invalid("lineage limits require LineageLimits", ErrorCode.CONFIG_INVALID) + return LineageLimits(value.max_nodes, value.max_edges, value.max_root_memberships, value.max_root_union_visits) + + +@dataclass(frozen=True, slots=True) +class LineageResourceUsage: + """Exact admitted work and the next rejected work unit, when exhausted.""" + admitted_node_count: int + admitted_edge_count: int + stored_root_membership_count: int + root_union_visit_count: int + limits: LineageLimits + exhausted_limit: str | None = None + attempted_value: int | None = None + + def __post_init__(self) -> None: + bounded = _limits(self.limits) + pairs = ( + ("max_nodes", self.admitted_node_count, bounded.max_nodes), + ("max_edges", self.admitted_edge_count, bounded.max_edges), + ("max_root_memberships", self.stored_root_membership_count, bounded.max_root_memberships), + ("max_root_union_visits", self.root_union_visit_count, bounded.max_root_union_visits), + ) + for _, count, maximum in pairs: + _integer(count) + if count > maximum: + raise _invalid("lineage usage exceeds its admitted limit") + if self.exhausted_limit is None: + if self.attempted_value is not None: + raise _invalid("completed lineage work cannot have a rejected attempt") + else: + limits_by_name = {name: (count, maximum) for name, count, maximum in pairs} + if type(self.exhausted_limit) is not str or self.exhausted_limit not in limits_by_name: + raise _invalid("lineage exhaustion names an unsupported limit") + _integer(self.attempted_value, positive=True) + count, maximum = limits_by_name[self.exhausted_limit] + if count != maximum or self.attempted_value != maximum + 1: + raise _invalid("lineage exhaustion must identify its next rejected admission") + object.__setattr__(self, "limits", bounded) + + +@dataclass(frozen=True, slots=True) +class LineageScope: + """Target records determine population size; all other nodes are context.""" + target_dataset_version: str | None + target_record_keys: tuple[RecordKey, ...] + target_record_count: int + loaded_record_count: int + context_record_count: int + loaded_dataset_versions: tuple[str, ...] + + def __post_init__(self) -> None: + keys = _keys(self.target_record_keys) + for value in (self.target_record_count, self.loaded_record_count, self.context_record_count): + _integer(value) + if self.target_record_count != len(keys) or self.loaded_record_count != len(keys) + self.context_record_count: + raise _invalid("lineage scope counts do not match their explicit population") + if self.target_dataset_version is None: + if keys: + raise _invalid("a populated lineage target requires an explicit version", ErrorCode.CONFIG_INVALID) + else: + _version(self.target_dataset_version) + if any(key.dataset_version != self.target_dataset_version for key in keys): + raise _invalid("lineage target keys do not match their selected version") + if type(self.loaded_dataset_versions) is not tuple: + raise _invalid("lineage version inventory requires an immutable tuple") + versions = tuple(_version(value) for value in self.loaded_dataset_versions) + if versions != tuple(sorted(set(versions))): + raise _invalid("lineage version inventory must be unique and lexical") + if bool(versions) != bool(self.loaded_record_count) or set(key.dataset_version for key in keys) - set(versions): + raise _invalid("lineage version inventory disagrees with its scope") + + +_DIAGNOSTIC_CODES = frozenset(code.value for code in ErrorCode) | frozenset(code.value for code in WarningCode) + + +def _safe_message(value: object) -> ValidationMessage: + if (type(value) is not ValidationMessage or type(value.code) is not str + or value.code not in _DIAGNOSTIC_CODES or type(value.severity) is not ValidationSeverity): + raise _invalid("lineage diagnostics require known typed validation codes") + key = None if value.record_key is None else _key(value.record_key) + return ValidationMessage(value.code, value.severity, + "Validation evidence reported this diagnostic.", record_key=key) + + +def _messages(values: object) -> tuple[ValidationMessage, ...]: + if type(values) is not tuple: + raise _invalid("lineage diagnostics require an immutable tuple") + messages = {_safe_message(value) for value in values} + return tuple(sorted(messages, key=lambda item: ( + "" if item.record_key is None else item.record_key.dataset_version, + "" if item.record_key is None else item.record_key.record_id, item.code, item.severity.value))) + + +def _safe_evidence(value: object, child: RecordKey) -> ParentRecordValidation: + if type(value) is not ParentRecordValidation or _key(value.child_key) != child: + raise _invalid("lineage parent evidence must match its record") + if type(value.provenance_available) is not bool: + raise _invalid("lineage provenance availability requires an explicit boolean") + if value.declared_reference_count is not None: + _integer(value.declared_reference_count) + _integer(value.resolved_reference_count) + _integer(value.invalid_self_reference_count) + if (value.declared_reference_count is not None + and value.resolved_reference_count + value.invalid_self_reference_count > value.declared_reference_count): + raise _invalid("lineage reference counts exceed their declarations") + result = value.result + if result is not None: + if (type(result) is not ParentValidationResult or _key(result.child_key) != child + or result.declaration_state not in ("absent", "null", "empty", "declared") + or type(result.references) is not tuple or type(result.graph_validation_deferred) is not bool): + raise _invalid("lineage requires typed parent declaration evidence") + refs = [] + for ref in result.references: + if (type(ref) is not ParentReference or type(ref.source_references) is not tuple + or not ref.source_references or any(type(text) is not str for text in ref.source_references) + or type(ref.resolution_status) is not ParentResolutionStatus + or ref.temporal_status not in ("same_version", "earlier_version", "unavailable")): + raise _invalid("lineage reference evidence is invalid") + parent = None if ref.parent_key is None else _key(ref.parent_key) + if ref.canonical_reference != (None if parent is None else str(parent)): + raise _invalid("lineage reference identity disagrees with its canonical spelling") + for text in ref.source_references: + try: + RecordKey.parse(text) if "::" in text else RecordKey(child.dataset_version, text) + except (TypeError, ValueError): + raise _invalid("lineage evidence contains an invalid reference spelling") from None + refs.append(ParentReference(tuple(ref.source_references), ref.canonical_reference, + parent, ref.resolution_status, ref.temporal_status)) + resolved = sum(len(ref.source_references) for ref in refs + if ref.resolution_status is ParentResolutionStatus.RESOLVED) + represented = sum(len(ref.source_references) for ref in refs) + value.invalid_self_reference_count + if (resolved != value.resolved_reference_count or value.declared_reference_count is None + or represented > value.declared_reference_count + or (result.declaration_state != "declared" and value.declared_reference_count != 0)): + raise _invalid("lineage reference evidence disagrees with its declared counts") + result = ParentValidationResult(child, result.declaration_state, tuple(refs), + result.graph_validation_deferred, _messages(result.messages)) + elif value.declared_reference_count is not None or value.resolved_reference_count or value.invalid_self_reference_count: + raise _invalid("missing parent result requires unknown declaration cardinality") + return ParentRecordValidation(child, value.provenance_available, result, + value.declared_reference_count, value.resolved_reference_count, + value.invalid_self_reference_count, _messages(value.messages)) + + +def _accepted_parents(evidence: ParentRecordValidation) -> tuple[RecordKey, ...]: + if evidence.result is None: + return () + return tuple(sorted({ref.parent_key for ref in evidence.result.references + if ref.resolution_status is ParentResolutionStatus.RESOLVED + and ref.temporal_status in ("same_version", "earlier_version") + and ref.parent_key is not None and ref.parent_key != evidence.child_key})) + + +@dataclass(frozen=True, slots=True) +class LineageGraph: + """Accepted edges and rejected-reference evidence, without an acyclic claim. + + Both adjacency mappings include isolated nodes. All nested containers are + detached immutable values; raw source locations are removed from diagnostics. + """ + scope: LineageScope + node_keys: tuple[RecordKey, ...] + parents_by_child: Mapping[RecordKey, tuple[RecordKey, ...]] + children_by_parent: Mapping[RecordKey, tuple[RecordKey, ...]] + parent_evidence_by_record: Mapping[RecordKey, ParentRecordValidation] + self_parent_record_keys: tuple[RecordKey, ...] + messages: tuple[ValidationMessage, ...] + resource_usage: LineageResourceUsage + + def __post_init__(self) -> None: + nodes = _keys(self.node_keys) + if type(self.scope) is not LineageScope: + raise _invalid("lineage graph requires an explicit typed scope") + scope = LineageScope(self.scope.target_dataset_version, self.scope.target_record_keys, + self.scope.target_record_count, self.scope.loaded_record_count, + self.scope.context_record_count, self.scope.loaded_dataset_versions) + node_set = set(nodes) + if (scope.loaded_record_count != len(nodes) + or scope.target_record_keys != tuple(key for key in nodes if key.dataset_version == scope.target_dataset_version) + or scope.loaded_dataset_versions != tuple(sorted({key.dataset_version for key in nodes}))): + raise _invalid("lineage graph nodes disagree with its scope") + mappings = (self.parents_by_child, self.children_by_parent, self.parent_evidence_by_record) + if any(type(value) not in (dict, MappingProxyType) or set(value) != node_set for value in mappings): + raise _invalid("lineage mappings must cover exactly the loaded nodes") + evidence = {key: _safe_evidence(self.parent_evidence_by_record[key], key) for key in nodes} + parents, children = {}, {key: [] for key in nodes} + for child in nodes: + adjacent = _keys(self.parents_by_child[child]) + if set(adjacent) - node_set or adjacent != _accepted_parents(evidence[child]): + raise _invalid("lineage adjacency disagrees with accepted reference evidence") + parents[child] = adjacent + for parent in adjacent: + children[parent].append(child) + for parent in nodes: + if _keys(self.children_by_parent[parent]) != tuple(children[parent]): + raise _invalid("lineage forward and reverse adjacency disagree") + self_keys = _keys(self.self_parent_record_keys) + if self_keys != tuple(key for key in nodes if evidence[key].invalid_self_reference_count): + raise _invalid("lineage self-reference diagnostics disagree with retained evidence") + usage = self.resource_usage + if type(usage) is not LineageResourceUsage: + raise _invalid("lineage graph requires typed resource usage") + usage = LineageResourceUsage(usage.admitted_node_count, usage.admitted_edge_count, + usage.stored_root_membership_count, usage.root_union_visit_count, + usage.limits, usage.exhausted_limit, usage.attempted_value) + if (usage.admitted_node_count != len(nodes) or usage.admitted_edge_count != sum(map(len, parents.values())) + or usage.stored_root_membership_count or usage.root_union_visit_count or usage.exhausted_limit is not None): + raise _invalid("completed parent graph has inconsistent resource usage") + object.__setattr__(self, "scope", scope) + object.__setattr__(self, "node_keys", nodes) + object.__setattr__(self, "parents_by_child", MappingProxyType(parents)) + object.__setattr__(self, "children_by_parent", MappingProxyType({key: tuple(values) for key, values in children.items()})) + object.__setattr__(self, "parent_evidence_by_record", MappingProxyType(evidence)) + object.__setattr__(self, "messages", _messages(self.messages)) + object.__setattr__(self, "resource_usage", usage) + + +def build_lineage_graph( + validation: BundleValidationResult, *, target_dataset_version: str | None, + limits: LineageLimits = LineageLimits(), +) -> LineageGraph: + """Revalidate the shared batch once, then admit bounded deduplicated edges. + + Older handoffs without retained ``parent_validation`` must be rebuilt through + ``validate_bundle``. Rechecking uses one identity lookup for the entire batch, + avoiding a per-record rebuild. Reference count evidence includes original + aliases; accepted graph edges additionally require known chronology. + """ + from ..io.validation import resolve_parent_batch + + bounded = _limits(limits) + if type(validation) is not BundleValidationResult or type(validation.records) is not tuple: + raise _invalid("lineage graph requires a typed bundle validation result") + if len(validation.records) > bounded.max_nodes: + raise LineageResourceLimitError(LineageResourceUsage(bounded.max_nodes, 0, 0, 0, + bounded, "max_nodes", bounded.max_nodes + 1)) + keys, primary_versions, roles_by_version = [], set(), {} + for row in validation.records: + if type(row) is not CanonicalRow or row.kind != "records": + raise _invalid("lineage graph requires canonical record rows") + key = _key(row.record_key) + if (type(row.values) not in (dict, MappingProxyType) + or row.values.get("dataset_version") != key.dataset_version + or row.values.get("record_id") != key.record_id): + raise _invalid("lineage record identity disagrees with its retained values") + if type(row.location) is not RowLocation or (row.location.file_role is not None and + (type(row.location.file_role) is not FileRole or row.location.file_role not in + (FileRole.RECORDS_PRIMARY, FileRole.RECORDS_COMPARE, FileRole.LINEAGE_CONTEXT))): + raise _invalid("lineage records have an unsupported input role") + role = row.location.file_role + roles_by_version.setdefault(key.dataset_version, set()).add(role) + if role is FileRole.RECORDS_PRIMARY: + primary_versions.add(key.dataset_version) + keys.append(key) + nodes = tuple(sorted(keys)) + if len(set(nodes)) != len(nodes): + raise _invalid("duplicate canonical record identity in lineage input", ErrorCode.RECORD_DUPLICATE_ID) + if len(primary_versions) > 1 or any( + FileRole.RECORDS_PRIMARY in roles and FileRole.RECORDS_COMPARE in roles + or FileRole.LINEAGE_CONTEXT in roles and bool( + roles & {FileRole.RECORDS_PRIMARY, FileRole.RECORDS_COMPARE}) + for roles in roles_by_version.values()): + raise _invalid("lineage primary and context version roles must be disjoint", ErrorCode.CONFIG_INVALID) + if target_dataset_version is None: + if primary_versions or (nodes and all(roles == {None} for roles in roles_by_version.values())): + raise _invalid("loaded primary records require an explicit target version", ErrorCode.CONFIG_INVALID) + else: + _version(target_dataset_version) + if primary_versions and target_dataset_version not in primary_versions: + raise _invalid("lineage target must select the primary version", ErrorCode.CONFIG_INVALID) + if not primary_versions and roles_by_version.get(target_dataset_version, set()) & { + FileRole.RECORDS_COMPARE, FileRole.LINEAGE_CONTEXT}: + raise _invalid("lineage context records cannot define the primary target", ErrorCode.CONFIG_INVALID) + retained = validation.parent_validation + if type(retained) is not ParentBatchValidationResult: + raise _invalid("lineage needs retained parent validation; call validate_bundle again") + promoted = retained.promoted_warning_codes + if type(promoted) is not tuple: + raise _invalid("lineage warning promotion evidence is invalid") + rebuilt = resolve_parent_batch(nodes, validation.provenance, version_order=validation.version_order, + strict_mode=bool(promoted), strict_warning_codes=promoted) + if retained != rebuilt: + raise _invalid("retained parent validation disagrees with its original declarations") + versions = tuple(sorted({key.dataset_version for key in nodes})) + if target_dataset_version is not None and target_dataset_version not in versions and target_dataset_version not in rebuilt.version_order.order: + raise _invalid("lineage target version is outside the declared input scope", ErrorCode.CONFIG_INVALID) + targets = tuple(key for key in nodes if key.dataset_version == target_dataset_version) + scope = LineageScope(target_dataset_version, targets, len(targets), len(nodes), len(nodes) - len(targets), versions) + evidence = {item.child_key: item for item in rebuilt.assessments} + if tuple(evidence) != nodes: + raise _invalid("retained parent validation does not cover the loaded scope") + parents, children = {}, {key: [] for key in nodes} + edges = 0 + for child in nodes: + adjacent = _accepted_parents(evidence[child]) + accepted = [] + for parent in adjacent: + if edges == bounded.max_edges: + raise LineageResourceLimitError(LineageResourceUsage(len(nodes), edges, 0, 0, + bounded, "max_edges", edges + 1)) + accepted.append(parent) + children[parent].append(child) + edges += 1 + parents[child] = tuple(accepted) + usage = LineageResourceUsage(len(nodes), edges, 0, 0, bounded) + if type(validation.validation_messages) is not tuple: + raise _invalid("lineage validation diagnostics require an immutable tuple") + return LineageGraph(scope, nodes, parents, {key: tuple(value) for key, value in children.items()}, + evidence, tuple(key for key in nodes if evidence[key].invalid_self_reference_count), + validation.validation_messages + rebuilt.messages, usage) diff --git a/src/recursive_integrity_toolkit/metrics/bounds.py b/src/recursive_integrity_toolkit/metrics/bounds.py index 80661dd..1b0077b 100644 --- a/src/recursive_integrity_toolkit/metrics/bounds.py +++ b/src/recursive_integrity_toolkit/metrics/bounds.py @@ -1,11 +1,12 @@ -"""Compute the direct closure-exposure interval, without lineage inference. +"""Compute separately labeled direct and lineage closure-exposure intervals. Owner IDs: - T3: F-009, F-010; Definitions 5.3-5.5; Phase 3 Step 6. + T3: F-009, F-010; Definitions 5.3-5.5 and 5.9; Phase 5 Step 6. Inputs: Explicit integer counts and a single-version scope, or the accepted Step 5 - ProvenanceCompositionResult. No source files or parent references are read. + ProvenanceCompositionResult for direct bounds. Explicit LineageAnalysisResult + for lineage bounds. No source files or parent references are read. Outputs: Immutable lower, upper and width calculations, denominator, operationalization @@ -17,17 +18,19 @@ has not been independently verified by this calculation. Limits: - No lineage bounds, midpoint, confidence discount, risk threshold, universal - score, empirical truth checking, simulation, report or file/network access. + No ancestry inference, midpoint, confidence discount, risk threshold, + universal score, empirical truth checking, simulation, report or file/network access. A count envelope alone never certifies usable dataset provenance. Current phase status: - Phase 3 Step 6 direct bounds only. Pure explicit calls; import-safe. + Phase 5 Step 6 adds explicit lineage bounds. Direct behavior is unchanged. + Pure explicit calls; direct-only users do not import lineage modules. """ from __future__ import annotations from dataclasses import dataclass, field, replace from math import isclose +from typing import TYPE_CHECKING from ..errors import CanonicalValidationError, ErrorCode from ..models import ( @@ -41,6 +44,11 @@ ProvenanceCompositionResult, ) +if TYPE_CHECKING: + from ..lineage.ancestry import LineageAnalysisResult + from ..lineage.graph import LineageScope + from ..result import ExecutionStatus, ReportStatus + @dataclass(frozen=True, slots=True) class DirectClosureExposureBounds: @@ -338,3 +346,121 @@ def direct_closure_exposure(result: ProvenanceCompositionResult) -> DirectClosur grounding_field_coverage=grounding_cov, confidence_field_coverage=confidence_cov, confidence_status=confidence_status, confidence_counts=confidence_counts, input_has_errors=result.input_has_errors, validation_messages=messages) + + +@dataclass(frozen=True, slots=True) +class LineageClosureExposureBounds: + """Exact conservative envelope from a complete target G/C/U partition. + + Interval availability is separate from input validity and ancestry coverage. + A nonempty all-unresolved partition supplies [0, 1], retaining all errors. + An aborted partition or an empty population cannot supply numeric bounds. + """ + source: LineageAnalysisResult = field(repr=False) + + def __post_init__(self) -> None: + from ..lineage.ancestry import LineageAnalysisResult + if type(self.source) is not LineageAnalysisResult: + raise CanonicalValidationError( + ErrorCode.SCHEMA_TYPE, "lineage bounds require a typed lineage analysis", + field="lineage_closure_exposure") + object.__setattr__(self, "source", replace(self.source)) + + @property + def status(self) -> ReportStatus: + from ..result import ReportStatus + return ReportStatus.UNAVAILABLE if self.reason_codes else ReportStatus.AVAILABLE + + @property + def reason_codes(self) -> tuple[str, ...]: + if self.source.records is None: + return ("LINEAGE_RESOURCE_LIMIT_EXCEEDED",) + return () if self.denominator else ("EMPTY_TARGET_SCOPE",) + + @property + def lower_bound(self) -> float | None: + return None if self.reason_codes else self.closed_record_count / self.denominator + + @property + def upper_bound(self) -> float | None: + return None if self.reason_codes else (self.closed_record_count + self.unresolved_record_count) / self.denominator + + @property + def interval_width(self) -> float | None: + return None if self.reason_codes else self.unresolved_record_count / self.denominator + + @property + def scope(self) -> LineageScope: + return self.source.scope + + @property + def denominator(self) -> int: + return self.scope.target_record_count + + @property + def denominator_basis(self) -> str: + return "all_valid_records_in_selected_dataset_scope" + + @property + def grounded_record_count(self) -> int | None: + return self.source.grounded_record_count + + @property + def closed_record_count(self) -> int | None: + return self.source.closed_record_count + + @property + def unresolved_record_count(self) -> int | None: + return self.source.unresolved_record_count + + @property + def resolved_lineage_coverage(self) -> float | None: + return self.source.resolved_lineage_coverage + + @property + def external_ancestry_coverage(self) -> float | None: + return self.source.external_ancestry_coverage + + @property + def ancestry_coverage_reason_codes(self) -> tuple[str, ...]: + return self.source.ancestry_coverage_reason_codes + + @property + def unresolved_reason_codes(self) -> tuple[str, ...]: + return tuple(sorted({reason for record in self.source.records or () for reason in record.reason_codes})) + + @property + def input_execution_status(self) -> ExecutionStatus: + return self.source.execution_status + + @property + def input_execution_reason_codes(self) -> tuple[str, ...]: + return self.source.execution_reason_codes + + @property + def input_has_errors(self) -> bool: + return any(message.severity in (ValidationSeverity.ERROR, ValidationSeverity.FATAL) + for message in self.source.messages) + + @property + def validation_messages(self) -> tuple[ValidationMessage, ...]: + return self.source.messages + + @property + def classification_basis(self) -> str: + return "toolkit_operationalization" + + @property + def operationalization_label(self) -> str: + return "toolkit_operationalization" + + @property + def limitations(self) -> tuple[str, ...]: + return ("Lineage exposure is relative to the audited loop and supplied metadata.", + "Unresolved ancestry remains in the conservative interval; availability does not certify input validity.", + "No midpoint, calibrated risk threshold or causal claim.") + + +def lineage_closure_exposure(result: LineageAnalysisResult) -> LineageClosureExposureBounds: + """Explicit pure lineage interval request; keep direct bounds unchanged.""" + return LineageClosureExposureBounds(result) diff --git a/src/recursive_integrity_toolkit/models.py b/src/recursive_integrity_toolkit/models.py index 53d58d0..2666969 100644 --- a/src/recursive_integrity_toolkit/models.py +++ b/src/recursive_integrity_toolkit/models.py @@ -40,6 +40,7 @@ class FileRole(StrEnum): RECORDS_PRIMARY = "records_primary" RECORDS_COMPARE = "records_compare" + LINEAGE_CONTEXT = "lineage_context" PROVENANCE_MANIFEST = "provenance_manifest" SCHEMA_MAPPING = "schema_mapping" CONFIG = "config" @@ -473,6 +474,86 @@ class ParentValidationResult: messages: tuple[ValidationMessage, ...] = () +@dataclass(frozen=True, slots=True) +class ParentRecordValidation: + """Retained immediate evidence for one loaded record, including failures. + + Missing and null declarations have zero declared references, while an + unparseable container has unknown cardinality. Identity resolution counts + remain distinct from chronological eligibility for ancestry edges. + """ + + child_key: RecordKey + provenance_available: bool + result: ParentValidationResult | None = field(repr=False) + declared_reference_count: int | None + resolved_reference_count: int + invalid_self_reference_count: int + messages: tuple[ValidationMessage, ...] = () + + def __post_init__(self) -> None: + if type(self.child_key) is not RecordKey or type(self.provenance_available) is not bool: + raise TypeError("parent evidence requires a canonical key and explicit provenance flag") + if self.result is not None and (type(self.result) is not ParentValidationResult + or self.result.child_key != self.child_key): + raise ValueError("parent result must belong to its loaded record") + if self.result is not None: + if (self.result.declaration_state not in ("absent", "null", "empty", "declared") + or type(self.result.graph_validation_deferred) is not bool): + raise ValueError("parent evidence requires an explicit declaration state") + if type(self.result.references) is not tuple: + raise TypeError("parent references require an immutable tuple") + for reference in self.result.references: + if (type(reference) is not ParentReference + or type(reference.source_references) is not tuple + or any(type(text) is not str for text in reference.source_references)): + raise TypeError("parent source references require immutable literal spellings") + if type(self.result.messages) is not tuple or any(type(item) is not ValidationMessage + for item in self.result.messages): + raise TypeError("parent result messages require an immutable tuple") + counts = (self.resolved_reference_count, self.invalid_self_reference_count) + if self.declared_reference_count is not None: + counts += (self.declared_reference_count,) + for value in counts: + if type(value) is not int or value < 0: + raise ValueError("parent reference counts require nonnegative integers") + if self.declared_reference_count is not None and ( + self.resolved_reference_count + self.invalid_self_reference_count + > self.declared_reference_count): + raise ValueError("parent evidence counts exceed the declaration count") + if type(self.messages) is not tuple or any(type(item) is not ValidationMessage for item in self.messages): + raise TypeError("parent messages require an immutable tuple") + + +@dataclass(frozen=True, slots=True) +class ParentBatchValidationResult: + """Input-only immediate evidence retained for every loaded canonical key.""" + + record_keys: tuple[RecordKey, ...] + assessments: tuple[ParentRecordValidation, ...] = field(repr=False) + version_order: VersionOrderResult + messages: tuple[ValidationMessage, ...] + promoted_warning_codes: tuple[str, ...] = () + + def __post_init__(self) -> None: + if type(self.record_keys) is not tuple or any(type(key) is not RecordKey for key in self.record_keys): + raise TypeError("parent batch keys require an immutable canonical tuple") + if tuple(sorted(set(self.record_keys))) != self.record_keys: + raise ValueError("parent batch keys must be unique and sorted") + if type(self.assessments) is not tuple or any(type(item) is not ParentRecordValidation + for item in self.assessments): + raise TypeError("parent batch assessments require an immutable tuple") + if tuple(item.child_key for item in self.assessments) != self.record_keys: + raise ValueError("parent batch assessments must cover every loaded key exactly once") + if type(self.version_order) is not VersionOrderResult: + raise TypeError("parent batch chronology requires typed evidence") + if type(self.messages) is not tuple or any(type(item) is not ValidationMessage for item in self.messages): + raise TypeError("parent batch messages require an immutable tuple") + if type(self.promoted_warning_codes) is not tuple or any(type(code) is not str + for code in self.promoted_warning_codes): + raise TypeError("parent warning policy requires an immutable tuple") + + @dataclass(frozen=True, slots=True) class GenerationAssessment: """PR-009 declaration versus safely established non-grounding step count.""" @@ -555,6 +636,7 @@ class BundleValidationResult: mapping_traces: tuple[RowMappingEvidence, ...] = field(repr=False) content_read_keys: tuple[RecordKey, ...] validation_messages: tuple[ValidationMessage, ...] + parent_validation: ParentBatchValidationResult | None = field(default=None, repr=False) @property def has_errors(self) -> bool: diff --git a/src/recursive_integrity_toolkit/reports/assembly.py b/src/recursive_integrity_toolkit/reports/assembly.py index 6b3621f..30d8749 100644 --- a/src/recursive_integrity_toolkit/reports/assembly.py +++ b/src/recursive_integrity_toolkit/reports/assembly.py @@ -2,16 +2,16 @@ Owner IDs: PR-012, PR-014, PR-015, PR-016, PR-018; each public field retains its registered owner. Theory Map IDs: inherited through the frozen public field registry. -Inputs: already validated BundleValidationResult and explicitly supplied Phase 3 results. +Inputs: validated BundleValidationResult and explicitly supplied metric and lineage results. Outputs: immutable CanonicalReport and explicitly selected SafeReportView. Assumptions: supplied typed results are evidence handoffs, not authenticity certificates. Limits: no ingestion, classification, metric execution, graph traversal, simulation, filesystem access, rendering or CLI orchestration. -Current phase status: Phase 4 Step 4 additive privacy views and explicit run metadata. +Current phase status: Phase 5 Step 7 canonical lineage evidence and privacy views. """ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, fields from math import isfinite, isclose from types import MappingProxyType from pathlib import PosixPath, WindowsPath @@ -25,7 +25,10 @@ FileRole, FileFormat, VersionOrderResult, ProvenanceJoinResult, ProvenanceMatch, ProvenanceAssessment, ExplicitPairContext, RowMappingEvidence, RowLocation, GenerationValidationResult, RecordStateAssignment, TailSelectionOptions, ) -from ..metrics.bounds import DirectClosureExposureBounds +from ..metrics.bounds import DirectClosureExposureBounds, LineageClosureExposureBounds +from ..lineage.ancestry import LineageAnalysisResult, SharedAncestryDependence, RootContribution, RecordAncestry, _lineage_input_signature +from ..lineage.cycles import CycleAnalysis +from ..lineage.graph import LineageScope, LineageResourceUsage, LineageLimits from ..metrics.diversity import DistributionMetrics, StateDistributionResult, SupportComparison, StateFrequency from ..metrics.duplicates import ExactDuplicateResult, ExactDuplicateGroup from ..metrics.provenance import ProvenanceCompositionResult, DeclaredComposition, WeightedSourceComposition, DirectGroundingBasis, DirectGroundingAssignment @@ -45,6 +48,7 @@ class FamilyFailure: capability: CapabilityKey messages: tuple[ValidationMessage, ...] + lineage_resource_usage: LineageResourceUsage | None = None def __post_init__(self): if type(self.capability) is not CapabilityKey or type(self.messages) is not tuple: @@ -52,6 +56,15 @@ def __post_init__(self): if not self.messages or any(type(item) is not ValidationMessage or item.severity not in (ValidationSeverity.ERROR, ValidationSeverity.FATAL) for item in self.messages): raise ReportAssemblyError("family failure must retain error or fatal diagnostics") + if self.lineage_resource_usage is not None: + _require(self.capability is CapabilityKey.LINEAGE, + "lineage resource usage belongs only to the lineage family") + usage = _lineage_usage(self.lineage_resource_usage) + _require(usage["exhausted_limit"] in ("max_nodes", "max_edges") + and usage["stored_root_membership_count"] == usage["root_union_visit_count"] == 0, + "family failure resource usage must describe graph admission exhaustion") + _require(any(message.code == "E_LINEAGE_RESOURCE_LIMIT_EXCEEDED" for message in self.messages), + "resource admission failure requires its matching error diagnostic") def _require(condition, message): @@ -1066,7 +1079,7 @@ def _inputs(payload, bundle): _diagnostics(payload, bundle.generation.messages) -def _capabilities(payload, bundle, operations, failures): +def _capabilities(payload, bundle, operations, failures, lineage=None): matrix = {} for key in CapabilityKey: original = bundle.observability.capabilities[key] @@ -1081,12 +1094,14 @@ def _capabilities(payload, bundle, operations, failures): reasons.append("R_ANALYSIS_NOT_REQUESTED") if key is CapabilityKey.DATASET_LONGITUDINAL and completed: execution, reasons = "partial", ["R_LONGITUDINAL_FAMILIES_DEFERRED"] - if key in (CapabilityKey.LINEAGE, CapabilityKey.MODEL_LONGITUDINAL): - execution, reasons = "deferred", ["R_LINEAGE_EXECUTION_DEFERRED" if key is CapabilityKey.LINEAGE else "R_MODEL_ANALYSIS_DEFERRED"] + if key is CapabilityKey.MODEL_LONGITUDINAL: + execution, reasons = "deferred", ["R_MODEL_ANALYSIS_DEFERRED"] + if key is CapabilityKey.LINEAGE and lineage is not None: + execution, reasons = lineage.execution_status.value, list(lineage.execution_reason_codes) family_codes = [error["code"] for error in payload["errors"] if key.value in error["effect_on_capabilities"]] - if family_codes and key not in (CapabilityKey.LINEAGE, CapabilityKey.MODEL_LONGITUDINAL): + if family_codes and key is not CapabilityKey.MODEL_LONGITUDINAL and (key is not CapabilityKey.LINEAGE or key.value in failures): execution = "partial" if completed else "failed" - reasons = list(dict.fromkeys(reasons + family_codes)) + reasons = list(dict.fromkeys(([] if key is CapabilityKey.LINEAGE else reasons) + family_codes)) if key is CapabilityKey.INGESTION and any("ingestion" in error["effect_on_capabilities"] for error in payload["errors"]): execution = "partial" if bundle.records else "failed" reasons = ["R_INPUT_VALIDATION_ERROR"] @@ -1106,6 +1121,14 @@ def _capabilities(payload, bundle, operations, failures): entry["coverage_details"][name] = _coverage(value) if key is CapabilityKey.LINEAGE and original.coverage is not None: entry["coverage_details"]["resolved_parent_edge_coverage"] = _coverage(original.coverage) + if key is CapabilityKey.LINEAGE and (lineage is not None or key.value in failures): + replacements = { + "Input eligibility only; downstream analytical implementations remain deferred.": + "Input eligibility describes supplied metadata; execution status records the explicit lineage request.", + "General graph validation is deferred; no roots or ancestors are traced.": + "Graph and ancestry execution is reported separately from immediate-reference eligibility.", + } + entry["notes"] = [replacements.get(note, note) for note in entry["notes"]] if key is CapabilityKey.DATASET_LONGITUDINAL and completed: entry["notes"].append("Executed only the supplied explicit-pair support/diversity comparison; no adjacent-pair discovery, lineage/provenance trajectory or relative-change calculation.") if key is CapabilityKey.INTERVENTION_SIMULATION and completed: @@ -1114,7 +1137,7 @@ def _capabilities(payload, bundle, operations, failures): payload["capabilities"] = matrix payload["observability"] = {"maximum_level": bundle.observability.maximum_level, "level_label": LEVEL_LABELS[bundle.observability.maximum_level], "basis": list(bundle.observability.basis), - "limitations": list(dict.fromkeys(bundle.observability.limitations + ("Input eligibility is distinct from executed analysis; Phase 5 lineage remains deferred.",))), + "limitations": list(dict.fromkeys(bundle.observability.limitations + ("Input eligibility is distinct from explicitly supplied executed analysis.",))), "partial_evidence": list(dict.fromkeys(reason for capability in matrix.values() for reason in capability["reason_codes"])), "capabilities": matrix} @@ -1144,11 +1167,205 @@ def _lineage_observations(payload, bundle): "all_resolved_edges_follow_order": True}, scope, limitations=("This retained earlier-version ordering certificate is not general graph-cycle traversal.",)) target["cycle_status"] = _envelope("observed_facts.lineage.cycle_status", None, scope, - status="unavailable", reasons=("R_GRAPH_EXECUTION_DEFERRED",), required=("Phase_5_graph_analysis",), - limitations=("No general graph-cycle traversal executes in Phase 4.",)) + status="unavailable", reasons=("R_ANALYSIS_NOT_REQUESTED",), required=("explicit_lineage_result",), + limitations=("No executed graph-cycle result was supplied to this report.",)) payload["observed_facts"]["lineage"] = target +def _lineage_scope(value, bundle): + """Bind population identities without rerunning the graph or mathematical owners.""" + _typed(value, LineageScope, "lineage scope") + LineageScope(**{field.name: getattr(value, field.name) for field in fields(LineageScope)}) + loaded = tuple(sorted(row.record_key for row in bundle.records)) + target = tuple(key for key in loaded if key.dataset_version == value.target_dataset_version) + primary = {row.record_key.dataset_version for row in bundle.records + if row.location.file_role is FileRole.RECORDS_PRIMARY} + context = {row.record_key.dataset_version for row in bundle.records + if row.location.file_role is FileRole.LINEAGE_CONTEXT} + selected = {row.record_key.dataset_version for row in bundle.records + if row.location.file_role in (FileRole.RECORDS_PRIMARY, FileRole.RECORDS_COMPARE)} + _require(not context & selected, + "lineage context versions overlap primary or comparison versions") + _require(not primary or primary == {value.target_dataset_version}, + "lineage target differs from the validated primary version") + _require(primary or not any(row.record_key.dataset_version == value.target_dataset_version and + row.location.file_role in (FileRole.RECORDS_COMPARE, FileRole.LINEAGE_CONTEXT) + for row in bundle.records), + "loaded context cannot become a lineage target") + _require(value.target_dataset_version is not None or not bundle.records or + any(row.location.file_role is not None for row in bundle.records), + "unassigned loaded records require an explicit lineage target version") + _require(value.target_record_keys == target and value.loaded_record_count == len(loaded) + and value.loaded_dataset_versions == tuple(sorted({key.dataset_version for key in loaded})), + "lineage scope differs from the validated target or loaded context") + _require(value.target_dataset_version is None or value.target_dataset_version in + set(value.loaded_dataset_versions) | set(bundle.version_order.order), + "lineage target version was not declared by the validated bundle") + return {"dataset_versions": [] if value.target_dataset_version is None else [value.target_dataset_version], + "record_count": value.target_record_count, "excluded_record_count": 0, + "denominator_basis": "all_valid_records_in_selected_dataset_scope", "scope_id": "lineage_target", + "included_record_keys": [_key(key) for key in value.target_record_keys], "excluded_record_keys": []} + + +def _lineage_detail(items, total=None): + """Limit presentation only; values and rank were computed by the owner.""" + total = len(items) if total is None else total + shown = list(items[:100]) + omitted = total - len(shown) + return {"items": shown, "total_count": total, "returned_count": len(shown), + "omitted_count": omitted, "limit": 100, + "detail_status": "truncated" if omitted else "complete", + "omission_reasons": ["diagnostic_limit"] if omitted else []} + + +def _lineage_usage(value): + _typed(value, LineageResourceUsage, "lineage resource usage") + LineageResourceUsage(**{field.name: getattr(value, field.name) for field in fields(LineageResourceUsage)}) + result = {field.name: getattr(value, field.name) for field in fields(LineageResourceUsage)} + result["limits"] = {field.name: getattr(value.limits, field.name) for field in fields(LineageLimits)} + return result + + +def _lineage_result(payload, value, bundle, bounds, proxy): + """Serialize supplied owner results; no graph traversal or formula evaluation.""" + _typed(value, LineageAnalysisResult, "lineage analysis") + _require(value.input_signature == _lineage_input_signature(bundle), + "lineage result was computed from different retained input evidence") + scope = _lineage_scope(value.scope, bundle) + _typed(value.cycles, CycleAnalysis, "lineage cycle result") + _require(value.cycles.scope == value.scope and + set(value.cycles.depths_by_record) == {row.record_key for row in bundle.records}, + "lineage cycle observations differ from the validated loaded identities") + _require(value.records is None or type(value.records) is tuple and + all(type(record) is RecordAncestry for record in value.records) and + tuple(record.record_key for record in value.records) == value.scope.target_record_keys, + "lineage records differ from the selected target population") + _require(value.root_contributions is None or type(value.root_contributions) is tuple and + all(type(item) is RootContribution for item in value.root_contributions), + "root contributions require immutable typed rows") + total = value.scope.target_record_count + coverage = value.resolved_lineage_coverage + aborted = value.records is None + root_reasons = ("LINEAGE_RESOURCE_LIMIT_EXCEEDED",) if aborted else ("UNRESOLVED_ANCESTRY",) if value.unresolved_record_count else () + assumptions = ("theory_guided_operationalization", "Only explicit supplied parent and grounding evidence defines ancestry.") + limitations = ("Root allocation describes supplied topology and does not establish causal contribution or source independence.",) + graph_scope = _bundle_scope(bundle) + observed = {} + graph_value = {field.name: getattr(value.scope, field.name) for field in fields(LineageScope) + if field.name != "target_record_keys"} + graph_value["loaded_dataset_versions"] = list(graph_value["loaded_dataset_versions"]) + observed["graph_scope"] = _envelope("observed_facts.lineage.graph_scope", graph_value, + graph_scope, denominator=value.scope.loaded_record_count, + limitations=("Context supplies ancestry evidence without entering target denominators.",)) + cycles = value.cycles + components = [{"component_index": item.component_index, "member_count": item.member_count, + "target_member_count": item.target_member_count, + "witness_record_keys": None if item.witness_record_keys is None else [_key(key) for key in item.witness_record_keys], + "witness_edge_count": item.witness_edge_count, "witness_reason": item.witness_reason} + for item in cycles.component_details] + cycle_value = {name: getattr(cycles, name) for name in ( + "detected", "cycle_count", "counting_method", "cycle_member_count", "target_cycle_member_count", + "affected_record_count", "target_affected_record_count")} + cycle_value.update({"components": _lineage_detail(components, cycles.cycle_count), + "cycle_member_record_keys": _lineage_detail([_key(key) for key in cycles.cycle_member_record_keys[:100]], cycles.cycle_member_count), + "affected_record_keys": _lineage_detail([_key(key) for key in cycles.affected_record_keys[:100]], cycles.affected_record_count)}) + cycle_limits = ("Engineering graph-validity observation over accepted edges and explicit self references.", + "An acyclic accepted subgraph does not certify rejected parent declarations.") + cycle_reasons = tuple(sorted({message.code for message in cycles.messages + if message.severity in (ValidationSeverity.ERROR, ValidationSeverity.FATAL)})) + for name, result in (("cycle_analysis", cycle_value), ("cycle_status", cycles.cycle_status)): + observed[name] = _envelope("observed_facts.lineage." + name, result, graph_scope, + status="partial" if cycle_reasons else "available", reasons=cycle_reasons, + assumptions=(cycles.cycle_status_scope,), limitations=cycle_limits) + observed["depth_summary"] = _envelope("observed_facts.lineage.depth_summary", { + "depth_resolved_record_count": value.depth_resolved_record_count, + "maximum_resolved_target_depth": value.maximum_resolved_target_depth, + "target_record_count": total}, scope, denominator=total, + limitations=("Resolved structural depth does not establish resolved external ancestry.",)) + unresolved = [] + for record in value.records or (): + if record.classification == "unresolved": + unresolved.append({"record_key": _key(record.record_key), "reason_codes": list(record.reason_codes)}) + if len(unresolved) == 100: + break + observed["unresolved_record_details"] = _envelope("observed_facts.lineage.unresolved_record_details", + None if aborted else _lineage_detail(unresolved, value.unresolved_record_count), scope, denominator=total, coverage=coverage, + status="unavailable" if aborted else "available", reasons=root_reasons if aborted else (), + required=("completed_target_partition",) if aborted else ()) + observed["resource_usage"] = _envelope("observed_facts.lineage.resource_usage", _lineage_usage(value.resource_usage), + graph_scope, limitations=("Defined admission and root-work units do not measure peak memory or scientific thresholds.",)) + for name, count in (("declared_parent_edge_count", value.declared_parent_reference_count), + ("resolved_parent_edge_count", value.resolved_parent_reference_count), + ("unresolved_parent_edge_count", value.unresolved_parent_reference_count)): + observed[name] = _envelope("observed_facts.lineage." + name, count, scope, + denominator=value.declared_parent_reference_count, coverage=value.resolved_parent_edge_coverage, + status="unavailable" if count is None else "available", reasons=value.reference_coverage_reason_codes, + required=("countable_original_parent_references",) if count is None else (), + limitations=("Counts preserve original target reference multiplicity; graph adjacency deduplicates canonical edges.",)) + payload["observed_facts"]["lineage"] = observed + metrics = {} + for name in ("grounded_record_count", "closed_record_count", "unresolved_record_count", "records_with_resolved_external_ancestry"): + metrics[name] = _envelope("derived_metrics.lineage." + name, getattr(value, name), scope, + denominator=total, coverage=coverage, status="unavailable" if aborted else "available", + reasons=root_reasons if aborted else (), required=("completed_target_partition",) if aborted else (), + assumptions=assumptions, limitations=limitations) + for name in ("resolved_lineage_coverage", "external_ancestry_coverage"): + number = getattr(value, name) + metrics[name] = _envelope("derived_metrics.lineage." + name, number, scope, + denominator=total, coverage=coverage, status="unavailable" if number is None else "available", + reasons=value.ancestry_coverage_reason_codes, required=("nonempty_completed_target_partition",) if number is None else (), + assumptions=assumptions, limitations=limitations) + metrics["resolved_parent_edge_coverage"] = _envelope("derived_metrics.lineage.resolved_parent_edge_coverage", + value.resolved_parent_edge_coverage, scope, denominator=value.declared_parent_reference_count, + coverage=value.resolved_parent_edge_coverage, status="unavailable" if value.resolved_parent_edge_coverage is None else "available", + reasons=value.reference_coverage_reason_codes, + required=("countable_original_parent_references",) if value.resolved_parent_edge_coverage is None else (), + limitations=("Identity resolution coverage remains distinct from accepted graph edges and complete ancestry.",)) + metrics["resolved_parent_edge_coverage"]["no_declared_parents"] = value.no_declared_parents + roots = None if aborted else [{field.name: _key(item.record_key) if field.name == "record_key" else getattr(item, field.name) + for field in fields(RootContribution)} for item in value.root_contributions[:100]] + for name, result in (("distinct_external_root_count", value.distinct_external_root_count), + ("top_shared_ancestors", None if aborted else _lineage_detail(roots, len(value.root_contributions)))): + metrics[name] = _envelope("derived_metrics.lineage." + name, result, scope, denominator=total, + coverage=value.external_ancestry_coverage, status=value.root_metrics_status.value, + reasons=root_reasons, required=("completed_root_resolution",) if aborted else (), + assumptions=assumptions, limitations=limitations) + for name in ("ancestry_concentration_hhi", "effective_external_root_count"): + metrics[name] = _envelope("derived_metrics.lineage." + name, getattr(value, name), scope, + denominator=value.grounded_record_count, coverage=value.external_ancestry_coverage, + status=value.concentration_status.value, reasons=value.concentration_reason_codes or root_reasons, + required=("nonempty_resolved_external_root_population",) if getattr(value, name) is None else (), + assumptions=assumptions, limitations=limitations + ("Concentration uses the complete grounded subset and is computed before detail limits.",)) + metrics["lineage_depth"] = _envelope("derived_metrics.lineage.lineage_depth", value.lineage_depth, scope, + denominator=total, status="unavailable" if value.lineage_depth is None else "available", + reasons=cycles.depth_reason_codes, required=("complete_target_structural_depth",) if value.lineage_depth is None else (), + limitations=("Whole-target depth is unavailable when any required structural path is unresolved.",)) + payload["derived_metrics"]["lineage"] = metrics + if bounds is not None: + _typed(bounds, LineageClosureExposureBounds, "lineage bounds") + _require(bounds.source == value, "lineage bounds source differs from the supplied lineage analysis") + target = {} + for name in ("lower_bound", "upper_bound", "interval_width"): + target[name] = _envelope("derived_metrics.closure_exposure.lineage." + name, getattr(bounds, name), scope, + denominator=bounds.denominator, coverage=coverage, status=bounds.status.value, + reasons=bounds.reason_codes, required=("nonempty_completed_target_partition",) if bounds.reason_codes else (), + assumptions=(bounds.operationalization_label,), limitations=bounds.limitations) + payload["derived_metrics"].setdefault("closure_exposure", {})["lineage"] = target + if proxy is not None: + _typed(proxy, SharedAncestryDependence, "shared ancestry proxy") + _require(proxy.source == value, "shared ancestry source differs from the supplied lineage analysis") + signal = _base("proxy_signals.shared_ancestry_dependence", scope, denominator=total, coverage=coverage, + status=proxy.status.value, reasons=proxy.reason_codes, + required=("complete_ancestry_or_shared_root_witness",) if proxy.status.value == "unavailable" else (), + assumptions=(proxy.operationalization_label,), limitations=proxy.limitations) + signal.update({"signal": "shared_ancestry_dependence", "level": proxy.level, + "basis_fields": ["derived_metrics.lineage.top_shared_ancestors", "derived_metrics.lineage.resolved_lineage_coverage", + "derived_metrics.lineage.external_ancestry_coverage"], + "trigger_rule": "Present when a complete external root supports at least two targets; absence requires complete target ancestry."}) + payload["proxy_signals"]["shared_ancestry_dependence"] = signal + _diagnostics(payload, value.messages, "lineage") + + def _bundle_check(bundle): _typed(bundle, BundleValidationResult, "validated bundle") _typed(bundle.version_order, VersionOrderResult, "retained version order") @@ -1223,6 +1440,9 @@ def assemble_report(bundle: BundleValidationResult, *, run: dict, tail: TailSelectionResult | None = None, comparison: SupportComparison | None = None, closure: DirectClosureExposureBounds | None = None, + lineage: LineageAnalysisResult | None = None, + lineage_bounds: LineageClosureExposureBounds | None = None, + shared_ancestry: SharedAncestryDependence | None = None, expected_diversity: ExpectedDiversityResult | None = None, resampling: ResamplingSimulation | None = None, extinction: tuple[ExtinctionProbabilityResult, ...] = (), @@ -1243,17 +1463,25 @@ def assemble_report(bundle: BundleValidationResult, *, run: dict, _require(set(bundle.observability.capabilities) == set(CapabilityKey), "bundle capability matrix is incomplete") _require(type(distributions) is tuple and type(extinction) is tuple and type(family_errors) is tuple, "explicit result collections require immutable tuples") _require(expected_diversity is None or resampling is None, "one closed-resampling report slot cannot hold two different method results") + _require(lineage is not None or lineage_bounds is None and shared_ancestry is None, + "lineage derivatives require their explicit source lineage result") payload = {key: {} if index < 8 else [] for index, key in enumerate(SECTION_ORDER)} payload["run"] = dict(run) _inputs(payload, bundle) operations = {key.value: [] for key in CapabilityKey} operations["ingestion"] = ["existing_bundle_validation"] if bundle.inventory or bundle.records else [] failures = {} + admission_usage = None for failure in family_errors: _typed(failure, FamilyFailure, "family failure") - FamilyFailure(failure.capability, failure.messages) + FamilyFailure(failure.capability, failure.messages, failure.lineage_resource_usage) + if failure.lineage_resource_usage is not None: + _require(admission_usage is None, "one lineage execution cannot have competing admission failures") + admission_usage = failure.lineage_resource_usage failures.setdefault(failure.capability.value, []).extend(failure.messages) _diagnostics(payload, failure.messages, failure.capability.value) + _require(lineage is None or "lineage" not in failures, + "a supplied lineage result cannot also have a separate lineage family failure") empirical_representations = [] for supplied in distributions: if type(supplied) is StateDistributionResult: @@ -1305,8 +1533,24 @@ def assemble_report(bundle: BundleValidationResult, *, run: dict, payload["inputs"]["representation"] = _representation(empirical_representations[0]) elif empirical_representations: payload["inputs"]["limitations"].append("Supplied calculation families use multiple explicit representations; inspect each result envelope.") - _lineage_observations(payload, bundle) - _capabilities(payload, bundle, operations, failures) + if lineage is None: + _lineage_observations(payload, bundle) + if "lineage" in failures: + payload["observed_facts"]["lineage"]["cycle_status"]["reason_codes"] = ["R_LINEAGE_EXECUTION_FAILED"] + if admission_usage is not None: + payload["observed_facts"]["lineage"]["resource_usage"] = _envelope( + "observed_facts.lineage.resource_usage", _lineage_usage(admission_usage), _bundle_scope(bundle), + limitations=("Defined admission and root-work units do not measure peak memory or scientific thresholds.",)) + else: + _lineage_result(payload, lineage, bundle, lineage_bounds, shared_ancestry) + operations["lineage"].append("supplied_lineage_graph_and_depth") + if lineage.records is not None: + operations["lineage"].append("supplied_lineage_roots_and_concentration") + if lineage_bounds is not None: + operations["lineage"].append("supplied_lineage_closure_interval") + if shared_ancestry is not None: + operations["lineage"].append("supplied_shared_ancestry_proxy") + _capabilities(payload, bundle, operations, failures, lineage) if payload["errors"]: payload["run"]["run_status"] = "failed" if any(error["severity"] == "fatal" for error in payload["errors"]) else "partial" if bundle.records or distributions or payload["simulations"] else "failed" for error in payload["errors"]: @@ -1428,7 +1672,6 @@ def _disclosures(payload): "This signal reports unresolved supplied provenance evidence and does not estimate factual truth or source independence.", "A not_present signal is limited to the cited fields and does not certify universal integrity."], ) - # No shared_ancestry_dependence proxy: P4-D01 prohibits Phase 4 ancestry inference. _disclosure_unavailable( payload, "model_performance_decline", "PR-014", "model_longitudinal", "The supplied audit evidence cannot establish model-performance decline.", @@ -1439,8 +1682,8 @@ def _disclosures(payload): _disclosure_unavailable( payload, "causal_ancestor_effect", "T4", "lineage", "A causal effect of an ancestor is unavailable.", - ["R_CAUSAL_EVIDENCE_MISSING", "R_LINEAGE_EXECUTION_DEFERRED"], - ["Phase 4 does not compute ancestry; topology alone would not identify a causal contribution."], + ["R_CAUSAL_EVIDENCE_MISSING"], + ["Supplied parent topology and fractional root allocation do not identify a causal contribution."], ["An identified ancestor, measured outcomes and a controlled or otherwise justified causal design."], "Parent declarations and graph topology do not establish causal effects.", ) @@ -1476,20 +1719,31 @@ def _disclosures(payload): ) lineage = payload["capabilities"].get("lineage") if lineage is not None: - reasons = ["R_LINEAGE_EXECUTION_DEFERRED"] - if lineage["status"] != "available": - reasons.extend(reason for reason in lineage["reason_codes"] if reason not in reasons) - for name, owner, statement in ( - ("lineage_analysis", "PR-014", "General lineage graph analysis is deferred to Phase 5."), - ("lineage_closure_exposure", "T3", "Lineage closure exposure is deferred to Phase 5."), - ("external_ancestry", "T4", "External ancestry results are deferred to Phase 5."), - ): - _disclosure_unavailable( - payload, name, owner, "lineage", statement, list(reasons), - ["Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes."], - ["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."], - "Implementation is deferred even when the existing input classifier marks lineage available.", - ) + if lineage["execution_status"] in ("not_requested", "failed"): + _disclosure_unavailable(payload, "lineage_analysis", "PR-014", "lineage", + "A completed whole-target lineage analysis is unavailable in this report.", + list(lineage["execution_reason_codes"]), + ["No complete successful lineage execution was supplied; independently completed observations remain visible."], + ["Supply an explicit lineage result for the validated target and its loaded context."], + "Input eligibility alone does not establish executed ancestry analysis.") + lineage_metrics = derived.get("lineage", {}) + coverage = lineage_metrics.get("resolved_lineage_coverage", {}) + if coverage.get("value") is None or coverage["value"] < 1: + _disclosure_unavailable(payload, "external_ancestry", "T4", "lineage", + "Complete external ancestry for every target is unavailable.", + list(coverage.get("reason_codes") or lineage["execution_reason_codes"] or ["UNRESOLVED_ANCESTRY"]), + ["Missing, invalid or unknown ancestry remains unresolved; any complete subset retains its reported coverage."], + ["Complete the required parent references, chronology and external-grounding evidence."], + "Partial ancestry cannot be promoted to whole-target completeness.") + lineage_interval = derived.get("closure_exposure", {}).get("lineage", {}) + lower = lineage_interval.get("lower_bound") + if lower is None or lower["status"] == "unavailable": + _disclosure_unavailable(payload, "lineage_closure_exposure", "T3", "lineage", + "A lineage closure interval is unavailable in this report.", + list(lower["reason_codes"] if lower is not None else ["R_ANALYSIS_NOT_REQUESTED"]), + ["No available explicitly supplied lineage interval covers a nonempty completed target partition."], + ["Supply lineage closure bounds from the same explicit lineage result."], + "Unavailable intervals are never replaced by a guessed numeric value.") if payload["simulations"]: _disclosure_unavailable( payload, "empirical_intervention_effect", "T5", "intervention_simulation", @@ -1530,7 +1784,7 @@ def _disclosures(payload): "The supplied capability assessment identifies missing chronology.") if lineage is not None and any(reason in lineage["reason_codes"] for reason in ("R_PARENT_UNRESOLVED", "R_PARENT_AMBIGUOUS", "R_PARENT_DECLARATION_MISSING", "R_PARENT_INVALID")): _disclosure_recommend(payload, 3, "composite_parent_references", "records with unresolved or invalid parent declarations", - ["more complete immediate-parent validation; future Phase 5 analysis remains deferred"], + ["more complete parent validation and explicitly requested ancestry coverage"], "The existing classifier identifies incomplete or invalid parent evidence.") _disclosure_recommend(payload, 4, "versioned_model_outcomes", "models and comparable evaluation conditions", ["evidence needed by a future model-longitudinal implementation"], @@ -1541,6 +1795,76 @@ def _disclosures(payload): # capability owners. Unknown caller prose never becomes trusted by a prefix, # regular expression, or similarity to one of these reviewed literal strings. _PRIVACY_SAFE_TEXT = frozenset(( + 'R_LINEAGE_EXECUTION_FAILED', + 'Input eligibility is distinct from explicitly supplied executed analysis.', + 'No executed graph-cycle result was supplied to this report.', + 'explicit_lineage_result', + 'A completed whole-target lineage analysis is unavailable in this report.', + 'No complete successful lineage execution was supplied; independently completed observations remain visible.', + 'Supply an explicit lineage result for the validated target and its loaded context.', + 'Input eligibility alone does not establish executed ancestry analysis.', + 'Complete external ancestry for every target is unavailable.', + 'Missing, invalid or unknown ancestry remains unresolved; any complete subset retains its reported coverage.', + 'Complete the required parent references, chronology and external-grounding evidence.', + 'Partial ancestry cannot be promoted to whole-target completeness.', + 'A lineage closure interval is unavailable in this report.', + 'No available explicitly supplied lineage interval covers a nonempty completed target partition.', + 'Supply lineage closure bounds from the same explicit lineage result.', + 'Unavailable intervals are never replaced by a guessed numeric value.', + 'more complete parent validation and explicitly requested ancestry coverage', + 'Supplied parent topology and fractional root allocation do not identify a causal contribution.', + 'supplied_lineage_graph_and_depth', + 'supplied_lineage_roots_and_concentration', + 'supplied_lineage_closure_interval', + 'supplied_shared_ancestry_proxy', + 'Only explicit supplied parent and grounding evidence defines ancestry.', + 'Root allocation describes supplied topology and does not establish causal contribution or source independence.', + 'Context supplies ancestry evidence without entering target denominators.', + 'Engineering graph-validity observation over accepted edges and explicit self references.', + 'An acyclic accepted subgraph does not certify rejected parent declarations.', + 'Resolved structural depth does not establish resolved external ancestry.', + 'Defined admission and root-work units do not measure peak memory or scientific thresholds.', + 'Counts preserve original target reference multiplicity; graph adjacency deduplicates canonical edges.', + 'Identity resolution coverage remains distinct from accepted graph edges and complete ancestry.', + 'Concentration uses the complete grounded subset and is computed before detail limits.', + 'Whole-target depth is unavailable when any required structural path is unresolved.', + 'Present when a complete external root supports at least two targets; absence requires complete target ancestry.', + 'completed_target_partition', + 'countable_original_parent_references', + 'nonempty_completed_target_partition', + 'completed_root_resolution', + 'nonempty_resolved_external_root_population', + 'complete_target_structural_depth', + 'complete_ancestry_or_shared_root_witness', + 'accepted_edges_and_explicit_self_references', + 'theory_guided_operationalization', + 'INCOMPLETE_TARGET_DEPTH', + 'MISSING_PROVENANCE', + 'MISSING_REQUIRED_PROVENANCE', + 'UNKNOWN_GROUNDING', + 'PARENT_DECLARATION_UNAVAILABLE', + 'INVALID_PARENT_REFERENCE', + 'UNRESOLVED_PARENT_REFERENCE', + 'VERSION_ORDER_UNAVAILABLE', + 'CYCLE_AFFECTED', + 'INCOMPLETE_PARENT_ANCESTRY', + 'INCOMPLETE_PARENT_DEPTH', + 'GROUNDED_PARENT_RULE_UNSUPPORTED', + 'INVALID_LINEAGE_INPUT', + 'UNRESOLVED_ANCESTRY', + 'LINEAGE_RESOURCE_LIMIT_EXCEEDED', + 'E_LINEAGE_RESOURCE_LIMIT_EXCEEDED', + 'EMPTY_TARGET_SCOPE', + 'PARENT_REFERENCE_COUNT_UNAVAILABLE', + 'NO_RESOLVED_EXTERNAL_ROOTS', + 'root_contributions.incidence_count', + 'resolved_lineage_coverage', + 'external_ancestry_coverage', + 'Descriptive shared-root topology under supplied metadata; no calibrated risk level.', + 'Does not establish causal contribution, semantic error, independent information, biological relatedness, universal integrity or collapse.', + 'Lineage exposure is relative to the audited loop and supplied metadata.', + 'Unresolved ancestry remains in the conservative interval; availability does not certify input validity.', + 'No midpoint, calibrated risk threshold or causal claim.', 'earlier_positive_mass_support_in_harmonized_representation', 'explicit_version_order', 'identifier_secret_file_path', @@ -1672,12 +1996,14 @@ def _disclosures(payload): 'Fixed finite declared state space and constant positive integer resample size.', 'Floating-point and pseudorandom sampling are numerical realizations of the declared model.', 'General graph validation is deferred; no roots or ancestors are traced.', + 'Graph and ancestry execution is reported separately from immediate-reference eligibility.', 'General lineage graph analysis is deferred to Phase 5.', 'Immediate-reference validation coverage does not establish resolved ancestry or external roots.', 'Implementation is deferred even when the existing input classifier marks lineage available.', 'Incomplete required provenance remains unresolved with original errors retained.', 'Input eligibility is distinct from executed analysis; Phase 5 lineage remains deferred.', 'Input eligibility only; downstream analytical implementations remain deferred.', + 'Input eligibility describes supplied metadata; execution status records the explicit lineage request.', 'Input hashes identify supplied bytes and do not certify authenticity.', 'Lineage closure exposure is deferred to Phase 5.', 'Literal declared field labels do not certify semantic validity.', @@ -2657,6 +2983,20 @@ def _privacy_value(value, contract, schema, *, mode, record_id_mode, protection, if "const" in contract or "enum" in contract or value is None or type(value) in (int, float, bool): return value if type(value) is dict: + # Lineage detail collections contain identities in several required row + # shapes. Omit the collection as a whole, retaining presentation counts, + # rather than making individual rows invalid by deleting identity keys. + if record_id_mode == "omit" and set(value) == { + "items", "total_count", "returned_count", "omitted_count", + "limit", "detail_status", "omission_reasons"}: + reasons = list(value["omission_reasons"]) + if "redacted_identity_details" not in reasons: + reasons.append("redacted_identity_details") + omissions.append(("field", ".".join(path), "")) + return {"items": None, "total_count": value["total_count"], + "returned_count": 0, "omitted_count": value["total_count"], + "limit": value["limit"], "detail_status": "omitted", + "omission_reasons": reasons} if set(value) == {"dataset_version", "record_id"}: return {"dataset_version": _privacy_alias(protection, "dataset_version", value["dataset_version"]) if mode == "redacted" else value["dataset_version"], "record_id": _privacy_alias(protection, "record_id", [value["dataset_version"], value["record_id"]]) if record_id_mode == "hash" else value["record_id"]} @@ -2700,7 +3040,8 @@ def _privacy_value(value, contract, schema, *, mode, record_id_mode, protection, if type(value) is list: return [_privacy_value(item, contract.get("items", {}), schema, mode=mode, record_id_mode=record_id_mode, protection=protection, path=path, omissions=omissions) for item in value] - version_fields = ("dataset_version", "dataset_versions", "version_order", "earlier_version", "later_version") + version_fields = ("dataset_version", "dataset_versions", "version_order", "earlier_version", "later_version", + "target_dataset_version", "loaded_dataset_versions") state_fields = ("state_id", "state_ids", "missing_state_id", "state_order", "source_state", "source_states", "target_state", "original_earlier_support", "original_later_support", "harmonized_earlier_support", "harmonized_later_support", "support") identity_fields = ("run_id", "scope_id", "group_id", "representation_name", "representation_source", "representation_version", @@ -2839,7 +3180,7 @@ def build_run_metadata(*, options, run_id: str, operation="python_api", started_ if type(run_id) is not str or not run_id or "\x00" in run_id: raise ValueError("run identifier must be nonempty literal text") command = None if operation == "python_api" else "rit " + operation + (" --strict" if options.strict_mode else "") + (" --redacted" if options.privacy_mode == "redacted" else "") - result = {"run_id": run_id, "toolkit_version": __version__, "report_schema_version": "1.0", + result = {"run_id": run_id, "toolkit_version": __version__, "report_schema_version": "1.1", "started_at": started_at, "completed_at": completed_at, "duration_seconds": duration_seconds, "python_version": python_version, "platform": platform, "command": command, "config_hash": phase4_config_hash(options), "random_seed": random_seed, diff --git a/src/recursive_integrity_toolkit/reports/markdown_report.py b/src/recursive_integrity_toolkit/reports/markdown_report.py index 1b3efad..88e089e 100644 --- a/src/recursive_integrity_toolkit/reports/markdown_report.py +++ b/src/recursive_integrity_toolkit/reports/markdown_report.py @@ -102,6 +102,10 @@ def _null_reason(key: str | int, owner: dict) -> str: return "Withheld by the selected privacy view." if key == "record_keys" and owner.get("redaction"): return "Identity details omitted by the declared redaction policy." + if key == "items" and owner.get("omission_reasons"): + return "Detail omitted: " + _code(owner["omission_reasons"]) + if key in {"witness_record_keys", "witness_edge_count"} and owner.get("witness_reason"): + return "Cycle witness omitted: " + _code(owner["witness_reason"]) if key == "representation": return "No applicable representation supplied; consult status and required evidence." if key in {"record_count", "excluded_record_count"}: diff --git a/src/recursive_integrity_toolkit/result.py b/src/recursive_integrity_toolkit/result.py index 9d6cfad..1efea73 100644 --- a/src/recursive_integrity_toolkit/result.py +++ b/src/recursive_integrity_toolkit/result.py @@ -14,7 +14,7 @@ No ingestion, classifier, metric, simulation, adapter, renderer or CLI runs. A valid contract does not certify the truth of supplied evidence. Current phase status: - Phase 4 Step 4 adds a safe-view wrapper; the Step 2 schema remains frozen. + Phase 5 Step 7 adds bounded lineage evidence under schema 1.1. """ from __future__ import annotations @@ -144,6 +144,31 @@ def _ref(name: str) -> dict: return {"$ref": "#/$defs/" + name} +def _lineage_details(item: dict) -> dict: + """Bound identity-bearing diagnostics independently of analytical values.""" + items = _array(item) + items["maxItems"] = 100 + result = _object({ + "items": _nullable(items), "total_count": _integer(), + "returned_count": {**_integer(), "maximum": 100}, "omitted_count": _integer(), + "limit": {"const": 100}, "detail_status": _enum("complete", "truncated", "omitted"), + "omission_reasons": _array(_enum("diagnostic_limit", "redacted_identity_details"), unique=True), + }) + result["allOf"] = [ + {"if": {"properties": {"detail_status": {"const": "omitted"}}}, + "then": {"properties": {"items": {"type": "null"}, "returned_count": {"const": 0}, + "omission_reasons": {"minItems": 1}}}, + "else": {"properties": {"items": items}}}, + {"if": {"properties": {"detail_status": {"const": "complete"}}}, + "then": {"properties": {"omitted_count": {"const": 0}, "omission_reasons": {"maxItems": 0}}}}, + {"if": {"properties": {"detail_status": {"const": "truncated"}}}, + "then": {"properties": {"omitted_count": {"minimum": 1}, + "returned_count": {"const": 100}, + "omission_reasons": {"const": ["diagnostic_limit"]}}}}, + ] + return result + + def _base_envelope(evidence: str, unit: str, owner: str, method: str) -> dict: return { "unit": {"const": unit}, "evidence_class": {"const": evidence}, @@ -239,7 +264,7 @@ def _build_contract() -> dict: }, ("status", "reason_codes", "coverage", "coverage_reason", "requirements_met", "requirements_missing", "notes", "execution_status", "execution_scope", "execution_reason_codes")) run = _object({ - "run_id": _text(), "toolkit_version": _text(), "report_schema_version": {"const": "1.0"}, + "run_id": _text(), "toolkit_version": _text(), "report_schema_version": {"const": "1.1"}, "started_at": _nullable(_text()), "completed_at": _nullable(_text()), "duration_seconds": _nullable(_number(minimum=0)), "python_version": _nullable(_text()), "platform": _nullable(_text()), "command": _nullable(_text()), @@ -260,7 +285,7 @@ def _build_contract() -> dict: }, ("run_id", "toolkit_version", "report_schema_version", *RUN_NULLABLE_FIELDS, "strict_mode", "redacted_mode", "network_call_count", "deterministic", "privacy_mode", "run_status", "null_reasons")) artifact = _object({ - "role": _enum("records_primary", "records_compare", "provenance_manifest", "schema_mapping", "config", + "role": _enum("records_primary", "records_compare", "lineage_context", "provenance_manifest", "schema_mapping", "config", "version_order", "embedding_data", "external_reference"), "path": _nullable(_text()), "path_redacted": {"type": "boolean"}, "format": _enum("csv", "jsonl", "parquet", "json", "toml", "npy"), @@ -325,7 +350,49 @@ def _build_contract() -> dict: observed_lineage = {name: _observed("observed_facts.lineage." + name, _integer(), "PR-008", "edges", "PR-008." + name, 3, "integer") for name in ("declared_parent_edge_count", "resolved_parent_edge_count", "unresolved_parent_edge_count")} observed_lineage["cycle_status"] = _observed("observed_facts.lineage.cycle_status", _enum("acyclic", "cyclic"), - "T6", "status", "T6.graph_cycle_check", 3, "enum", boundary="deferred_phase5") + "T6", "status", "T6.graph_cycle_check", 3, "enum") + graph_scope = _object({ + "target_dataset_version": _nullable(_text()), "target_record_count": _integer(), + "loaded_record_count": _integer(), "context_record_count": _integer(), + "loaded_dataset_versions": _strings(), + }) + witness = _array(_ref("record_key"), minimum=2) + witness["maxItems"] = 65 + component = _object({ + "component_index": _integer(minimum=1), "member_count": _integer(minimum=1), + "target_member_count": _integer(), "witness_record_keys": _nullable(witness), + "witness_edge_count": _nullable({**_integer(minimum=1), "maximum": 64}), + "witness_reason": _nullable({"const": "diagnostic_limit"}), + }) + cycle_analysis = _object({ + "detected": {"type": "boolean"}, "cycle_count": _integer(), + "counting_method": {"const": "cyclic_strongly_connected_components"}, + "cycle_member_count": _integer(), "target_cycle_member_count": _integer(), + "affected_record_count": _integer(), "target_affected_record_count": _integer(), + "components": _lineage_details(component), + "cycle_member_record_keys": _lineage_details(_ref("record_key")), + "affected_record_keys": _lineage_details(_ref("record_key")), + }) + depth_summary = _object({"depth_resolved_record_count": _integer(), + "maximum_resolved_target_depth": _nullable(_integer()), "target_record_count": _integer()}) + resource_usage = _object({ + "admitted_node_count": _integer(), "admitted_edge_count": _integer(), + "stored_root_membership_count": _integer(), "root_union_visit_count": _integer(), + "limits": _object({name: _integer(minimum=1) for name in ( + "max_nodes", "max_edges", "max_root_memberships", "max_root_union_visits")}), + "exhausted_limit": _nullable(_enum("max_nodes", "max_edges", "max_root_memberships", "max_root_union_visits")), + "attempted_value": _nullable(_integer(minimum=1)), + }) + for name, value, owner, unit, method, value_type in ( + ("graph_scope", graph_scope, "T4", "scope", "T4.graph_scope", "lineage_scope"), + ("cycle_analysis", cycle_analysis, "T6", "cycles", "T6.cyclic_strongly_connected_components", "cycle_analysis"), + ("depth_summary", depth_summary, "PR-009", "edges", "PR-009.resolved_depth_summary", "depth_summary"), + ("unresolved_record_details", _lineage_details(_object({"record_key": _ref("record_key"), + "reason_codes": _strings(minimum=1)})), "T4", "records", "T4.unresolved_record_details", "unresolved_record_details"), + ("resource_usage", resource_usage, "PR-015", "work_units", "PR-015.lineage_resource_usage", "lineage_resource_usage"), + ): + observed_lineage[name] = _observed("observed_facts.lineage." + name, value, + owner, unit, method, 3, value_type) observed_lineage["ordering_certificate"] = _observed("observed_facts.lineage.ordering_certificate", _object({ "method": {"const": "declared_earlier_version_order"}, "version_order": _strings(), "all_resolved_edges_follow_order": {"type": "boolean"}, @@ -416,20 +483,30 @@ def _build_contract() -> dict: ("lower_bound", "F-009"), ("upper_bound", "F-010"), ("interval_width", "T3.upper_minus_lower"))} direct["classification_basis"] = {"const": "toolkit_operationalization"} direct["confidence_disclosure"] = {"const": "provenance_confidence_is_separate_and_does_not_discount_grounding"} - lineage_bounds = {name: _metric("derived_metrics.closure_exposure.lineage." + name, _number(minimum=0, maximum=1), "T3", "ratio", "T3.lineage_closure", 3, "ratio", boundary="deferred_phase5") for name in ("lower_bound", "upper_bound", "interval_width")} + lineage_bounds = {name: _metric("derived_metrics.closure_exposure.lineage." + name, _number(minimum=0, maximum=1), "T3", "ratio", "T3.lineage_closure", 3, "ratio") for name in ("lower_bound", "upper_bound", "interval_width")} + root_contribution = _object({ + "record_key": _ref("record_key"), "incidence_count": _integer(minimum=1), + "incidence_share": _number(minimum=0, maximum=1), "incidence_denominator": _integer(minimum=1), + "fractional_mass": _number(minimum=0), "normalized_weight": _number(minimum=0, maximum=1), + "weight_denominator": _integer(minimum=1), + }) lineage = {} for name, owner, unit, method, value, value_type in ( ("resolved_parent_edge_coverage", "PR-008", "ratio", "PR-008.resolved_edges_over_declared", _number(minimum=0, maximum=1), "ratio"), ("resolved_lineage_coverage", "T4", "ratio", "T4.resolved_records_over_scope", _number(minimum=0, maximum=1), "ratio"), ("external_ancestry_coverage", "T4", "ratio", "T4.external_roots_over_scope", _number(minimum=0, maximum=1), "ratio"), ("distinct_external_root_count", "T4", "roots", "T4.root_set_union", _integer(), "integer"), - ("top_shared_ancestors", "T4", "records", "T4.incidence_ranking", _array(_object({"record_key": _ref("record_key"), "incidence_count": _integer()})), "ancestor_incidence[]"), + ("top_shared_ancestors", "T4", "records", "T4.incidence_ranking", _lineage_details(root_contribution), "root_contribution_details"), + ("grounded_record_count", "T4", "records", "T4.grounded_record_count", _integer(), "integer"), + ("closed_record_count", "T4", "records", "T4.closed_record_count", _integer(), "integer"), + ("unresolved_record_count", "T4", "records", "T4.unresolved_record_count", _integer(), "integer"), + ("records_with_resolved_external_ancestry", "T4", "records", "T4.records_with_resolved_external_ancestry", _integer(), "integer"), ("ancestry_concentration_hhi", "T4", "ratio", "F-012", _number(minimum=0, maximum=1), "ratio"), ("effective_external_root_count", "T4", "roots", "F-013", _number(minimum=0), "number"), ("lineage_depth", "PR-009", "edges", "PR-009.maximum_resolved_parent_depth", _integer(), "integer"), ): - lineage[name] = _metric("derived_metrics.lineage." + name, value, owner, unit, method, 3, value_type, - boundary="current" if name == "resolved_parent_edge_coverage" else "deferred_phase5") + lineage[name] = _metric("derived_metrics.lineage." + name, value, owner, unit, method, 3, value_type) + lineage["resolved_parent_edge_coverage"]["properties"]["no_declared_parents"] = {"type": "boolean"} derived = _object({"support": _object(support, ()), "diversity": diversity, "tail": _object(tail, ()), "provenance": _object(provenance_metrics, ()), "closure_exposure": _object({"direct": _object(direct, ()), "lineage": _object(lineage_bounds, ())}, ()), @@ -441,19 +518,13 @@ def _build_contract() -> dict: props.update({"signal": {"const": name}, "level": _enum("present", "not_present", "indeterminate"), "basis_fields": _strings(minimum=1), "trigger_rule": _text()}) props["limitations"] = _strings(minimum=1) - if name == "shared_ancestry_dependence": - props["status"] = {"const": "unavailable"} - props["level"] = {"const": "indeterminate"} - props["reason_codes"] = _strings(minimum=1) - props["required_evidence"] = _strings(minimum=1) proxies[name] = _object(props) proxies[name]["allOf"] = [{ "if": {"properties": {"status": {"const": "unavailable"}}}, "then": {"properties": {"level": {"const": "indeterminate"}, "reason_codes": {"minItems": 1}, "required_evidence": {"minItems": 1}}}, }] - _field_entries.append(FieldDefinition("proxy_signals." + name, owner, "proxy_signal", "signal", owner + "." + name, level, "proxy", False, - "deferred_phase5" if name == "shared_ancestry_dependence" else "current")) + _field_entries.append(FieldDefinition("proxy_signals." + name, owner, "proxy_signal", "signal", owner + "." + name, level, "proxy")) simulations = {} for name, owner, level in (("closed_resampling", "T1", 5), ("tail_extinction", "T2", 1)): props = _base_envelope("simulation", "scenario", owner, owner + "." + name) @@ -535,7 +606,8 @@ def _build_contract() -> dict: "record_id_mode": _enum("preserve", "hash", "omit"), "representation": _nullable(_ref("representation")), "tail_selection": _nullable(_ref("tail_selection")), "version_order": _strings(), "state_meaning": _nullable(_text()), "weighted": {"type": "boolean"}, - "comparison_requested": {"type": "boolean"}, "scenario_requested": {"type": "boolean"}}, ()), + "comparison_requested": {"type": "boolean"}, "scenario_requested": {"type": "boolean"}, + "lineage_requested": {"type": "boolean"}}, ()), "state_probability": _object({"state_id": _text(empty=True), "probability": _number(minimum=0, maximum=1)}), "simulation_parameters": _object({ "resample_size": _integer(minimum=1), "simulation_horizon": _integer(), @@ -571,8 +643,8 @@ def _build_contract() -> dict: "warnings": _array(warning), "errors": _array(error)}) return {"$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://recursive-integrity-toolkit.example/schemas/report.schema.json", - "title": "Recursive Integrity Toolkit canonical report 1.0", - "description": "Phase 4 Step 2 public contract. Product metadata and five analytical evidence classes remain separate.", + "title": "Recursive Integrity Toolkit canonical report 1.1", + "description": "Phase 5 public contract with bounded lineage evidence. Product metadata and five analytical evidence classes remain separate.", **schema, "$defs": defs} @@ -724,6 +796,8 @@ def _check(value, schema: dict, path: str) -> None: if type(value) is list: if len(value) < schema.get("minItems", 0): _fail(path, "too few array items") + if "maxItems" in schema and len(value) > schema["maxItems"]: + _fail(path, "too many array items") if schema.get("uniqueItems"): identities = {_equality_key(item) for item in value} if len(identities) != len(value): @@ -784,6 +858,275 @@ def _check_nullable_reason(value: dict, name: str, reason_name: str, path: str) _fail(path, "null requires its reason; a present value cannot carry a null reason") +def _lineage_detail_checks(detail: dict) -> None: + path = "$.lineage.details" + for name in ("total_count", "returned_count", "omitted_count", "limit"): + if type(detail[name]) is not int: + _fail(path, "detail counts require exact integers") + total, returned, omitted = (detail[name] for name in ("total_count", "returned_count", "omitted_count")) + if total != returned + omitted: + _fail(path, "detail counts do not partition the total") + status, items, reasons = detail["detail_status"], detail["items"], detail["omission_reasons"] + if status == "omitted": + expected = {"redacted_identity_details"} + if total > 100: + expected.add("diagnostic_limit") + if items is not None or returned or omitted != total or set(reasons) != expected: + _fail(path, "privacy omission must retain the complete aggregate and applicable cap reason") + elif items is None or returned != len(items) or returned != min(total, 100): + _fail(path, "visible detail must contain the bounded prefix of the supplied total") + elif (status == "complete") != (omitted == 0): + _fail(path, "detail status disagrees with omitted count") + if items: + identities = [row.get("record_key", row) for row in items] + if len({_equality_key(key) for key in identities}) != len(identities): + _fail(path, "detail identities must be unique") + + +def _lineage_usage_checks(usage: dict, execution: str) -> None: + """Validate exact supplied work accounting for admitted and aborted graphs.""" + counter_names = { + "max_nodes": "admitted_node_count", "max_edges": "admitted_edge_count", + "max_root_memberships": "stored_root_membership_count", "max_root_union_visits": "root_union_visit_count", + } + for limit, counter in counter_names.items(): + if (type(usage[counter]) is not int or type(usage["limits"][limit]) is not int + or not 0 <= usage[counter] <= usage["limits"][limit] or usage["limits"][limit] <= 0): + _fail("$.lineage.resource_usage", "consumed work must remain within its positive integer limit") + exhausted = usage["exhausted_limit"] + if exhausted is None: + if usage["attempted_value"] is not None: + _fail("$.lineage.resource_usage", "unexhausted work cannot contain a rejected attempt") + elif (execution != "failed" or type(usage["attempted_value"]) is not int + or usage["attempted_value"] != usage["limits"][exhausted] + 1 + or usage[counter_names[exhausted]] != usage["limits"][exhausted]): + _fail("$.lineage.resource_usage", "resource exhaustion requires failed execution and its next rejected unit") + + +def _lineage_semantic_checks(payload: dict) -> None: + """Check supplied lineage evidence consistency without traversing a graph.""" + observed = payload["observed_facts"].get("lineage", {}) + metrics = payload["derived_metrics"].get("lineage", {}) + bounds = payload["derived_metrics"].get("closure_exposure", {}).get("lineage", {}) + proxy = payload["proxy_signals"].get("shared_ancestry_dependence") + capability = payload["capabilities"].get("lineage", {}) + execution = capability.get("execution_status", "not_requested") + new_observed = ("graph_scope", "cycle_analysis", "depth_summary", "unresolved_record_details", "resource_usage", "cycle_status") + new_metrics = tuple(name for name in metrics if name != "resolved_parent_edge_coverage") + + def value(group, name): + return group.get(name, {}).get("value") + + active = (any(value(observed, name) is not None for name in new_observed) + or any(value(metrics, name) is not None for name in new_metrics) + or any(entry["value"] is not None for entry in bounds.values()) + or (proxy is not None and proxy["status"] != "unavailable")) + if execution in ("not_requested", "deferred"): + if active or "no_declared_parents" in metrics.get("resolved_parent_edge_coverage", {}): + _fail("$.capabilities.lineage", "unexecuted lineage cannot contain analytical results") + return + scope = value(observed, "graph_scope") + if scope is None: + if execution == "failed" and not active: + return + usage = value(observed, "resource_usage") + if execution == "failed" and usage is not None: + _lineage_usage_checks(usage, execution) + if (any(value(observed, name) is not None for name in new_observed if name != "resource_usage") + or any(value(metrics, name) is not None for name in new_metrics) + or any(entry["value"] is not None for entry in bounds.values()) + or (proxy is not None and proxy["status"] != "unavailable")): + _fail("$.lineage", "failed graph admission cannot contain subsequent analytical results") + loaded_scope = payload["inputs"].get("scope") + if loaded_scope is None or observed["resource_usage"]["scope"] != loaded_scope: + _fail("$.lineage.resource_usage", "admission accounting must retain the complete loaded input scope") + loaded = loaded_scope["record_count"] + exhausted = usage["exhausted_limit"] + if (exhausted not in ("max_nodes", "max_edges") + or usage["stored_root_membership_count"] or usage["root_union_visit_count"] + or type(loaded) is not int + or exhausted == "max_nodes" and (usage["admitted_node_count"] >= loaded or usage["admitted_edge_count"]) + or exhausted == "max_edges" and usage["admitted_node_count"] != loaded): + _fail("$.lineage.resource_usage", "admission exhaustion is inconsistent with the loaded records or later-stage work") + if not any(error["code"] == "E_LINEAGE_RESOURCE_LIMIT_EXCEEDED" + and "lineage" in error["effect_on_capabilities"] for error in payload["errors"]): + _fail("$.lineage.resource_usage", "admission exhaustion requires its matching lineage error") + return + _fail("$.observed_facts.lineage.graph_scope", "executed lineage requires its explicit target and loaded scope") + target, loaded, context = (scope[name] for name in ("target_record_count", "loaded_record_count", "context_record_count")) + if any(type(number) is not int for number in (target, loaded, context)) or loaded != target + context: + _fail("$.observed_facts.lineage.graph_scope", "loaded scope must partition into target and context") + version = scope["target_dataset_version"] + versions = scope["loaded_dataset_versions"] + if (target and (version is None or version not in versions)) or (loaded and not versions): + _fail("$.observed_facts.lineage.graph_scope", "scope counts require their declared versions") + target_versions = [] if version is None else [version] + graph_fields = {"graph_scope", "cycle_analysis", "cycle_status", "resource_usage"} + for name in graph_fields & observed.keys(): + entry_scope = observed[name]["scope"] + if entry_scope["record_count"] != loaded or set(entry_scope["dataset_versions"]) != set(versions): + _fail("$.lineage.scope", "graph diagnostics must retain the complete loaded population") + entries = [entry for name, entry in observed.items() if name not in graph_fields] + list(metrics.values()) + list(bounds.values()) + if proxy is not None: + entries.append(proxy) + for entry in entries: + if entry["scope"]["record_count"] != target or entry["scope"]["dataset_versions"] != target_versions: + _fail("$.lineage.scope", "lineage evidence must retain the selected target population") + usage = value(observed, "resource_usage") + if usage is None: + _fail("$.observed_facts.lineage.resource_usage", "executed lineage requires work accounting") + _lineage_usage_checks(usage, execution) + exhausted = usage["exhausted_limit"] + cycles, depth = value(observed, "cycle_analysis"), value(observed, "depth_summary") + if execution in ("completed", "partial") and (cycles is None or depth is None): + _fail("$.capabilities.lineage", "completed traversal requires cycle and depth observations") + if cycles is None: + if value(observed, "cycle_status") is not None: + _fail("$.lineage.cycle_status", "cycle status requires a completed cycle analysis") + else: + if usage["admitted_node_count"] != loaded: + _fail("$.lineage.resource_usage", "completed graph evidence requires every loaded node") + if any(type(cycles[name]) is not int for name in ( + "cycle_count", "cycle_member_count", "target_cycle_member_count", + "affected_record_count", "target_affected_record_count")): + _fail("$.lineage.cycle_analysis", "cycle counts require exact integers") + total_members, target_members = cycles["cycle_member_count"], cycles["target_cycle_member_count"] + affected, target_affected = cycles["affected_record_count"], cycles["target_affected_record_count"] + if (cycles["detected"] != (cycles["cycle_count"] > 0) + or not cycles["cycle_count"] <= total_members <= affected <= loaded + or not target_members <= total_members or not target_members <= target_affected <= target + or target_affected > affected or (not cycles["detected"] and affected)): + _fail("$.lineage.cycle_analysis", "cycle counts disagree with their loaded and target scopes") + if execution == "completed" and cycles["detected"]: + _fail("$.capabilities.lineage", "cyclic input cannot claim completed valid lineage") + if value(observed, "cycle_status") != ("cyclic" if cycles["detected"] else "acyclic"): + _fail("$.lineage.cycle_status", "cycle status disagrees with cycle diagnostics") + for name, expected in (("components", cycles["cycle_count"]), ("cycle_member_record_keys", total_members), ("affected_record_keys", affected)): + if cycles[name]["total_count"] != expected: + _fail("$.lineage.cycle_analysis", "detail total disagrees with its aggregate") + rows = cycles["components"]["items"] + if rows is not None: + for index, row in enumerate(rows, 1): + if (any(type(row[name]) is not int for name in ("component_index", "member_count", "target_member_count")) + or row["component_index"] != index or row["target_member_count"] > row["member_count"]): + _fail("$.lineage.cycle_analysis", "component order or member counts are inconsistent") + witness = row["witness_record_keys"] + if witness is None: + if row["witness_edge_count"] is not None or row["witness_reason"] != "diagnostic_limit": + _fail("$.lineage.cycle_analysis", "omitted witness requires a diagnostic-limit reason") + elif (type(row["witness_edge_count"]) is not int or witness[0] != witness[-1] + or row["witness_edge_count"] != len(witness) - 1 + or row["witness_reason"] is not None or len(witness) - 1 > row["member_count"]): + _fail("$.lineage.cycle_analysis", "witness must be closed with a consistent bounded edge count") + if cycles["components"]["detail_status"] == "complete" and ( + sum(row["member_count"] for row in rows) != total_members + or sum(row["target_member_count"] for row in rows) != target_members): + _fail("$.lineage.cycle_analysis", "component rows disagree with aggregate membership") + if depth is not None: + resolved_depth = depth["depth_resolved_record_count"] + maximum = depth["maximum_resolved_target_depth"] + if (type(resolved_depth) is not int or type(depth["target_record_count"]) is not int + or (maximum is not None and type(maximum) is not int) + or depth["target_record_count"] != target or resolved_depth > target + or (maximum is None) != (resolved_depth == 0)): + _fail("$.lineage.depth_summary", "depth subset must retain its actual target coverage") + expected_depth = maximum if target and resolved_depth == target else None + if value(metrics, "lineage_depth") != expected_depth: + _fail("$.derived_metrics.lineage.lineage_depth", "whole-target depth cannot claim a subset maximum") + count_names = ("grounded_record_count", "closed_record_count", "unresolved_record_count", "records_with_resolved_external_ancestry") + counts = [value(metrics, name) for name in count_names] + if any(count is None for count in counts): + if any(count is not None for count in counts) or execution in ("completed", "partial"): + _fail("$.derived_metrics.lineage", "an exact target partition requires all G/C/U counts") + if any(value(metrics, name) is not None for name in new_metrics if name != "lineage_depth") or any(entry["value"] is not None for entry in bounds.values()): + _fail("$.derived_metrics.lineage", "incomplete root traversal cannot supply dependent values") + else: + grounded, closed, unresolved, resolved = counts + if any(type(count) is not int for count in counts) or grounded + closed + unresolved != target or resolved != grounded + closed: + _fail("$.derived_metrics.lineage", "G/C/U counts must exactly partition the target") + if any(metrics[name]["denominator"] != target for name in count_names): + _fail("$.derived_metrics.lineage", "partition counts must disclose the complete target denominator") + if exhausted is not None or (execution == "completed" and unresolved): + _fail("$.capabilities.lineage", "execution status disagrees with root completeness") + for name, numerator in (("resolved_lineage_coverage", resolved), ("external_ancestry_coverage", grounded)): + entry = metrics.get(name) + if entry is None or entry["denominator"] != target: + _fail("$.derived_metrics.lineage", "ancestry coverage must disclose the whole target denominator") + ratio = entry["value"] + if (target == 0 and ratio is not None) or (target and (ratio is None or not math.isclose(ratio * target, numerator, rel_tol=1e-12, abs_tol=1e-12))): + _fail("$.derived_metrics.lineage", "ancestry coverage disagrees with the supplied partition") + details = value(observed, "unresolved_record_details") + if details is None or details["total_count"] != unresolved: + _fail("$.lineage.unresolved_record_details", "unresolved detail total disagrees with the target partition") + root_details = value(metrics, "top_shared_ancestors") + distinct = value(metrics, "distinct_external_root_count") + if type(distinct) is not int or root_details is None or root_details["total_count"] != distinct: + _fail("$.lineage.top_shared_ancestors", "root detail total disagrees with distinct roots") + if grounded == 0 and (distinct != 0 or any(value(metrics, name) is not None for name in ("ancestry_concentration_hhi", "effective_external_root_count"))): + _fail("$.derived_metrics.lineage", "empty grounded population has no concentration value") + if grounded and (not distinct or any(value(metrics, name) is None for name in ("ancestry_concentration_hhi", "effective_external_root_count"))): + _fail("$.derived_metrics.lineage", "grounded population requires its root distribution and concentration") + if grounded and (value(metrics, "ancestry_concentration_hhi") <= 0 + or value(metrics, "effective_external_root_count") <= 0 + or not math.isclose(value(metrics, "ancestry_concentration_hhi") + * value(metrics, "effective_external_root_count"), + 1.0, rel_tol=1e-12, abs_tol=1e-12)): + _fail("$.derived_metrics.lineage", "effective roots and positive concentration must be reciprocal") + for name in ("distinct_external_root_count", "top_shared_ancestors", "ancestry_concentration_hhi", "effective_external_root_count"): + entry = metrics[name] + concentration = name in ("ancestry_concentration_hhi", "effective_external_root_count") + expected_status = "unavailable" if concentration and not grounded else "partial" if unresolved else "available" + if (entry["status"] != expected_status or entry["denominator"] != (grounded if concentration else target) + or entry["coverage"] != value(metrics, "external_ancestry_coverage")): + _fail("$.derived_metrics.lineage", "root metrics must disclose their grounded subset and unresolved coverage") + rows = root_details["items"] + if rows is not None: + for row in rows: + if (any(type(row[name]) is not int for name in ("incidence_count", "incidence_denominator", "weight_denominator")) + or row["incidence_denominator"] != target or row["weight_denominator"] != grounded + or row["incidence_count"] > grounded or row["fractional_mass"] <= 0 + or row["fractional_mass"] > row["incidence_count"] + or row["incidence_share"] <= 0 or row["normalized_weight"] <= 0 + or not math.isclose(row["incidence_share"] * target, row["incidence_count"], rel_tol=1e-12, abs_tol=1e-12) + or not math.isclose(row["normalized_weight"] * grounded, row["fractional_mass"], rel_tol=1e-12, abs_tol=1e-12)): + _fail("$.lineage.top_shared_ancestors", "root contribution contradicts its explicit count basis") + if any(left["incidence_count"] < right["incidence_count"] for left, right in zip(rows, rows[1:])): + _fail("$.lineage.top_shared_ancestors", "root contributions must retain decreasing incidence rank") + for name, numerator in (("lower_bound", closed), ("upper_bound", closed + unresolved), ("interval_width", unresolved)): + if name not in bounds: + continue + entry = bounds[name] + invalid_ratio = (entry["value"] is not None) if target == 0 else ( + entry["value"] is None or not math.isclose( + entry["value"] * target, numerator, rel_tol=1e-12, abs_tol=1e-12)) + if entry["denominator"] != target or invalid_ratio: + _fail("$.derived_metrics.closure_exposure.lineage", "lineage interval contradicts its target partition") + if proxy is not None: + required_basis = {"derived_metrics.lineage.top_shared_ancestors", "derived_metrics.lineage.resolved_lineage_coverage", "derived_metrics.lineage.external_ancestry_coverage"} + if not required_basis.issubset(proxy["basis_fields"]): + _fail("$.proxy_signals.shared_ancestry_dependence", "shared-root signal must cite incidence and coverage evidence") + if proxy["denominator"] != target or proxy["coverage"] != value(metrics, "resolved_lineage_coverage"): + _fail("$.proxy_signals.shared_ancestry_dependence", "shared-root signal must disclose its target coverage") + if proxy["level"] == "not_present" and (not target or unresolved or proxy["status"] != "available"): + _fail("$.proxy_signals.shared_ancestry_dependence", "absence requires complete nonempty target evidence") + if proxy["level"] == "not_present" and rows and rows[0]["incidence_count"] >= 2: + _fail("$.proxy_signals.shared_ancestry_dependence", "absence contradicts a supplied shared-root witness") + if proxy["level"] == "present" and (grounded < 2 or proxy["status"] != ("partial" if unresolved else "available") or (rows is not None and (not rows or rows[0]["incidence_count"] < 2))): + _fail("$.proxy_signals.shared_ancestry_dependence", "presence requires a shared-root witness with honest coverage") + reference_counts = [value(observed, name) for name in ("declared_parent_edge_count", "resolved_parent_edge_count", "unresolved_parent_edge_count")] + reference = metrics.get("resolved_parent_edge_coverage") + if reference is None or "no_declared_parents" not in reference: + _fail("$.derived_metrics.lineage.resolved_parent_edge_coverage", "executed reference coverage requires its explicit zero-reference state") + if any(count is None for count in reference_counts): + if any(count is not None for count in reference_counts) or reference["value"] is not None or reference["no_declared_parents"]: + _fail("$.lineage.reference_coverage", "unknown reference cardinality cannot become a zero declaration") + else: + declared, resolved, unresolved = reference_counts + ratio = reference["value"] + if any(type(count) is not int for count in reference_counts) or declared != resolved + unresolved or reference["no_declared_parents"] != (declared == 0) or reference["denominator"] != declared or ratio is None or (declared == 0 and ratio != 1) or (declared and not math.isclose(ratio * declared, resolved, rel_tol=1e-12, abs_tol=1e-12)): + _fail("$.lineage.reference_coverage", "reference coverage contradicts its declared count basis") + + def _semantic_checks(payload: dict) -> None: run = payload["run"] null_fields = {key for key in RUN_NULLABLE_FIELDS if run[key] is None} @@ -805,8 +1148,7 @@ def _semantic_checks(payload: dict) -> None: _fail("$.capabilities", "noncompleted execution needs explicit reasons") if capability["execution_status"] in ("completed", "partial") and not capability["execution_scope"]: _fail("$.capabilities", "executed work must name its operation scope") - if key == "lineage" and capability["execution_status"] in ("completed", "partial"): - _fail("$.capabilities.lineage", "Phase 5 lineage execution is deferred") + _lineage_semantic_checks(payload) tail = payload["derived_metrics"].get("tail", {}) if "tail_rule" in tail and "selection" in tail and tail["tail_rule"] != tail["selection"]["rule"]: _fail("$.derived_metrics.tail", "tail rule declarations disagree") @@ -824,6 +1166,8 @@ def _semantic_checks(payload: dict) -> None: _fail("$.table", "typed table identity must be unique") if type(item) is not dict: continue + if "detail_status" in contract.get("properties", {}): + _lineage_detail_checks(item) if contract is _REPORT_SCHEMA["$defs"]["coverage"]: if item["numerator"] > item["denominator"]: _fail("$.coverage", "coverage numerator exceeds denominator") @@ -912,7 +1256,7 @@ def _semantic_checks(payload: dict) -> None: lower, upper = interval["lower_bound"]["value"], interval["upper_bound"]["value"] if lower > upper: _fail("$.derived_metrics.closure_exposure", "interval lower bound exceeds upper bound") - if family == "direct" and not math.isclose(interval["interval_width"]["value"], upper - lower, rel_tol=1e-12, abs_tol=1e-12): + if not math.isclose(interval["interval_width"]["value"], upper - lower, rel_tol=1e-12, abs_tol=1e-12): _fail("$.derived_metrics.closure_exposure", "interval width disagrees with endpoints") for name, scenario in payload["simulations"].items(): parameters = scenario["parameters"] diff --git a/src/recursive_integrity_toolkit/utils/logging.py b/src/recursive_integrity_toolkit/utils/logging.py index 4b6ad48..a865561 100644 --- a/src/recursive_integrity_toolkit/utils/logging.py +++ b/src/recursive_integrity_toolkit/utils/logging.py @@ -50,6 +50,7 @@ "E_PARENT_AMBIGUOUS": ("A parent reference is ambiguous.", "Supply an explicit dataset-version and record-ID pair."), "E_PARENT_FUTURE_VERSION": ("A parent reference conflicts with declared version order.", "Check the parent reference against the declared version order."), "E_LINEAGE_CYCLE": ("Declared parent references contain a validated cycle.", "Correct the parent declarations before requesting dependent work."), + "E_LINEAGE_RESOURCE_LIMIT_EXCEEDED": ("Lineage analysis reached a configured resource limit.", "Review the reported work counters and configured limit before reducing the input scope or explicitly adjusting that limit."), "E_VERSION_ORDER_CONFLICT": ("Version-order declarations conflict.", "Supply one consistent explicit version order."), "E_REPRESENTATION_INCOMPATIBLE": ("The supplied representations are incompatible.", "Supply compatible explicit representation declarations."), "E_MAPPING_SOURCE_FIELD_MISSING": ("A declared mapping source field is missing.", "Correct the explicit mapping or supply its source field."), diff --git a/tests/conftest.py b/tests/conftest.py index 8b0e0fb..1fe3fe5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,25 +1,10 @@ -"""Shared scaffold fixtures and explicit Phase 3 Step 10 test-only composition. - -Owner IDs: - Technical Maintainer, Phase 1 acceptance gate - -Current scope: - Repository structure, safe import, CLI startup, JSON parsing, owner IDs, - prohibited paths, optional dependency isolation, no-network import, and - absence of analytical implementation. - -Limits: - Existing scaffold fixtures stay unchanged. New helpers use only approved Hero - or synthetic data. Nothing here is an importable product orchestrator. -""" +"""Shared fixtures for current product behavior and bounded performance runs.""" from __future__ import annotations -import ast import os import sys from pathlib import Path -from typing import Callable import pytest @@ -61,35 +46,6 @@ def subprocess_env() -> dict[str, str]: return env -@pytest.fixture(scope="session") -def owner_checker(package_root: Path) -> Callable[[str, str], None]: - """Return a checker for approved owner IDs in module docstrings.""" - def check(relative_path: str, owner_id: str) -> None: - path = package_root / relative_path - assert path.is_file(), f"Missing approved module: {relative_path}" - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - doc = ast.get_docstring(tree) or "" - assert "Owner IDs:" in doc, f"Owner section missing in {relative_path}" - assert owner_id in doc, f"{owner_id} missing from {relative_path} owner section" - assert "Current phase status:" in doc, f"Phase status missing in {relative_path}" - assert "No analytical" in doc or "No " in doc, f"No-implementation limit missing in {relative_path}" - return check - - -@pytest.fixture(scope="session") -def placeholder_checker(package_root: Path) -> Callable[[str], None]: - """Return a checker that requires a module to contain only its docstring.""" - def check(relative_path: str) -> None: - path = package_root / relative_path - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - assert len(tree.body) == 1, f"Unexpected executable body in {relative_path}" - node = tree.body[0] - assert isinstance(node, ast.Expr) - assert isinstance(node.value, ast.Constant) - assert isinstance(node.value.value, str) - return check - - @pytest.fixture def phase3_hero_pipeline(repo_root): """Return a test-only callable; ordinary validation still performs no metrics.""" @@ -169,669 +125,6 @@ def measure(name, operation, **details): return measure -@pytest.fixture(scope="session") -def phase3_step10_snapshot(repo_root, tmp_path_factory): - """Materialize immutable Step 10 source for its historical stage assertions.""" - import io - import subprocess - import zipfile - root = tmp_path_factory.mktemp("phase3-step10") - commit = "150a2a105e01883672ef0c2300b41b0de3be352e" - tree = subprocess.check_output(["git", "-C", str(repo_root), "rev-parse", commit+"^{tree}"], text=True).strip() - assert tree == "b609cb75913fc6352729086819a24795caae1dd8" - raw = subprocess.check_output(["git", "-C", str(repo_root), "archive", "--format=zip", commit]) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - archive.extractall(root) - return root - - -@pytest.fixture(scope="session") -def phase3_final_snapshot(repo_root, tmp_path_factory): - """Provide verified final Phase 3 bytes only to listed historical tests. - - The source checkout is never rebound. The private archive has no Git working - tree, accepts only pinned regular-file blobs, and is checked again after use. - Files remain writable so copies in inherited mutation tests work on Windows; - changing this snapshot itself fails the closing byte and file-set check. - """ - import hashlib - import io - import subprocess - import zipfile - from pathlib import PurePosixPath - - commit = "e3ffb8c0a88bfe31f669f9662d9b5213da628b3a" - tree = "e2a25f8cfdc66c3317809c479f80fdae162e6ba9" - tests_tree = "6ab22cb9197a8f094f455b29a07a851062bb26c9" - - def git(*arguments): - return subprocess.check_output(["git", "-C", str(repo_root), *arguments]) - - assert git("rev-parse", commit + "^{commit}").decode().strip() == commit - assert git("rev-parse", commit + "^{tree}").decode().strip() == tree - assert git("rev-parse", commit + ":tests").decode().strip() == tests_tree - blobs = {} - for entry in git("ls-tree", "-rz", commit).split(b"\0"): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - relative = name.decode("utf-8") - parts = PurePosixPath(relative).parts - assert mode in (b"100644", b"100755") and kind == b"blob" - assert parts and not relative.startswith("/") and ".." not in parts and ".git" not in parts - assert relative not in blobs - blobs[relative] = oid.decode("ascii") - - def blob_oid(raw): - return hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - - root = tmp_path_factory.mktemp("phase3-final") - raw = git("archive", "--format=zip", commit) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = [member for member in archive.infolist() if not member.is_dir()] - assert len(members) == len(blobs) - assert {member.filename for member in members} == set(blobs) - for member in members: - content = archive.read(member) - assert blob_oid(content) == blobs[member.filename], member.filename - destination = root.joinpath(*PurePosixPath(member.filename).parts) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(content) - - def verify_unchanged(): - paths = list(root.rglob("*")) - assert not any(path.is_symlink() for path in paths), "Historical snapshot gained a symlink" - files = {path.relative_to(root).as_posix(): path for path in paths if path.is_file()} - assert set(files) == set(blobs), "Historical snapshot file set changed" - for name, path in files.items(): - assert blob_oid(path.read_bytes()) == blobs[name], f"Historical snapshot changed: {name}" - - verify_unchanged() - yield root - verify_unchanged() - - -@pytest.fixture(scope="session") -def phase3_final_package_root(phase3_final_snapshot): - """Return the pinned package only to explicitly migrated historical checks.""" - return phase3_final_snapshot / "src/recursive_integrity_toolkit" - - -@pytest.fixture(scope="session") -def phase3_final_owner_checker(phase3_final_package_root): - """Run the inherited ownership assertions against pinned Phase 3 source.""" - def check(relative_path, owner_id): - path = phase3_final_package_root / relative_path - assert path.is_file(), f"Missing approved module: {relative_path}" - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - doc = ast.get_docstring(tree) or "" - assert "Owner IDs:" in doc, f"Owner section missing in {relative_path}" - assert owner_id in doc, f"{owner_id} missing from {relative_path} owner section" - assert "Current phase status:" in doc, f"Phase status missing in {relative_path}" - assert "No analytical" in doc or "No " in doc, f"No-implementation limit missing in {relative_path}" - return check - - -@pytest.fixture(scope="session") -def phase3_final_placeholder_checker(phase3_final_package_root): - """Run the inherited empty-module assertions against pinned Phase 3 source.""" - def check(relative_path): - path = phase3_final_package_root / relative_path - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - assert len(tree.body) == 1, f"Unexpected executable body in {relative_path}" - node = tree.body[0] - assert isinstance(node, ast.Expr) - assert isinstance(node.value, ast.Constant) - assert isinstance(node.value.value, str) - return check - - -@pytest.fixture(scope="session") -def phase3_final_subprocess_env(phase3_final_snapshot): - """Import only the pinned src tree for historical help, without bytecode writes.""" - env = os.environ.copy() - env["PYTHONPATH"] = str(phase3_final_snapshot / "src") - env["PYTHONDONTWRITEBYTECODE"] = "1" - return env - - -@pytest.fixture(scope="session") -def phase4_step1_snapshot(repo_root, tmp_path_factory): - """Provide verified accepted Phase 4 Step 1 bytes only to listed historical tests. - - The source checkout is never rebound. The private archive has no Git working - tree, accepts only pinned regular-file blobs, and is checked again after use. - Files remain writable so copies in inherited mutation tests work on Windows; - changing this snapshot itself fails the closing byte and file-set check. - """ - import hashlib - import io - import subprocess - import zipfile - from pathlib import PurePosixPath - - commit = "a7f3c46d6ca05de36bfcb60f496ebb2d5ab4a37c" - tree = "7f8ef492568456bf5d46fb1b7e2631cb9b94e3fd" - tests_tree = "2ce77e331a0fc377387735bb55dffd3be630b59c" - - def git(*arguments): - return subprocess.check_output(["git", "-C", str(repo_root), *arguments]) - - assert git("rev-parse", commit + "^{commit}").decode().strip() == commit - assert git("rev-parse", commit + "^{tree}").decode().strip() == tree - assert git("rev-parse", commit + ":tests").decode().strip() == tests_tree - blobs = {} - for entry in git("ls-tree", "-rz", commit).split(b"\0"): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - relative = name.decode("utf-8") - parts = PurePosixPath(relative).parts - assert mode in (b"100644", b"100755") and kind == b"blob" - assert parts and not relative.startswith("/") and ".." not in parts and ".git" not in parts - assert relative not in blobs - blobs[relative] = oid.decode("ascii") - - def blob_oid(raw): - return hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - - root = tmp_path_factory.mktemp("phase4-step1") - raw = git("archive", "--format=zip", commit) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = [member for member in archive.infolist() if not member.is_dir()] - assert len(members) == len(blobs) - assert {member.filename for member in members} == set(blobs) - for member in members: - content = archive.read(member) - assert blob_oid(content) == blobs[member.filename], member.filename - destination = root.joinpath(*PurePosixPath(member.filename).parts) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(content) - - def verify_unchanged(): - paths = list(root.rglob("*")) - assert not any(path.is_symlink() for path in paths), "Historical snapshot gained a symlink" - files = {path.relative_to(root).as_posix(): path for path in paths if path.is_file()} - assert set(files) == set(blobs), "Historical snapshot file set changed" - for name, path in files.items(): - assert blob_oid(path.read_bytes()) == blobs[name], f"Historical snapshot changed: {name}" - - verify_unchanged() - yield root - verify_unchanged() - - -@pytest.fixture(scope="session") -def phase4_step2_snapshot(repo_root, tmp_path_factory): - """Provide verified accepted Phase 4 Step 2 bytes only to listed historical tests. - - The source checkout is never rebound. The private archive has no Git working - tree, accepts only pinned regular-file blobs, and is checked again after use. - Files remain writable so copies in inherited mutation tests work on Windows; - changing this snapshot itself fails the closing byte and file-set check. - """ - import hashlib - import io - import subprocess - import zipfile - from pathlib import PurePosixPath - - commit = "fcea74e2b1858e83cdbfd1b8212d15343d63ff08" - tree = "c3ce887fd393cfc6c18364546005f2b74a63b746" - tests_tree = "9d6ac0cbf3827ab54c7852503d99d442a3796259" - - def git(*arguments): - return subprocess.check_output(["git", "-C", str(repo_root), *arguments]) - - assert git("rev-parse", commit + "^{commit}").decode().strip() == commit - assert git("rev-parse", commit + "^{tree}").decode().strip() == tree - assert git("rev-parse", commit + ":tests").decode().strip() == tests_tree - blobs = {} - for entry in git("ls-tree", "-rz", commit).split(b"\0"): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - relative = name.decode("utf-8") - parts = PurePosixPath(relative).parts - assert mode in (b"100644", b"100755") and kind == b"blob" - assert parts and not relative.startswith("/") and ".." not in parts and ".git" not in parts - assert relative not in blobs - blobs[relative] = oid.decode("ascii") - - def blob_oid(raw): - return hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - - root = tmp_path_factory.mktemp("phase4-step2") - raw = git("archive", "--format=zip", commit) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = [member for member in archive.infolist() if not member.is_dir()] - assert len(members) == len(blobs) - assert {member.filename for member in members} == set(blobs) - for member in members: - content = archive.read(member) - assert blob_oid(content) == blobs[member.filename], member.filename - destination = root.joinpath(*PurePosixPath(member.filename).parts) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(content) - - def verify_unchanged(): - paths = list(root.rglob("*")) - assert not any(path.is_symlink() for path in paths), "Historical snapshot gained a symlink" - files = {path.relative_to(root).as_posix(): path for path in paths if path.is_file()} - assert set(files) == set(blobs), "Historical snapshot file set changed" - for name, path in files.items(): - assert blob_oid(path.read_bytes()) == blobs[name], f"Historical snapshot changed: {name}" - - verify_unchanged() - yield root - verify_unchanged() - - -@pytest.fixture(scope="session") -def phase4_step3_snapshot(repo_root, tmp_path_factory): - """Provide verified accepted Phase 4 Step 3 bytes only to listed historical tests. - - The source checkout is never rebound. The private archive has no Git working - tree, accepts only pinned regular-file blobs, and is checked again after use. - Files remain writable so copies in inherited mutation tests work on Windows; - changing this snapshot itself fails the closing byte and file-set check. - """ - import hashlib - import io - import subprocess - import zipfile - from pathlib import PurePosixPath - - commit = "974545c456e535ba1e1c5b6bf4ae0ce25bc04b57" - tree = "15498f3f00d0ce048c3cba2156eb7a44f096ae88" - tests_tree = "17b87858f77ee4413c98cfd76f16f52170246741" - - def git(*arguments): - return subprocess.check_output(["git", "-C", str(repo_root), *arguments]) - - assert git("rev-parse", commit + "^{commit}").decode().strip() == commit - assert git("rev-parse", commit + "^{tree}").decode().strip() == tree - assert git("rev-parse", commit + ":tests").decode().strip() == tests_tree - blobs = {} - for entry in git("ls-tree", "-rz", commit).split(b"\0"): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - relative = name.decode("utf-8") - parts = PurePosixPath(relative).parts - assert mode in (b"100644", b"100755") and kind == b"blob" - assert parts and not relative.startswith("/") and ".." not in parts and ".git" not in parts - assert relative not in blobs - blobs[relative] = oid.decode("ascii") - - def blob_oid(raw): - return hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - - root = tmp_path_factory.mktemp("phase4-step3") - raw = git("archive", "--format=zip", commit) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = [member for member in archive.infolist() if not member.is_dir()] - assert len(members) == len(blobs) - assert {member.filename for member in members} == set(blobs) - for member in members: - content = archive.read(member) - assert blob_oid(content) == blobs[member.filename], member.filename - destination = root.joinpath(*PurePosixPath(member.filename).parts) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(content) - - def verify_unchanged(): - paths = list(root.rglob("*")) - assert not any(path.is_symlink() for path in paths), "Historical snapshot gained a symlink" - files = {path.relative_to(root).as_posix(): path for path in paths if path.is_file()} - assert set(files) == set(blobs), "Historical snapshot file set changed" - for name, path in files.items(): - assert blob_oid(path.read_bytes()) == blobs[name], f"Historical snapshot changed: {name}" - - verify_unchanged() - yield root - verify_unchanged() - - -@pytest.fixture(scope="session") -def phase4_step4_snapshot(repo_root, tmp_path_factory): - """Provide verified accepted Phase 4 Step 4 bytes only to listed historical tests. - - The source checkout is never rebound. The private archive has no Git working - tree, accepts only pinned regular-file blobs, and is checked again after use. - Files remain writable so copies in inherited mutation tests work on Windows; - changing this snapshot itself fails the closing byte and file-set check. - """ - import hashlib - import io - import subprocess - import zipfile - from pathlib import PurePosixPath - - commit = "49454a9b162cb8d35e38d1cb1ae32cb208d3a01c" - tree = "79264d845e08f73ee68e160dea04f7786f18fd0a" - tests_tree = "4aa4ca0b22f8b8c25afe27feeaba713d6f0830bf" - - def git(*arguments): - return subprocess.check_output(["git", "-C", str(repo_root), *arguments]) - - assert git("rev-parse", commit + "^{commit}").decode().strip() == commit - assert git("rev-parse", commit + "^{tree}").decode().strip() == tree - assert git("rev-parse", commit + ":tests").decode().strip() == tests_tree - blobs = {} - for entry in git("ls-tree", "-rz", commit).split(b"\0"): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - relative = name.decode("utf-8") - parts = PurePosixPath(relative).parts - assert mode in (b"100644", b"100755") and kind == b"blob" - assert parts and not relative.startswith("/") and ".." not in parts and ".git" not in parts - assert relative not in blobs - blobs[relative] = oid.decode("ascii") - - def blob_oid(raw): - return hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - - root = tmp_path_factory.mktemp("phase4-step4") - raw = git("archive", "--format=zip", commit) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = [member for member in archive.infolist() if not member.is_dir()] - assert len(members) == len(blobs) - assert {member.filename for member in members} == set(blobs) - for member in members: - content = archive.read(member) - assert blob_oid(content) == blobs[member.filename], member.filename - destination = root.joinpath(*PurePosixPath(member.filename).parts) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(content) - - def verify_unchanged(): - paths = list(root.rglob("*")) - assert not any(path.is_symlink() for path in paths), "Historical snapshot gained a symlink" - files = {path.relative_to(root).as_posix(): path for path in paths if path.is_file()} - assert set(files) == set(blobs), "Historical snapshot file set changed" - for name, path in files.items(): - assert blob_oid(path.read_bytes()) == blobs[name], f"Historical snapshot changed: {name}" - - verify_unchanged() - yield root - verify_unchanged() - - -@pytest.fixture(scope="session") -def phase4_step5_snapshot(repo_root, tmp_path_factory): - """Provide verified accepted Phase 4 Step 5 bytes only to listed historical tests. - - The source checkout is never rebound. The private archive has no Git working - tree, accepts only pinned regular-file blobs, and is checked again after use. - Files remain writable so copies in inherited mutation tests work on Windows; - changing this snapshot itself fails the closing byte and file-set check. - """ - import hashlib - import io - import subprocess - import zipfile - from pathlib import PurePosixPath - - commit = "c1667179e895a798bba2349960162542efa5ef87" - tree = "d83f9fc8e0b569f2d796bd2a83b87422fe350fba" - tests_tree = "974f8e65904c24ed4e226996eaf571aab6123965" - - def git(*arguments): - return subprocess.check_output(["git", "-C", str(repo_root), *arguments]) - - assert git("rev-parse", commit + "^{commit}").decode().strip() == commit - assert git("rev-parse", commit + "^{tree}").decode().strip() == tree - assert git("rev-parse", commit + ":tests").decode().strip() == tests_tree - blobs = {} - for entry in git("ls-tree", "-rz", commit).split(b"\0"): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - relative = name.decode("utf-8") - parts = PurePosixPath(relative).parts - assert mode in (b"100644", b"100755") and kind == b"blob" - assert parts and not relative.startswith("/") and ".." not in parts and ".git" not in parts - assert relative not in blobs - blobs[relative] = oid.decode("ascii") - - def blob_oid(raw): - return hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - - root = tmp_path_factory.mktemp("phase4-step5") - raw = git("archive", "--format=zip", commit) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = [member for member in archive.infolist() if not member.is_dir()] - assert len(members) == len(blobs) - assert {member.filename for member in members} == set(blobs) - for member in members: - content = archive.read(member) - assert blob_oid(content) == blobs[member.filename], member.filename - destination = root.joinpath(*PurePosixPath(member.filename).parts) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(content) - - def verify_unchanged(): - paths = list(root.rglob("*")) - assert not any(path.is_symlink() for path in paths), "Historical snapshot gained a symlink" - files = {path.relative_to(root).as_posix(): path for path in paths if path.is_file()} - assert set(files) == set(blobs), "Historical snapshot file set changed" - for name, path in files.items(): - assert blob_oid(path.read_bytes()) == blobs[name], f"Historical snapshot changed: {name}" - - verify_unchanged() - yield root - verify_unchanged() - - -@pytest.fixture(scope="session") -def phase4_step6_snapshot(repo_root, tmp_path_factory): - """Provide verified accepted Phase 4 Step 6 bytes only to listed historical tests. - - The source checkout is never rebound. The private archive has no Git working - tree, accepts only pinned regular-file blobs, and is checked again after use. - Files remain writable so copies in inherited mutation tests work on Windows; - changing this snapshot itself fails the closing byte and file-set check. - """ - import hashlib - import io - import subprocess - import zipfile - from pathlib import PurePosixPath - - commit = "aa2355f2359c3af4fc05345c792f9903b0d5858c" - tree = "d9b6910079f49a2c739f86eceec05040918a41a5" - tests_tree = "8d3a03c61da807d1fef8d26fc4af0d6f72e4546e" - - def git(*arguments): - return subprocess.check_output(["git", "-C", str(repo_root), *arguments]) - - assert git("rev-parse", commit + "^{commit}").decode().strip() == commit - assert git("rev-parse", commit + "^{tree}").decode().strip() == tree - assert git("rev-parse", commit + ":tests").decode().strip() == tests_tree - blobs = {} - for entry in git("ls-tree", "-rz", commit).split(b"\0"): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - relative = name.decode("utf-8") - parts = PurePosixPath(relative).parts - assert mode in (b"100644", b"100755") and kind == b"blob" - assert parts and not relative.startswith("/") and ".." not in parts and ".git" not in parts - assert relative not in blobs - blobs[relative] = oid.decode("ascii") - - def blob_oid(raw): - return hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - - root = tmp_path_factory.mktemp("phase4-step6") - raw = git("archive", "--format=zip", commit) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = [member for member in archive.infolist() if not member.is_dir()] - assert len(members) == len(blobs) - assert {member.filename for member in members} == set(blobs) - for member in members: - content = archive.read(member) - assert blob_oid(content) == blobs[member.filename], member.filename - destination = root.joinpath(*PurePosixPath(member.filename).parts) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(content) - - def verify_unchanged(): - paths = list(root.rglob("*")) - assert not any(path.is_symlink() for path in paths), "Historical snapshot gained a symlink" - files = {path.relative_to(root).as_posix(): path for path in paths if path.is_file()} - assert set(files) == set(blobs), "Historical snapshot file set changed" - for name, path in files.items(): - assert blob_oid(path.read_bytes()) == blobs[name], f"Historical snapshot changed: {name}" - - verify_unchanged() - yield root - verify_unchanged() - - -@pytest.fixture(scope="session") -def phase4_step7_snapshot(repo_root, tmp_path_factory): - """Provide verified accepted Phase 4 Step 7 bytes only to listed historical tests. - - The source checkout is never rebound. The private archive has no Git working - tree, accepts only pinned regular-file blobs, and is checked again after use. - Files remain writable so copies in inherited mutation tests work on Windows; - changing this snapshot itself fails the closing byte and file-set check. - """ - import hashlib - import io - import subprocess - import zipfile - from pathlib import PurePosixPath - - commit = "bacd33ae65e392872776c6d976401fdc3d565f63" - tree = "4c16914bb0f0c5281efc36624efb8b1a1d4a174e" - tests_tree = "96317cb7b25236b79efc0f2d93e1e0f5f132fdb2" - - def git(*arguments): - return subprocess.check_output(["git", "-C", str(repo_root), *arguments]) - - assert git("rev-parse", commit + "^{commit}").decode().strip() == commit - assert git("rev-parse", commit + "^{tree}").decode().strip() == tree - assert git("rev-parse", commit + ":tests").decode().strip() == tests_tree - blobs = {} - for entry in git("ls-tree", "-rz", commit).split(b"\0"): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - relative = name.decode("utf-8") - parts = PurePosixPath(relative).parts - assert mode in (b"100644", b"100755") and kind == b"blob" - assert parts and not relative.startswith("/") and ".." not in parts and ".git" not in parts - assert relative not in blobs - blobs[relative] = oid.decode("ascii") - - def blob_oid(raw): - return hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - - root = tmp_path_factory.mktemp("phase4-step7") - raw = git("archive", "--format=zip", commit) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = [member for member in archive.infolist() if not member.is_dir()] - assert len(members) == len(blobs) - assert {member.filename for member in members} == set(blobs) - for member in members: - content = archive.read(member) - assert blob_oid(content) == blobs[member.filename], member.filename - destination = root.joinpath(*PurePosixPath(member.filename).parts) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(content) - - def verify_unchanged(): - paths = list(root.rglob("*")) - assert not any(path.is_symlink() for path in paths), "Historical snapshot gained a symlink" - files = {path.relative_to(root).as_posix(): path for path in paths if path.is_file()} - assert set(files) == set(blobs), "Historical snapshot file set changed" - for name, path in files.items(): - assert blob_oid(path.read_bytes()) == blobs[name], f"Historical snapshot changed: {name}" - - verify_unchanged() - yield root - verify_unchanged() - - -@pytest.fixture(scope="session") -def phase4_step8_snapshot(repo_root, tmp_path_factory): - """Provide verified accepted Phase 4 Step 8 bytes only to listed historical tests. - - The source checkout is never rebound. The private archive has no Git working - tree, accepts only pinned regular-file blobs, and is checked again after use. - Files remain writable so copies in inherited mutation tests work on Windows; - changing this snapshot itself fails the closing byte and file-set check. - """ - import hashlib - import io - import subprocess - import zipfile - from pathlib import PurePosixPath - - commit = "63e347a785838b64a9c32e636ea48c664458cee4" - tree = "3ac450b5f67cad738849de18601999fefee98f63" - tests_tree = "ddaaab24d83080b7c1a27229c15e6e4a3f5b3cd8" - - def git(*arguments): - return subprocess.check_output(["git", "-C", str(repo_root), *arguments]) - - assert git("rev-parse", commit + "^{commit}").decode().strip() == commit - assert git("rev-parse", commit + "^{tree}").decode().strip() == tree - assert git("rev-parse", commit + ":tests").decode().strip() == tests_tree - blobs = {} - for entry in git("ls-tree", "-rz", commit).split(b"\0"): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - mode, kind, oid = metadata.split() - relative = name.decode("utf-8") - parts = PurePosixPath(relative).parts - assert mode in (b"100644", b"100755") and kind == b"blob" - assert parts and not relative.startswith("/") and ".." not in parts and ".git" not in parts - assert relative not in blobs - blobs[relative] = oid.decode("ascii") - - def blob_oid(raw): - return hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() - - root = tmp_path_factory.mktemp("phase4-step8") - raw = git("archive", "--format=zip", commit) - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = [member for member in archive.infolist() if not member.is_dir()] - assert len(members) == len(blobs) - assert {member.filename for member in members} == set(blobs) - for member in members: - content = archive.read(member) - assert blob_oid(content) == blobs[member.filename], member.filename - destination = root.joinpath(*PurePosixPath(member.filename).parts) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(content) - - def verify_unchanged(): - paths = list(root.rglob("*")) - assert not any(path.is_symlink() for path in paths), "Historical snapshot gained a symlink" - files = {path.relative_to(root).as_posix(): path for path in paths if path.is_file()} - assert set(files) == set(blobs), "Historical snapshot file set changed" - for name, path in files.items(): - assert blob_oid(path.read_bytes()) == blobs[name], f"Historical snapshot changed: {name}" - - verify_unchanged() - yield root - verify_unchanged() - - @pytest.fixture def phase4_step10_measure_reports(request, tmp_path, subprocess_env): """Measure complete CLI publication in fresh processes; retain every attempt. diff --git a/tests/fixtures/lineage_complete/README.md b/tests/fixtures/lineage_complete/README.md index 9ea078c..67019ac 100644 --- a/tests/fixtures/lineage_complete/README.md +++ b/tests/fixtures/lineage_complete/README.md @@ -1,7 +1,38 @@ -# lineage_complete fixture placeholder +# Complete and boundary lineage acceptance inputs -Phase 1 status: structure only. +Phase 5 Step 1 specifies synthetic inputs and independent future acceptance +values. `cases.json` contains canonical record/provenance rows, explicit version +order, selected target keys, current input-validation expectations, and future +lineage expectations. These files are test data, not report-schema documents. -This directory is reserved for the approved `lineage_complete` validation fixture. -It contains no real user data and no analytical expected values in Phase 1. -Data may be added only in the phase that owns the corresponding behavior. +The cases separate root support from strict ancestry, preserve known-empty root +sets, distinguish unknown grounding from absent/null parent declarations, and +cover single-parent carryover, duplicate aliases, diamond paths, and conservative +grounded-parent handling. The empty-target case selects no observations from a +loaded context; it concerns the future analysis API and makes no claim about +current empty-file CLI support. + +`hero_expected.json` transcribes the approved Hero arithmetic into exact rational +values for later lineage tests. The six canonical files in `examples/hero` remain +unchanged. The three roots with incidence one are ordered by canonical key. + +Conventions shared by these three lineage fixture directories: + +- Rational strings are exact values. `lineage_bounds` lists lower, upper, width. +- `complete_root_sets` omits unresolved targets. An empty array is a completely + resolved known-empty set. Roots on incomplete paths contribute no exact mass. +- Counts distinguish selected targets from loaded supporting records. +- Conceptual unresolved reasons are explanatory fixture values; later runtime + tests map them to the approved typed reason codes. +- `shared_ancestry_proxy` uses existing report levels: `present`, `not_present`, + or `indeterminate`. The last includes unavailable empty-target results. +- The Step 1 loader test validates only `expected_input`. It uses the existing + comparison input role to load supporting records until the dedicated context + role is implemented. It executes no graph or lineage formula. +- `expected_lineage` becomes a product oracle when Steps 2 through 8 implement + the corresponding behavior. Its presence is not a passing lineage result. + +Arithmetic was derived from the approved P5-D01 through P5-D07 contracts before +graph implementation. No output from the future implementation generated these +expected values. General permutation, renaming, long-chain and resource-boundary +cases belong in the later direct tests instead of expanded static fixture copies. diff --git a/tests/fixtures/lineage_complete/cases.json b/tests/fixtures/lineage_complete/cases.json new file mode 100644 index 0000000..ed59ac2 --- /dev/null +++ b/tests/fixtures/lineage_complete/cases.json @@ -0,0 +1,566 @@ +{ + "purpose": "Independent Phase 5 acceptance inputs and future lineage oracles; no lineage execution is claimed by Step 1.", + "fraction_encoding": "Exact rational strings; null means unavailable and requires the adjacent reason.", + "cases": [ + { + "case_id": "external_root_supports_itself", + "records": [ + { + "dataset_version": "v2", + "record_id": "a", + "content": "Synthetic fixture a" + } + ], + "provenance": [ + { + "dataset_version": "v2", + "record_id": "a", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + } + ], + "version_order": [ + "v2" + ], + "target_record_keys": [ + "v2::a" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [], + "target_declared_references": 0, + "target_resolved_references": 0 + }, + "expected_lineage": { + "complete_root_sets": { + "v2::a": [ + "v2::a" + ] + }, + "strict_ancestors": { + "v2::a": [] + }, + "counts": { + "target": 1, + "grounded": 1, + "closed": 0, + "unresolved": 0 + }, + "depth_by_record": { + "v2::a": 0 + }, + "lineage_depth": 0, + "no_declared_parents": true, + "edge_coverage": "1", + "ancestry_hhi": "1", + "effective_external_root_count": "1", + "shared_ancestry_proxy": "not_present", + "lineage_bounds": [ + "0", + "0", + "0" + ] + } + }, + { + "case_id": "all_closed", + "records": [ + { + "dataset_version": "v2", + "record_id": "a", + "content": "Synthetic fixture a" + }, + { + "dataset_version": "v2", + "record_id": "b", + "content": "Synthetic fixture b" + } + ], + "provenance": [ + { + "dataset_version": "v2", + "record_id": "a", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v2", + "record_id": "b", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::a" + ] + } + ], + "version_order": [ + "v2" + ], + "target_record_keys": [ + "v2::a", + "v2::b" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [], + "target_declared_references": 1, + "target_resolved_references": 1 + }, + "expected_lineage": { + "complete_root_sets": { + "v2::a": [], + "v2::b": [] + }, + "counts": { + "target": 2, + "grounded": 0, + "closed": 2, + "unresolved": 0 + }, + "depth_by_record": { + "v2::a": 0, + "v2::b": 1 + }, + "lineage_depth": 1, + "resolved_lineage_coverage": "1", + "external_ancestry_coverage": "0", + "distinct_external_roots": 0, + "ancestry_hhi": null, + "effective_external_root_count": null, + "concentration_unavailable_reason": "no_completely_resolved_nonempty_root_sets", + "shared_ancestry_proxy": "not_present", + "lineage_bounds": [ + "1", + "1", + "0" + ] + } + }, + { + "case_id": "unknown_absent_and_null_boundaries", + "records": [ + { + "dataset_version": "v2", + "record_id": "unknown", + "content": "Synthetic fixture unknown" + }, + { + "dataset_version": "v2", + "record_id": "absent", + "content": "Synthetic fixture absent" + }, + { + "dataset_version": "v2", + "record_id": "null", + "content": "Synthetic fixture null" + }, + { + "dataset_version": "v2", + "record_id": "grounded_absent", + "content": "Synthetic fixture grounded_absent" + } + ], + "provenance": [ + { + "dataset_version": "v2", + "record_id": "unknown", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "unknown", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v2", + "record_id": "absent", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate" + }, + { + "dataset_version": "v2", + "record_id": "null", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": null + }, + { + "dataset_version": "v2", + "record_id": "grounded_absent", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate" + } + ], + "version_order": [ + "v2" + ], + "target_record_keys": [ + "v2::absent", + "v2::grounded_absent", + "v2::null", + "v2::unknown" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [ + "W_GROUNDING_UNKNOWN" + ], + "target_declared_references": 0, + "target_resolved_references": 0, + "parent_field_states": { + "v2::unknown": "value", + "v2::absent": "absent", + "v2::null": "null", + "v2::grounded_absent": "absent" + } + }, + "expected_lineage": { + "complete_root_sets": {}, + "unresolved_reasons": { + "v2::unknown": [ + "unknown_grounding" + ], + "v2::absent": [ + "undeclared_parent_boundary" + ], + "v2::null": [ + "undeclared_parent_boundary" + ], + "v2::grounded_absent": [ + "undeclared_parent_boundary" + ] + }, + "counts": { + "target": 4, + "grounded": 0, + "closed": 0, + "unresolved": 4 + }, + "depth_by_record": { + "v2::unknown": 0, + "v2::absent": null, + "v2::null": null, + "v2::grounded_absent": null + }, + "lineage_depth": null, + "depth_unavailable_reason": "incomplete_target_parent_paths", + "no_declared_parents": true, + "edge_coverage": "1", + "resolved_lineage_coverage": "0", + "external_ancestry_coverage": "0", + "ancestry_hhi": null, + "effective_external_root_count": null, + "concentration_unavailable_reason": "no_completely_resolved_nonempty_root_sets", + "shared_ancestry_proxy": "indeterminate", + "lineage_bounds": [ + "0", + "1", + "1" + ] + } + }, + { + "case_id": "carryover_diamond_and_grounded_parent_limits", + "records": [ + { + "dataset_version": "v1", + "record_id": "a", + "content": "Synthetic fixture a" + }, + { + "dataset_version": "v1", + "record_id": "b", + "content": "Synthetic fixture b" + }, + { + "dataset_version": "v2", + "record_id": "carry", + "content": "Synthetic fixture carry" + }, + { + "dataset_version": "v2", + "record_id": "left", + "content": "Synthetic fixture left" + }, + { + "dataset_version": "v2", + "record_id": "right", + "content": "Synthetic fixture right" + }, + { + "dataset_version": "v2", + "record_id": "diamond", + "content": "Synthetic fixture diamond" + }, + { + "dataset_version": "v2", + "record_id": "grounded_transform", + "content": "Synthetic fixture grounded_transform" + }, + { + "dataset_version": "v2", + "record_id": "multi_carry", + "content": "Synthetic fixture multi_carry" + }, + { + "dataset_version": "v2", + "record_id": "incomplete_carry", + "content": "Synthetic fixture incomplete_carry" + } + ], + "provenance": [ + { + "dataset_version": "v1", + "record_id": "a", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v1", + "record_id": "b", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v2", + "record_id": "carry", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "carryover", + "parent_ids": [ + "v1::a", + "a" + ] + }, + { + "dataset_version": "v2", + "record_id": "left", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "right", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "diamond", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::left", + "v2::right" + ] + }, + { + "dataset_version": "v2", + "record_id": "grounded_transform", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [ + "v1::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "multi_carry", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "carryover", + "parent_ids": [ + "v1::a", + "v1::b" + ] + }, + { + "dataset_version": "v2", + "record_id": "incomplete_carry", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "carryover", + "parent_ids": [ + "v1::a", + "v1::missing" + ] + } + ], + "version_order": [ + "v1", + "v2" + ], + "target_record_keys": [ + "v2::carry", + "v2::diamond", + "v2::grounded_transform", + "v2::incomplete_carry", + "v2::left", + "v2::multi_carry", + "v2::right" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [ + "W_PARENT_BARE_COMPATIBILITY", + "W_PARENT_DUPLICATE_REFERENCE", + "W_PARENT_UNRESOLVED" + ], + "target_declared_references": 11, + "target_resolved_references": 10 + }, + "expected_lineage": { + "complete_root_sets": { + "v2::carry": [ + "v1::a" + ], + "v2::diamond": [ + "v1::a" + ], + "v2::left": [ + "v1::a" + ], + "v2::right": [ + "v1::a" + ] + }, + "unresolved_reasons": { + "v2::grounded_transform": [ + "unsupported_grounded_parent_rule" + ], + "v2::multi_carry": [ + "unsupported_grounded_parent_rule" + ], + "v2::incomplete_carry": [ + "missing_parent" + ] + }, + "counts": { + "target": 7, + "grounded": 4, + "closed": 0, + "unresolved": 3 + }, + "carryover_generation": 0, + "carryover_depth": 1, + "diamond_depth": 2, + "distinct_external_roots": 1, + "root_order": [ + "v1::a" + ], + "incidence": [ + 4 + ], + "incidence_shares": [ + "4/7" + ], + "fractional_masses": [ + "4" + ], + "normalized_weights": [ + "1" + ], + "ancestry_hhi": "1", + "effective_external_root_count": "1", + "resolved_lineage_coverage": "4/7", + "external_ancestry_coverage": "4/7", + "edge_coverage": "10/11", + "lineage_bounds": [ + "0", + "3/7", + "3/7" + ], + "shared_ancestry_proxy": "present" + }, + "note": "Carryover has two original declarations for one canonical parent. Diamond paths both reach a. Loaded root b has no complete grounded target and contributes no exact root mass." + }, + { + "case_id": "empty_selected_scope", + "records": [ + { + "dataset_version": "v1", + "record_id": "context", + "content": "Synthetic fixture context" + } + ], + "provenance": [ + { + "dataset_version": "v1", + "record_id": "context", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + } + ], + "version_order": [ + "v1" + ], + "target_record_keys": [], + "expected_input": { + "error_codes": [], + "required_warning_codes": [], + "target_declared_references": 0, + "target_resolved_references": 0 + }, + "expected_lineage": { + "target_dataset_version": null, + "counts": { + "target": 0, + "grounded": 0, + "closed": 0, + "unresolved": 0 + }, + "complete_root_sets": {}, + "distinct_external_roots": 0, + "no_declared_parents": true, + "edge_coverage": "1", + "resolved_lineage_coverage": null, + "external_ancestry_coverage": null, + "ancestry_hhi": null, + "effective_external_root_count": null, + "lineage_bounds": null, + "ratios_unavailable_reason": "empty_target", + "shared_ancestry_proxy": "indeterminate" + }, + "note": "Future analysis API empty selection over a loaded context. This case makes no claim that the current CLI accepts an empty primary input file." + } + ] +} diff --git a/tests/fixtures/lineage_complete/hero_expected.json b/tests/fixtures/lineage_complete/hero_expected.json new file mode 100644 index 0000000..dfdfb62 --- /dev/null +++ b/tests/fixtures/lineage_complete/hero_expected.json @@ -0,0 +1,60 @@ +{ + "purpose": "Future lineage oracle copied from the approved Hero arithmetic; canonical Hero files are unchanged.", + "canonical_source": "examples/hero/EXPECTED_OUTPUTS.md", + "target_dataset_version": "v2", + "target_records": 8, + "loaded_records": 16, + "declared_target_references": 8, + "resolved_target_references": 8, + "root_order": [ + "v1::v1_01", + "v1::v1_02", + "v1::v1_03", + "v1::v1_04", + "v1::v1_07" + ], + "incidence": [ + 3, + 2, + 1, + 1, + 1 + ], + "incidence_shares": [ + "3/8", + "1/4", + "1/8", + "1/8", + "1/8" + ], + "fractional_masses": [ + "3", + "2", + "1", + "1", + "1" + ], + "normalized_weights": [ + "3/8", + "1/4", + "1/8", + "1/8", + "1/8" + ], + "ancestry_hhi": "1/4", + "effective_external_root_count": "4", + "lineage_bounds": [ + "0", + "0", + "0" + ], + "direct_bounds": [ + "1/2", + "1/2", + "0" + ], + "distinct_external_roots": 5, + "cycle_detected": false, + "shared_ancestry_proxy": "present", + "lineage_execution": "future_completed_when_requested" +} diff --git a/tests/fixtures/lineage_cycles/README.md b/tests/fixtures/lineage_cycles/README.md index 87291d0..49249ce 100644 --- a/tests/fixtures/lineage_cycles/README.md +++ b/tests/fixtures/lineage_cycles/README.md @@ -1,7 +1,20 @@ -# lineage_cycles fixture placeholder +# Cycle components and affected descendants -Phase 1 status: structure only. +These Phase 5 Step 1 fixtures distinguish cycle members from descendants affected +along parent-to-child edges and from independent target records. A disconnected +cycle in the loaded context remains a whole-graph lineage error even when every +selected target has complete ancestry. General cycle detection starts in Step 3. -This directory is reserved for the approved `lineage_cycles` validation fixture. -It contains no real user data and no analytical expected values in Phase 1. -Data may be added only in the phase that owns the corresponding behavior. +Current input validation accepts valid same-version references in the general +cycle cases, without establishing acyclicity. A direct self-parent already +produces `E_LINEAGE_CYCLE`; its expected input error is recorded separately. +Future graph construction must retain self-reference evidence for diagnosis while +rejecting it as a valid ancestry edge. + +`cycle_count` counts cyclic strongly connected components. It never counts all +simple cycles, individual members or affected descendants. Future cycle failures +must preserve independent results and return an unsuccessful audit exit status. +Step 1 does not execute or claim any of those future graph results. + +Shared fixture format and execution limits are documented in +`../lineage_complete/README.md`. diff --git a/tests/fixtures/lineage_cycles/cases.json b/tests/fixtures/lineage_cycles/cases.json new file mode 100644 index 0000000..bd2499d --- /dev/null +++ b/tests/fixtures/lineage_cycles/cases.json @@ -0,0 +1,530 @@ +{ + "purpose": "Independent Phase 5 acceptance inputs and future lineage oracles; no lineage execution is claimed by Step 1.", + "fraction_encoding": "Exact rational strings; null means unavailable and requires the adjacent reason.", + "cases": [ + { + "case_id": "two_node_cycle_and_descendants", + "records": [ + { + "dataset_version": "v2", + "record_id": "a", + "content": "Synthetic fixture a" + }, + { + "dataset_version": "v2", + "record_id": "b", + "content": "Synthetic fixture b" + }, + { + "dataset_version": "v2", + "record_id": "descendant", + "content": "Synthetic fixture descendant" + }, + { + "dataset_version": "v2", + "record_id": "independent", + "content": "Synthetic fixture independent" + } + ], + "provenance": [ + { + "dataset_version": "v2", + "record_id": "a", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::b" + ] + }, + { + "dataset_version": "v2", + "record_id": "b", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "descendant", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "independent", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + } + ], + "version_order": [ + "v2" + ], + "target_record_keys": [ + "v2::a", + "v2::b", + "v2::descendant", + "v2::independent" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [], + "target_declared_references": 3, + "target_resolved_references": 3 + }, + "expected_lineage": { + "cycle_detected": true, + "cycle_count": 1, + "cycle_count_method": "cyclic_strongly_connected_components", + "cyclic_components": [ + [ + "v2::a", + "v2::b" + ] + ], + "cycle_member_keys": [ + "v2::a", + "v2::b" + ], + "affected_record_keys": [ + "v2::a", + "v2::b", + "v2::descendant" + ], + "complete_root_sets": { + "v2::independent": [ + "v2::independent" + ] + }, + "counts": { + "target": 4, + "grounded": 1, + "closed": 0, + "unresolved": 3 + }, + "lineage_depth": null, + "depth_unavailable_reason": "cycle_affected_target_paths", + "lineage_error": true, + "cli_audit_success": false, + "edge_coverage": "1", + "lineage_bounds": [ + "0", + "3/4", + "3/4" + ] + }, + "note": "Current immediate-reference validation accepts these references. General cycle detection remains Phase 5 Step 3." + }, + { + "case_id": "disconnected_context_cycle", + "records": [ + { + "dataset_version": "v1", + "record_id": "a", + "content": "Synthetic fixture a" + }, + { + "dataset_version": "v1", + "record_id": "b", + "content": "Synthetic fixture b" + }, + { + "dataset_version": "v2", + "record_id": "root", + "content": "Synthetic fixture root" + } + ], + "provenance": [ + { + "dataset_version": "v1", + "record_id": "a", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::b" + ] + }, + { + "dataset_version": "v1", + "record_id": "b", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "root", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + } + ], + "version_order": [ + "v1", + "v2" + ], + "target_record_keys": [ + "v2::root" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [], + "target_declared_references": 0, + "target_resolved_references": 0 + }, + "expected_lineage": { + "cycle_detected": true, + "cycle_count": 1, + "cycle_count_method": "cyclic_strongly_connected_components", + "cyclic_components": [ + [ + "v1::a", + "v1::b" + ] + ], + "cycle_member_keys": [ + "v1::a", + "v1::b" + ], + "affected_record_keys": [ + "v1::a", + "v1::b" + ], + "affected_target_record_keys": [], + "complete_root_sets": { + "v2::root": [ + "v2::root" + ] + }, + "counts": { + "target": 1, + "grounded": 1, + "closed": 0, + "unresolved": 0 + }, + "lineage_depth": 0, + "lineage_error": true, + "cli_audit_success": false, + "no_declared_parents": true, + "edge_coverage": "1", + "lineage_bounds": [ + "0", + "0", + "0" + ] + }, + "note": "A loaded disconnected cycle remains a lineage error even though target ancestry is fully resolved." + }, + { + "case_id": "self_cycle_and_descendant", + "records": [ + { + "dataset_version": "v2", + "record_id": "self", + "content": "Synthetic fixture self" + }, + { + "dataset_version": "v2", + "record_id": "descendant", + "content": "Synthetic fixture descendant" + }, + { + "dataset_version": "v2", + "record_id": "closed", + "content": "Synthetic fixture closed" + } + ], + "provenance": [ + { + "dataset_version": "v2", + "record_id": "self", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::self" + ] + }, + { + "dataset_version": "v2", + "record_id": "descendant", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::self" + ] + }, + { + "dataset_version": "v2", + "record_id": "closed", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [] + } + ], + "version_order": [ + "v2" + ], + "target_record_keys": [ + "v2::closed", + "v2::descendant", + "v2::self" + ], + "expected_input": { + "error_codes": [ + "E_LINEAGE_CYCLE" + ], + "required_warning_codes": [] + }, + "expected_lineage": { + "cycle_detected": true, + "cycle_count": 1, + "cycle_count_method": "cyclic_strongly_connected_components", + "cyclic_components": [ + [ + "v2::self" + ] + ], + "cycle_member_keys": [ + "v2::self" + ], + "affected_record_keys": [ + "v2::descendant", + "v2::self" + ], + "complete_root_sets": { + "v2::closed": [] + }, + "counts": { + "target": 3, + "grounded": 0, + "closed": 1, + "unresolved": 2 + }, + "lineage_error": true, + "cli_audit_success": false, + "lineage_bounds": [ + "1/3", + "1", + "2/3" + ] + }, + "note": "Current validator rejects the direct self-parent. Future graph construction must retain explicit self-reference evidence without accepting it as a valid ancestry edge." + }, + { + "case_id": "two_components_with_multiple_simple_cycles", + "records": [ + { + "dataset_version": "v2", + "record_id": "a", + "content": "Synthetic fixture a" + }, + { + "dataset_version": "v2", + "record_id": "b", + "content": "Synthetic fixture b" + }, + { + "dataset_version": "v2", + "record_id": "c", + "content": "Synthetic fixture c" + }, + { + "dataset_version": "v2", + "record_id": "d", + "content": "Synthetic fixture d" + }, + { + "dataset_version": "v2", + "record_id": "e", + "content": "Synthetic fixture e" + }, + { + "dataset_version": "v2", + "record_id": "f", + "content": "Synthetic fixture f" + }, + { + "dataset_version": "v2", + "record_id": "root", + "content": "Synthetic fixture root" + } + ], + "provenance": [ + { + "dataset_version": "v2", + "record_id": "a", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::b", + "v2::c" + ] + }, + { + "dataset_version": "v2", + "record_id": "b", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::a", + "v2::c" + ] + }, + { + "dataset_version": "v2", + "record_id": "c", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::a", + "v2::b" + ] + }, + { + "dataset_version": "v2", + "record_id": "d", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::e" + ] + }, + { + "dataset_version": "v2", + "record_id": "e", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::d" + ] + }, + { + "dataset_version": "v2", + "record_id": "f", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v2::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "root", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + } + ], + "version_order": [ + "v2" + ], + "target_record_keys": [ + "v2::a", + "v2::b", + "v2::c", + "v2::d", + "v2::e", + "v2::f", + "v2::root" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [], + "target_declared_references": 9, + "target_resolved_references": 9 + }, + "expected_lineage": { + "cycle_detected": true, + "cycle_count": 2, + "cycle_count_method": "cyclic_strongly_connected_components", + "cyclic_components": [ + [ + "v2::a", + "v2::b", + "v2::c" + ], + [ + "v2::d", + "v2::e" + ] + ], + "cycle_member_keys": [ + "v2::a", + "v2::b", + "v2::c", + "v2::d", + "v2::e" + ], + "affected_record_keys": [ + "v2::a", + "v2::b", + "v2::c", + "v2::d", + "v2::e", + "v2::f" + ], + "complete_root_sets": { + "v2::root": [ + "v2::root" + ] + }, + "counts": { + "target": 7, + "grounded": 1, + "closed": 0, + "unresolved": 6 + }, + "lineage_error": true, + "cli_audit_success": false, + "edge_coverage": "1", + "lineage_bounds": [ + "0", + "6/7", + "6/7" + ] + }, + "note": "The three-node component contains several simple cycles but counts once; the second component counts once." + } + ] +} diff --git a/tests/fixtures/lineage_multi_root/README.md b/tests/fixtures/lineage_multi_root/README.md index 91bde3d..16da13f 100644 --- a/tests/fixtures/lineage_multi_root/README.md +++ b/tests/fixtures/lineage_multi_root/README.md @@ -1,7 +1,23 @@ -# lineage_multi_root fixture placeholder +# Multi-root and independent uncertainty inputs -Phase 1 status: structure only. +These three Phase 5 Step 1 cases each load seven records and select four targets. +Targets t1 and t2 have complete roots {a} and {a,b}; t3 is known-empty; t4 remains +unresolved. Context records a, b and u never enlarge the target denominator. -This directory is reserved for the approved `lineage_multi_root` validation fixture. -It contains no real user data and no analytical expected values in Phase 1. -Data may be added only in the phase that owns the corresponding behavior. +The mixed case gives t4 both an unknown-grounding ancestor and a missing parent. +The two variants isolate each failure: remove the missing reference while keeping +u unknown, or ground u while retaining the missing reference. Either defect alone +keeps t4 out of exact root allocation. Grounding u in the latter variant does not +make u a contributing root while its only target remains unresolved. + +Hand arithmetic: incidence is (2,1), with target shares (1/2,1/4). Fractional mass +is (3/2,1/2); normalization by two externally rooted targets yields (3/4,1/4). +Thus HHI is 9/16 + 1/16 = 5/8 and effective roots are 8/5. Resolved coverage is +3/4, external ancestry coverage is 1/2, and closure bounds are [1/4,1/2]. Width +is 1/4. Reference coverage is 5/6 with a missing reference and 1 in the +unknown-only variant. Perfect reference coverage alone does not resolve unknown +grounding. + +`cases.json` is future acceptance data. Step 1 executes only its input-validation +expectations. Shared format and execution limits are documented in +`../lineage_complete/README.md`. diff --git a/tests/fixtures/lineage_multi_root/cases.json b/tests/fixtures/lineage_multi_root/cases.json new file mode 100644 index 0000000..7901641 --- /dev/null +++ b/tests/fixtures/lineage_multi_root/cases.json @@ -0,0 +1,583 @@ +{ + "purpose": "Independent Phase 5 acceptance inputs and future lineage oracles; no lineage execution is claimed by Step 1.", + "fraction_encoding": "Exact rational strings; null means unavailable and requires the adjacent reason.", + "cases": [ + { + "case_id": "mixed_missing_and_unknown", + "records": [ + { + "dataset_version": "v1", + "record_id": "a", + "content": "Synthetic fixture a" + }, + { + "dataset_version": "v1", + "record_id": "b", + "content": "Synthetic fixture b" + }, + { + "dataset_version": "v1", + "record_id": "u", + "content": "Synthetic fixture u" + }, + { + "dataset_version": "v2", + "record_id": "t1", + "content": "Synthetic fixture t1" + }, + { + "dataset_version": "v2", + "record_id": "t2", + "content": "Synthetic fixture t2" + }, + { + "dataset_version": "v2", + "record_id": "t3", + "content": "Synthetic fixture t3" + }, + { + "dataset_version": "v2", + "record_id": "t4", + "content": "Synthetic fixture t4" + } + ], + "provenance": [ + { + "dataset_version": "v1", + "record_id": "a", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v1", + "record_id": "b", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v1", + "record_id": "u", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "unknown", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v2", + "record_id": "t1", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "t2", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a", + "v1::b" + ] + }, + { + "dataset_version": "v2", + "record_id": "t3", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v2", + "record_id": "t4", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a", + "v1::u", + "v1::missing" + ] + } + ], + "version_order": [ + "v1", + "v2" + ], + "target_record_keys": [ + "v2::t1", + "v2::t2", + "v2::t3", + "v2::t4" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [ + "W_GROUNDING_UNKNOWN", + "W_PARENT_UNRESOLVED" + ], + "target_declared_references": 6, + "target_resolved_references": 5 + }, + "expected_lineage": { + "complete_root_sets": { + "v2::t1": [ + "v1::a" + ], + "v2::t2": [ + "v1::a", + "v1::b" + ], + "v2::t3": [] + }, + "unresolved_reasons": { + "v2::t4": [ + "unknown_grounding", + "missing_parent" + ] + }, + "counts": { + "target": 4, + "grounded": 2, + "closed": 1, + "unresolved": 1 + }, + "loaded_records": 7, + "records_with_resolved_external_ancestry": 3, + "distinct_external_roots": 2, + "root_order": [ + "v1::a", + "v1::b" + ], + "incidence": [ + 2, + 1 + ], + "incidence_shares": [ + "1/2", + "1/4" + ], + "incidence_denominator": 4, + "fractional_masses": [ + "3/2", + "1/2" + ], + "normalized_weights": [ + "3/4", + "1/4" + ], + "mass_denominator": 2, + "ancestry_hhi": "5/8", + "effective_external_root_count": "8/5", + "resolved_lineage_coverage": "3/4", + "external_ancestry_coverage": "1/2", + "edge_coverage": "5/6", + "lineage_bounds": [ + "1/4", + "1/2", + "1/4" + ], + "shared_ancestry_proxy": "present", + "cycle_detected": false + } + }, + { + "case_id": "unknown_without_missing", + "records": [ + { + "dataset_version": "v1", + "record_id": "a", + "content": "Synthetic fixture a" + }, + { + "dataset_version": "v1", + "record_id": "b", + "content": "Synthetic fixture b" + }, + { + "dataset_version": "v1", + "record_id": "u", + "content": "Synthetic fixture u" + }, + { + "dataset_version": "v2", + "record_id": "t1", + "content": "Synthetic fixture t1" + }, + { + "dataset_version": "v2", + "record_id": "t2", + "content": "Synthetic fixture t2" + }, + { + "dataset_version": "v2", + "record_id": "t3", + "content": "Synthetic fixture t3" + }, + { + "dataset_version": "v2", + "record_id": "t4", + "content": "Synthetic fixture t4" + } + ], + "provenance": [ + { + "dataset_version": "v1", + "record_id": "a", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v1", + "record_id": "b", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v1", + "record_id": "u", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "unknown", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v2", + "record_id": "t1", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "t2", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a", + "v1::b" + ] + }, + { + "dataset_version": "v2", + "record_id": "t3", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v2", + "record_id": "t4", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a", + "v1::u" + ] + } + ], + "version_order": [ + "v1", + "v2" + ], + "target_record_keys": [ + "v2::t1", + "v2::t2", + "v2::t3", + "v2::t4" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [ + "W_GROUNDING_UNKNOWN" + ], + "target_declared_references": 5, + "target_resolved_references": 5 + }, + "expected_lineage": { + "complete_root_sets": { + "v2::t1": [ + "v1::a" + ], + "v2::t2": [ + "v1::a", + "v1::b" + ], + "v2::t3": [] + }, + "unresolved_reasons": { + "v2::t4": [ + "unknown_grounding" + ] + }, + "counts": { + "target": 4, + "grounded": 2, + "closed": 1, + "unresolved": 1 + }, + "loaded_records": 7, + "records_with_resolved_external_ancestry": 3, + "distinct_external_roots": 2, + "root_order": [ + "v1::a", + "v1::b" + ], + "incidence": [ + 2, + 1 + ], + "incidence_shares": [ + "1/2", + "1/4" + ], + "incidence_denominator": 4, + "fractional_masses": [ + "3/2", + "1/2" + ], + "normalized_weights": [ + "3/4", + "1/4" + ], + "mass_denominator": 2, + "ancestry_hhi": "5/8", + "effective_external_root_count": "8/5", + "resolved_lineage_coverage": "3/4", + "external_ancestry_coverage": "1/2", + "edge_coverage": "1", + "lineage_bounds": [ + "1/4", + "1/2", + "1/4" + ], + "shared_ancestry_proxy": "present", + "cycle_detected": false + } + }, + { + "case_id": "missing_without_unknown", + "records": [ + { + "dataset_version": "v1", + "record_id": "a", + "content": "Synthetic fixture a" + }, + { + "dataset_version": "v1", + "record_id": "b", + "content": "Synthetic fixture b" + }, + { + "dataset_version": "v1", + "record_id": "u", + "content": "Synthetic fixture u" + }, + { + "dataset_version": "v2", + "record_id": "t1", + "content": "Synthetic fixture t1" + }, + { + "dataset_version": "v2", + "record_id": "t2", + "content": "Synthetic fixture t2" + }, + { + "dataset_version": "v2", + "record_id": "t3", + "content": "Synthetic fixture t3" + }, + { + "dataset_version": "v2", + "record_id": "t4", + "content": "Synthetic fixture t4" + } + ], + "provenance": [ + { + "dataset_version": "v1", + "record_id": "a", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v1", + "record_id": "b", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v1", + "record_id": "u", + "source_type": "human", + "provenance_confidence": "confirmed", + "external_grounding": "yes", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v2", + "record_id": "t1", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a" + ] + }, + { + "dataset_version": "v2", + "record_id": "t2", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a", + "v1::b" + ] + }, + { + "dataset_version": "v2", + "record_id": "t3", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [] + }, + { + "dataset_version": "v2", + "record_id": "t4", + "source_type": "synthetic", + "provenance_confidence": "confirmed", + "external_grounding": "no", + "transformation": "generate", + "parent_ids": [ + "v1::a", + "v1::u", + "v1::missing" + ] + } + ], + "version_order": [ + "v1", + "v2" + ], + "target_record_keys": [ + "v2::t1", + "v2::t2", + "v2::t3", + "v2::t4" + ], + "expected_input": { + "error_codes": [], + "required_warning_codes": [ + "W_PARENT_UNRESOLVED" + ], + "target_declared_references": 6, + "target_resolved_references": 5 + }, + "expected_lineage": { + "complete_root_sets": { + "v2::t1": [ + "v1::a" + ], + "v2::t2": [ + "v1::a", + "v1::b" + ], + "v2::t3": [] + }, + "unresolved_reasons": { + "v2::t4": [ + "missing_parent" + ] + }, + "counts": { + "target": 4, + "grounded": 2, + "closed": 1, + "unresolved": 1 + }, + "loaded_records": 7, + "records_with_resolved_external_ancestry": 3, + "distinct_external_roots": 2, + "root_order": [ + "v1::a", + "v1::b" + ], + "incidence": [ + 2, + 1 + ], + "incidence_shares": [ + "1/2", + "1/4" + ], + "incidence_denominator": 4, + "fractional_masses": [ + "3/2", + "1/2" + ], + "normalized_weights": [ + "3/4", + "1/4" + ], + "mass_denominator": 2, + "ancestry_hhi": "5/8", + "effective_external_root_count": "8/5", + "resolved_lineage_coverage": "3/4", + "external_ancestry_coverage": "1/2", + "edge_coverage": "5/6", + "lineage_bounds": [ + "1/4", + "1/2", + "1/4" + ], + "shared_ancestry_proxy": "present", + "cycle_detected": false + } + } + ] +} diff --git a/tests/golden/phase4_hero_redacted.json b/tests/golden/phase4_hero_redacted.json index 5ca0a7d..27eeef7 100644 --- a/tests/golden/phase4_hero_redacted.json +++ b/tests/golden/phase4_hero_redacted.json @@ -33,7 +33,7 @@ "python_version": null, "random_seed": null, "redacted_mode": true, - "report_schema_version": "1.0", + "report_schema_version": "1.1", "resolved_options": { "comparison_requested": true, "privacy_mode": "redacted", @@ -47,7 +47,7 @@ "run_status": "complete", "started_at": "2000-01-01T00:00:00+00:00", "strict_mode": false, - "toolkit_version": "0.1.0.dev2" + "toolkit_version": "0.1.0.dev4" }, "inputs": { "artifacts": [ @@ -401,10 +401,10 @@ }, "coverage_reason": null, "execution_reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "execution_scope": [], - "execution_status": "deferred", + "execution_status": "not_requested", "notes": [ "Input eligibility only; downstream analytical implementations remain deferred.", "Reference-entry resolution coverage is not ancestry or external-root coverage.", @@ -486,7 +486,7 @@ "limitations": [ "R_MODEL_EVIDENCE_MISSING", "R_SCENARIO_NOT_CONFIGURED", - "Input eligibility is distinct from executed analysis; Phase 5 lineage remains deferred." + "Input eligibility is distinct from explicitly supplied executed analysis." ], "maximum_level": 4, "partial_evidence": [ @@ -607,10 +607,10 @@ }, "coverage_reason": null, "execution_reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "execution_scope": [], - "execution_status": "deferred", + "execution_status": "not_requested", "notes": [ "Input eligibility only; downstream analytical implementations remain deferred.", "Reference-entry resolution coverage is not ancestry or external-root coverage.", @@ -698,18 +698,18 @@ "denominator_reason": "denominator_not_applicable_or_unavailable", "evidence_class": "observed_fact", "limitations": [ - "No general graph-cycle traversal executes in Phase 4." + "No executed graph-cycle result was supplied to this report." ], "method_id": "T6.graph_cycle_check", "owner_ids": [ "T6" ], "reason_codes": [ - "R_GRAPH_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "representation": null, "required_evidence": [ - "Phase_5_graph_analysis" + "explicit_lineage_result" ], "scope": { "dataset_versions": [ @@ -5650,7 +5650,7 @@ { "assumptions": [], "blocking_evidence": [ - "Phase 4 does not compute ancestry; topology alone would not identify a causal contribution." + "Supplied parent topology and fractional root allocation do not identify a causal contribution." ], "conclusion": "causal_ancestor_effect", "coverage": null, @@ -5666,8 +5666,7 @@ "T4" ], "reason_codes": [ - "R_CAUSAL_EVIDENCE_MISSING", - "R_LINEAGE_EXECUTION_DEFERRED" + "R_CAUSAL_EVIDENCE_MISSING" ], "related_capability": "lineage", "representation": null, @@ -6222,7 +6221,7 @@ { "assumptions": [], "blocking_evidence": [ - "Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes." + "No complete successful lineage execution was supplied; independently completed observations remain visible." ], "conclusion": "lineage_analysis", "coverage": null, @@ -6238,15 +6237,15 @@ "PR-014" ], "reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "related_capability": "lineage", "representation": null, "required_evidence": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Supply an explicit lineage result for the validated target and its loaded context." ], "required_next_metadata": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Supply an explicit lineage result for the validated target and its loaded context." ], "scope": { "dataset_versions": [ @@ -6325,19 +6324,19 @@ "record_count": 16, "scope_id": "hmac-sha256:fe3080b69d042a9eca2d392f8429412803f06dd3eb9e71fd16178d05e5971ac3" }, - "statement": "General lineage graph analysis is deferred to Phase 5.", + "statement": "A completed whole-target lineage analysis is unavailable in this report.", "status": "unavailable", "theory_map_ids": [], - "theory_or_product_limit": "Implementation is deferred even when the existing input classifier marks lineage available.", + "theory_or_product_limit": "Input eligibility alone does not establish executed ancestry analysis.", "trace_ids": [], "unit": "conclusion" }, { "assumptions": [], "blocking_evidence": [ - "Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes." + "Missing, invalid or unknown ancestry remains unresolved; any complete subset retains its reported coverage." ], - "conclusion": "lineage_closure_exposure", + "conclusion": "external_ancestry", "coverage": null, "coverage_reason": "This conclusion has no measured evidence coverage.", "denominator": null, @@ -6346,20 +6345,20 @@ "limitations": [ "Unavailable means the supplied evidence does not support this conclusion; it does not establish that the conclusion is false." ], - "method_id": "T3.unavailable_conclusion", + "method_id": "T4.unavailable_conclusion", "owner_ids": [ - "T3" + "T4" ], "reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "related_capability": "lineage", "representation": null, "required_evidence": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Complete the required parent references, chronology and external-grounding evidence." ], "required_next_metadata": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Complete the required parent references, chronology and external-grounding evidence." ], "scope": { "dataset_versions": [ @@ -6438,21 +6437,21 @@ "record_count": 16, "scope_id": "hmac-sha256:fe3080b69d042a9eca2d392f8429412803f06dd3eb9e71fd16178d05e5971ac3" }, - "statement": "Lineage closure exposure is deferred to Phase 5.", + "statement": "Complete external ancestry for every target is unavailable.", "status": "unavailable", "theory_map_ids": [], - "theory_or_product_limit": "Implementation is deferred even when the existing input classifier marks lineage available.", + "theory_or_product_limit": "Partial ancestry cannot be promoted to whole-target completeness.", "trace_ids": [ - "T3" + "T4" ], "unit": "conclusion" }, { "assumptions": [], "blocking_evidence": [ - "Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes." + "No available explicitly supplied lineage interval covers a nonempty completed target partition." ], - "conclusion": "external_ancestry", + "conclusion": "lineage_closure_exposure", "coverage": null, "coverage_reason": "This conclusion has no measured evidence coverage.", "denominator": null, @@ -6461,20 +6460,20 @@ "limitations": [ "Unavailable means the supplied evidence does not support this conclusion; it does not establish that the conclusion is false." ], - "method_id": "T4.unavailable_conclusion", + "method_id": "T3.unavailable_conclusion", "owner_ids": [ - "T4" + "T3" ], "reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "related_capability": "lineage", "representation": null, "required_evidence": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Supply lineage closure bounds from the same explicit lineage result." ], "required_next_metadata": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Supply lineage closure bounds from the same explicit lineage result." ], "scope": { "dataset_versions": [ @@ -6553,12 +6552,12 @@ "record_count": 16, "scope_id": "hmac-sha256:fe3080b69d042a9eca2d392f8429412803f06dd3eb9e71fd16178d05e5971ac3" }, - "statement": "External ancestry results are deferred to Phase 5.", + "statement": "A lineage closure interval is unavailable in this report.", "status": "unavailable", "theory_map_ids": [], - "theory_or_product_limit": "Implementation is deferred even when the existing input classifier marks lineage available.", + "theory_or_product_limit": "Unavailable intervals are never replaced by a guessed numeric value.", "trace_ids": [ - "T4" + "T3" ], "unit": "conclusion" } diff --git a/tests/golden/phase4_hero_redacted.md b/tests/golden/phase4_hero_redacted.md index ff78d13..b95f9b7 100644 --- a/tests/golden/phase4_hero_redacted.md +++ b/tests/golden/phase4_hero_redacted.md @@ -4,7 +4,7 @@ - Run ID: `"phase4-step9-golden"`. - Maximum observability level: 4; `"longitudinal_dataset_observability"`. -- Capability statuses: Ingestion: `"available"` (execution `"completed"`); Content diagnostics: `"available"` (execution `"completed"`); Provenance: `"available"` (execution `"completed"`); Lineage: `"available"` (execution `"deferred"`); Dataset longitudinal: `"available"` (execution `"partial"`); Model longitudinal: `"unavailable"` (execution `"deferred"`); Intervention simulation: `"unavailable"` (execution `"not_requested"`). +- Capability statuses: Ingestion: `"available"` (execution `"completed"`); Content diagnostics: `"available"` (execution `"completed"`); Provenance: `"available"` (execution `"completed"`); Lineage: `"available"` (execution `"not_requested"`); Dataset longitudinal: `"available"` (execution `"partial"`); Model longitudinal: `"unavailable"` (execution `"deferred"`); Intervention simulation: `"unavailable"` (execution `"not_requested"`). - Record scope: `"hmac-sha256:fe3080b69d042a9eca2d392f8429412803f06dd3eb9e71fd16178d05e5971ac3"`; dataset versions `["hmac-sha256:4e02dbb7e57fd003c1b4a2d16e058bca5c848e94f5b8730261ad64c1175dfef4", "hmac-sha256:70696b56c6ef6dbd4145752d88c4da6f71498a8f35a521e3946d89acc2a56a5d"]`; records 16; excluded records 0. - Representation: `"hmac-sha256:220b787af129ef50eab4ee9f3c90a00a7bce0a525c6e49bfc40fe9bb8af1d0b8"`; source `"hmac-sha256:1faa75c58dd3f727bec66d81de5cab7c2e64afbebcd03c32653ba5432a5c4369"`; version `"hmac-sha256:b6d6724490a6778f4b6e234f43fff3602f2707d6b9ab74acb0cc8e83c96d3da8"`. Complete mapping metadata appears in Input inventory. - Warning entries: 0; error entries: 0. @@ -37,7 +37,7 @@ Quoted code literals represent supplied data, including identifiers and labels. | `["python_version"]` | Unavailable (null): `"private_or_nonstandard_metadata_omitted"` | | `["random_seed"]` | Unavailable (null): `"not_recorded"` | | `["redacted_mode"]` | true | -| `["report_schema_version"]` | `"1.0"` | +| `["report_schema_version"]` | `"1.1"` | | `["resolved_options"]["comparison_requested"]` | true | | `["resolved_options"]["privacy_mode"]` | `"redacted"` | | `["resolved_options"]["record_id_mode"]` | `"hash"` | @@ -49,7 +49,7 @@ Quoted code literals represent supplied data, including identifiers and labels. | `["run_status"]` | `"complete"` | | `["started_at"]` | `"2000-01-01T00:00:00+00:00"` | | `["strict_mode"]` | false | -| `["toolkit_version"]` | `"0.1.0.dev2"` | +| `["toolkit_version"]` | `"0.1.0.dev4"` | **`["identifier_protection"]["limitations"]`** @@ -193,7 +193,7 @@ Quoted code literals represent supplied data, including identifiers and labels. - `"R_MODEL_EVIDENCE_MISSING"` - `"R_SCENARIO_NOT_CONFIGURED"` -- `"Input eligibility is distinct from executed analysis; Phase 5 lineage remains deferred."` +- `"Input eligibility is distinct from explicitly supplied executed analysis."` The equal observability.capabilities compatibility mirror is represented once in the Capability matrix section. @@ -204,7 +204,7 @@ The equal observability.capabilities compatibility mirror is represented once in | Ingestion | `"available"` | 1.0 | `[]` | `"completed"` | `[]` | | Content diagnostics | `"available"` | 1.0 | `[]` | `"completed"` | `[]` | | Provenance | `"available"` | 1.0 | `[]` | `"completed"` | `[]` | -| Lineage | `"available"` | 1.0 | `[]` | `"deferred"` | `["R_LINEAGE_EXECUTION_DEFERRED"]` | +| Lineage | `"available"` | 1.0 | `[]` | `"not_requested"` | `["R_ANALYSIS_NOT_REQUESTED"]` | | Dataset longitudinal | `"available"` | 1.0 | `[]` | `"partial"` | `["R_LONGITUDINAL_FAMILIES_DEFERRED"]` | | Model longitudinal | `"unavailable"` | Unavailable (null): `"coverage_not_supplied_or_empty_denominator"` | `["R_MODEL_EVIDENCE_MISSING"]` | `"deferred"` | `["R_MODEL_ANALYSIS_DEFERRED"]` | | Intervention simulation | `"unavailable"` | Unavailable (null): `"coverage_not_supplied_or_empty_denominator"` | `["R_SCENARIO_NOT_CONFIGURED"]` | `"not_requested"` | `["R_ANALYSIS_NOT_REQUESTED"]` | @@ -323,7 +323,7 @@ Coverage details and unresolved counts, when supplied, follow below. Related dia Status: `"unavailable"`; evidence class: `"observed_fact"`; unit: `"status"`; method: `"T6.graph_cycle_check"`. -Value: Unavailable (null): reason codes `["R_GRAPH_EXECUTION_DEFERRED"]`. +Value: Unavailable (null): reason codes `["R_ANALYSIS_NOT_REQUESTED"]`. Denominator: Unavailable (null): `"denominator_not_applicable_or_unavailable"`; coverage (ratio): Unavailable (null): `"coverage_not_supplied_for_this_result"`. @@ -337,9 +337,9 @@ Denominator: Unavailable (null): `"denominator_not_applicable_or_unavailable"`; | `["evidence_class"]` | `"observed_fact"` | | `["method_id"]` | `"T6.graph_cycle_check"` | | `["owner_ids"]` | `["T6"]` | -| `["reason_codes"]` | `["R_GRAPH_EXECUTION_DEFERRED"]` | +| `["reason_codes"]` | `["R_ANALYSIS_NOT_REQUESTED"]` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | -| `["required_evidence"]` | `["Phase_5_graph_analysis"]` | +| `["required_evidence"]` | `["explicit_lineage_result"]` | | `["scope"]["dataset_versions"]` | `["hmac-sha256:4e02dbb7e57fd003c1b4a2d16e058bca5c848e94f5b8730261ad64c1175dfef4", "hmac-sha256:70696b56c6ef6dbd4145752d88c4da6f71498a8f35a521e3946d89acc2a56a5d"]` | | `["scope"]["denominator_basis"]` | `"all_validated_bundle_records"` | | `["scope"]["excluded_record_count"]` | 0 | @@ -350,11 +350,11 @@ Denominator: Unavailable (null): `"denominator_not_applicable_or_unavailable"`; | `["theory_map_ids"]` | `[]` | | `["trace_ids"]` | `["T6"]` | | `["unit"]` | `"status"` | -| `["value"]` | Unavailable (null): reason codes `["R_GRAPH_EXECUTION_DEFERRED"]` | +| `["value"]` | Unavailable (null): reason codes `["R_ANALYSIS_NOT_REQUESTED"]` | **`["limitations"]`** -- `"No general graph-cycle traversal executes in Phase 4."` +- `"No executed graph-cycle result was supplied to this report."` **`["scope"]["included_record_keys"]`** @@ -3763,7 +3763,7 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; | Field | Value | |---|---| | `["assumptions"]` | `[]` | -| `["blocking_evidence"]` | `["Phase 4 does not compute ancestry; topology alone would not identify a causal contribution."]` | +| `["blocking_evidence"]` | `["Supplied parent topology and fractional root allocation do not identify a causal contribution."]` | | `["conclusion"]` | `"causal_ancestor_effect"` | | `["coverage"]` | Unavailable (null): `"This conclusion has no measured evidence coverage."` | | `["coverage_reason"]` | `"This conclusion has no measured evidence coverage."` | @@ -3772,7 +3772,7 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; | `["evidence_class"]` | `"unavailable_conclusion"` | | `["method_id"]` | `"T4.unavailable_conclusion"` | | `["owner_ids"]` | `["T4"]` | -| `["reason_codes"]` | `["R_CAUSAL_EVIDENCE_MISSING", "R_LINEAGE_EXECUTION_DEFERRED"]` | +| `["reason_codes"]` | `["R_CAUSAL_EVIDENCE_MISSING"]` | | `["related_capability"]` | `"lineage"` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | | `["required_evidence"]` | `["An identified ancestor, measured outcomes and a controlled or otherwise justified causal design."]` | @@ -4078,7 +4078,7 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; | Field | Value | |---|---| | `["assumptions"]` | `[]` | -| `["blocking_evidence"]` | `["Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes."]` | +| `["blocking_evidence"]` | `["No complete successful lineage execution was supplied; independently completed observations remain visible."]` | | `["conclusion"]` | `"lineage_analysis"` | | `["coverage"]` | Unavailable (null): `"This conclusion has no measured evidence coverage."` | | `["coverage_reason"]` | `"This conclusion has no measured evidence coverage."` | @@ -4087,21 +4087,21 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; | `["evidence_class"]` | `"unavailable_conclusion"` | | `["method_id"]` | `"PR-014.unavailable_conclusion"` | | `["owner_ids"]` | `["PR-014"]` | -| `["reason_codes"]` | `["R_LINEAGE_EXECUTION_DEFERRED"]` | +| `["reason_codes"]` | `["R_ANALYSIS_NOT_REQUESTED"]` | | `["related_capability"]` | `"lineage"` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | -| `["required_evidence"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | -| `["required_next_metadata"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | +| `["required_evidence"]` | `["Supply an explicit lineage result for the validated target and its loaded context."]` | +| `["required_next_metadata"]` | `["Supply an explicit lineage result for the validated target and its loaded context."]` | | `["scope"]["dataset_versions"]` | `["hmac-sha256:4e02dbb7e57fd003c1b4a2d16e058bca5c848e94f5b8730261ad64c1175dfef4", "hmac-sha256:70696b56c6ef6dbd4145752d88c4da6f71498a8f35a521e3946d89acc2a56a5d"]` | | `["scope"]["denominator_basis"]` | `"all_validated_bundle_records"` | | `["scope"]["excluded_record_count"]` | 0 | | `["scope"]["excluded_record_keys"]` | `[]` | | `["scope"]["record_count"]` | 16 | | `["scope"]["scope_id"]` | `"hmac-sha256:fe3080b69d042a9eca2d392f8429412803f06dd3eb9e71fd16178d05e5971ac3"` | -| `["statement"]` | `"General lineage graph analysis is deferred to Phase 5."` | +| `["statement"]` | `"A completed whole-target lineage analysis is unavailable in this report."` | | `["status"]` | `"unavailable"` | | `["theory_map_ids"]` | `[]` | -| `["theory_or_product_limit"]` | `"Implementation is deferred even when the existing input classifier marks lineage available."` | +| `["theory_or_product_limit"]` | `"Input eligibility alone does not establish executed ancestry analysis."` | | `["trace_ids"]` | `[]` | | `["unit"]` | `"conclusion"` | @@ -4134,38 +4134,38 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; `["unavailable_conclusions"][7]` -Status: `"unavailable"`; evidence class: `"unavailable_conclusion"`; unit: `"conclusion"`; method: `"T3.unavailable_conclusion"`. +Status: `"unavailable"`; evidence class: `"unavailable_conclusion"`; unit: `"conclusion"`; method: `"T4.unavailable_conclusion"`. Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; coverage (ratio): Unavailable (null): `"This conclusion has no measured evidence coverage."`. | Field | Value | |---|---| | `["assumptions"]` | `[]` | -| `["blocking_evidence"]` | `["Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes."]` | -| `["conclusion"]` | `"lineage_closure_exposure"` | +| `["blocking_evidence"]` | `["Missing, invalid or unknown ancestry remains unresolved; any complete subset retains its reported coverage."]` | +| `["conclusion"]` | `"external_ancestry"` | | `["coverage"]` | Unavailable (null): `"This conclusion has no measured evidence coverage."` | | `["coverage_reason"]` | `"This conclusion has no measured evidence coverage."` | | `["denominator"]` | Unavailable (null): `"This conclusion has no scalar denominator."` | | `["denominator_reason"]` | `"This conclusion has no scalar denominator."` | | `["evidence_class"]` | `"unavailable_conclusion"` | -| `["method_id"]` | `"T3.unavailable_conclusion"` | -| `["owner_ids"]` | `["T3"]` | -| `["reason_codes"]` | `["R_LINEAGE_EXECUTION_DEFERRED"]` | +| `["method_id"]` | `"T4.unavailable_conclusion"` | +| `["owner_ids"]` | `["T4"]` | +| `["reason_codes"]` | `["R_ANALYSIS_NOT_REQUESTED"]` | | `["related_capability"]` | `"lineage"` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | -| `["required_evidence"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | -| `["required_next_metadata"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | +| `["required_evidence"]` | `["Complete the required parent references, chronology and external-grounding evidence."]` | +| `["required_next_metadata"]` | `["Complete the required parent references, chronology and external-grounding evidence."]` | | `["scope"]["dataset_versions"]` | `["hmac-sha256:4e02dbb7e57fd003c1b4a2d16e058bca5c848e94f5b8730261ad64c1175dfef4", "hmac-sha256:70696b56c6ef6dbd4145752d88c4da6f71498a8f35a521e3946d89acc2a56a5d"]` | | `["scope"]["denominator_basis"]` | `"all_validated_bundle_records"` | | `["scope"]["excluded_record_count"]` | 0 | | `["scope"]["excluded_record_keys"]` | `[]` | | `["scope"]["record_count"]` | 16 | | `["scope"]["scope_id"]` | `"hmac-sha256:fe3080b69d042a9eca2d392f8429412803f06dd3eb9e71fd16178d05e5971ac3"` | -| `["statement"]` | `"Lineage closure exposure is deferred to Phase 5."` | +| `["statement"]` | `"Complete external ancestry for every target is unavailable."` | | `["status"]` | `"unavailable"` | | `["theory_map_ids"]` | `[]` | -| `["theory_or_product_limit"]` | `"Implementation is deferred even when the existing input classifier marks lineage available."` | -| `["trace_ids"]` | `["T3"]` | +| `["theory_or_product_limit"]` | `"Partial ancestry cannot be promoted to whole-target completeness."` | +| `["trace_ids"]` | `["T4"]` | | `["unit"]` | `"conclusion"` | **`["limitations"]`** @@ -4197,38 +4197,38 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; `["unavailable_conclusions"][8]` -Status: `"unavailable"`; evidence class: `"unavailable_conclusion"`; unit: `"conclusion"`; method: `"T4.unavailable_conclusion"`. +Status: `"unavailable"`; evidence class: `"unavailable_conclusion"`; unit: `"conclusion"`; method: `"T3.unavailable_conclusion"`. Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; coverage (ratio): Unavailable (null): `"This conclusion has no measured evidence coverage."`. | Field | Value | |---|---| | `["assumptions"]` | `[]` | -| `["blocking_evidence"]` | `["Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes."]` | -| `["conclusion"]` | `"external_ancestry"` | +| `["blocking_evidence"]` | `["No available explicitly supplied lineage interval covers a nonempty completed target partition."]` | +| `["conclusion"]` | `"lineage_closure_exposure"` | | `["coverage"]` | Unavailable (null): `"This conclusion has no measured evidence coverage."` | | `["coverage_reason"]` | `"This conclusion has no measured evidence coverage."` | | `["denominator"]` | Unavailable (null): `"This conclusion has no scalar denominator."` | | `["denominator_reason"]` | `"This conclusion has no scalar denominator."` | | `["evidence_class"]` | `"unavailable_conclusion"` | -| `["method_id"]` | `"T4.unavailable_conclusion"` | -| `["owner_ids"]` | `["T4"]` | -| `["reason_codes"]` | `["R_LINEAGE_EXECUTION_DEFERRED"]` | +| `["method_id"]` | `"T3.unavailable_conclusion"` | +| `["owner_ids"]` | `["T3"]` | +| `["reason_codes"]` | `["R_ANALYSIS_NOT_REQUESTED"]` | | `["related_capability"]` | `"lineage"` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | -| `["required_evidence"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | -| `["required_next_metadata"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | +| `["required_evidence"]` | `["Supply lineage closure bounds from the same explicit lineage result."]` | +| `["required_next_metadata"]` | `["Supply lineage closure bounds from the same explicit lineage result."]` | | `["scope"]["dataset_versions"]` | `["hmac-sha256:4e02dbb7e57fd003c1b4a2d16e058bca5c848e94f5b8730261ad64c1175dfef4", "hmac-sha256:70696b56c6ef6dbd4145752d88c4da6f71498a8f35a521e3946d89acc2a56a5d"]` | | `["scope"]["denominator_basis"]` | `"all_validated_bundle_records"` | | `["scope"]["excluded_record_count"]` | 0 | | `["scope"]["excluded_record_keys"]` | `[]` | | `["scope"]["record_count"]` | 16 | | `["scope"]["scope_id"]` | `"hmac-sha256:fe3080b69d042a9eca2d392f8429412803f06dd3eb9e71fd16178d05e5971ac3"` | -| `["statement"]` | `"External ancestry results are deferred to Phase 5."` | +| `["statement"]` | `"A lineage closure interval is unavailable in this report."` | | `["status"]` | `"unavailable"` | | `["theory_map_ids"]` | `[]` | -| `["theory_or_product_limit"]` | `"Implementation is deferred even when the existing input classifier marks lineage available."` | -| `["trace_ids"]` | `["T4"]` | +| `["theory_or_product_limit"]` | `"Unavailable intervals are never replaced by a guessed numeric value."` | +| `["trace_ids"]` | `["T3"]` | | `["unit"]` | `"conclusion"` | **`["limitations"]`** @@ -4281,7 +4281,7 @@ No entries supplied. Empty list: `[]`. **Report footer** -Toolkit version: `"0.1.0.dev2"`; report schema version: `"1.0"`. +Toolkit version: `"0.1.0.dev4"`; report schema version: `"1.1"`. Evidence-class legend: diff --git a/tests/golden/phase4_hero_report.json b/tests/golden/phase4_hero_report.json index e0af097..7c45d30 100644 --- a/tests/golden/phase4_hero_report.json +++ b/tests/golden/phase4_hero_report.json @@ -34,7 +34,7 @@ "python_version": null, "random_seed": null, "redacted_mode": false, - "report_schema_version": "1.0", + "report_schema_version": "1.1", "resolved_options": { "comparison_requested": true, "privacy_mode": "standard", @@ -48,7 +48,7 @@ "run_status": "complete", "started_at": "2000-01-01T00:00:00+00:00", "strict_mode": false, - "toolkit_version": "0.1.0.dev2" + "toolkit_version": "0.1.0.dev4" }, "inputs": { "artifacts": [ @@ -402,10 +402,10 @@ }, "coverage_reason": null, "execution_reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "execution_scope": [], - "execution_status": "deferred", + "execution_status": "not_requested", "notes": [ "Input eligibility only; downstream analytical implementations remain deferred.", "Reference-entry resolution coverage is not ancestry or external-root coverage.", @@ -487,7 +487,7 @@ "limitations": [ "R_MODEL_EVIDENCE_MISSING", "R_SCENARIO_NOT_CONFIGURED", - "Input eligibility is distinct from executed analysis; Phase 5 lineage remains deferred." + "Input eligibility is distinct from explicitly supplied executed analysis." ], "maximum_level": 4, "partial_evidence": [ @@ -608,10 +608,10 @@ }, "coverage_reason": null, "execution_reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "execution_scope": [], - "execution_status": "deferred", + "execution_status": "not_requested", "notes": [ "Input eligibility only; downstream analytical implementations remain deferred.", "Reference-entry resolution coverage is not ancestry or external-root coverage.", @@ -699,18 +699,18 @@ "denominator_reason": "denominator_not_applicable_or_unavailable", "evidence_class": "observed_fact", "limitations": [ - "No general graph-cycle traversal executes in Phase 4." + "No executed graph-cycle result was supplied to this report." ], "method_id": "T6.graph_cycle_check", "owner_ids": [ "T6" ], "reason_codes": [ - "R_GRAPH_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "representation": null, "required_evidence": [ - "Phase_5_graph_analysis" + "explicit_lineage_result" ], "scope": { "dataset_versions": [ @@ -5651,7 +5651,7 @@ { "assumptions": [], "blocking_evidence": [ - "Phase 4 does not compute ancestry; topology alone would not identify a causal contribution." + "Supplied parent topology and fractional root allocation do not identify a causal contribution." ], "conclusion": "causal_ancestor_effect", "coverage": null, @@ -5667,8 +5667,7 @@ "T4" ], "reason_codes": [ - "R_CAUSAL_EVIDENCE_MISSING", - "R_LINEAGE_EXECUTION_DEFERRED" + "R_CAUSAL_EVIDENCE_MISSING" ], "related_capability": "lineage", "representation": null, @@ -6223,7 +6222,7 @@ { "assumptions": [], "blocking_evidence": [ - "Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes." + "No complete successful lineage execution was supplied; independently completed observations remain visible." ], "conclusion": "lineage_analysis", "coverage": null, @@ -6239,15 +6238,15 @@ "PR-014" ], "reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "related_capability": "lineage", "representation": null, "required_evidence": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Supply an explicit lineage result for the validated target and its loaded context." ], "required_next_metadata": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Supply an explicit lineage result for the validated target and its loaded context." ], "scope": { "dataset_versions": [ @@ -6326,19 +6325,19 @@ "record_count": 16, "scope_id": "validated_bundle" }, - "statement": "General lineage graph analysis is deferred to Phase 5.", + "statement": "A completed whole-target lineage analysis is unavailable in this report.", "status": "unavailable", "theory_map_ids": [], - "theory_or_product_limit": "Implementation is deferred even when the existing input classifier marks lineage available.", + "theory_or_product_limit": "Input eligibility alone does not establish executed ancestry analysis.", "trace_ids": [], "unit": "conclusion" }, { "assumptions": [], "blocking_evidence": [ - "Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes." + "Missing, invalid or unknown ancestry remains unresolved; any complete subset retains its reported coverage." ], - "conclusion": "lineage_closure_exposure", + "conclusion": "external_ancestry", "coverage": null, "coverage_reason": "This conclusion has no measured evidence coverage.", "denominator": null, @@ -6347,20 +6346,20 @@ "limitations": [ "Unavailable means the supplied evidence does not support this conclusion; it does not establish that the conclusion is false." ], - "method_id": "T3.unavailable_conclusion", + "method_id": "T4.unavailable_conclusion", "owner_ids": [ - "T3" + "T4" ], "reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "related_capability": "lineage", "representation": null, "required_evidence": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Complete the required parent references, chronology and external-grounding evidence." ], "required_next_metadata": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Complete the required parent references, chronology and external-grounding evidence." ], "scope": { "dataset_versions": [ @@ -6439,21 +6438,21 @@ "record_count": 16, "scope_id": "validated_bundle" }, - "statement": "Lineage closure exposure is deferred to Phase 5.", + "statement": "Complete external ancestry for every target is unavailable.", "status": "unavailable", "theory_map_ids": [], - "theory_or_product_limit": "Implementation is deferred even when the existing input classifier marks lineage available.", + "theory_or_product_limit": "Partial ancestry cannot be promoted to whole-target completeness.", "trace_ids": [ - "T3" + "T4" ], "unit": "conclusion" }, { "assumptions": [], "blocking_evidence": [ - "Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes." + "No available explicitly supplied lineage interval covers a nonempty completed target partition." ], - "conclusion": "external_ancestry", + "conclusion": "lineage_closure_exposure", "coverage": null, "coverage_reason": "This conclusion has no measured evidence coverage.", "denominator": null, @@ -6462,20 +6461,20 @@ "limitations": [ "Unavailable means the supplied evidence does not support this conclusion; it does not establish that the conclusion is false." ], - "method_id": "T4.unavailable_conclusion", + "method_id": "T3.unavailable_conclusion", "owner_ids": [ - "T4" + "T3" ], "reason_codes": [ - "R_LINEAGE_EXECUTION_DEFERRED" + "R_ANALYSIS_NOT_REQUESTED" ], "related_capability": "lineage", "representation": null, "required_evidence": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Supply lineage closure bounds from the same explicit lineage result." ], "required_next_metadata": [ - "Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis." + "Supply lineage closure bounds from the same explicit lineage result." ], "scope": { "dataset_versions": [ @@ -6554,12 +6553,12 @@ "record_count": 16, "scope_id": "validated_bundle" }, - "statement": "External ancestry results are deferred to Phase 5.", + "statement": "A lineage closure interval is unavailable in this report.", "status": "unavailable", "theory_map_ids": [], - "theory_or_product_limit": "Implementation is deferred even when the existing input classifier marks lineage available.", + "theory_or_product_limit": "Unavailable intervals are never replaced by a guessed numeric value.", "trace_ids": [ - "T4" + "T3" ], "unit": "conclusion" } diff --git a/tests/golden/phase4_hero_report.md b/tests/golden/phase4_hero_report.md index cf26dfc..9e4a125 100644 --- a/tests/golden/phase4_hero_report.md +++ b/tests/golden/phase4_hero_report.md @@ -4,7 +4,7 @@ - Run ID: `"phase4-step9-golden"`. - Maximum observability level: 4; `"longitudinal_dataset_observability"`. -- Capability statuses: Ingestion: `"available"` (execution `"completed"`); Content diagnostics: `"available"` (execution `"completed"`); Provenance: `"available"` (execution `"completed"`); Lineage: `"available"` (execution `"deferred"`); Dataset longitudinal: `"available"` (execution `"partial"`); Model longitudinal: `"unavailable"` (execution `"deferred"`); Intervention simulation: `"unavailable"` (execution `"not_requested"`). +- Capability statuses: Ingestion: `"available"` (execution `"completed"`); Content diagnostics: `"available"` (execution `"completed"`); Provenance: `"available"` (execution `"completed"`); Lineage: `"available"` (execution `"not_requested"`); Dataset longitudinal: `"available"` (execution `"partial"`); Model longitudinal: `"unavailable"` (execution `"deferred"`); Intervention simulation: `"unavailable"` (execution `"not_requested"`). - Record scope: `"validated_bundle"`; dataset versions `["v1", "v2"]`; records 16; excluded records 0. - Representation: `"topic"`; source `"topic_field"`; version `"hero-topic-v1"`. Complete mapping metadata appears in Input inventory. - Warning entries: 0; error entries: 0. @@ -37,7 +37,7 @@ Quoted code literals represent supplied data, including identifiers and labels. | `["python_version"]` | Unavailable (null): `"private_or_nonstandard_metadata_omitted"` | | `["random_seed"]` | Unavailable (null): `"not_recorded"` | | `["redacted_mode"]` | false | -| `["report_schema_version"]` | `"1.0"` | +| `["report_schema_version"]` | `"1.1"` | | `["resolved_options"]["comparison_requested"]` | true | | `["resolved_options"]["privacy_mode"]` | `"standard"` | | `["resolved_options"]["record_id_mode"]` | `"preserve"` | @@ -49,7 +49,7 @@ Quoted code literals represent supplied data, including identifiers and labels. | `["run_status"]` | `"complete"` | | `["started_at"]` | `"2000-01-01T00:00:00+00:00"` | | `["strict_mode"]` | false | -| `["toolkit_version"]` | `"0.1.0.dev2"` | +| `["toolkit_version"]` | `"0.1.0.dev4"` | **`["identifier_protection"]["limitations"]`** @@ -194,7 +194,7 @@ Quoted code literals represent supplied data, including identifiers and labels. - `"R_MODEL_EVIDENCE_MISSING"` - `"R_SCENARIO_NOT_CONFIGURED"` -- `"Input eligibility is distinct from executed analysis; Phase 5 lineage remains deferred."` +- `"Input eligibility is distinct from explicitly supplied executed analysis."` The equal observability.capabilities compatibility mirror is represented once in the Capability matrix section. @@ -205,7 +205,7 @@ The equal observability.capabilities compatibility mirror is represented once in | Ingestion | `"available"` | 1.0 | `[]` | `"completed"` | `[]` | | Content diagnostics | `"available"` | 1.0 | `[]` | `"completed"` | `[]` | | Provenance | `"available"` | 1.0 | `[]` | `"completed"` | `[]` | -| Lineage | `"available"` | 1.0 | `[]` | `"deferred"` | `["R_LINEAGE_EXECUTION_DEFERRED"]` | +| Lineage | `"available"` | 1.0 | `[]` | `"not_requested"` | `["R_ANALYSIS_NOT_REQUESTED"]` | | Dataset longitudinal | `"available"` | 1.0 | `[]` | `"partial"` | `["R_LONGITUDINAL_FAMILIES_DEFERRED"]` | | Model longitudinal | `"unavailable"` | Unavailable (null): `"coverage_not_supplied_or_empty_denominator"` | `["R_MODEL_EVIDENCE_MISSING"]` | `"deferred"` | `["R_MODEL_ANALYSIS_DEFERRED"]` | | Intervention simulation | `"unavailable"` | Unavailable (null): `"coverage_not_supplied_or_empty_denominator"` | `["R_SCENARIO_NOT_CONFIGURED"]` | `"not_requested"` | `["R_ANALYSIS_NOT_REQUESTED"]` | @@ -324,7 +324,7 @@ Coverage details and unresolved counts, when supplied, follow below. Related dia Status: `"unavailable"`; evidence class: `"observed_fact"`; unit: `"status"`; method: `"T6.graph_cycle_check"`. -Value: Unavailable (null): reason codes `["R_GRAPH_EXECUTION_DEFERRED"]`. +Value: Unavailable (null): reason codes `["R_ANALYSIS_NOT_REQUESTED"]`. Denominator: Unavailable (null): `"denominator_not_applicable_or_unavailable"`; coverage (ratio): Unavailable (null): `"coverage_not_supplied_for_this_result"`. @@ -338,9 +338,9 @@ Denominator: Unavailable (null): `"denominator_not_applicable_or_unavailable"`; | `["evidence_class"]` | `"observed_fact"` | | `["method_id"]` | `"T6.graph_cycle_check"` | | `["owner_ids"]` | `["T6"]` | -| `["reason_codes"]` | `["R_GRAPH_EXECUTION_DEFERRED"]` | +| `["reason_codes"]` | `["R_ANALYSIS_NOT_REQUESTED"]` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | -| `["required_evidence"]` | `["Phase_5_graph_analysis"]` | +| `["required_evidence"]` | `["explicit_lineage_result"]` | | `["scope"]["dataset_versions"]` | `["v1", "v2"]` | | `["scope"]["denominator_basis"]` | `"all_validated_bundle_records"` | | `["scope"]["excluded_record_count"]` | 0 | @@ -351,11 +351,11 @@ Denominator: Unavailable (null): `"denominator_not_applicable_or_unavailable"`; | `["theory_map_ids"]` | `[]` | | `["trace_ids"]` | `["T6"]` | | `["unit"]` | `"status"` | -| `["value"]` | Unavailable (null): reason codes `["R_GRAPH_EXECUTION_DEFERRED"]` | +| `["value"]` | Unavailable (null): reason codes `["R_ANALYSIS_NOT_REQUESTED"]` | **`["limitations"]`** -- `"No general graph-cycle traversal executes in Phase 4."` +- `"No executed graph-cycle result was supplied to this report."` **`["scope"]["included_record_keys"]`** @@ -3764,7 +3764,7 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; | Field | Value | |---|---| | `["assumptions"]` | `[]` | -| `["blocking_evidence"]` | `["Phase 4 does not compute ancestry; topology alone would not identify a causal contribution."]` | +| `["blocking_evidence"]` | `["Supplied parent topology and fractional root allocation do not identify a causal contribution."]` | | `["conclusion"]` | `"causal_ancestor_effect"` | | `["coverage"]` | Unavailable (null): `"This conclusion has no measured evidence coverage."` | | `["coverage_reason"]` | `"This conclusion has no measured evidence coverage."` | @@ -3773,7 +3773,7 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; | `["evidence_class"]` | `"unavailable_conclusion"` | | `["method_id"]` | `"T4.unavailable_conclusion"` | | `["owner_ids"]` | `["T4"]` | -| `["reason_codes"]` | `["R_CAUSAL_EVIDENCE_MISSING", "R_LINEAGE_EXECUTION_DEFERRED"]` | +| `["reason_codes"]` | `["R_CAUSAL_EVIDENCE_MISSING"]` | | `["related_capability"]` | `"lineage"` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | | `["required_evidence"]` | `["An identified ancestor, measured outcomes and a controlled or otherwise justified causal design."]` | @@ -4079,7 +4079,7 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; | Field | Value | |---|---| | `["assumptions"]` | `[]` | -| `["blocking_evidence"]` | `["Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes."]` | +| `["blocking_evidence"]` | `["No complete successful lineage execution was supplied; independently completed observations remain visible."]` | | `["conclusion"]` | `"lineage_analysis"` | | `["coverage"]` | Unavailable (null): `"This conclusion has no measured evidence coverage."` | | `["coverage_reason"]` | `"This conclusion has no measured evidence coverage."` | @@ -4088,21 +4088,21 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; | `["evidence_class"]` | `"unavailable_conclusion"` | | `["method_id"]` | `"PR-014.unavailable_conclusion"` | | `["owner_ids"]` | `["PR-014"]` | -| `["reason_codes"]` | `["R_LINEAGE_EXECUTION_DEFERRED"]` | +| `["reason_codes"]` | `["R_ANALYSIS_NOT_REQUESTED"]` | | `["related_capability"]` | `"lineage"` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | -| `["required_evidence"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | -| `["required_next_metadata"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | +| `["required_evidence"]` | `["Supply an explicit lineage result for the validated target and its loaded context."]` | +| `["required_next_metadata"]` | `["Supply an explicit lineage result for the validated target and its loaded context."]` | | `["scope"]["dataset_versions"]` | `["v1", "v2"]` | | `["scope"]["denominator_basis"]` | `"all_validated_bundle_records"` | | `["scope"]["excluded_record_count"]` | 0 | | `["scope"]["excluded_record_keys"]` | `[]` | | `["scope"]["record_count"]` | 16 | | `["scope"]["scope_id"]` | `"validated_bundle"` | -| `["statement"]` | `"General lineage graph analysis is deferred to Phase 5."` | +| `["statement"]` | `"A completed whole-target lineage analysis is unavailable in this report."` | | `["status"]` | `"unavailable"` | | `["theory_map_ids"]` | `[]` | -| `["theory_or_product_limit"]` | `"Implementation is deferred even when the existing input classifier marks lineage available."` | +| `["theory_or_product_limit"]` | `"Input eligibility alone does not establish executed ancestry analysis."` | | `["trace_ids"]` | `[]` | | `["unit"]` | `"conclusion"` | @@ -4135,38 +4135,38 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; `["unavailable_conclusions"][7]` -Status: `"unavailable"`; evidence class: `"unavailable_conclusion"`; unit: `"conclusion"`; method: `"T3.unavailable_conclusion"`. +Status: `"unavailable"`; evidence class: `"unavailable_conclusion"`; unit: `"conclusion"`; method: `"T4.unavailable_conclusion"`. Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; coverage (ratio): Unavailable (null): `"This conclusion has no measured evidence coverage."`. | Field | Value | |---|---| | `["assumptions"]` | `[]` | -| `["blocking_evidence"]` | `["Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes."]` | -| `["conclusion"]` | `"lineage_closure_exposure"` | +| `["blocking_evidence"]` | `["Missing, invalid or unknown ancestry remains unresolved; any complete subset retains its reported coverage."]` | +| `["conclusion"]` | `"external_ancestry"` | | `["coverage"]` | Unavailable (null): `"This conclusion has no measured evidence coverage."` | | `["coverage_reason"]` | `"This conclusion has no measured evidence coverage."` | | `["denominator"]` | Unavailable (null): `"This conclusion has no scalar denominator."` | | `["denominator_reason"]` | `"This conclusion has no scalar denominator."` | | `["evidence_class"]` | `"unavailable_conclusion"` | -| `["method_id"]` | `"T3.unavailable_conclusion"` | -| `["owner_ids"]` | `["T3"]` | -| `["reason_codes"]` | `["R_LINEAGE_EXECUTION_DEFERRED"]` | +| `["method_id"]` | `"T4.unavailable_conclusion"` | +| `["owner_ids"]` | `["T4"]` | +| `["reason_codes"]` | `["R_ANALYSIS_NOT_REQUESTED"]` | | `["related_capability"]` | `"lineage"` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | -| `["required_evidence"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | -| `["required_next_metadata"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | +| `["required_evidence"]` | `["Complete the required parent references, chronology and external-grounding evidence."]` | +| `["required_next_metadata"]` | `["Complete the required parent references, chronology and external-grounding evidence."]` | | `["scope"]["dataset_versions"]` | `["v1", "v2"]` | | `["scope"]["denominator_basis"]` | `"all_validated_bundle_records"` | | `["scope"]["excluded_record_count"]` | 0 | | `["scope"]["excluded_record_keys"]` | `[]` | | `["scope"]["record_count"]` | 16 | | `["scope"]["scope_id"]` | `"validated_bundle"` | -| `["statement"]` | `"Lineage closure exposure is deferred to Phase 5."` | +| `["statement"]` | `"Complete external ancestry for every target is unavailable."` | | `["status"]` | `"unavailable"` | | `["theory_map_ids"]` | `[]` | -| `["theory_or_product_limit"]` | `"Implementation is deferred even when the existing input classifier marks lineage available."` | -| `["trace_ids"]` | `["T3"]` | +| `["theory_or_product_limit"]` | `"Partial ancestry cannot be promoted to whole-target completeness."` | +| `["trace_ids"]` | `["T4"]` | | `["unit"]` | `"conclusion"` | **`["limitations"]`** @@ -4198,38 +4198,38 @@ Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; `["unavailable_conclusions"][8]` -Status: `"unavailable"`; evidence class: `"unavailable_conclusion"`; unit: `"conclusion"`; method: `"T4.unavailable_conclusion"`. +Status: `"unavailable"`; evidence class: `"unavailable_conclusion"`; unit: `"conclusion"`; method: `"T3.unavailable_conclusion"`. Denominator: Unavailable (null): `"This conclusion has no scalar denominator."`; coverage (ratio): Unavailable (null): `"This conclusion has no measured evidence coverage."`. | Field | Value | |---|---| | `["assumptions"]` | `[]` | -| `["blocking_evidence"]` | `["Only existing reference validation observations can be reported; no graph traversal, root tracing or ancestry metric executes."]` | -| `["conclusion"]` | `"external_ancestry"` | +| `["blocking_evidence"]` | `["No available explicitly supplied lineage interval covers a nonempty completed target partition."]` | +| `["conclusion"]` | `"lineage_closure_exposure"` | | `["coverage"]` | Unavailable (null): `"This conclusion has no measured evidence coverage."` | | `["coverage_reason"]` | `"This conclusion has no measured evidence coverage."` | | `["denominator"]` | Unavailable (null): `"This conclusion has no scalar denominator."` | | `["denominator_reason"]` | `"This conclusion has no scalar denominator."` | | `["evidence_class"]` | `"unavailable_conclusion"` | -| `["method_id"]` | `"T4.unavailable_conclusion"` | -| `["owner_ids"]` | `["T4"]` | -| `["reason_codes"]` | `["R_LINEAGE_EXECUTION_DEFERRED"]` | +| `["method_id"]` | `"T3.unavailable_conclusion"` | +| `["owner_ids"]` | `["T3"]` | +| `["reason_codes"]` | `["R_ANALYSIS_NOT_REQUESTED"]` | | `["related_capability"]` | `"lineage"` | | `["representation"]` | Unavailable (null): No applicable representation supplied; consult status and required evidence. | -| `["required_evidence"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | -| `["required_next_metadata"]` | `["Preserve explicit composite parent references, chronology and external-grounding metadata for the future Phase 5 analysis."]` | +| `["required_evidence"]` | `["Supply lineage closure bounds from the same explicit lineage result."]` | +| `["required_next_metadata"]` | `["Supply lineage closure bounds from the same explicit lineage result."]` | | `["scope"]["dataset_versions"]` | `["v1", "v2"]` | | `["scope"]["denominator_basis"]` | `"all_validated_bundle_records"` | | `["scope"]["excluded_record_count"]` | 0 | | `["scope"]["excluded_record_keys"]` | `[]` | | `["scope"]["record_count"]` | 16 | | `["scope"]["scope_id"]` | `"validated_bundle"` | -| `["statement"]` | `"External ancestry results are deferred to Phase 5."` | +| `["statement"]` | `"A lineage closure interval is unavailable in this report."` | | `["status"]` | `"unavailable"` | | `["theory_map_ids"]` | `[]` | -| `["theory_or_product_limit"]` | `"Implementation is deferred even when the existing input classifier marks lineage available."` | -| `["trace_ids"]` | `["T4"]` | +| `["theory_or_product_limit"]` | `"Unavailable intervals are never replaced by a guessed numeric value."` | +| `["trace_ids"]` | `["T3"]` | | `["unit"]` | `"conclusion"` | **`["limitations"]`** @@ -4282,7 +4282,7 @@ No entries supplied. Empty list: `[]`. **Report footer** -Toolkit version: `"0.1.0.dev2"`; report schema version: `"1.0"`. +Toolkit version: `"0.1.0.dev4"`; report schema version: `"1.1"`. Evidence-class legend: diff --git a/tests/golden/phase4_report_cases.md b/tests/golden/phase4_report_cases.md index 65888ac..ab43981 100644 --- a/tests/golden/phase4_report_cases.md +++ b/tests/golden/phase4_report_cases.md @@ -61,7 +61,7 @@ observations may state 8 declared, 8 resolved and 0 unresolved references and th sufficient declared-earlier-version ordering certificate. They cannot claim graph traversal or a general cycle verdict. -Maximum input observability stays Level 4. Lineage execution remains deferred; +Maximum input observability stays Level 4. Lineage execution is not requested; dataset-longitudinal execution is partial with the sole explicit-pair support and diversity operation identified. The successful run remains complete. Model longitudinal is unavailable/deferred, default intervention simulation is @@ -161,7 +161,7 @@ Earlier minus later is battery,lizard,turtle; intersection contains five states. The selected v2 scope has eight matching valid provenance rows: four human/yes and four synthetic/no. Each coverage is 8/8; each nonzero source share is 4/8. Direct interval=(closed/N,(closed+unresolved)/N)=[4/8,4/8]. -All eight supplied v2 parent references resolve to explicitly earlier v1 records. This supports reference counts and a declared-order certificate only. Graph traversal and all ancestry metrics remain deferred. +All eight supplied v2 parent references resolve to explicitly earlier v1 records. This supports reference counts and a declared-order certificate only. This CLI invocation does not request graph traversal or ancestry metrics. ### hero_redacted @@ -281,9 +281,9 @@ Required retained errors: `E_CONFIG_INVALID`. Driver: `cli`. Expected exit: `0`. -Of two later parent references one resolves and one is absent: 2 declared,1 resolved,1 unresolved. Missing references retain a warning and cannot become external roots. One valid resolved path and one unresolved reference give partial input capability under the frozen lineage classification; execution remains deferred. +Of two later parent references one resolves and one is absent: 2 declared,1 resolved,1 unresolved. Missing references retain a warning and cannot become external roots. One valid resolved path and one unresolved reference give partial input capability under the frozen lineage classification; execution is not requested. -Independent ordered supports {A,B} and {A,C} both have size2, intersection1, retention1/2; lineage execution remains deferred. +Independent ordered supports {A,B} and {A,C} both have size2, intersection1, retention1/2; lineage execution is not requested. Required retained warnings: `W_PARENT_UNRESOLVED`. @@ -398,3 +398,11 @@ Acceptance requires every case, unchanged original math cases, literal JSON and Markdown review, normalization mutation rejection, privacy/format/reordering metamorphic checks and the governing regression gates. This oracle alone makes no claim that current implementation, installed artifacts or remote jobs have passed. + +## Phase 5 Step 7 current-schema consolidation + +Current canonical fixtures use report schema 1.1 and package 0.1.0.dev4. +Ordinary CLI calls report lineage execution as `not_requested`; explicit lineage +report assembly is covered by the current integration suite. Original numeric +expectations are retained. Prior schema 1.0 fixtures and the original authority +hashes remain auditable in Git history. No source-binding migration is added. diff --git a/tests/golden/phase4_report_expected.json b/tests/golden/phase4_report_expected.json index 53bfc7d..5a4ecb7 100644 --- a/tests/golden/phase4_report_expected.json +++ b/tests/golden/phase4_report_expected.json @@ -156,8 +156,8 @@ }, "expected_exit": 0, "expected": { - "/run/report_schema_version": "1.0", - "/run/toolkit_version": "0.1.0.dev2", + "/run/report_schema_version": "1.1", + "/run/toolkit_version": "0.1.0.dev4", "/run/run_status": "complete", "/run/network_call_count": 0, "/run/network_count_scope": "toolkit_managed_outbound_operations", @@ -244,7 +244,7 @@ "/capabilities/ingestion/execution_status": "completed", "/capabilities/content_diagnostics/execution_status": "completed", "/capabilities/provenance/execution_status": "completed", - "/capabilities/lineage/execution_status": "deferred", + "/capabilities/lineage/execution_status": "not_requested", "/capabilities/dataset_longitudinal/execution_status": "partial", "/capabilities/dataset_longitudinal/execution_scope": [ "supplied_explicit_pair_support_and_diversity" @@ -468,7 +468,7 @@ "v2 topic counts cat=3,dog=2,bird=fish=refund=1 give support 5; sum squared frequencies=16/64=1/4 and D=3/4.", "Earlier minus later is battery,lizard,turtle; intersection contains five states. Delta=5-8=-3, retention=5/8, diversity delta=3/4-7/8=-1/8.", "The selected v2 scope has eight matching valid provenance rows: four human/yes and four synthetic/no. Each coverage is 8/8; each nonzero source share is 4/8. Direct interval=(closed/N,(closed+unresolved)/N)=[4/8,4/8].", - "All eight supplied v2 parent references resolve to explicitly earlier v1 records. This supports reference counts and a declared-order certificate only. Graph traversal and all ancestry metrics remain deferred." + "All eight supplied v2 parent references resolve to explicitly earlier v1 records. This supports reference counts and a declared-order certificate only. This CLI invocation does not request graph traversal or ancestry metrics." ] }, { @@ -485,8 +485,8 @@ }, "expected_exit": 0, "expected": { - "/run/report_schema_version": "1.0", - "/run/toolkit_version": "0.1.0.dev2", + "/run/report_schema_version": "1.1", + "/run/toolkit_version": "0.1.0.dev4", "/run/run_status": "complete", "/run/network_call_count": 0, "/run/network_count_scope": "toolkit_managed_outbound_operations", @@ -545,7 +545,7 @@ "/capabilities/ingestion/execution_status": "completed", "/capabilities/content_diagnostics/execution_status": "completed", "/capabilities/provenance/execution_status": "completed", - "/capabilities/lineage/execution_status": "deferred", + "/capabilities/lineage/execution_status": "not_requested", "/capabilities/dataset_longitudinal/execution_status": "partial", "/capabilities/dataset_longitudinal/execution_scope": [ "supplied_explicit_pair_support_and_diversity" @@ -650,8 +650,8 @@ }, "expected_exit": 0, "expected": { - "/run/report_schema_version": "1.0", - "/run/toolkit_version": "0.1.0.dev2", + "/run/report_schema_version": "1.1", + "/run/toolkit_version": "0.1.0.dev4", "/run/run_status": "complete", "/run/network_call_count": 0, "/run/network_count_scope": "toolkit_managed_outbound_operations", @@ -738,7 +738,7 @@ "/capabilities/ingestion/execution_status": "completed", "/capabilities/content_diagnostics/execution_status": "completed", "/capabilities/provenance/execution_status": "completed", - "/capabilities/lineage/execution_status": "deferred", + "/capabilities/lineage/execution_status": "not_requested", "/capabilities/dataset_longitudinal/execution_status": "partial", "/capabilities/dataset_longitudinal/execution_scope": [ "supplied_explicit_pair_support_and_diversity" @@ -2412,7 +2412,7 @@ "/run/run_status": "complete", "/observability/maximum_level": 4, "/capabilities/lineage/status": "partial", - "/capabilities/lineage/execution_status": "deferred", + "/capabilities/lineage/execution_status": "not_requested", "/observed_facts/lineage/declared_parent_edge_count/value": 2, "/observed_facts/lineage/resolved_parent_edge_count/value": 1, "/observed_facts/lineage/unresolved_parent_edge_count/value": 1, @@ -2442,8 +2442,8 @@ "json_value_evidence_status_parity" ], "numeric_rationale": [ - "Of two later parent references one resolves and one is absent: 2 declared,1 resolved,1 unresolved. Missing references retain a warning and cannot become external roots. One valid resolved path and one unresolved reference give partial input capability under the frozen lineage classification; execution remains deferred.", - "Independent ordered supports {A,B} and {A,C} both have size2, intersection1, retention1/2; lineage execution remains deferred." + "Of two later parent references one resolves and one is absent: 2 declared,1 resolved,1 unresolved. Missing references retain a warning and cannot become external roots. One valid resolved path and one unresolved reference give partial input capability under the frozen lineage classification; execution is not requested.", + "Independent ordered supports {A,B} and {A,C} both have size2, intersection1, retention1/2; lineage execution is not requested." ] }, { @@ -2574,7 +2574,7 @@ "/derived_metrics/support/by_version/later/support_size/scope/record_count": 2, "/run/run_status": "partial", "/observability/maximum_level": 4, - "/capabilities/lineage/execution_status": "deferred" + "/capabilities/lineage/execution_status": "not_requested" }, "required_error_codes": [ "E_LINEAGE_CYCLE" @@ -3550,7 +3550,7 @@ "/capabilities/ingestion/execution_status": "completed", "/capabilities/content_diagnostics/execution_status": "completed", "/capabilities/provenance/execution_status": "completed", - "/capabilities/lineage/execution_status": "deferred", + "/capabilities/lineage/execution_status": "not_requested", "/capabilities/dataset_longitudinal/execution_status": "partial", "/capabilities/dataset_longitudinal/execution_scope": [ "supplied_explicit_pair_support_and_diversity" diff --git a/tests/golden/test_phase4_reports.py b/tests/golden/test_phase4_reports.py index 0fc3ad0..7d37cb0 100644 --- a/tests/golden/test_phase4_reports.py +++ b/tests/golden/test_phase4_reports.py @@ -90,22 +90,13 @@ def test_phase4_step9_literal_json_and_markdown_goldens(redacted, phase4_step9_h stem = "phase4_hero_redacted" if redacted else "phase4_hero_report" json_path, markdown_path = GOLDEN / (stem + ".json"), GOLDEN / (stem + ".md") before = {path: path.read_bytes() for path in (json_path, markdown_path)} - # Step 11 synchronizes only the expected package-version metadata. Frozen - # fixture bytes and the actual report's closed normalization stay unchanged. + # Current schema fixtures retain the independently authored numeric oracle. + # Prior schema fixtures remain recoverable from Git history. expected_json = json.loads(before[json_path]) - assert expected_json["run"]["toolkit_version"] == "0.1.0.dev2" + assert expected_json["run"]["toolkit_version"] == __version__ assert actual["report"]["run"]["toolkit_version"] == __version__ assert actual["normalized"]["run"]["toolkit_version"] == __version__ - expected_json["run"]["toolkit_version"] = __version__ expected_markdown = before[markdown_path].decode("utf-8") - for old, new in ( - ('| `["toolkit_version"]` | `"0.1.0.dev2"` |', - '| `["toolkit_version"]` | `"' + __version__ + '"` |'), - ('Toolkit version: `"0.1.0.dev2"`; report schema version:', - 'Toolkit version: `"' + __version__ + '"`; report schema version:'), - ): - assert expected_markdown.count(old) == 1 - expected_markdown = expected_markdown.replace(old, new, 1) NORMALIZER["assert_golden_pair"](actual["normalized"], actual["normalized_markdown"], expected_json, expected_markdown) assert {path: path.read_bytes() for path in before} == before @@ -498,9 +489,6 @@ def phase4_step9_assert_oracle(case, actual, schema): assert markdown.startswith("# Recursive Integrity Audit Report\n") assert [line for line in markdown.splitlines() if line.startswith("## ")] == ["## " + value for value in headings] for pointer, expected in case.get("expected", {}).items(): - if pointer == "/run/toolkit_version": - assert expected == "0.1.0.dev2" - expected = __version__ got = phase4_step9_pointer(payload, pointer) assert got == expected, (case["case_id"], pointer, got, expected) if isinstance(expected, bool): @@ -660,7 +648,7 @@ def denied(*args, **kwargs): installed = Path(package.__file__).resolve().parent assert not installed.is_relative_to(repository), installed distribution = importlib.metadata.distribution('recursive-integrity-toolkit') -assert distribution.version == package.__version__ == '0.1.0.dev3' +assert distribution.version == package.__version__ == '0.1.0.dev4' members = {str(p).replace('\\', '/') for p in distribution.files or ()} modules = sorted(p for p in installed.rglob('*.py')) assert len(modules) == 40 diff --git a/tests/integration/test_ci_workflows.py b/tests/integration/test_ci_workflows.py index e3ac3a9..439d5dd 100644 --- a/tests/integration/test_ci_workflows.py +++ b/tests/integration/test_ci_workflows.py @@ -1,112 +1,93 @@ -"""Preserve workflow guarantees while moving active gates to Phase 3 Step 11.""" - -from pathlib import Path - -EXPECTED = {"ci.yml", "golden.yml", "security.yml", "release.yml"} -FORBIDDEN_WORKFLOW_TERMS = {"services:", "docker run", "fastapi", "streamlit", "telemetry", "publish-package"} - - -def test_workflow_file_set_and_basic_contract(repo_root: Path) -> None: - workflow_root = repo_root / ".github" / "workflows" - assert {path.name for path in workflow_root.glob("*.yml")} == EXPECTED - for path in sorted(workflow_root.glob("*.yml")): - text = path.read_text(encoding="utf-8") - assert text.startswith("name:") - assert "permissions:\n contents: read" in text - assert "actions/checkout@v4" in text - assert "actions/setup-python@v5" in text - assert "\t" not in text - - -def test_workflows_preserve_phase_boundary(repo_root: Path,phase3_final_snapshot) -> None: - repo_root = phase3_final_snapshot - combined = "\n".join(p.read_text(encoding="utf-8").lower() for p in (repo_root / ".github/workflows").glob("*.yml")) - for term in FORBIDDEN_WORKFLOW_TERMS: - assert term not in combined - assert "test_no_algorithms.py" in combined - assert "test_no_network.py" in combined - assert "python -m build" in combined - assert "rit version" in combined - assert "--phase 3 --step 11" in combined - assert "release_check.py --diff" not in combined +"""Current workflow scheduling, frozen specifications and release result integrity. +Complete compatibility and delivery gates run for an explicitly selected candidate. +Historical dispatch and source-body migration tests remain in Git history. +""" import ast import hashlib import re import runpy import tomllib -import pytest - -def _release_tools(repo_root): - return runpy.run_path(str(repo_root / "scripts/release_check.py"), run_name="phase2_release_test") +import pytest -def test_phase2_workflows_never_mask_failed_commands(repo_root): - for path in (repo_root / ".github/workflows").glob("*.yml"): +def test_workflow_file_set_and_read_only_permissions(repo_root): + paths = list((repo_root / ".github/workflows").glob("*.yml")) + assert {p.name for p in paths} == {"ci.yml", "golden.yml", "security.yml", "release.yml"} + for path in paths: text = path.read_text(encoding="utf-8") - for unsafe in ("continue-on-error:", "|| true", "allow-failure:", "contents: write", - "id-token: write", "pull_request_target:", "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION"): - assert unsafe not in text, (path.name, unsafe) - assert "shell: bash" in text - assert "timeout-minutes:" in text + assert "permissions:\n contents: read" in text assert "persist-credentials: false" in text + assert "--phase " not in text and "--step " not in text + assert "twine upload" not in text and "git push" not in text -def test_phase2_core_and_real_parquet_have_independent_complete_runs(repo_root): +def test_full_matrix_requires_explicit_candidate_selection(repo_root): text = (repo_root / ".github/workflows/ci.yml").read_text(encoding="utf-8") + assert "pull_request:" in text + assert "docs/**" not in text and "docs/lineage_contract.md" not in text.split(" workflow_dispatch:", 1)[0] + assert "default: focused" in text and "options: [focused, candidate]" in text + assert "types: [opened, synchronize, reopened, labeled]" in text + for job in ("core", "parquet", "performance", "hero", "security"): + body = re.search(r"^ " + job + r":\n(.*?)(?=^ [a-z_]+:|\Z)", text, re.M | re.S).group(1) + assert "github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate'" in body + assert "github.event.action == 'labeled' && github.event.label.name == 'verification:candidate'" in body + focused = text.split(" focused:\n", 1)[1].split("\n core:", 1)[0] + assert "inputs.gate != 'candidate'" in focused + assert "test_current_verification.py" in focused and "test_lineage_fixture_inputs.py" in focused + assert "--ignore=tests/performance" not in focused and "tests/performance " not in focused assert 'os: [ubuntu-latest, windows-latest]' in text assert 'python-version: ["3.11", "3.12"]' in text - assert 'RIT_TEST_PARQUET: "0"' in text and 'RIT_TEST_PARQUET: "1"' in text - assert 'python -m pip install ".[test,parquet]"' in text - assert "--require-parquet" in text - assert "find_spec('pyarrow') is None" in text - assert text.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in text assert 'dependencies: [current, minimum]' in text assert '"numpy==2.0.0" "pandas==2.2.2"' in text - assert 'python -m pip check' in text - assert 'dependencies.txt' in text - - -def test_phase2_delivery_builds_both_formats_and_tests_installed_wheel(repo_root,phase3_final_snapshot): - repo_root = phase3_final_snapshot - # Same identity and safety guarantees; only the active artifact stage changes. - text = (repo_root / ".github/workflows/release.yml").read_text(encoding="utf-8") - for required in ("python -m build", "python -m twine check --strict", "--dist", "--delivery", - "--require-parquet", "actions/upload-artifact@v4", "retention-days: 90", - "if-no-files-found: error", "phase3_test_results.log", "phase3_build_results.log"): - assert required in text - assert "twine upload" not in text and "git push" not in text - script = (repo_root / "scripts/release_check.py").read_text(encoding="utf-8") - for required in ('"--no-index", "--no-deps"', '"-I"', '"--prefix=recursive-integrity-toolkit/"', - '"recursive-integrity-toolkit-phase3.zip"', '"phase3_artifacts.sha256"'): - assert required in script + assert "find_spec('pyarrow') is None" in text and "--require-parquet" in text + assert text.count("--ignore=tests/performance") == 2 + assert text.count("-q tests/performance --junitxml=") == 1 + assert "needs: [core, parquet, performance, hero, security]" in text + + +def test_candidate_checks_preserve_behavior_and_package_roles(repo_root): + workflows = repo_root / ".github/workflows" + for filename in ("golden.yml", "security.yml"): + text = (workflows / filename).read_text(encoding="utf-8") + assert "workflow_call:" in text and "workflow_dispatch:" in text + assert "pull_request:" not in text + golden = (workflows / "golden.yml").read_text(encoding="utf-8") + assert "test_phase3_math.py" in golden and "test_phase4_reports.py" in golden + assert "build_golden.py" not in golden + security = (workflows / "security.yml").read_text(encoding="utf-8") + for name in ("test_no_network.py", "test_PR015_redaction.py", "test_PR017_content_refs.py", "test_phase4_output_safety.py"): + assert name in security + release = (workflows / "release.yml").read_text(encoding="utf-8") + for required in ("python -m build", "python -m twine check --strict", "--candidate", "SOURCE_DATE_EPOCH", "assert same_bytes", "assert payload_equal", "actions/download-artifact@v4", "source-commit.txt"): + assert required in release + assert "python -m pytest" not in release - -def test_phase2_hero_workflow_preserves_scaffold_only_golden_boundary(repo_root,phase3_final_snapshot): - repo_root = phase3_final_snapshot - text = (repo_root / ".github/workflows/golden.yml").read_text(encoding="utf-8") - body = "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#")) - assert "test_hero_structure.py" in body - assert "build_golden.py" not in body and "normalize_golden.py" not in body +def _release_tools(repo_root): + return runpy.run_path(str(repo_root / "scripts/release_check.py"), run_name="phase2_release_test") -def test_phase2_completion_records_exist_and_do_not_authorize_phase3(repo_root): - for name in ("PHASE_2_COMPLETION.md", "PHASE_2_VALIDATION_REPORT.md", "PHASE_2_ARCHITECTURE_COMPLIANCE_REPORT.md"): - text = (repo_root / name).read_text(encoding="utf-8") - assert "Phase 3" in text and "not authorized" in text +def test_current_workflows_never_mask_failed_commands(repo_root): + for path in (repo_root / ".github/workflows").glob("*.yml"): + text = path.read_text(encoding="utf-8") + for unsafe in ("continue-on-error:", "|| true", "allow-failure:", "contents: write", + "id-token: write", "pull_request_target:", "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION"): + assert unsafe not in text, (path.name, unsafe) + assert "shell: bash" in text + assert "timeout-minutes:" in text + assert "persist-credentials: false" in text -def test_phase2_approved_phase0_hashes_match_current_sources(repo_root): +def test_current_approved_phase0_hashes_match_current_sources(repo_root): entries = re.findall(r"\| `([^`]+\.md)` \| `([0-9a-f]{64})` \|", (repo_root / "PHASE_0_APPROVAL.md").read_text(encoding="utf-8")) assert len(entries) == 16 for name, expected in entries: assert hashlib.sha256((repo_root / name).read_bytes()).hexdigest() == expected, name -def test_phase2_package_version_is_consistent_without_protected_module_edit(repo_root): +def test_current_package_version_is_consistent_without_protected_module_edit(repo_root): project = tomllib.loads((repo_root / "pyproject.toml").read_text(encoding="utf-8"))["project"] tree = ast.parse((repo_root / "src/recursive_integrity_toolkit/__init__.py").read_text(encoding="utf-8")) value = [ast.literal_eval(n.value) for n in tree.body if isinstance(n, ast.Assign) @@ -121,84 +102,49 @@ def _junit(tmp_path, body, counts='tests="1" errors="0" failures="0" skipped="0" return path -def test_phase2_junit_gate_counts_passes(repo_root, tmp_path): +def test_current_junit_gate_counts_passes(repo_root, tmp_path): result = _release_tools(repo_root)["verify_junit"](_junit(tmp_path, '')) assert result == {"tests": 1, "passed": 1, "failed": 0, "skipped": 0, "real_parquet_cases": 0} @pytest.mark.parametrize("tag", ["failure", "error", "skipped"]) -def test_phase2_junit_gate_rejects_hidden_failures_and_skips(repo_root, tmp_path, tag): +def test_current_junit_gate_rejects_hidden_failures_and_skips(repo_root, tmp_path, tag): with pytest.raises(ValueError): _release_tools(repo_root)["verify_junit"](_junit(tmp_path, f'<{tag}/>')) @pytest.mark.parametrize("counter", ["failures", "errors", "skipped"]) -def test_phase2_junit_gate_checks_suite_counters(repo_root, tmp_path, counter): +def test_current_junit_gate_checks_suite_counters(repo_root, tmp_path, counter): with pytest.raises(ValueError): _release_tools(repo_root)["verify_junit"](_junit(tmp_path, '', f'{counter}="1"')) -def test_phase2_junit_gate_requires_collected_cases(repo_root, tmp_path): +def test_current_junit_gate_requires_collected_cases(repo_root, tmp_path): with pytest.raises(ValueError): _release_tools(repo_root)["verify_junit"](_junit(tmp_path, "")) -def test_phase2_junit_gate_rejects_duplicate_case_identity(repo_root, tmp_path): +def test_current_junit_gate_rejects_duplicate_case_identity(repo_root, tmp_path): with pytest.raises(ValueError): _release_tools(repo_root)["verify_junit"](_junit(tmp_path, '')) -def test_phase2_junit_gate_requires_full_core_count(repo_root, tmp_path): +def test_current_junit_gate_rejects_insufficient_requested_count(repo_root, tmp_path): with pytest.raises(ValueError): - _release_tools(repo_root)["verify_junit"](_junit(tmp_path, ''), minimum=1131) + _release_tools(repo_root)["verify_junit"](_junit(tmp_path, ''), minimum=2) -def test_phase2_junit_gate_requires_actual_parquet_cases(repo_root, tmp_path): +def test_current_junit_gate_requires_actual_parquet_cases(repo_root, tmp_path): with pytest.raises(ValueError): _release_tools(repo_root)["verify_junit"](_junit(tmp_path, ''), require_parquet=True) -def test_phase2_junit_gate_accepts_three_real_case_names(repo_root, tmp_path): - names = ("test_PR002_parquet_real_roundtrip", "test_PR002_parquet_real_row_limit", "test_PR002_parquet_real_invalid_file") +def test_current_junit_gate_accepts_real_case_names(repo_root, tmp_path): + names = ( + "test_PR002_parquet_real_roundtrip", "test_PR002_parquet_real_row_limit", "test_PR002_parquet_real_invalid_file", + "test_context_table_uses_existing_local_parsers_and_normalizer[parquet]", + "test_context_loaders_and_repeated_inputs_keep_primary_only_denominators[parquet]", + ) body = "".join(f'' for name in names) - result = _release_tools(repo_root)["verify_junit"](_junit(tmp_path, body, 'tests="3"'), require_parquet=True) - assert result["real_parquet_cases"] == 3 - - -def test_phase2_runtime_gates_remain_exact_allowlists(repo_root): - script = (repo_root / "scripts/check_traceability.py").read_text(encoding="utf-8") - assert "ALLOWED_FUNCTIONS" in script and "ALLOWED_CORE_IMPORTS" in script - for term in ("numpy", "pandas", "networkx", "transformers"): - assert term in script - assert "protected docstring-only modules" in script - - -def _restoration_bytes(repo_root): - after = (repo_root / "VALIDATION_PLAN.md").read_bytes() - before = after.replace(b"| Status | APPROVED PHASE 0 BASELINE |", b"| Status | DRAFT VALIDATION BASELINE |", 1) - before = before.replace(b"Definitions marked `APPROVED DECISION`", b"Definitions marked `PENDING DECISION`", 1) - return before, after - - -def test_phase2_restoration_accepts_only_original_approved_transition(repo_root): - tools = _release_tools(repo_root) - before, after = _restoration_bytes(repo_root) - result = tools["verify_approved_restoration"]("VALIDATION_PLAN.md", "M", before, after) - assert result["after_sha256"] == "16f5fe539da2b3cff1c3e0a2854208bd7fbc1e4c5b332684265b06a03a90cf2f" - assert "VALIDATION_PLAN.md" not in tools["STEP10_ALLOWED"] - - -@pytest.mark.parametrize("change", ["path", "added", "deleted", "before", "after", "unchanged", "type"]) -def test_phase2_restoration_rejects_wider_changes(repo_root, change): - verify = _release_tools(repo_root)["verify_approved_restoration"] - before, after = _restoration_bytes(repo_root) - path, status = "VALIDATION_PLAN.md", "M" - if change == "path": path = "DEFINITIONS_AND_UNITS.md" - elif change == "added": status = "A" - elif change == "deleted": status = "D" - elif change == "before": before += b"\n" - elif change == "after": after += b"\n" - elif change == "unchanged": after = before - elif change == "type": after = after.decode("utf-8") - with pytest.raises(ValueError): - verify(path, status, before, after) + result = _release_tools(repo_root)["verify_junit"](_junit(tmp_path, body, 'tests="5"'), require_parquet=True) + assert result["real_parquet_cases"] == 5 diff --git a/tests/integration/test_cli_validation.py b/tests/integration/test_cli_validation.py index 527d90d..2cf1744 100644 --- a/tests/integration/test_cli_validation.py +++ b/tests/integration/test_cli_validation.py @@ -1,24 +1,9 @@ -"""Check the Phase 1 help and version startup commands.""" +"""Check help and version startup against the current package.""" import subprocess import sys -def test_cli_help_runs(subprocess_env,phase3_final_subprocess_env) -> None: - subprocess_env = phase3_final_subprocess_env - result = subprocess.run( - [sys.executable, "-m", "recursive_integrity_toolkit", "--help"], - capture_output=True, - text=True, - env=subprocess_env, - check=False, - ) - assert result.returncode == 0, result.stderr - normalized = " ".join(result.stdout.split()) - assert "Phase 1 scaffold" in normalized - assert "Analytical audit functionality is not implemented" in normalized - - def test_cli_version_runs(subprocess_env) -> None: result = subprocess.run( [sys.executable, "-m", "recursive_integrity_toolkit", "version"], @@ -28,11 +13,11 @@ def test_cli_version_runs(subprocess_env) -> None: check=False, ) assert result.returncode == 0, result.stderr - assert result.stdout.strip() == "recursive-integrity-toolkit 0.1.0.dev3" + assert result.stdout.strip() == "recursive-integrity-toolkit 0.1.0.dev4" def test_phase4_cli_help_uses_current_source(subprocess_env) -> None: - """Current startup remains exercised after preserving the historical help text.""" + """The current CLI starts with a useful help response and no diagnostics.""" result = subprocess.run( [sys.executable, "-m", "recursive_integrity_toolkit", "--help"], capture_output=True, diff --git a/tests/integration/test_current_verification.py b/tests/integration/test_current_verification.py new file mode 100644 index 0000000..3859961 --- /dev/null +++ b/tests/integration/test_current_verification.py @@ -0,0 +1,166 @@ +"""Current integrity controls protect source scope, specifications and resources. + +These negatives exercise the live controls against disposable copies. Historical +phase tests and their source-binding migrations remain recoverable in Git. +""" + +import json +import runpy +import shutil + +import pytest + + +@pytest.fixture(scope="module") +def current_tools(repo_root): + return runpy.run_path(str(repo_root / "scripts/release_check.py")) + + +@pytest.fixture +def protected_copy(repo_root, tmp_path): + for relative in ("src", "schemas", "examples/hero"): + shutil.copytree(repo_root / relative, tmp_path / relative, + ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + shutil.copy2(repo_root / "pyproject.toml", tmp_path / "pyproject.toml") + return tmp_path + + +@pytest.mark.parametrize("mutation", ["formula", "new_module", "deleted_module", + "nested_metadata", "similar_metadata"]) +def test_current_scope_rejects_unauthorized_product_changes(current_tools, protected_copy, mutation): + """Lineage report integration does not authorize unrelated product changes.""" + verify = current_tools["verify_source_scope"] + generated = protected_copy / "src/recursive_integrity_toolkit.egg-info" + generated.mkdir(exist_ok=True) + (generated / "PKG-INFO").write_text("Generated wheel metadata\n", encoding="utf-8") + verify(protected_copy) + target = protected_copy / "src/recursive_integrity_toolkit/metrics/diversity.py" + if mutation == "formula": + source = target.read_text(encoding="utf-8") + assert "1.0 -" in source + target.write_text(source.replace("1.0 -", "1.0 +", 1), encoding="utf-8") + elif mutation == "new_module": + target.with_name("unauthorized_metric.py").write_text("def score(): return 1\n", encoding="utf-8") + elif mutation == "deleted_module": + target.unlink() + else: + relative = ("src/recursive_integrity_toolkit/recursive_integrity_toolkit.egg-info" + if mutation == "nested_metadata" else + "src/recursive_integrity_toolkit_other.egg-info") + extra = protected_copy / relative + extra.mkdir() + (extra / "unauthorized.py").write_text("def score(): return 1\n", encoding="utf-8") + with pytest.raises(ValueError): + verify(protected_copy) + + +def test_specification_changes_cannot_self_authorize(current_tools, repo_root, tmp_path): + """A changed specification and forged working manifest still fail the gate.""" + control = json.loads((repo_root / "PHASE_4_BASELINE.json").read_text(encoding="utf-8")) + for relative in control["phase0_sha256"]: + shutil.copy2(repo_root / relative, tmp_path / relative) + current_tools["verify_frozen_specifications"](tmp_path) + specification = tmp_path / "DEFINITIONS_AND_UNITS.md" + specification.write_bytes(specification.read_bytes() + b"\nUnauthorized formula change.\n") + control["phase0_sha256"][specification.name] = "0" * 64 + (tmp_path / "PHASE_4_BASELINE.json").write_text(json.dumps(control), encoding="utf-8") + with pytest.raises(ValueError): + current_tools["verify_frozen_specifications"](tmp_path) + + +@pytest.mark.parametrize("relative", [ + "src/recursive_integrity_toolkit/data/hero/config.json", + "src/recursive_integrity_toolkit/data/report.schema.json", +]) +def test_packaged_resource_corruption_is_rejected(current_tools, protected_copy, relative): + verify = current_tools["verify_resources"] + verify(protected_copy) + (protected_copy / relative).write_text("{}\n", encoding="utf-8") + with pytest.raises(ValueError): + verify(protected_copy) + + +def test_unlisted_packaged_resource_is_rejected(current_tools, protected_copy): + current_tools["verify_resources"](protected_copy) + (protected_copy / "src/recursive_integrity_toolkit/data/unlisted.json").write_text("{}", encoding="utf-8") + with pytest.raises(ValueError): + current_tools["verify_resources"](protected_copy) + + +def _write_test_archives(directory, wheel_entries, sdist_entries): + """Create validator inputs only; these are never release or install evidence.""" + import io + import tarfile + import warnings + import zipfile + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="Duplicate name:.*", category=UserWarning) + with zipfile.ZipFile(directory / "synthetic.whl", "w") as archive: + for name, content in wheel_entries: + archive.writestr(name, content) + with tarfile.open(directory / "synthetic.tar.gz", "w:gz") as archive: + for name, content in sdist_entries: + member = tarfile.TarInfo("synthetic/" + name) + member.size = len(content) + archive.addfile(member, io.BytesIO(content)) + + +@pytest.mark.parametrize("kind", ["wheel", "sdist"]) +@pytest.mark.parametrize("mutation", ["code", "resource", "missing_resource", "duplicate_member"]) +def test_distribution_integrity_rejects_changed_payloads(current_tools, repo_root, tmp_path, kind, mutation): + runtime = {path.relative_to(repo_root / "src").as_posix(): path.read_bytes() + for path in (repo_root / "src/recursive_integrity_toolkit").rglob("*.py")} + resources = {name.removeprefix("src/"): (repo_root / source).read_bytes() + for name, source in current_tools["RESOURCES"].items()} + wheel_entries = list(runtime.items()) + list(resources.items()) + [ + ("synthetic.dist-info/METADATA", b"Version: 0.1.0.test\n"), + ("synthetic.dist-info/LICENSE", b"synthetic notice"), + ("synthetic.dist-info/NOTICE", b"synthetic notice"), + ] + sdist_entries = [("src/" + name, raw) for name, raw in (*runtime.items(), *resources.items())] + sdist_entries += [("LICENSE", b"synthetic notice"), ("NOTICE", b"synthetic notice")] + verify = current_tools["verify_distributions"] + _write_test_archives(tmp_path, wheel_entries, sdist_entries) + verify(tmp_path, expected_version="0.1.0.test") + entries = wheel_entries if kind == "wheel" else sdist_entries + target = "recursive_integrity_toolkit/metrics/diversity.py" if mutation == "code" else "recursive_integrity_toolkit/data/report.schema.json" + if kind == "sdist": + target = "src/" + target + original = next(raw for name, raw in entries if name == target) + if mutation == "duplicate_member": + entries.append((target, original)) + elif mutation == "missing_resource": + entries[:] = [(name, raw) for name, raw in entries if name != target] + else: + entries[:] = [(name, b"corrupted payload" if name == target else raw) for name, raw in entries] + _write_test_archives(tmp_path, wheel_entries, sdist_entries) + with pytest.raises(ValueError): + verify(tmp_path, expected_version="0.1.0.test") + + +@pytest.mark.parametrize("mutation", ["parent", "absolute", "backslash", "drive", "symlink", "hardlink"]) +def test_sdist_extraction_rejects_unsafe_members_before_writes(current_tools, tmp_path, mutation): + import io + import tarfile + + path = tmp_path / "unsafe.tar.gz" + names = {"parent": "root/../outside", "absolute": "/outside", + "backslash": "root\\outside", "drive": "C:/outside"} + with tarfile.open(path, "w:gz") as archive: + first = tarfile.TarInfo("root/valid.txt") + first.size = 5 + archive.addfile(first, io.BytesIO(b"valid")) + member = tarfile.TarInfo(names.get(mutation, "root/link")) + if mutation in {"symlink", "hardlink"}: + member.type = tarfile.SYMTYPE if mutation == "symlink" else tarfile.LNKTYPE + member.linkname = "../outside" + archive.addfile(member) + else: + member.size = 6 + archive.addfile(member, io.BytesIO(b"unsafe")) + destination = tmp_path / "extracted" + destination.mkdir() + with pytest.raises(ValueError): + current_tools["extract_sdist"](path, destination) + assert list(destination.iterdir()) == [] + assert not (tmp_path / "outside").exists() diff --git a/tests/integration/test_hero_end_to_end.py b/tests/integration/test_hero_end_to_end.py index 25e1ec2..2292db9 100644 --- a/tests/integration/test_hero_end_to_end.py +++ b/tests/integration/test_hero_end_to_end.py @@ -284,7 +284,7 @@ def phase4_step8_hero_values(report): direct = report["derived_metrics"]["closure_exposure"]["direct"] assert [direct[n]["value"] for n in ("lower_bound", "upper_bound", "interval_width")] == [0.5, 0.5, 0] assert report["observability"]["maximum_level"] == 4 and report["capabilities"] == report["observability"]["capabilities"] - assert report["capabilities"]["lineage"]["execution_status"] == "deferred" + assert report["capabilities"]["lineage"]["execution_status"] == "not_requested" assert report["capabilities"]["dataset_longitudinal"]["execution_status"] == "partial" assert report["capabilities"]["model_longitudinal"]["status"] == "unavailable" assert report["capabilities"]["intervention_simulation"]["status"] == "unavailable" @@ -312,7 +312,7 @@ def denied(*args, **kwargs): report = json.loads((output / "reports/report.json").read_bytes()) phase4_step8_hero_values(report) assert report["run"]["command"] == "rit example" + (" --redacted" if redacted else "") - assert "full-product reference" in streams.err and "lineage execution is deferred" in streams.err + assert "Lineage was not requested" in streams.err and "rit example --lineage" in streams.err assert len(list((output / "inputs").iterdir())) == 6 for path in (output / "inputs").iterdir(): assert path.read_bytes() == (repo_root / "examples/hero" / path.name).read_bytes() diff --git a/tests/integration/test_no_algorithms.py b/tests/integration/test_no_algorithms.py index 73d816d..34f40a1 100644 --- a/tests/integration/test_no_algorithms.py +++ b/tests/integration/test_no_algorithms.py @@ -1,49 +1,13 @@ -"""Protect Phase 3 Step 9 scoped metrics from premature analytical implementation.""" +"""Current package dependency boundary. + +Direct product suites protect mathematical behavior. Historical staged module +allowlists and exact implementation forms are recoverable from Git history. +""" import ast -from pathlib import Path -STEP1_EXECUTABLE = {"__init__.py", "__main__.py", "cli.py", "config.py", "errors.py", "models.py"} -STEP2_EXECUTABLE = {"io/loaders.py", "utils/hashing.py", "utils/paths.py"} -STEP3_EXECUTABLE = {"io/schema_mapping.py"} -STEP8_EXECUTABLE = {"observability/levels.py"} -PHASE3_FIELD_EXECUTABLE = {"representations/base.py", "representations/field.py"} -PHASE3_DISTRIBUTION_EXECUTABLE = {"metrics/diversity.py"} -PHASE3_PROVENANCE_EXECUTABLE = {"metrics/provenance.py"} -PHASE3_BOUNDS_EXECUTABLE = {"metrics/bounds.py"} -PHASE3_TAIL_EXECUTABLE = {"metrics/tail.py"} -PHASE3_RESAMPLING_EXECUTABLE = {"metrics/resampling.py"} -PHASE3_PAIR_EXECUTABLE = {"representations/compatibility.py"} -PHASE3_EXACT_EXECUTABLE = {"representations/content_hash.py", "metrics/duplicates.py"} -STEP4_EXECUTABLE = {"io/normalization.py", "io/validation.py", "utils/ordering.py"} -PROTECTED_PREFIXES = {"io", "lineage", "metrics", "observability", "reports", "representations", "utils"} FORBIDDEN_ANALYTICAL_IMPORT_ROOTS = {"networkx", "scipy", "sklearn", "torch", "tensorflow", "transformers"} -FORBIDDEN_ANALYTICAL_NAMES = {"support_size", "gini_simpson", "tail_fragility", "source_type_shares", "closure_bounds", "ancestry_hhi", "effective_external_roots", "resample", "reopening"} - - -def _is_docstring_only(path: Path) -> bool: - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - return len(tree.body) == 1 and isinstance(tree.body[0], ast.Expr) and isinstance(tree.body[0].value, ast.Constant) and isinstance(tree.body[0].value.value, str) - - -def test_only_step8_authorized_modules_gain_behavior(package_root,phase3_final_package_root) -> None: - package_root = phase3_final_package_root - for path in sorted(package_root.rglob("*.py")): - relative = path.relative_to(package_root) - if relative.as_posix() in STEP2_EXECUTABLE | STEP3_EXECUTABLE | STEP4_EXECUTABLE | STEP8_EXECUTABLE | PHASE3_FIELD_EXECUTABLE | PHASE3_EXACT_EXECUTABLE | PHASE3_DISTRIBUTION_EXECUTABLE | PHASE3_PROVENANCE_EXECUTABLE | PHASE3_BOUNDS_EXECUTABLE | PHASE3_TAIL_EXECUTABLE | PHASE3_RESAMPLING_EXECUTABLE | PHASE3_PAIR_EXECUTABLE or (len(relative.parts) == 1 and path.name in STEP1_EXECUTABLE): - continue - assert _is_docstring_only(path), f"Premature executable body: {relative}" - - -def test_protected_phase3_plus_modules_remain_placeholders(package_root,phase3_final_package_root) -> None: - package_root = phase3_final_package_root - for prefix in PROTECTED_PREFIXES: - for path in sorted((package_root / prefix).rglob("*.py")): - if path.relative_to(package_root).as_posix() in STEP2_EXECUTABLE | STEP3_EXECUTABLE | STEP4_EXECUTABLE | STEP8_EXECUTABLE | PHASE3_FIELD_EXECUTABLE | PHASE3_EXACT_EXECUTABLE | PHASE3_DISTRIBUTION_EXECUTABLE | PHASE3_PROVENANCE_EXECUTABLE | PHASE3_BOUNDS_EXECUTABLE | PHASE3_TAIL_EXECUTABLE | PHASE3_RESAMPLING_EXECUTABLE | PHASE3_PAIR_EXECUTABLE: - continue - assert _is_docstring_only(path), f"Protected module changed after Step 8: {path}" - assert _is_docstring_only(package_root / "result.py") def test_no_prohibited_analytical_dependency_imports(package_root) -> None: @@ -56,563 +20,3 @@ def test_no_prohibited_analytical_dependency_imports(package_root) -> None: elif isinstance(node, ast.ImportFrom) and node.module: imported_roots.add(node.module.split(".")[0]) assert not (imported_roots & FORBIDDEN_ANALYTICAL_IMPORT_ROOTS) - - -def test_no_analytical_function_names_exist(package_root) -> None: - names = set() - for path in sorted(package_root.rglob("*.py")): - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - names.update(node.name for node in ast.walk(tree) if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))) - assert not (names & FORBIDDEN_ANALYTICAL_NAMES) - - -def test_PR003_traceability_script_enforces_step8_scope(repo_root,phase3_final_snapshot) -> None: - repo_root = phase3_final_snapshot - import subprocess - import sys - result = subprocess.run([sys.executable, "scripts/check_traceability.py"], - cwd=repo_root, capture_output=True, text=True, check=False) - assert result.returncode == 0, result.stdout + result.stderr - assert "protected docstring-only modules: 16" in result.stdout - - -# The checker itself must reject new work outside the current Step 8 boundary. -import shutil -import subprocess -import sys -import pytest - - -@pytest.mark.parametrize("relative,injected", [ - ("io/validation.py", '_read_source(None, None)\n'), - ("io/validation.py", 'load_table(None)\n'), - ("io/validation.py", 'inventory_source(None)\n'), - ("io/validation.py", "load_content_reference('x')\n"), - ("io/validation.py", 'validate_bundle(None)\n'), - ("io/validation.py", '_bundle_control(None, None)\n'), - ("io/validation.py", 'from .loaders import load_table\n'), - ("io/validation.py", 'from .normalization import normalize_row\n'), - ("io/validation.py", 'from .loaders import _parse_csv\n'), - ("io/validation.py", "Path('x').write_text('data')\n"), - - ("observability/levels.py", "import socket\n"), - ("observability/levels.py", "from ..metrics import diversity\n"), - ("observability/levels.py", "from ..lineage import graph\n"), - ("observability/levels.py", "from ..io.loaders import load_table\n"), - ("observability/levels.py", "eval('1')\n"), - ("observability/levels.py", "open('unexpected')\n"), - ("observability/levels.py", "lambda: None\n"), - ("observability/levels.py", "math.sqrt(4)\n"), - ("observability/levels.py", "math.fsum((0.5, 0.5))\n"), - ("observability/levels.py", "1 - 0.5 ** 2\n"), - ("observability/levels.py", "1 / 2\n"), - ("observability/levels.py", "def universal_score(): pass\n"), - - ("utils/paths.py", "import socket\n"), - ("utils/paths.py", "eval('1')\n"), - ("utils/paths.py", "os.system('unapproved')\n"), - ("utils/paths.py", "Path('x').write_text('unapproved')\n"), - ("io/loaders.py", "__import__('socket')\n"), - ("io/loaders.py", "os.remove('unapproved')\n"), - ("io/schema_mapping.py", "def premature(): pass\n"), - ("io/normalization.py", "def premature(): pass\n"), - ("io/validation.py", "def premature(): pass\n"), - ("observability/levels.py", "def premature(): pass\n"), - ("metrics/diversity.py", "def premature(): pass\n"), - ("lineage/graph.py", "def premature(): pass\n"), - ("representations/field.py", "def premature(): pass\n"), - ("reports/assembly.py", "def premature(): pass\n"), - ("utils/ordering.py", "def premature(): pass\n"), - ("result.py", "def premature(): pass\n"), - ("io/loaders.py", "def premature(): pass\n"), - ("io/loaders.py", "import pyarrow\n"), - ("io/loaders.py", "import pandas\n"), - ("models.py", "import math\n"), - ("io/normalization.py", "import socket\n"), - ("io/normalization.py", "import pandas\n"), - ("io/normalization.py", "eval('1')\n"), - ("io/normalization.py", "open('unapproved')\n"), - ("io/validation.py", "def derive_generation(): pass\n"), - ("io/validation.py", "import networkx\n"), - ("io/validation.py", "open('unapproved')\n"), - ("io/validation.py", "def source_type_shares(): pass\n"), - ("io/validation.py", "def closure_bounds(): pass\n"), - ("io/validation.py", "def resolve_parents(): pass\n"), - ("models.py", "class UniversalScore: pass\n"), - ("io/validation.py", "def detect_cycles(): pass\n"), - ("io/validation.py", "def lineage_depth(): pass\n"), - ("io/validation.py", "import socket\n"), - ("models.py", "__import__('socket')\n"), - ("models.py", "eval('1')\n"), - ("models.py", "lambda: None\n"), - ("utils/ordering.py", "def version_rank(): pass\n"), - ("io/schema_mapping.py", "import os\n"), - ("io/schema_mapping.py", "import socket\n"), - ("io/schema_mapping.py", "eval('1')\n"), - ("io/schema_mapping.py", "exec('pass')\n"), - ("io/schema_mapping.py", "__import__('os')\n"), - ("io/schema_mapping.py", "lambda: None\n"), - ("io/schema_mapping.py", "Path('unexpected').write_text('x')\n"), -]) -def test_PR002_checker_rejects_unauthorized_mutations(repo_root, package_root, tmp_path, relative, injected): - target = tmp_path / "repo" - script = target / "scripts/check_traceability.py" - script.parent.mkdir(parents=True) - script.write_bytes((repo_root / "scripts/check_traceability.py").read_bytes()) - copied = target / "src/recursive_integrity_toolkit" - shutil.copytree(package_root, copied, ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) - path = copied / relative - path.write_text(path.read_text(encoding="utf-8") + "\n" + injected, encoding="utf-8") - result = subprocess.run([sys.executable, str(script)], capture_output=True, text=True, check=False) - assert result.returncode != 0, f"Unauthorized implementation accepted: {relative}" - assert any(word in result.stderr for word in ("Protected", "Unexpected", "Dependency", "Import")), result.stderr - - -# Phase 3 Step 2: retain all prior negative identities and add pure-field gates. -@pytest.mark.parametrize("relative", ["representations/base.py", "representations/field.py"]) -@pytest.mark.parametrize("injection", [ - "import socket\n", "from ..metrics import diversity\n", "from ..models import __builtins__\n", - "from ..config import RepresentationConfig as eval\n", "value = 1\n", - "open('private')\n", "eval('1')\n", "lambda: None\n", - "def support_size(): return 1\n", "from ..io.loaders import load_table\n", -]) -def test_phase3_step2_injected_later_feature_is_rejected(repo_root, tmp_path, relative, injection): - target = tmp_path / "gate" - shutil.copytree(repo_root / "src", target / "src", ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) - (target / "scripts").mkdir() - shutil.copy2(repo_root / "scripts/check_traceability.py", target / "scripts/check_traceability.py") - file = target / "src/recursive_integrity_toolkit" / relative - file.write_text(file.read_text(encoding="utf-8") + "\n" + injection, encoding="utf-8") - result = subprocess.run([sys.executable, str(target / "scripts/check_traceability.py")], - cwd=target, text=True, capture_output=True) - assert result.returncode != 0, result.stdout + result.stderr - - -@pytest.mark.parametrize("body", [ - "return sum(values)", "return values ** 2", "return value / 2", "return value * value", - "return open(value)", "return values.pop()", "return callback(values)", - "return __import__('numpy')", "return (lambda: value)()", "return value.lower()", -]) -def test_phase3_step2_hidden_computation_is_rejected(repo_root, body): - import runpy - checker = runpy.run_path(str(repo_root / "scripts/check_traceability.py"), run_name="field_gate") - with pytest.raises(SystemExit): - checker["_phase3_representation_boundary"]( - ast.parse("def _selection():\n " + body + "\n"), "representations/field.py") - - -@pytest.mark.parametrize("path", ["PHASE_3_PLAN.md", "tests/golden/phase3_math_cases.json", - "src/recursive_integrity_toolkit/metrics/diversity.py", "src/recursive_integrity_toolkit/io/validation.py", - "VALIDATION_PLAN.md", "pyproject.toml"]) -def test_phase3_step2_incremental_scope_rejects_unapproved_edits(repo_root, path): - import runpy - checker = runpy.run_path(str(repo_root / "scripts/release_check.py"), run_name="step2_paths") - with pytest.raises(ValueError): - checker["verify_prior_step_changes"]([("M", path)], step=2) - - -@pytest.mark.parametrize("change", ["none", "extra_newline", "skip", "numeric_expectation", "wrong_type"]) -def test_phase3_step2_contract_test_exception_is_exact(repo_root, change): - import runpy - checker = runpy.run_path(str(repo_root / "scripts/release_check.py"), run_name="step2_exception") - after = (repo_root / "tests/unit/test_phase3_contracts.py").read_bytes() - before = after.replace(b"control[key]+=1", b"control[key]=2", 1) - checker["verify_step2_contract_test_migration"](before, after) - if change == "none": bad = before - elif change == "extra_newline": bad = after + b"\n" - elif change == "skip": bad = b"# skipped\n" + after - elif change == "numeric_expectation": bad = after.replace(b'=="5/8"', b'=="1/2"') - else: bad = after.decode() - with pytest.raises(ValueError): - checker["verify_step2_contract_test_migration"](before, bad) - - -# Phase 3 Step 3: inherited identities stay; exact-content modules have bounded calls. -@pytest.mark.parametrize("relative", ["representations/content_hash.py", "metrics/duplicates.py"]) -@pytest.mark.parametrize("injection", [ - "import socket\n", "from ..io.loaders import load_table\n", "from ..metrics import diversity\n", - "from ..utils.hashing import sha256_bytes as eval\n", "from ..models import __builtins__\n", - "open('private')\n", "def semantic_support(): return 1\n", "x = 1\n", - "lambda: None\n", "eval('1')\n", -]) -def test_phase3_step3_later_feature_injection_is_rejected(repo_root, tmp_path, relative, injection): - target = tmp_path / "gate" - shutil.copytree(repo_root / "src", target / "src", ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) - (target / "scripts").mkdir() - shutil.copy2(repo_root / "scripts/check_traceability.py", target / "scripts/check_traceability.py") - file = target / "src/recursive_integrity_toolkit" / relative - file.write_text(file.read_text(encoding="utf-8") + "\n" + injection, encoding="utf-8") - result = subprocess.run([sys.executable, str(target / "scripts/check_traceability.py")], - cwd=target, text=True, capture_output=True) - assert result.returncode != 0, result.stdout + result.stderr - - -@pytest.mark.parametrize("relative", ["representations/content_hash.py", "metrics/duplicates.py"]) -@pytest.mark.parametrize("body", [ - "return value ** 2", "return value / 2", "return sum(values)", "return callback(values)", - "return text.lower()", "return text.casefold()", "return text.replace('a', 'b')", - "return __import__('numpy')", "return open(value)", "return members.pop()", "value += 1", -]) -def test_phase3_step3_hidden_operation_is_rejected(repo_root, relative, body): - import runpy - checker = runpy.run_path(str(repo_root / "scripts/check_traceability.py"), run_name="exact_gate") - with pytest.raises(SystemExit): - checker["_phase3_exact_boundary"](ast.parse("def helper():\n " + body + "\n"), relative) - - -@pytest.mark.parametrize("path", ["tests/unit/test_phase3_contracts.py", - "src/recursive_integrity_toolkit/representations/base.py", - "src/recursive_integrity_toolkit/representations/field.py", - "src/recursive_integrity_toolkit/utils/hashing.py", "tests/golden/phase3_math_cases.json"]) -def test_phase3_step3_incremental_scope_keeps_prior_implementations_frozen(repo_root, path): - import runpy - checker = runpy.run_path(str(repo_root / "scripts/release_check.py"), run_name="step3_paths") - with pytest.raises(ValueError): - checker["verify_prior_step_changes"]([("M", path)], step=3) - - -# Step 4 adds a reviewed-body pin to all inherited phase boundaries. -@pytest.mark.parametrize("injection", [ - "import socket", "from math import log", "from ..io.loaders import load_table", - "from ..metrics.resampling import resample", "from ..lineage import graph", - "from math import fsum as eval", "open('private')", "x=1", - "def shannon_entropy(): return 1", "def support_delta(): return 1", - "def closure_bounds(): return 1", -]) -def test_phase3_step4_later_injection_rejected(repo_root,tmp_path,injection): - target=tmp_path/"step4-gate" - shutil.copytree(repo_root/"src",target/"src",ignore=shutil.ignore_patterns("__pycache__","*.pyc")) - (target/"scripts").mkdir() - shutil.copy2(repo_root/"scripts/check_traceability.py",target/"scripts/check_traceability.py") - p=target/"src/recursive_integrity_toolkit/metrics/diversity.py" - p.write_text(p.read_text(encoding="utf-8")+"\n"+injection+"\n",encoding="utf-8") - result=subprocess.run([sys.executable,str(target/"scripts/check_traceability.py")],cwd=target,capture_output=True,text=True) - assert result.returncode!=0 - - -@pytest.mark.parametrize("old,new", [ - ("value * value", "value ** 3"), ("1.0 - concentration", "1.0 - concentration / 2"), - ("count / total", "count / (total + 1)"), ("if value > 0", "if value >= 0"), - ("counts.get(state, 0) + 1", "counts.get(state, 0) + 2"), - ("return value", "return eval(value)"), -]) -def test_phase3_step4_hidden_formula_changes_rejected(repo_root,old,new): - import runpy - gate=runpy.run_path(str(repo_root/"scripts/check_traceability.py"),run_name="step4-ast") - source=(repo_root/"src/recursive_integrity_toolkit/metrics/diversity.py").read_text(encoding="utf-8") - assert old in source - gate["_phase3_distribution_boundary"](ast.parse(source)) - with pytest.raises(SystemExit): gate["_phase3_distribution_boundary"](ast.parse(source.replace(old,new,1))) - - -@pytest.mark.parametrize("path", [ - "src/recursive_integrity_toolkit/metrics/resampling.py", "src/recursive_integrity_toolkit/metrics/provenance.py", - "src/recursive_integrity_toolkit/metrics/duplicates.py", "src/recursive_integrity_toolkit/io/validation.py", - "src/recursive_integrity_toolkit/representations/base.py", "docs/data_schema.md", "docs/privacy.md", - "pyproject.toml", "PHASE_3_PLAN.md", "tests/golden/phase3_math_cases.json", -]) -def test_phase3_step4_unapproved_paths_rejected(repo_root,path): - import runpy - gate=runpy.run_path(str(repo_root/"scripts/release_check.py"),run_name="step4-path") - with pytest.raises(ValueError): gate["verify_prior_step_changes"]([("M",path)],step=4) - - -def test_phase3_step4_current_and_historical_permissions_separate(repo_root): - import runpy - gate=runpy.run_path(str(repo_root/"scripts/release_check.py"),run_name="step4-authorization") - path="src/recursive_integrity_toolkit/metrics/diversity.py" - gate["verify_prior_step_changes"]([("M",path)],step=4) - for step in (1,2,3): - with pytest.raises(ValueError): gate["verify_prior_step_changes"]([("M",path)],step=step) - - -# Step 5 retains the reviewed positive control and rejects future behavior. -@pytest.mark.parametrize('injection',[ - 'import socket', 'from ..io.loaders import load_table', 'from ..lineage import graph', - 'from .bounds import direct_closure_bounds', 'from math import log', 'from math import fsum as eval', - "open('private')", 'x = 1', 'def confidence_score(): return 1', - 'def effective_source_diversity(): return 1', 'def closure_bounds(): return 1', 'def resample(): return 1', -]) -def test_phase3_step5_rejects_later_features(repo_root,injection): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py'),run_name='step5-boundary') - source=(repo_root/'src/recursive_integrity_toolkit/metrics/provenance.py').read_text(encoding='utf-8') - gate['_phase3_provenance_boundary'](ast.parse(source)) - with pytest.raises(SystemExit):gate['_phase3_provenance_boundary'](ast.parse(source+'\n'+injection+'\n')) - - -@pytest.mark.parametrize('old,new',[ - ('counts[value] += 1','counts[value] += 2'), - ('counts[c] / len(entries)','counts[c] / (len(entries) + 1)'), - ('elif not row.required_fields_valid:', 'elif False:'), - ('mass / total','mass / (total + 1)'), - ('if row is None:\n continue','if row is None:\n counts["unknown"] += 1\n continue'), -]) -def test_phase3_step5_rejects_in_body_changes(repo_root,old,new): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py'),run_name='step5-formula') - source=(repo_root/'src/recursive_integrity_toolkit/metrics/provenance.py').read_text(encoding='utf-8') - assert old in source - gate['_phase3_provenance_boundary'](ast.parse(source)) - with pytest.raises(SystemExit):gate['_phase3_provenance_boundary'](ast.parse(source.replace(old,new,1))) - - -@pytest.mark.parametrize('path',[ - 'src/recursive_integrity_toolkit/metrics/bounds.py','src/recursive_integrity_toolkit/metrics/diversity.py', - 'src/recursive_integrity_toolkit/metrics/resampling.py','src/recursive_integrity_toolkit/metrics/duplicates.py', - 'src/recursive_integrity_toolkit/io/validation.py','src/recursive_integrity_toolkit/lineage/graph.py', - 'docs/data_schema.md','docs/privacy.md','pyproject.toml','PHASE_3_PLAN.md', - 'tests/unit/test_phase3_contracts.py','tests/golden/phase3_math_cases.json', -]) -def test_phase3_step5_unapproved_paths_rejected(repo_root,path): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/release_check.py'),run_name='step5-path') - with pytest.raises(ValueError):gate['verify_prior_step_changes']([('M',path)],step=5) - - -def test_phase3_step5_permissions_keep_prior_boundaries(repo_root): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/release_check.py'),run_name='step5-permissions') - path='src/recursive_integrity_toolkit/metrics/provenance.py' - gate['verify_prior_step_changes']([('M',path)],step=5) - for step in (1,2,3,4): - with pytest.raises(ValueError):gate['verify_prior_step_changes']([('M',path)],step=step) - assert gate['STEP5_NEW']=={ - 'tests/fixtures/provenance_partial/phase3_composition.json', - 'tests/fixtures/provenance_unknown/phase3_grounding_crossed.json', - 'tests/fixtures/weighted/phase3_source_weights.json'} - - -# Step 6 historical gates remain; only direct interval computation is newly opened. -@pytest.mark.parametrize('injection',[ - 'import socket','from ..lineage import graph','from .resampling import resample', - 'open("private")','x = 1','def midpoint(): return 0.5','def risk_score(): return 1', - 'def classify_lineage_grounding(): return 1','def ancestry_hhi(): return 1', -]) -def test_phase3_step6_rejects_later_bounds_features(repo_root,injection): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/bounds.py').read_text() - gate['_phase3_bounds_boundary'](ast.parse(source)) - with pytest.raises(SystemExit):gate['_phase3_bounds_boundary'](ast.parse(source+'\n'+injection+'\n')) - - -@pytest.mark.parametrize('old,new',[ - ('lower = known_closed / total','lower = 0.0'), - ('upper = (known_closed + unresolved) / total','upper = known_closed / total'), - ('width = unresolved / total','width = 0.0'), - ('required_cov.numerator == 0','False'), -]) -def test_phase3_step6_rejects_formula_and_availability_edits(repo_root,old,new): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/bounds.py').read_text() - assert old in source - with pytest.raises(SystemExit):gate['_phase3_bounds_boundary'](ast.parse(source.replace(old,new,1))) - - -@pytest.mark.parametrize('path',[ - 'src/recursive_integrity_toolkit/metrics/provenance.py','src/recursive_integrity_toolkit/metrics/diversity.py', - 'src/recursive_integrity_toolkit/metrics/resampling.py','src/recursive_integrity_toolkit/lineage/graph.py', - 'tests/unit/test_phase3_contracts.py','docs/data_schema.md','docs/privacy.md','PHASE_3_PLAN.md','pyproject.toml', -]) -def test_phase3_step6_unapproved_file_changes_fail(repo_root,path): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/release_check.py')) - with pytest.raises(ValueError):gate['verify_prior_step_changes']([('M',path)],step=6) - - -def test_phase3_step6_requires_reviewed_source_bytes(repo_root,tmp_path): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/bounds.py').read_text() - (tmp_path/'metrics').mkdir();(tmp_path/'metrics/bounds.py').write_text(source+'\nx = 1\n') - gate['_phase3_bounds_boundary'].__globals__['PACKAGE']=tmp_path - with pytest.raises(SystemExit):gate['_phase3_bounds_boundary'](ast.parse(source+'\nx = 1\n')) - - -# Step 7: only observed tail/rank and analytic one-step F-014 are authorized. -@pytest.mark.parametrize('injection',[ - 'import socket','from ..lineage import graph','from .resampling import resample', - 'import numpy.random','open("private")','x = 1','def risk_score(): return 1', - 'def reopen(): return 1','def tail_fragility_signal(): return "high"', - 'def bottom_frequency_quantile(): return 1','def simulate(): return 1', -]) -def test_phase3_step7_rejects_later_tail_features(repo_root,injection): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/tail.py').read_text() - gate['_phase3_tail_boundary'](ast.parse(source)) - with pytest.raises(SystemExit):gate['_phase3_tail_boundary'](ast.parse(source+'\n'+injection+'\n')) - - -@pytest.mark.parametrize('old,new',[ - ('probability = exp(resample_size * log1p(-p))','probability = 0.5'), - ('row.state_count == 1','row.state_count <= 1'), - ('row.state_frequency <= options.frequency_threshold','row.state_frequency < options.frequency_threshold'), - ('row.state_count > 0','row.state_count >= 0'), - ('CalculationEvidenceClass.SIMULATION if scenario','CalculationEvidenceClass.DERIVED_METRIC if scenario'), -]) -def test_phase3_step7_rejects_formula_or_rule_rewrite(repo_root,old,new): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/tail.py').read_text() - assert old in source - with pytest.raises(SystemExit):gate['_phase3_tail_boundary'](ast.parse(source.replace(old,new,1))) - - -@pytest.mark.parametrize('path',[ - 'src/recursive_integrity_toolkit/metrics/provenance.py','src/recursive_integrity_toolkit/metrics/diversity.py', - 'src/recursive_integrity_toolkit/metrics/bounds.py','src/recursive_integrity_toolkit/metrics/resampling.py', - 'src/recursive_integrity_toolkit/lineage/graph.py','src/recursive_integrity_toolkit/io/validation.py', - 'docs/data_schema.md','docs/privacy.md','PHASE_3_PLAN.md','pyproject.toml', - 'tests/golden/phase3_math_cases.json', -]) -def test_phase3_step7_unapproved_file_changes_fail(repo_root,path): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/release_check.py')) - with pytest.raises(ValueError):gate['verify_prior_step_changes']([('M',path)],step=7) - - -def test_phase3_step7_requires_reviewed_source_bytes(repo_root,tmp_path): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/tail.py').read_text() - (tmp_path/'metrics').mkdir();(tmp_path/'metrics/tail.py').write_text(source+'\nx = 1\n') - gate['_phase3_tail_boundary'].__globals__['PACKAGE']=tmp_path - with pytest.raises(SystemExit):gate['_phase3_tail_boundary'](ast.parse(source+'\nx = 1\n')) - - -# Step 8 authorizes only F-015 and the explicit fixed-state closed sampler. -@pytest.mark.parametrize('injection',[ - 'import socket', 'import numpy as np', 'np.random.seed(1)', - 'from ..lineage import graph', 'from ..io.loaders import load_table', - 'open("private")', 'def simulate_reopening(): return 1', - 'def external_reference_loss(): return 1', 'def report(): return "ok"', - 'def auto_simulate(): return 1', 'def universal_score(): return 1', -]) -def test_phase3_step8_rejects_unapproved_resampling_features(repo_root,injection): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/resampling.py').read_text() - gate['_phase3_resampling_boundary'](ast.parse(source)) - with pytest.raises(SystemExit):gate['_phase3_resampling_boundary'](ast.parse(source+'\n'+injection+'\n')) - - -@pytest.mark.parametrize('old,new',[ - ('factor = 1.0 - 1.0 / resample_size','factor = 1.0'), - ('masses = counts','masses = probabilities'), - ('np.random.PCG64(seed)','np.random.PCG64()'), - ('mass == 0 or remaining == 0','remaining == 0'), - ('corrected = sampled and total != 1.0','corrected = True'), - ('abs(total - 1.0) > tolerance','abs(total - 1.0) > .1'), - ('CalculationEvidenceClass.SIMULATION','CalculationEvidenceClass.OBSERVED_FACT'), - ('replicates: int, scope:', 'replicates: int = 1, scope:'), -]) -def test_phase3_step8_rejects_formula_seed_and_evidence_edits(repo_root,old,new): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/resampling.py').read_text() - assert old in source - with pytest.raises(SystemExit):gate['_phase3_resampling_boundary'](ast.parse(source.replace(old,new,1))) - - -@pytest.mark.parametrize('path',[ - 'src/recursive_integrity_toolkit/metrics/tail.py','src/recursive_integrity_toolkit/metrics/diversity.py', - 'src/recursive_integrity_toolkit/metrics/provenance.py','src/recursive_integrity_toolkit/metrics/bounds.py', - 'src/recursive_integrity_toolkit/io/validation.py','src/recursive_integrity_toolkit/lineage/graph.py', - 'src/recursive_integrity_toolkit/representations/compatibility.py', - 'PHASE_3_PLAN.md','pyproject.toml','tests/golden/phase3_math_cases.json','docs/data_schema.md', -]) -def test_phase3_step8_unapproved_file_changes_fail(repo_root,path): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/release_check.py')) - with pytest.raises(ValueError):gate['verify_prior_step_changes']([('M',path)],step=8) - - -def test_phase3_step8_current_permission_does_not_change_prior_permission(repo_root): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/release_check.py')) - path='src/recursive_integrity_toolkit/metrics/resampling.py' - gate['verify_prior_step_changes']([('M',path)],step=8) - for step in range(1,8): - with pytest.raises(ValueError):gate['verify_prior_step_changes']([('M',path)],step=step) - assert gate['STEP8_NEW']=={'tests/fixtures/resampling/phase3_closed_cases.json'} - - -def test_phase3_step8_reviewed_source_bytes_required(repo_root,tmp_path): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/resampling.py').read_text() - (tmp_path/'metrics').mkdir();(tmp_path/'metrics/resampling.py').write_text(source+'\nx = 1\n') - gate['_phase3_resampling_boundary'].__globals__['PACKAGE']=tmp_path - with pytest.raises(SystemExit):gate['_phase3_resampling_boundary'](ast.parse(source+'\nx = 1\n')) - - -# Step 9 opens only compatibility declarations and an explicitly selected pair. -@pytest.mark.parametrize('path,boundary',[ - ('representations/compatibility.py','_phase3_pair_boundary'), - ('metrics/diversity.py','_phase3_distribution_boundary'), -]) -@pytest.mark.parametrize('injection',[ - 'import socket', 'from ..io.loaders import load_table', 'from ..lineage import graph', - 'open("private")', 'def compare_all_versions(): return []', - 'def model_performance_delta(): return 1', 'def render_report(): return "ok"', - 'def infer_state_mapping(): return {}', -]) -def test_phase3_step9_rejects_unapproved_pair_features(repo_root,path,boundary,injection): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit'/path).read_text(encoding='utf-8') - gate[boundary](ast.parse(source)) - with pytest.raises(SystemExit):gate[boundary](ast.parse(source+'\n'+injection+'\n')) - - -@pytest.mark.parametrize('old,new',[ - ('len(right) - len(left)','len(left) - len(right)'), - ('len(left & right) / len(left)','len(left & right) / len(right)'), - ('b.gini_simpson_diversity.value - a.gini_simpson_diversity.value','0.0'), - ('a.scope != context.earlier_scope','False'), - ('if any(row.state_id not in mapping for row in value.states):','if False:'), -]) -def test_phase3_step9_rejects_formula_or_scope_bypass(repo_root,old,new): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/metrics/diversity.py').read_text(encoding='utf-8') - assert old in source - with pytest.raises(SystemExit):gate['_phase3_distribution_boundary'](ast.parse(source.replace(old,new,1))) - - -@pytest.mark.parametrize('path',[ - 'src/recursive_integrity_toolkit/metrics/resampling.py','src/recursive_integrity_toolkit/metrics/tail.py', - 'src/recursive_integrity_toolkit/metrics/provenance.py','src/recursive_integrity_toolkit/metrics/bounds.py', - 'src/recursive_integrity_toolkit/io/validation.py','src/recursive_integrity_toolkit/lineage/graph.py', - 'src/recursive_integrity_toolkit/reports/json_report.py','src/recursive_integrity_toolkit/cli.py', - 'tests/unit/test_phase3_contracts.py','PHASE_3_PLAN.md','tests/golden/phase3_math_cases.json', - 'docs/data_schema.md','docs/privacy.md','pyproject.toml', -]) -def test_phase3_step9_prohibits_unlisted_modifications(repo_root,path): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/release_check.py')) - with pytest.raises(ValueError):gate['verify_prior_step_changes']([('M',path)],step=9) - - -def test_phase3_step9_current_scope_does_not_change_step8_scope(repo_root): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/release_check.py')) - path='src/recursive_integrity_toolkit/representations/compatibility.py' - gate['verify_prior_step_changes']([('M',path)],step=9) - with pytest.raises(ValueError):gate['verify_prior_step_changes']([('M',path)],step=8) - assert gate['STEP9_NEW']=={ - 'tests/unit/test_T1_compatibility.py','tests/fixtures/representation_compatible/phase3_pair.json', - 'tests/fixtures/representation_incompatible/phase3_pair.json'} - - -def test_phase3_step9_compatibility_source_bytes_must_be_reviewed(repo_root,tmp_path): - import runpy - gate=runpy.run_path(str(repo_root/'scripts/check_traceability.py')) - source=(repo_root/'src/recursive_integrity_toolkit/representations/compatibility.py').read_text(encoding='utf-8') - (tmp_path/'representations').mkdir() - (tmp_path/'representations/compatibility.py').write_text(source+'\nx = 1\n',encoding='utf-8') - gate['_phase3_pair_boundary'].__globals__['PACKAGE']=tmp_path - with pytest.raises(SystemExit):gate['_phase3_pair_boundary'](ast.parse(source+'\nx = 1\n')) diff --git a/tests/integration/test_optional_dependency.py b/tests/integration/test_optional_dependency.py index a25b3fa..c736752 100644 --- a/tests/integration/test_optional_dependency.py +++ b/tests/integration/test_optional_dependency.py @@ -29,7 +29,7 @@ def test_core_import_without_optional_pyarrow(subprocess_env) -> None: check=False, ) assert result.returncode == 0, result.stderr - assert result.stdout.strip() == "0.1.0.dev3" + assert result.stdout.strip() == "0.1.0.dev4" def test_PR002_csv_and_jsonl_work_with_pyarrow_blocked(subprocess_env, tmp_path) -> None: diff --git a/tests/integration/test_package_install.py b/tests/integration/test_package_install.py index e9dd885..60064bd 100644 --- a/tests/integration/test_package_install.py +++ b/tests/integration/test_package_install.py @@ -10,7 +10,7 @@ def test_package_metadata_and_entry_points(repo_root: Path) -> None: config = tomllib.loads((repo_root / "pyproject.toml").read_text(encoding="utf-8")) project = config["project"] assert project["name"] == "recursive-integrity-toolkit" - assert project["version"] == "0.1.0.dev3" + assert project["version"] == "0.1.0.dev4" assert project["requires-python"] == ">=3.11" assert project["license"] == "Apache-2.0" assert project["scripts"] == { diff --git a/tests/integration/test_partial_lineage_report.py b/tests/integration/test_partial_lineage_report.py index 1a63873..c13aae2 100644 --- a/tests/integration/test_partial_lineage_report.py +++ b/tests/integration/test_partial_lineage_report.py @@ -96,7 +96,7 @@ def test_phase2_step9_content_limits_and_invalid_utf8(tmp_path): def phase4_step3_run(): return { "run_id": "step3-independent-case", "toolkit_version": "0.1.0.dev2", - "report_schema_version": "1.0", "started_at": None, "completed_at": None, + "report_schema_version": "1.1", "started_at": None, "completed_at": None, "duration_seconds": None, "python_version": None, "platform": None, "command": None, "config_hash": None, "random_seed": None, "strict_mode": False, "redacted_mode": False, "network_call_count": 0, @@ -130,14 +130,14 @@ def phase4_step3_schema(report, repo_root): return payload -def test_phase4_step3_level_four_keeps_lineage_input_available_and_execution_deferred(tmp_path, repo_root): +def test_phase4_step3_level_four_keeps_lineage_input_available_and_execution_not_requested(tmp_path, repo_root): from recursive_integrity_toolkit.reports.assembly import assemble_report bundle = validate_bundle(_bundle(tmp_path, ["v1::p"]), configuration=_config()) report = phase4_step3_schema(assemble_report(bundle, run=phase4_step3_run()), repo_root) assert report["observability"]["maximum_level"] == 4 lineage = report["capabilities"]["lineage"] - assert lineage["status"] == "available" and lineage["execution_status"] == "deferred" + assert lineage["status"] == "available" and lineage["execution_status"] == "not_requested" assert lineage["execution_reason_codes"] assert report["capabilities"]["dataset_longitudinal"]["status"] == "available" assert report["capabilities"]["dataset_longitudinal"]["execution_status"] == "not_requested" @@ -171,7 +171,7 @@ def test_phase4_step3_unresolved_parent_warning_survives_without_observability_d report = phase4_step3_schema(assemble_report(bundle, run=phase4_step3_run()), repo_root) assert report["observability"]["maximum_level"] == 4 assert report["capabilities"]["lineage"]["status"] == "unavailable" - assert report["capabilities"]["lineage"]["execution_status"] == "deferred" + assert report["capabilities"]["lineage"]["execution_status"] == "not_requested" assert any(item["code"] == WarningCode.PARENT_UNRESOLVED.value for item in report["warnings"]) assert not report["errors"] and report["run"]["run_status"] == "complete" @@ -204,7 +204,7 @@ def phase4_step3_forbid(*args, **kwargs): monkeypatch.setattr(assembly, name, phase4_step3_forbid) report = phase4_step3_schema(assembly.assemble_report(bundle, run=phase4_step3_run()), repo_root) assert report["observability"]["maximum_level"] == 4 - assert report["capabilities"]["lineage"]["execution_status"] == "deferred" + assert report["capabilities"]["lineage"]["execution_status"] == "not_requested" @pytest.mark.parametrize("redacted", [False, True]) @@ -240,7 +240,7 @@ def test_phase4_step9_parent_failure_survives_pair_cli_and_both_formats( assert all(path.read_bytes() == raw for path, raw in before.items()) assert report["observability"]["maximum_level"] == 4 lineage = report["capabilities"]["lineage"] - assert lineage["status"] == "unavailable" and lineage["execution_status"] == "deferred" + assert lineage["status"] == "unavailable" and lineage["execution_status"] == "not_requested" assert lineage == report["observability"]["capabilities"]["lineage"] support = report["derived_metrics"]["support"] assert support["support_delta"]["value"] == 0 diff --git a/tests/integration/test_partial_provenance_report.py b/tests/integration/test_partial_provenance_report.py index 5c2e8fe..f784865 100644 --- a/tests/integration/test_partial_provenance_report.py +++ b/tests/integration/test_partial_provenance_report.py @@ -391,7 +391,7 @@ def test_PR004_pipeline_incomplete_assessment_preserves_failure_evidence(repo_ro def phase4_step3_run(): return { "run_id": "step3-independent-case", "toolkit_version": "0.1.0.dev2", - "report_schema_version": "1.0", "started_at": None, "completed_at": None, + "report_schema_version": "1.1", "started_at": None, "completed_at": None, "duration_seconds": None, "python_version": None, "platform": None, "command": None, "config_hash": None, "random_seed": None, "strict_mode": False, "redacted_mode": False, "network_call_count": 0, diff --git a/tests/integration/test_phase3_metric_pipeline.py b/tests/integration/test_phase3_metric_pipeline.py index 7575361..628d2bd 100644 --- a/tests/integration/test_phase3_metric_pipeline.py +++ b/tests/integration/test_phase3_metric_pipeline.py @@ -6,12 +6,10 @@ from dataclasses import replace from fractions import Fraction from pathlib import Path -import ast import builtins import hashlib import importlib import json -import runpy import socket import pytest @@ -218,186 +216,3 @@ def test_phase3_hero_source_and_oracle_bytes_are_not_rewritten(repo_root,phase3_ assert before == {p:hashlib.sha256(p.read_bytes()).hexdigest() for p in before} assert len(result["bundle"].records) == 16 assert not any((repo_root/name).exists() for name in ("hero_report.json","hero_report.md","lineage_report.json","cycle_report.json")) - - -@pytest.mark.parametrize("module",["lineage/graph.py","lineage/cycles.py","lineage/ancestry.py","reports/assembly.py","reports/json_report.py","reports/markdown_report.py","reports/html_report.py","result.py"]) -def test_phase3_later_implementations_remain_empty(package_root,module,phase3_final_package_root): - package_root = phase3_final_package_root - tree = ast.parse((package_root/module).read_bytes()) - assert len(tree.body) == 1 and isinstance(tree.body[0],ast.Expr) and isinstance(tree.body[0].value,ast.Constant) - assert isinstance(tree.body[0].value.value,str) - - -@pytest.mark.parametrize("path",[ - "src/recursive_integrity_toolkit/metrics/diversity.py", - "src/recursive_integrity_toolkit/metrics/provenance.py", - "src/recursive_integrity_toolkit/io/validation.py", - "src/recursive_integrity_toolkit/reports/assembly.py", - "src/recursive_integrity_toolkit/cli.py", - "tests/unit/test_T1_compatibility.py","tests/unit/test_phase3_contracts.py", - "PHASE_3_PLAN.md","DEFINITIONS_AND_UNITS.md","pyproject.toml", - "examples/hero/EXPECTED_OUTPUTS.md","schemas/report.schema.json", -]) -def test_phase3_step10_rejects_unopened_file_changes(repo_root,path): - gate = runpy.run_path(str(repo_root/"scripts/release_check.py")) - with pytest.raises(ValueError): - gate["verify_prior_step_changes"]([("M",path)],step=10) - - -def test_phase3_step10_scope_is_fixed_not_self_authorized(repo_root,phase3_step10_snapshot): - gate = runpy.run_path(str(repo_root/"scripts/release_check.py")) - control = json.loads((phase3_step10_snapshot/"PHASE_3_BASELINE.json").read_bytes()) - gate["verify_step10_control"](control) - assert gate["PHASE3_STEP10_NEW"] == {"tests/integration/test_phase3_metric_pipeline.py","tests/golden/test_phase3_math.py"} - assert not any(p.startswith("src/") for p in gate["PHASE3_STEP10_ALLOWED"]) - for key,value in (("active_step",11),("previous_step_commit","0"*40),("phase_complete",True),("main_merge_authorized",True)): - altered = dict(control,**{key:value}) - with pytest.raises(ValueError): - gate["verify_step10_control"](altered) - altered = dict(control,permitted_paths=control["permitted_paths"]+["src/new.py"]) - with pytest.raises(ValueError): - gate["verify_step10_control"](altered) - assert gate["verify_step10_snapshot"](phase3_step10_snapshot)["runtime_modules_unchanged"] == 40 - - -@pytest.mark.parametrize("path",["src/recursive_integrity_toolkit/metrics/diversity.py","tests/golden/phase3_math_cases.json","examples/hero/records_v2.csv","pyproject.toml"]) -def test_phase3_step10_snapshot_rejects_byte_changes(repo_root,tmp_path,path,phase3_step10_snapshot): - import shutil - gate = runpy.run_path(str(repo_root/"scripts/release_check.py")) - shutil.copytree(phase3_step10_snapshot/"src/recursive_integrity_toolkit",tmp_path/"src/recursive_integrity_toolkit",ignore=shutil.ignore_patterns("__pycache__")) - for name in gate["STEP10_FROZEN_FILES"]: - target = tmp_path/name - target.parent.mkdir(parents=True,exist_ok=True) - target.write_bytes((phase3_step10_snapshot/name).read_bytes()) - gate["verify_step10_snapshot"](tmp_path) - target = tmp_path/path - target.write_bytes(target.read_bytes()+b"\n") - with pytest.raises(ValueError): - gate["verify_step10_snapshot"](tmp_path) - - -def _synthetic_evidence(repo_root,tmp_path,change=None): - # Synthetic XML solely for testing the evidence validator, never run evidence. - import xml.etree.ElementTree as ET - root = ET.Element("testsuites") - suite = ET.SubElement(root,"testsuite",tests="21",failures="0",errors="0",skipped="0") - cases = json.loads((repo_root/"tests/golden/phase3_math_cases.json").read_bytes())["cases"] - for c in cases: - ET.SubElement(suite,"testcase",classname="tests.golden.test_phase3_math",name="test_phase3_frozen_case["+c["case_id"]+"]") - test = ET.SubElement(suite,"testcase",classname="synthetic",name="synthetic_performance_evidence") - properties = ET.SubElement(test,"properties") - names = ["hero_input_plus_calculations","metadata_100k_setup","metadata_100k_field_representation","metadata_100k_support_diversity", - "metadata_100k_provenance_join","metadata_100k_composition","metadata_100k_exact_duplicates"] - for name in names: - d = dict(name=name,elapsed_seconds=0.0,traced_current_bytes=0,traced_peak_bytes=0, - record_count=16 if name.startswith("hero") else 100000,whole_product_report_target_certified=False, - memory_method="synthetic validator test",timing_method="synthetic validator test",python="synthetic",platform="synthetic") - if change and name == names[-1]: - d.update(change) - ET.SubElement(properties,"property",name="phase3_performance",value=json.dumps(d)) - path = tmp_path/"synthetic-evidence.xml" - ET.ElementTree(root).write(path,encoding="utf-8") - return path - - -@pytest.mark.parametrize("change",[{"elapsed_seconds":-1},{"elapsed_seconds":True},{"traced_peak_bytes":float("nan")}, - {"record_count":1000},{"whole_product_report_target_certified":True},{"memory_method":""},{"name":"metadata_100k_setup"}]) -def test_phase3_step10_rejects_invalid_performance_evidence(repo_root,tmp_path,change): - gate = runpy.run_path(str(repo_root/"scripts/release_check.py")) - with pytest.raises(ValueError): - gate["verify_step10_evidence"](_synthetic_evidence(repo_root,tmp_path,change)) - - -def test_phase3_step10_evidence_requires_executed_cases_and_all_measurements(repo_root,tmp_path): - import xml.etree.ElementTree as ET - gate = runpy.run_path(str(repo_root/"scripts/release_check.py")) - path = _synthetic_evidence(repo_root,tmp_path) - assert gate["verify_step10_evidence"](path)["frozen_cases_executed"] == 20 - tree = ET.parse(path) - suite = tree.getroot().find("testsuite") - suite.remove(suite.find("testcase")) - tree.write(path) - with pytest.raises(ValueError): - gate["verify_step10_evidence"](path) - path = _synthetic_evidence(repo_root,tmp_path) - tree = ET.parse(path) - props = tree.getroot().find(".//properties") - props.remove(props.find("property")) - tree.write(path) - with pytest.raises(ValueError): - gate["verify_step10_evidence"](path) - - -@pytest.mark.parametrize("path",[ - "src/recursive_integrity_toolkit/metrics/diversity.py", - "src/recursive_integrity_toolkit/models.py", - "src/recursive_integrity_toolkit/io/validation.py", - "src/recursive_integrity_toolkit/cli.py", - "tests/unit/test_T1_compatibility.py", - "tests/golden/phase3_math_cases.json", - "tests/golden/phase3_math_cases.md", - "PHASE_3_PLAN.md","PHASE_2_COMPLETION.md", - "schemas/report.schema.json","examples/hero/EXPECTED_OUTPUTS.md", -]) -def test_phase3_step11_rejects_unopened_changes(repo_root,path): - gate = runpy.run_path(str(repo_root/"scripts/release_check.py")) - with pytest.raises(ValueError): - gate["verify_phase3_changes"]([("M",path)],incremental=True) - - -def test_phase3_step11_control_and_new_files_are_fixed(repo_root): - gate = runpy.run_path(str(repo_root/"scripts/release_check.py")) - control = json.loads((repo_root/"PHASE_3_BASELINE.json").read_bytes()) - gate["verify_phase3_control"](control,11) - assert gate["PHASE3_STEP11_NEW"] == {"PHASE_3_COMPLETION.md","PHASE_3_VALIDATION_REPORT.md","PHASE_3_ARCHITECTURE_COMPLIANCE_REPORT.md"} - assert gate["STEP11_TEST_EXCEPTIONS"] == {"tests/unit/test_phase3_contracts.py","tests/integration/test_phase3_metric_pipeline.py"} - for key,value in (("active_step",12),("phase_complete",not control["phase_complete"]), - ("previous_step_commit","0"*40),("next_step_authorized",True), - ("main_merge_authorized",True),("publication_authorized",True)): - with pytest.raises(ValueError): - gate["verify_phase3_control"](dict(control,**{key:value}),11) - for status,path in (("A","src/new.py"),("A","tests/unit/test_phase3_contracts.py"), - ("D","README.md"),("R100","docs/privacy.md")): - with pytest.raises(ValueError): - gate["verify_phase3_changes"]([(status,path)],incremental=True) - - -@pytest.mark.parametrize("path",[ - "src/recursive_integrity_toolkit/__init__.py", - "src/recursive_integrity_toolkit/metrics/resampling.py", - "pyproject.toml","tests/golden/phase3_math_cases.json", -]) -def test_phase3_step11_rejects_changes_beyond_version_literals(repo_root,tmp_path,path,phase3_final_snapshot): - repo_root = phase3_final_snapshot - import shutil - gate = runpy.run_path(str(repo_root/"scripts/release_check.py")) - shutil.copytree(repo_root/"src/recursive_integrity_toolkit",tmp_path/"src/recursive_integrity_toolkit",ignore=shutil.ignore_patterns("__pycache__")) - for name in gate["STEP10_FROZEN_FILES"]: - target = tmp_path/name - target.parent.mkdir(parents=True,exist_ok=True) - target.write_bytes((repo_root/name).read_bytes()) - assert gate["verify_step11_snapshot"](tmp_path)["approved_version_literals"] == 2 - target = tmp_path/path - original = target.read_bytes() - for altered in (original+b"\n", original.replace(b"0.1.0.dev2",b"0.1.0.dev3")): - if altered == original: - continue - target.write_bytes(altered) - with pytest.raises(ValueError): - gate["verify_step11_snapshot"](tmp_path) - - -def test_phase3_step11_test_exceptions_are_exact(repo_root,phase3_step10_snapshot,phase3_final_snapshot): - repo_root = phase3_final_snapshot - gate = runpy.run_path(str(repo_root/"scripts/release_check.py")) - unit = "tests/unit/test_phase3_contracts.py" - pipeline = "tests/integration/test_phase3_metric_pipeline.py" - for name,verify in ((unit,gate["verify_step11_contract_test_migration"]), - (pipeline,gate["verify_step11_pipeline_test_migration"])): - before = (phase3_step10_snapshot/name).read_bytes() - current = (repo_root/name).read_bytes() - verify(before,current) - with pytest.raises(ValueError): - verify(before,b"# unauthorized inherited edit\n"+current) - with pytest.raises(ValueError): - verify(before,current+b"\nFORGED_TEST = True\n") diff --git a/tests/integration/test_phase4_cli.py b/tests/integration/test_phase4_cli.py index 8de3968..3b26857 100644 --- a/tests/integration/test_phase4_cli.py +++ b/tests/integration/test_phase4_cli.py @@ -51,7 +51,7 @@ def test_phase4_step7_explicit_single_version_metrics_match_hand_counts(tmp_path assert report["run"]["resolved_options"]["weighted"] is False assert "weighted_support_size" not in report["derived_metrics"]["support"]["by_version"]["v1"] assert report["simulations"] == {} and "tail" not in report["derived_metrics"] - assert report["capabilities"]["lineage"]["execution_status"] == "deferred" + assert report["capabilities"]["lineage"]["execution_status"] == "not_requested" assert report["capabilities"] == report["observability"]["capabilities"] assert all(path.read_bytes() == data for path, data in before.items()) assert {artifact["role"] for artifact in report["inputs"]["artifacts"]} == {"config", "records_primary", "provenance_manifest"} @@ -202,10 +202,13 @@ def test_phase4_step7_validate_blocks_every_calculation_and_content_reader(tmp_p def denied(*args, **kwargs): touched.append(True) raise AssertionError("calculation or content-reference reader executed") - for name in ("metrics.diversity", "metrics.provenance", "metrics.bounds", "metrics.duplicates", "metrics.tail", - "metrics.resampling", "representations.base", "representations.field", "representations.content_hash", - "representations.compatibility"): - module = importlib.import_module("recursive_integrity_toolkit." + name) + names = ("metrics.diversity", "metrics.provenance", "metrics.bounds", "metrics.duplicates", "metrics.tail", + "metrics.resampling", "representations.base", "representations.field", "representations.content_hash", + "representations.compatibility", "lineage.graph", "lineage.cycles", "lineage.ancestry") + # Import dependencies before patching so late imports cannot retain a stub + # after monkeypatch restores the original owner module. + modules = [importlib.import_module("recursive_integrity_toolkit." + name) for name in names] + for module in modules: for key, value in vars(module).copy().items(): if inspect.isfunction(value) and value.__module__ == module.__name__: monkeypatch.setattr(module, key, denied) @@ -348,7 +351,7 @@ def find_spec(self,fullname,path=None,target=None): ''' result = subprocess.run([sys.executable, "-c", program], cwd=tmp_path, env=subprocess_env, capture_output=True, text=True) assert result.returncode == 0, result.stderr - assert "0.1.0.dev3" in result.stdout and "--records" in result.stdout + assert "0.1.0.dev4" in result.stdout and "--records" in result.stdout assert not list(tmp_path.iterdir()) @@ -388,7 +391,7 @@ def test_phase4_step7_existing_lineage_error_retains_metrics_and_precedence(tmp_ assert "E_PARENT_FORMAT" in {item["code"] for item in report["errors"]} assert report["observed_facts"]["record_counts"]["v1"]["value"] == 4 assert "closure_exposure" in report["derived_metrics"] - assert report["capabilities"]["lineage"]["execution_status"] == "deferred" + assert report["capabilities"]["lineage"]["execution_status"] == "not_requested" if not unsupported: assert report["derived_metrics"]["support"]["by_version"]["v1"]["support_size"]["value"] == 3 @@ -602,8 +605,9 @@ def test_phase4_step8_pair_does_not_activate_other_calculation_families(tmp_path def denied(*args, **kwargs): calls.append(True) raise AssertionError("unopened operation") - for name in ("metrics.resampling", "lineage.graph", "lineage.ancestry", "lineage.cycles"): - module = importlib.import_module("recursive_integrity_toolkit." + name) + names = ("metrics.resampling", "lineage.graph", "lineage.ancestry", "lineage.cycles") + modules = [importlib.import_module("recursive_integrity_toolkit." + name) for name in names] + for module in modules: for key, value in vars(module).copy().items(): if inspect.isfunction(value): monkeypatch.setattr(module, key, denied) for owner, name in ((socket, "socket"), (socket, "getaddrinfo"), (urllib.request, "urlopen"), (loaders, "load_content_reference")): diff --git a/tests/integration/test_phase4_gates.py b/tests/integration/test_phase4_gates.py deleted file mode 100644 index f2bbfac..0000000 --- a/tests/integration/test_phase4_gates.py +++ /dev/null @@ -1,1862 +0,0 @@ -"""Current Phase 4 governance, immutable-source mutations and active workflows.""" -import hashlib -import runpy -import shutil -import subprocess -import sys - -import pytest - - -@pytest.fixture(scope="module") -def phase4_gate_tools(repo_root): - return runpy.run_path(str(repo_root / "scripts/release_check.py"), run_name="phase4_gate_tests") - - -@pytest.fixture(scope="module") -def phase4_mutation_tree(repo_root, tmp_path_factory, phase4_gate_tools, phase4_step1_snapshot): - """One source copy; each individual mutation is restored in a finally block.""" - root = tmp_path_factory.mktemp("phase4-active-mutations") - paths = {path.relative_to(phase4_step1_snapshot).as_posix() - for path in phase4_step1_snapshot.rglob("*") if path.is_file()} - paths.update(phase4_gate_tools["PHASE4_STEP1_NEW"]) - for relative in sorted(paths): - destination = root / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(phase4_step1_snapshot / relative, destination) - return root - - -def test_phase4_current_tree_passes_governance_snapshot(phase4_gate_tools, phase4_mutation_tree): - phase4_gate_tools["verify_phase4_snapshot"](phase4_mutation_tree) - - -@pytest.mark.parametrize("relative", [ - "PROJECT_INSTRUCTIONS.md", - "SPEC_AUDIT.md", - "PHASE_3_BASELINE.json", - "PHASE_3_COMPLETION.md", - "PHASE_4_PLAN.md", - "pyproject.toml", - "schemas/report.schema.json", - "tests/golden/phase3_math_cases.json", - "examples/hero/records_v2.csv", - "examples/hero/EXPECTED_OUTPUTS.md", - "src/recursive_integrity_toolkit/models.py", - "src/recursive_integrity_toolkit/metrics/diversity.py", - "src/recursive_integrity_toolkit/io/validation.py", - "src/recursive_integrity_toolkit/reports/assembly.py", - "src/recursive_integrity_toolkit/cli.py", -]) -def test_phase4_snapshot_rejects_protected_byte_changes(phase4_gate_tools, phase4_mutation_tree, relative): - path = phase4_mutation_tree / relative - original = path.read_bytes() - # Exercise actual formula/oracle/dependency changes as well as any-byte - # preservation. These mutations never enter the source checkout. - replacements = { - "src/recursive_integrity_toolkit/metrics/diversity.py": - (b"diversity = 1.0 - concentration", b"diversity = 0.5 - concentration"), - "tests/golden/phase3_math_cases.json": - (b'"v2_support":5', b'"v2_support":3'), - "pyproject.toml": (b"numpy>=2.0", b"numpy>=3.0"), - "examples/hero/records_v2.csv": (b"v2_08,v2,", b"v2_99,v2,"), - } - if relative in replacements: - before, after = replacements[relative] - assert original.count(before) == 1 - changed = original.replace(before, after, 1) - else: - changed = original + b"\nunauthorized Phase 4 Step 1 mutation\n" - try: - path.write_bytes(changed) - with pytest.raises(ValueError): - phase4_gate_tools["verify_phase4_snapshot"](phase4_mutation_tree) - finally: - path.write_bytes(original) - - -@pytest.mark.parametrize("relative", [ - "PHASE_4_COMPLETION.md", "PHASE_4_VALIDATION_REPORT.md", - "PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md", "report.json", "report.md", - "src/recursive_integrity_toolkit/reports/unauthorized.py", -]) -def test_phase4_snapshot_rejects_premature_outputs_and_new_runtime(phase4_gate_tools, phase4_mutation_tree, relative): - path = phase4_mutation_tree / relative - assert not path.exists() - try: - path.write_text("Unauthorized later-stage artifact\n", encoding="utf-8") - with pytest.raises(ValueError): - phase4_gate_tools["verify_phase4_snapshot"](phase4_mutation_tree) - finally: - path.unlink() - - -def test_phase4_snapshot_rejects_missing_protected_runtime(phase4_gate_tools, phase4_mutation_tree): - path = phase4_mutation_tree / "src/recursive_integrity_toolkit/metrics/bounds.py" - original = path.read_bytes() - try: - path.unlink() - with pytest.raises(ValueError): - phase4_gate_tools["verify_phase4_snapshot"](phase4_mutation_tree) - finally: - path.write_bytes(original) - - -def test_phase4_current_snapshot_enforces_migrated_historical_assertions(phase4_gate_tools, phase4_mutation_tree): - path = phase4_mutation_tree / "tests/integration/test_cli_validation.py" - original = path.read_bytes() - assertion = b"assert result.returncode == 0, result.stderr" - assert assertion in original - try: - path.write_bytes(original.replace(assertion, b"assert True", 1)) - with pytest.raises(ValueError): - phase4_gate_tools["verify_phase4_snapshot"](phase4_mutation_tree) - finally: - path.write_bytes(original) - - -def test_phase4_final_snapshot_preserves_all_current_runtime_bytes(repo_root, phase3_final_snapshot, phase4_step1_snapshot): - repo_root = phase4_step1_snapshot - package = "src/recursive_integrity_toolkit" - current = {path.relative_to(repo_root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (repo_root / package).rglob("*.py")} - frozen = {path.relative_to(phase3_final_snapshot).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (phase3_final_snapshot / package).rglob("*.py")} - assert len(current) == len(frozen) == 40 - assert current == frozen - - -def test_phase4_active_workflows_preserve_full_matrix_and_use_current_dispatch(repo_root, phase4_step1_snapshot): - repo_root = phase4_step1_snapshot - names = {"ci.yml", "golden.yml", "security.yml", "release.yml"} - root = repo_root / ".github/workflows" - assert {path.name for path in root.glob("*.yml")} == names - for name in sorted(names): - text = (root / name).read_text(encoding="utf-8") - assert "--phase 4 --step 1" in text, name - assert "--phase 3 --step 11" not in text, name - assert "permissions:\n contents: read" in text - assert "persist-credentials: false" in text - assert "timeout-minutes:" in text and "set -euo pipefail" in text - assert "actions/upload-artifact@v4" in text - assert "if-no-files-found: error" in text - for forbidden in ("continue-on-error:", "|| true", "contents: write", "id-token: write", "twine upload", "git push"): - assert forbidden not in text, (name, forbidden) - ci = (root / "ci.yml").read_text(encoding="utf-8") - for required in ('os: [ubuntu-latest, windows-latest]', 'python-version: ["3.11", "3.12"]', - 'dependencies: [current, minimum]', '"numpy==2.0.0" "pandas==2.2.2"', - 'RIT_TEST_PARQUET: "0"', 'RIT_TEST_PARQUET: "1"', '--require-parquet', - '--baseline-evidence', 'python -m pip check'): - assert required in ci - assert ci.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in ci - release = (root / "release.yml").read_text(encoding="utf-8") - assert "python -m build" in release and "python -m twine check --strict" in release - assert "--dist" in release and "--candidate" in release - - -def test_phase4_step1_keeps_current_help_and_future_commands_unopened(subprocess_env, tmp_path, phase4_step6_snapshot): - subprocess_env = dict(subprocess_env, PYTHONPATH=str(phase4_step6_snapshot / "src")) - before = set(tmp_path.iterdir()) - result = subprocess.run( - [sys.executable, "-m", "recursive_integrity_toolkit", "--help"], - cwd=tmp_path, env=subprocess_env, text=True, capture_output=True, check=False, - ) - assert result.returncode == 0 and "version" in result.stdout - assert "Analytical audit functionality is not implemented" in " ".join(result.stdout.split()) - rejected = subprocess.run( - [sys.executable, "-m", "recursive_integrity_toolkit", "audit"], - cwd=tmp_path, env=subprocess_env, text=True, capture_output=True, check=False, - ) - assert rejected.returncode == 2 - assert set(tmp_path.iterdir()) == before - - -def test_phase4_step1_has_no_completion_records_or_reports(repo_root, phase4_step1_snapshot): - repo_root = phase4_step1_snapshot - for relative in ("PHASE_4_COMPLETION.md", "PHASE_4_VALIDATION_REPORT.md", - "PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md", "report.json", "report.md"): - assert not (repo_root / relative).exists(), relative - - -def test_phase4_step2_current_runtime_opens_only_the_canonical_result(repo_root, phase3_final_snapshot, phase4_step2_snapshot): - repo_root = phase4_step2_snapshot - package = "src/recursive_integrity_toolkit" - current = {path.relative_to(repo_root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (repo_root / package).rglob("*.py")} - frozen = {path.relative_to(phase3_final_snapshot).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (phase3_final_snapshot / package).rglob("*.py")} - assert len(current) == len(frozen) == 40 - assert current.keys() == frozen.keys() - opened = "src/recursive_integrity_toolkit/result.py" - assert current[opened] != frozen[opened] - assert {path: digest for path, digest in current.items() if path != opened} == { - path: digest for path, digest in frozen.items() if path != opened - } - - -def test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step2_snapshot): - repo_root = phase4_step2_snapshot - names = {"ci.yml", "golden.yml", "security.yml", "release.yml"} - root = repo_root / ".github/workflows" - assert {path.name for path in root.glob("*.yml")} == names - for name in sorted(names): - text = (root / name).read_text(encoding="utf-8") - assert "Phase 4 Step 2" in text, name - assert "--phase 4 --step 2" in text, name - assert "--phase 4 --step 1" not in text, name - assert "--phase 3 --step 11" not in text, name - assert "permissions:\n contents: read" in text - assert "persist-credentials: false" in text - assert "timeout-minutes:" in text and "set -euo pipefail" in text - assert "actions/upload-artifact@v4" in text - assert "if-no-files-found: error" in text - for line in text.splitlines(): - if "python scripts/release_check.py" in line: - assert "--phase 4 --step 2" in line, (name, line) - for forbidden in ("continue-on-error:", "|| true", "contents: write", "id-token: write", "twine upload", "git push"): - assert forbidden not in text, (name, forbidden) - ci = (root / "ci.yml").read_text(encoding="utf-8") - for required in ('os: [ubuntu-latest, windows-latest]', 'python-version: ["3.11", "3.12"]', - 'dependencies: [current, minimum]', '"numpy==2.0.0" "pandas==2.2.2"', - 'RIT_TEST_PARQUET: "0"', 'RIT_TEST_PARQUET: "1"', '--require-parquet', - '--baseline-evidence', 'python -m pip check', - '--minimum-tests 2584', '--minimum-tests 2587', - "find_spec('pyarrow') is None", 'import pyarrow'): - assert required in ci - assert ci.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in ci - golden = (root / "golden.yml").read_text(encoding="utf-8") - assert "tests/golden/test_phase3_math.py" in golden - assert "tests/integration/test_hero_structure.py" in golden - assert "tests/integration/test_phase3_metric_pipeline.py" in golden - security = (root / "security.yml").read_text(encoding="utf-8") - for required in ('tests/integration/test_no_network.py', 'tests/integration/test_optional_dependency.py', - 'tests/unit/test_PR012_evidence_classes.py', 'tests/unit/test_PR013_report_schema.py', - 'tests/unit/test_phase4_contracts.py', 'tests/integration/test_phase4_gates.py'): - assert required in security - release = (root / "release.yml").read_text(encoding="utf-8") - assert "python -m build" in release and "python -m twine check --strict" in release - assert "--dist" in release and "--candidate" in release - assert "--delivery" not in release - assert "recursive-integrity-toolkit-phase4-step2-candidate" in release - assert "rit-phase4-step1" not in release - assert release.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - - -@pytest.mark.parametrize("mutation", [ - "unchanged", "formula", "math_oracle", "frozen_model", "frozen_schema", - "hero", "dependency", "plan", "assembly", "cli", "unknown_runtime", - "missing_runtime", "premature_completion", "premature_report", - "control_step", "control_scope", "control_schema_scope", - "schema_open_root", "schema_wrong_dialect", "model_computation_import", - "historical_assertion", "historical_tooling", -]) -def test_phase4_step2_current_snapshot_checks_valid_tree_before_mutations(repo_root, tmp_path, phase4_gate_tools, mutation, phase4_step2_snapshot): - import json - - paths = {path.relative_to(phase4_step2_snapshot).as_posix() - for path in phase4_step2_snapshot.rglob("*") if path.is_file()} - assert len(paths) == 227 - for relative in sorted(paths): - destination = tmp_path / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(phase4_step2_snapshot / relative, destination) - verify = phase4_gate_tools["verify_phase4_step2_snapshot"] - baseline = verify(tmp_path) - assert baseline["frozen_runtime_modules"] == 39 - assert baseline["frozen_schemas"] == 4 - assert baseline["result_contracts_enabled"] is True - assert baseline["adapters_enabled"] is False - assert baseline["cli_analysis_enabled"] is False - assert baseline["phase_complete"] is False - if mutation == "unchanged": - return - replacements = { - "formula": ("src/recursive_integrity_toolkit/metrics/diversity.py", - b"diversity = 1.0 - concentration", b"diversity = 0.5 - concentration"), - "math_oracle": ("tests/golden/phase3_math_cases.json", b'"v2_support":5', b'"v2_support":3'), - "hero": ("examples/hero/records_v2.csv", b"v2_08,v2,", b"v2_99,v2,"), - "dependency": ("pyproject.toml", b"numpy>=2.0", b"numpy>=3.0"), - "historical_assertion": ("tests/unit/test_phase4_contracts.py", - b' assert phase4_control["active_step"] == 1\n', b" assert True\n"), - "historical_tooling": ("scripts/release_check.py", - b"def verify_phase4_control(control: dict, step: int = 1) -> None:", - b"def verify_phase4_control(control: dict, step: int = 1) -> None:\n return"), - } - appended = { - "frozen_model": "src/recursive_integrity_toolkit/models.py", - "frozen_schema": "schemas/config.schema.json", - "plan": "PHASE_4_PLAN.md", - "assembly": "src/recursive_integrity_toolkit/reports/assembly.py", - "cli": "src/recursive_integrity_toolkit/cli.py", - } - added = { - "unknown_runtime": "src/recursive_integrity_toolkit/reports/unauthorized.py", - "premature_completion": "PHASE_4_COMPLETION.md", - "premature_report": "report.json", - } - if mutation in replacements: - relative, original, replacement = replacements[mutation] - path = tmp_path / relative - source = path.read_bytes() - assert source.count(original) == 1 - path.write_bytes(source.replace(original, replacement, 1)) - elif mutation in appended: - path = tmp_path / appended[mutation] - path.write_bytes(path.read_bytes() + b"\nUnauthorized Step 2 mutation\n") - elif mutation in added: - path = tmp_path / added[mutation] - assert not path.exists() - path.write_text("Unauthorized later-stage file\n", encoding="utf-8") - elif mutation == "missing_runtime": - (tmp_path / "src/recursive_integrity_toolkit/metrics/bounds.py").unlink() - elif mutation.startswith("control_"): - path = tmp_path / "PHASE_4_BASELINE.json" - control = json.loads(path.read_text(encoding="utf-8")) - if mutation == "control_step": - control["active_step"] = 3 - elif mutation == "control_scope": - control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/cli.py") - else: - control["schema_paths_authorized"].append("schemas/config.schema.json") - path.write_text(json.dumps(control), encoding="utf-8") - elif mutation.startswith("schema_"): - path = tmp_path / "schemas/report.schema.json" - schema = json.loads(path.read_text(encoding="utf-8")) - if mutation == "schema_open_root": - schema["additionalProperties"] = True - else: - schema["$schema"] = "http://json-schema.org/draft-07/schema#" - path.write_text(json.dumps(schema), encoding="utf-8") - elif mutation == "model_computation_import": - path = tmp_path / "src/recursive_integrity_toolkit/result.py" - path.write_bytes(path.read_bytes() + b"\nfrom .metrics.diversity import effective_state_diversity\n") - else: - raise AssertionError(f"Unknown active mutation case: {mutation}") - with pytest.raises(ValueError): - verify(tmp_path) - - -def test_phase4_step2_junit_dispatch_accepts_valid_subsets_and_rejects_bad_evidence(repo_root, tmp_path): - import json - - junit = tmp_path / "subset.xml" - command = [sys.executable, str(repo_root / "scripts/release_check.py"), - "--phase", "4", "--step", "2", "--junit", str(junit)] - valid = ('' - '' - '') - junit.write_text(valid, encoding="utf-8") - passed = subprocess.run(command, cwd=tmp_path, text=True, capture_output=True, check=False) - assert passed.returncode == 0, passed.stderr - assert json.loads(passed.stdout)["tests"] == 1 - assert json.loads(passed.stdout)["passed"] == 1 - too_small = subprocess.run(command + ["--minimum-tests", "2584"], cwd=tmp_path, - text=True, capture_output=True, check=False) - assert too_small.returncode != 0 - assert "Insufficient or duplicate test cases" in too_small.stderr - for outcome in ("failure", "skipped"): - invalid = ('' - '' - f'<{outcome} message="independent rejection case" />' - '') - junit.write_text(invalid, encoding="utf-8") - rejected = subprocess.run(command, cwd=tmp_path, text=True, capture_output=True, check=False) - assert rejected.returncode != 0 - assert "Failed, errored or skipped test" in rejected.stderr - - -def test_phase4_step3_current_runtime_opens_only_assembly_and_freezes_schema(repo_root, phase4_step2_snapshot, phase4_step3_snapshot): - repo_root = phase4_step3_snapshot - package = "src/recursive_integrity_toolkit" - current = {path.relative_to(repo_root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (repo_root / package).rglob("*.py")} - frozen = {path.relative_to(phase4_step2_snapshot).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (phase4_step2_snapshot / package).rglob("*.py")} - assert len(current) == len(frozen) == 40 - assert current.keys() == frozen.keys() - opened = "src/recursive_integrity_toolkit/reports/assembly.py" - assert current[opened] != frozen[opened] - assert {path: digest for path, digest in current.items() if path != opened} == { - path: digest for path, digest in frozen.items() if path != opened - } - schemas = {path.name: path.read_bytes() for path in (repo_root / "schemas").glob("*.json")} - assert len(schemas) == 5 - assert schemas == {path.name: path.read_bytes() for path in (phase4_step2_snapshot / "schemas").glob("*.json")} - - -def test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step3_snapshot): - repo_root = phase4_step3_snapshot - names = {"ci.yml", "golden.yml", "security.yml", "release.yml"} - root = repo_root / ".github/workflows" - assert {path.name for path in root.glob("*.yml")} == names - for name in sorted(names): - text = (root / name).read_text(encoding="utf-8") - assert "Phase 4 Step 3" in text, name - assert "--phase 4 --step 3" in text, name - assert "--phase 4 --step 2" not in text, name - assert "--phase 4 --step 1" not in text, name - assert "--phase 3 --step 11" not in text, name - assert "permissions:\n contents: read" in text - assert "persist-credentials: false" in text - assert "timeout-minutes:" in text and "set -euo pipefail" in text - assert "actions/upload-artifact@v4" in text - assert "if-no-files-found: error" in text - for line in text.splitlines(): - if "python scripts/release_check.py" in line: - assert "--phase 4 --step 3" in line, (name, line) - for forbidden in ("continue-on-error:", "|| true", "contents: write", "id-token: write", "twine upload", "git push"): - assert forbidden not in text, (name, forbidden) - ci = (root / "ci.yml").read_text(encoding="utf-8") - for required in ('os: [ubuntu-latest, windows-latest]', 'python-version: ["3.11", "3.12"]', - 'dependencies: [current, minimum]', '"numpy==2.0.0" "pandas==2.2.2"', - 'RIT_TEST_PARQUET: "0"', 'RIT_TEST_PARQUET: "1"', '--require-parquet', - '--baseline-evidence', 'python -m pip check', - '--minimum-tests 2728', '--minimum-tests 2731', - "find_spec('pyarrow') is None", 'import pyarrow'): - assert required in ci - assert ci.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in ci - golden = (root / "golden.yml").read_text(encoding="utf-8") - assert "tests/golden/test_phase3_math.py" in golden - assert "tests/integration/test_hero_structure.py" in golden - assert "tests/integration/test_phase3_metric_pipeline.py" in golden - security = (root / "security.yml").read_text(encoding="utf-8") - for required in ('tests/integration/test_no_network.py', 'tests/integration/test_optional_dependency.py', - 'tests/unit/test_PR012_evidence_classes.py', 'tests/unit/test_PR013_report_schema.py', - 'tests/unit/test_PR014_unavailable.py', 'tests/unit/test_PR018_language.py', - 'tests/integration/test_partial_provenance_report.py', - 'tests/integration/test_partial_lineage_report.py', - 'tests/unit/test_phase4_contracts.py', 'tests/integration/test_phase4_gates.py'): - assert required in security - release = (root / "release.yml").read_text(encoding="utf-8") - assert "python -m build" in release and "python -m twine check --strict" in release - assert "--dist" in release and "--candidate" in release - assert "--delivery" not in release - assert "recursive-integrity-toolkit-phase4-step3-candidate" in release - assert "rit-phase4-step2" not in release - assert "--minimum-tests 2728" in release and "--minimum-tests 2731" in release - assert release.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - - -@pytest.mark.parametrize("mutation", [ - "unchanged", "formula", "math_oracle", "frozen_model", "frozen_result", "frozen_schema", - "hero", "dependency", "plan", "cli", "renderer", "unknown_runtime", "missing_runtime", - "premature_completion", "premature_report", "control_step", "control_scope", "control_schema_scope", - "schema_open_root", "historical_assertion", "historical_tooling", "historical_binding", "historical_document", - "assembly_metric_import", "assembly_calculation_import", "assembly_numeric_import", - "assembly_network_import", "assembly_file_io", "assembly_dynamic_execution", "assembly_new_formula", -]) -def test_phase4_step3_current_snapshot_checks_valid_tree_before_mutations(repo_root, tmp_path, phase4_gate_tools, mutation, phase4_step3_snapshot): - import json - - paths = {path.relative_to(phase4_step3_snapshot).as_posix() - for path in phase4_step3_snapshot.rglob("*") if path.is_file()} - assert len(paths) == 227 - for relative in sorted(paths): - destination = tmp_path / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(phase4_step3_snapshot / relative, destination) - verify = phase4_gate_tools["verify_phase4_step3_snapshot"] - baseline = verify(tmp_path) - assert baseline["package_modules"] == 40 - assert baseline["frozen_runtime_modules"] == 39 - assert baseline["frozen_schemas"] == 5 - assert baseline["result_contracts_enabled"] is True - assert baseline["adapters_enabled"] is True - assert baseline["cli_analysis_enabled"] is False - assert baseline["phase_complete"] is False - if mutation == "unchanged": - return - replacements = { - "formula": ("src/recursive_integrity_toolkit/metrics/diversity.py", - b"diversity = 1.0 - concentration", b"diversity = 0.5 - concentration"), - "math_oracle": ("tests/golden/phase3_math_cases.json", b'"v2_support":5', b'"v2_support":3'), - "hero": ("examples/hero/records_v2.csv", b"v2_08,v2,", b"v2_99,v2,"), - "dependency": ("pyproject.toml", b"numpy>=2.0", b"numpy>=3.0"), - "historical_assertion": ("tests/unit/test_phase4_contracts.py", - b' assert control["active_phase"] == 4 and control["active_step"] == 2\n', - b" assert True\n"), - "historical_tooling": ("scripts/release_check.py", - b"def verify_phase4_step2_control(control: dict, step: int = 2) -> None:", - b"def verify_phase4_step2_control(control: dict, step: int = 2) -> None:\n return"), - "historical_binding": ("tests/integration/test_phase4_gates.py", - b"def test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step2_snapshot):\n repo_root = phase4_step2_snapshot\n", - b"def test_phase4_step2_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step2_snapshot):\n"), - "historical_document": ("docs/report_schema.md", b"# Report Schema\n", b"# Modified Report Schema\n"), - } - appended = { - "frozen_model": "src/recursive_integrity_toolkit/models.py", - "frozen_result": "src/recursive_integrity_toolkit/result.py", - "frozen_schema": "schemas/config.schema.json", - "plan": "PHASE_4_PLAN.md", - "cli": "src/recursive_integrity_toolkit/cli.py", - "renderer": "src/recursive_integrity_toolkit/reports/json_report.py", - } - added = { - "unknown_runtime": "src/recursive_integrity_toolkit/reports/unauthorized.py", - "premature_completion": "PHASE_4_COMPLETION.md", - "premature_report": "report.json", - } - assembly_injections = { - "assembly_metric_import": b"\nfrom ..metrics.diversity import effective_state_diversity\n", - "assembly_calculation_import": b"\nfrom ..metrics import coverage\n", - "assembly_numeric_import": b"\nimport numpy\n", - "assembly_network_import": b"\nimport socket\n", - "assembly_file_io": b"\nopen('unapproved.txt', 'w').write('data')\n", - "assembly_dynamic_execution": b"\nexec('forged = True')\n", - "assembly_new_formula": b"\ndef invented_metric(values):\n return sum(value * value for value in values)\n", - } - if mutation in replacements: - relative, original, replacement = replacements[mutation] - path = tmp_path / relative - source = path.read_bytes() - assert source.count(original) == 1 - path.write_bytes(source.replace(original, replacement, 1)) - elif mutation in appended: - path = tmp_path / appended[mutation] - path.write_bytes(path.read_bytes() + b"\nUnauthorized Step 3 mutation\n") - elif mutation in added: - path = tmp_path / added[mutation] - assert not path.exists() - path.write_text("Unauthorized later-stage file\n", encoding="utf-8") - elif mutation == "missing_runtime": - (tmp_path / "src/recursive_integrity_toolkit/metrics/bounds.py").unlink() - elif mutation.startswith("control_"): - path = tmp_path / "PHASE_4_BASELINE.json" - control = json.loads(path.read_text(encoding="utf-8")) - if mutation == "control_step": - control["active_step"] = 4 - elif mutation == "control_scope": - control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/result.py") - else: - control["schema_changes_authorized"] = True - control["schema_paths_authorized"].append("schemas/report.schema.json") - path.write_text(json.dumps(control), encoding="utf-8") - elif mutation == "schema_open_root": - path = tmp_path / "schemas/report.schema.json" - schema = json.loads(path.read_text(encoding="utf-8")) - schema["additionalProperties"] = True - path.write_text(json.dumps(schema), encoding="utf-8") - elif mutation in assembly_injections: - path = tmp_path / "src/recursive_integrity_toolkit/reports/assembly.py" - path.write_bytes(path.read_bytes() + assembly_injections[mutation]) - else: - raise AssertionError(f"Unknown active mutation case: {mutation}") - with pytest.raises(ValueError): - verify(tmp_path) - - -def test_phase4_step3_junit_dispatch_accepts_valid_subsets_and_rejects_bad_evidence(repo_root, tmp_path): - import json - - junit = tmp_path / "subset.xml" - command = [sys.executable, str(repo_root / "scripts/release_check.py"), - "--phase", "4", "--step", "3", "--junit", str(junit)] - valid = ('' - '' - '') - junit.write_text(valid, encoding="utf-8") - passed = subprocess.run(command, cwd=tmp_path, text=True, capture_output=True, check=False) - assert passed.returncode == 0, passed.stderr - assert json.loads(passed.stdout)["tests"] == 1 - assert json.loads(passed.stdout)["passed"] == 1 - too_small = subprocess.run(command + ["--minimum-tests", "2728"], cwd=tmp_path, - text=True, capture_output=True, check=False) - assert too_small.returncode != 0 - assert "Insufficient or duplicate test cases" in too_small.stderr - for outcome in ("failure", "skipped"): - invalid = ('' - '' - f'<{outcome} message="independent rejection case" />' - '') - junit.write_text(invalid, encoding="utf-8") - rejected = subprocess.run(command, cwd=tmp_path, text=True, capture_output=True, check=False) - assert rejected.returncode != 0 - assert "Failed, errored or skipped test" in rejected.stderr - - -def test_phase4_step4_current_runtime_opens_only_privacy_metadata_modules_and_freezes_schema(repo_root, phase4_step3_snapshot, phase4_step4_snapshot): - repo_root = phase4_step4_snapshot - package = "src/recursive_integrity_toolkit" - current = {path.relative_to(repo_root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (repo_root / package).rglob("*.py")} - frozen = {path.relative_to(phase4_step3_snapshot).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (phase4_step3_snapshot / package).rglob("*.py")} - assert len(current) == len(frozen) == 40 - assert current.keys() == frozen.keys() - opened = {package + "/" + relative for relative in ( - "result.py", "reports/assembly.py", "config.py", "utils/hashing.py", "utils/logging.py", - )} - assert {path for path in current if current[path] != frozen[path]} == opened - assert {path: digest for path, digest in current.items() if path not in opened} == { - path: digest for path, digest in frozen.items() if path not in opened - } - schemas = {path.name: path.read_bytes() for path in (repo_root / "schemas").glob("*.json")} - assert len(schemas) == 5 - assert schemas == {path.name: path.read_bytes() for path in (phase4_step3_snapshot / "schemas").glob("*.json")} - - -def test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step4_snapshot): - repo_root = phase4_step4_snapshot - names = {"ci.yml", "golden.yml", "security.yml", "release.yml"} - root = repo_root / ".github/workflows" - assert {path.name for path in root.glob("*.yml")} == names - for name in sorted(names): - text = (root / name).read_text(encoding="utf-8") - assert "Phase 4 Step 4" in text, name - assert "--phase 4 --step 4" in text, name - for older in ("--phase 4 --step 3", "--phase 4 --step 2", "--phase 4 --step 1", "--phase 3 --step 11"): - assert older not in text, (name, older) - assert "permissions:\n contents: read" in text - assert "persist-credentials: false" in text - assert "timeout-minutes:" in text and "set -euo pipefail" in text - assert "actions/upload-artifact@v4" in text and "if-no-files-found: error" in text - for line in text.splitlines(): - if "python scripts/release_check.py" in line: - assert "--phase 4 --step 4" in line, (name, line) - for forbidden in ("continue-on-error:", "|| true", "contents: write", "id-token: write", "twine upload", "git push"): - assert forbidden not in text, (name, forbidden) - ci = (root / "ci.yml").read_text(encoding="utf-8") - for required in ('os: [ubuntu-latest, windows-latest]', 'python-version: ["3.11", "3.12"]', - 'dependencies: [current, minimum]', '"numpy==2.0.0" "pandas==2.2.2"', - 'RIT_TEST_PARQUET: "0"', 'RIT_TEST_PARQUET: "1"', '--require-parquet', - '--baseline-evidence', 'python -m pip check', - '--minimum-tests 2876', '--minimum-tests 2879', - "find_spec('pyarrow') is None", 'import pyarrow'): - assert required in ci - assert ci.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in ci - golden = (root / "golden.yml").read_text(encoding="utf-8") - for required in ("tests/golden/test_phase3_math.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_phase3_metric_pipeline.py"): - assert required in golden - security = (root / "security.yml").read_text(encoding="utf-8") - for required in ('tests/integration/test_no_network.py', 'tests/integration/test_optional_dependency.py', - 'tests/unit/test_PR012_evidence_classes.py', 'tests/unit/test_PR013_report_schema.py', - 'tests/unit/test_PR014_unavailable.py', 'tests/unit/test_PR015_redaction.py', - 'tests/unit/test_PR016_determinism.py', 'tests/unit/test_PR018_language.py', - 'tests/integration/test_partial_provenance_report.py', - 'tests/integration/test_partial_lineage_report.py', - 'tests/unit/test_phase4_contracts.py', 'tests/integration/test_phase4_gates.py'): - assert required in security - release = (root / "release.yml").read_text(encoding="utf-8") - assert "python -m build" in release and "python -m twine check --strict" in release - assert "--dist" in release and "--candidate" in release - assert "--delivery" not in release - assert "recursive-integrity-toolkit-phase4-step4-candidate" in release - assert "rit-phase4-step3" not in release - assert "--minimum-tests 2876" in release and "--minimum-tests 2879" in release - assert release.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - - -@pytest.mark.parametrize("mutation", [ - "unchanged", "formula", "math_oracle", "frozen_model", "frozen_schema", "hero", "dependency", "plan", - "cli", "renderer", "output_paths", "unknown_runtime", "missing_runtime", "premature_completion", "premature_report", - "control_step", "control_scope", "control_schema_scope", "historical_assertion", "historical_tooling", - "historical_binding", "historical_document", "input_hash_helper", "config_resolver_helper", -]) -def test_phase4_step4_current_snapshot_checks_valid_tree_before_mutations(repo_root, tmp_path, phase4_gate_tools, mutation, phase4_step4_snapshot): - import json - - paths = {path.relative_to(phase4_step4_snapshot).as_posix() - for path in phase4_step4_snapshot.rglob("*") if path.is_file()} - assert len(paths) == 227 - for relative in sorted(paths): - destination = tmp_path / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(phase4_step4_snapshot / relative, destination) - verify = phase4_gate_tools["verify_phase4_step4_snapshot"] - baseline = verify(tmp_path) - assert baseline["package_modules"] == 40 - assert baseline["frozen_runtime_modules"] == 35 - assert baseline["frozen_schemas"] == 5 - assert baseline["result_contracts_enabled"] is True - assert baseline["adapters_enabled"] is True - assert baseline["privacy_views_enabled"] is True - assert baseline["cli_analysis_enabled"] is False - assert baseline["phase_complete"] is False - if mutation == "unchanged": - return - replacements = { - "formula": ("src/recursive_integrity_toolkit/metrics/diversity.py", - b"diversity = 1.0 - concentration", b"diversity = 0.5 - concentration"), - "math_oracle": ("tests/golden/phase3_math_cases.json", b'"v2_support":5', b'"v2_support":3'), - "hero": ("examples/hero/records_v2.csv", b"v2_08,v2,", b"v2_99,v2,"), - "dependency": ("pyproject.toml", b"numpy>=2.0", b"numpy>=3.0"), - "historical_assertion": ("tests/unit/test_phase4_contracts.py", - b' assert control["active_phase"] == 4 and control["active_step"] == 3\n', - b" assert True\n"), - "historical_tooling": ("scripts/release_check.py", - b"def verify_phase4_step3_control(control: dict, step: int = 3) -> None:", - b"def verify_phase4_step3_control(control: dict, step: int = 3) -> None:\n return"), - "historical_binding": ("tests/integration/test_phase4_gates.py", - b"def test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step3_snapshot):\n repo_root = phase4_step3_snapshot\n", - b"def test_phase4_step3_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step3_snapshot):\n"), - "historical_document": ("docs/privacy.md", b"# Privacy and Local Input Boundaries\n", - b"# Unauthorized Privacy and Local Input Boundaries\n"), - "input_hash_helper": ("src/recursive_integrity_toolkit/utils/hashing.py", - b"return hashlib.sha256(data).hexdigest()", b"return hashlib.sha256(b'changed').hexdigest()"), - "config_resolver_helper": ("src/recursive_integrity_toolkit/config.py", - b"def resolve_config(", b"def resolve_config_disabled("), - } - appended = { - "frozen_model": "src/recursive_integrity_toolkit/models.py", - "frozen_schema": "schemas/report.schema.json", - "plan": "PHASE_4_PLAN.md", - "cli": "src/recursive_integrity_toolkit/cli.py", - "renderer": "src/recursive_integrity_toolkit/reports/json_report.py", - "output_paths": "src/recursive_integrity_toolkit/utils/paths.py", - } - added = { - "unknown_runtime": "src/recursive_integrity_toolkit/reports/unauthorized.py", - "premature_completion": "PHASE_4_COMPLETION.md", "premature_report": "report.json", - } - if mutation in replacements: - relative, original, replacement = replacements[mutation] - path = tmp_path / relative - source = path.read_bytes() - assert source.count(original) == 1 - path.write_bytes(source.replace(original, replacement, 1)) - elif mutation in appended: - path = tmp_path / appended[mutation] - path.write_bytes(path.read_bytes() + b"\nUnauthorized Step 4 mutation\n") - elif mutation in added: - path = tmp_path / added[mutation] - assert not path.exists() - path.write_text("Unauthorized later-stage file\n", encoding="utf-8") - elif mutation == "missing_runtime": - (tmp_path / "src/recursive_integrity_toolkit/metrics/bounds.py").unlink() - elif mutation.startswith("control_"): - path = tmp_path / "PHASE_4_BASELINE.json" - control = json.loads(path.read_text(encoding="utf-8")) - if mutation == "control_step": - control["active_step"] = 5 - elif mutation == "control_scope": - control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/json_report.py") - else: - control["schema_changes_authorized"] = True - control["schema_paths_authorized"].append("schemas/report.schema.json") - path.write_text(json.dumps(control), encoding="utf-8") - else: - raise AssertionError(f"Unknown active mutation case: {mutation}") - with pytest.raises(ValueError): - verify(tmp_path) - - -@pytest.mark.parametrize("relative,injection", [ - ("result.py", b"\nimport socket\n"), - ("result.py", b"\nfrom .metrics.diversity import effective_state_diversity\n"), - ("reports/assembly.py", b"\nimport numpy\n"), - ("reports/assembly.py", b"\ndef invented_metric(values):\n return sum(value * value for value in values)\n"), - ("reports/assembly.py", b"\nfrom .json_report import render_json\n"), - ("config.py", b"\nfrom .metrics import coverage\n"), - ("config.py", b"\nresolve_config = lambda *args, **kwargs: None\n"), - ("utils/hashing.py", b"\nimport urllib.request\n"), - ("utils/hashing.py", b"\ndef expose_secret(secret):\n print(secret)\n"), - ("utils/hashing.py", b"\nsha256_bytes = lambda data: '0' * 64\n"), - ("utils/logging.py", b"\nopen('private.txt', 'w').write('raw content')\n"), - ("utils/logging.py", b"\nexec('forged = True')\n"), - ("utils/logging.py", b"\ndef dynamic_call(value):\n return eval(value)\n"), - ("utils/logging.py", b"\ndef effect_at_definition(value=globals().clear()):\n pass\n"), - ("utils/logging.py", b"\n@print('raw content')\ndef decorated_effect():\n pass\n"), -]) -def test_phase4_step4_privacy_ast_checks_current_source_before_rejecting_scope_injection(repo_root, tmp_path, relative, injection, phase4_step5_snapshot, phase4_step6_snapshot): - checker = runpy.run_path(str(repo_root / "scripts/check_traceability.py"), run_name="phase4_step4_ast_tests") - verify = checker["phase4_step4_runtime_boundary"] - repository_path = "src/recursive_integrity_toolkit/" + relative - current = (phase4_step5_snapshot if relative in ("utils/logging.py", "reports/assembly.py") else phase4_step6_snapshot if relative == "config.py" else repo_root) / repository_path - verify(current, repository_path) - copy = tmp_path / "source.py" - copy.write_bytes(current.read_bytes() + injection) - with pytest.raises(ValueError): - verify(copy, repository_path) - - -def test_phase4_step4_junit_dispatch_accepts_valid_subsets_and_rejects_bad_evidence(repo_root, tmp_path): - import json - - junit = tmp_path / "subset.xml" - command = [sys.executable, str(repo_root / "scripts/release_check.py"), - "--phase", "4", "--step", "4", "--junit", str(junit)] - valid = ('' - '' - '') - junit.write_text(valid, encoding="utf-8") - passed = subprocess.run(command, cwd=tmp_path, text=True, capture_output=True, check=False) - assert passed.returncode == 0, passed.stderr - assert json.loads(passed.stdout)["tests"] == 1 - assert json.loads(passed.stdout)["passed"] == 1 - too_small = subprocess.run(command + ["--minimum-tests", "2876"], cwd=tmp_path, - text=True, capture_output=True, check=False) - assert too_small.returncode != 0 - assert "Insufficient or duplicate test cases" in too_small.stderr - for outcome in ("failure", "error", "skipped"): - invalid = ('' - '' - f'<{outcome} message="independent rejection case" />' - '') - junit.write_text(invalid, encoding="utf-8") - rejected = subprocess.run(command, cwd=tmp_path, text=True, capture_output=True, check=False) - assert rejected.returncode != 0 - assert "Failed, errored or skipped test" in rejected.stderr - - -def test_phase4_step5_current_runtime_opens_only_renderers_and_freezes_schema(repo_root, phase4_step4_snapshot, phase4_step5_snapshot): - repo_root = phase4_step5_snapshot - package = "src/recursive_integrity_toolkit" - current = {path.relative_to(repo_root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (repo_root / package).rglob("*.py")} - frozen = {path.relative_to(phase4_step4_snapshot).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (phase4_step4_snapshot / package).rglob("*.py")} - assert len(current) == len(frozen) == 40 - assert current.keys() == frozen.keys() - opened = {package + "/" + relative for relative in ( - "reports/json_report.py", "reports/markdown_report.py", - )} - assert {path for path in current if current[path] != frozen[path]} == opened - assert {path: digest for path, digest in current.items() if path not in opened} == { - path: digest for path, digest in frozen.items() if path not in opened - } - schemas = {path.name: path.read_bytes() for path in (repo_root / "schemas").glob("*.json")} - assert len(schemas) == 5 - assert schemas == {path.name: path.read_bytes() for path in (phase4_step4_snapshot / "schemas").glob("*.json")} - - -def test_phase4_step5_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step5_snapshot): - repo_root = phase4_step5_snapshot - names = {"ci.yml", "golden.yml", "security.yml", "release.yml"} - root = repo_root / ".github/workflows" - expected_timeouts = {"ci.yml": [60, 20], "golden.yml": [15], "security.yml": [20], "release.yml": [25]} - assert {path.name for path in root.glob("*.yml")} == names - for name in sorted(names): - text = (root / name).read_text(encoding="utf-8") - assert [int(line.split(":", 1)[1]) for line in text.splitlines() - if line.strip().startswith("timeout-minutes:")] == expected_timeouts[name] - assert "Phase 4 Step 5" in text, name - assert "--phase 4 --step 5" in text, name - for older in ("--phase 4 --step 4", "--phase 4 --step 3", "--phase 4 --step 2", "--phase 4 --step 1", "--phase 3 --step 11"): - assert older not in text, (name, older) - assert "permissions:\n contents: read" in text - assert "persist-credentials: false" in text - assert "timeout-minutes:" in text and "set -euo pipefail" in text - assert "actions/upload-artifact@v4" in text and "if-no-files-found: error" in text - for line in text.splitlines(): - if "python scripts/release_check.py" in line: - assert "--phase 4 --step 5" in line, (name, line) - for forbidden in ("continue-on-error:", "|| true", "contents: write", "id-token: write", "twine upload", "git push"): - assert forbidden not in text, (name, forbidden) - ci = (root / "ci.yml").read_text(encoding="utf-8") - for required in ('os: [ubuntu-latest, windows-latest]', 'python-version: ["3.11", "3.12"]', - 'dependencies: [current, minimum]', '"numpy==2.0.0" "pandas==2.2.2"', - 'RIT_TEST_PARQUET: "0"', 'RIT_TEST_PARQUET: "1"', '--require-parquet', - '--baseline-evidence', 'python -m pip check', - '--minimum-tests 3071', '--minimum-tests 3074', - "find_spec('pyarrow') is None", 'import pyarrow'): - assert required in ci - assert ci.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in ci - golden = (root / "golden.yml").read_text(encoding="utf-8") - for required in ("tests/golden/test_phase3_math.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_phase3_metric_pipeline.py"): - assert required in golden - security = (root / "security.yml").read_text(encoding="utf-8") - for required in ('tests/integration/test_no_network.py', 'tests/integration/test_optional_dependency.py', - 'tests/unit/test_PR012_evidence_classes.py', 'tests/unit/test_PR013_report_schema.py', - 'tests/unit/test_PR014_unavailable.py', 'tests/unit/test_PR015_redaction.py', - 'tests/unit/test_PR016_determinism.py', 'tests/unit/test_PR018_language.py', - 'tests/integration/test_partial_provenance_report.py', - 'tests/integration/test_partial_lineage_report.py', - 'tests/unit/test_phase4_contracts.py', 'tests/integration/test_phase4_gates.py'): - assert required in security - release = (root / "release.yml").read_text(encoding="utf-8") - assert "python -m build" in release and "python -m twine check --strict" in release - assert "--dist" in release and "--candidate" in release - assert "--delivery" not in release - assert "recursive-integrity-toolkit-phase4-step5-candidate" in release - assert "rit-phase4-step4" not in release - assert "--minimum-tests 3071" in release and "--minimum-tests 3074" in release - assert release.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - - -@pytest.mark.parametrize("mutation", [ - "unchanged", "formula", "math_oracle", "frozen_model", "frozen_schema", "hero", "dependency", "plan", - "cli", "html", "assembly", "result", "config", "hashing", "logging", "output_paths", "unknown_runtime", "missing_runtime", "premature_completion", "premature_report", - "control_step", "control_scope", "control_schema_scope", "historical_assertion", "historical_tooling", - "historical_binding", "historical_document", "input_hash_helper", "config_resolver_helper", -]) -def test_phase4_step5_current_snapshot_checks_valid_tree_before_mutations(repo_root, tmp_path, phase4_gate_tools, mutation, phase4_step5_snapshot): - import json - - paths = {path.relative_to(phase4_step5_snapshot).as_posix() - for path in phase4_step5_snapshot.rglob("*") if path.is_file()} - assert len(paths) == 227 - for relative in sorted(paths): - destination = tmp_path / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(phase4_step5_snapshot / relative, destination) - verify = phase4_gate_tools["verify_phase4_step5_snapshot"] - baseline = verify(tmp_path) - assert baseline["package_modules"] == 40 - assert baseline["frozen_runtime_modules"] == 38 - assert baseline["frozen_schemas"] == 5 - assert baseline["result_contracts_enabled"] is True - assert baseline["adapters_enabled"] is True - assert baseline["privacy_views_enabled"] is True - assert baseline["renderers_enabled"] is True - assert baseline["cli_analysis_enabled"] is False - assert baseline["phase_complete"] is False - if mutation == "unchanged": - return - replacements = { - "formula": ("src/recursive_integrity_toolkit/metrics/diversity.py", - b"diversity = 1.0 - concentration", b"diversity = 0.5 - concentration"), - "math_oracle": ("tests/golden/phase3_math_cases.json", b'"v2_support":5', b'"v2_support":3'), - "hero": ("examples/hero/records_v2.csv", b"v2_08,v2,", b"v2_99,v2,"), - "dependency": ("pyproject.toml", b"numpy>=2.0", b"numpy>=3.0"), - "historical_assertion": ("tests/unit/test_phase4_contracts.py", - b' assert control["active_phase"] == 4 and control["active_step"] == 4\n', - b" assert True\n"), - "historical_tooling": ("scripts/release_check.py", - b"def verify_phase4_step4_control(control: dict, step: int = 4) -> None:", - b"def verify_phase4_step4_control(control: dict, step: int = 4) -> None:\n return"), - "historical_binding": ("tests/integration/test_phase4_gates.py", - b"def test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step4_snapshot):\n repo_root = phase4_step4_snapshot\n", - b"def test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step4_snapshot):\n"), - "historical_document": ("docs/report_schema.md", b"# Report Schema\n", - b"# Unauthorized historical document\n"), - "input_hash_helper": ("src/recursive_integrity_toolkit/utils/hashing.py", - b"return hashlib.sha256(data).hexdigest()", b"return hashlib.sha256(b'changed').hexdigest()"), - "config_resolver_helper": ("src/recursive_integrity_toolkit/config.py", - b"def resolve_config(", b"def resolve_config_disabled("), - } - appended = { - "frozen_model": "src/recursive_integrity_toolkit/models.py", - "frozen_schema": "schemas/report.schema.json", - "plan": "PHASE_4_PLAN.md", - "cli": "src/recursive_integrity_toolkit/cli.py", - "html": "src/recursive_integrity_toolkit/reports/html_report.py", - "assembly": "src/recursive_integrity_toolkit/reports/assembly.py", - "result": "src/recursive_integrity_toolkit/result.py", - "config": "src/recursive_integrity_toolkit/config.py", - "hashing": "src/recursive_integrity_toolkit/utils/hashing.py", - "logging": "src/recursive_integrity_toolkit/utils/logging.py", - "output_paths": "src/recursive_integrity_toolkit/utils/paths.py", - } - added = { - "unknown_runtime": "src/recursive_integrity_toolkit/reports/unauthorized.py", - "premature_completion": "PHASE_4_COMPLETION.md", "premature_report": "report.json", - } - if mutation in replacements: - relative, original, replacement = replacements[mutation] - path = tmp_path / relative - source = path.read_bytes() - assert source.count(original) == 1 - path.write_bytes(source.replace(original, replacement, 1)) - elif mutation in appended: - path = tmp_path / appended[mutation] - path.write_bytes(path.read_bytes() + b"\nUnauthorized Step 5 mutation\n") - elif mutation in added: - path = tmp_path / added[mutation] - assert not path.exists() - path.write_text("Unauthorized later-stage file\n", encoding="utf-8") - elif mutation == "missing_runtime": - (tmp_path / "src/recursive_integrity_toolkit/metrics/bounds.py").unlink() - elif mutation.startswith("control_"): - path = tmp_path / "PHASE_4_BASELINE.json" - control = json.loads(path.read_text(encoding="utf-8")) - if mutation == "control_step": - control["active_step"] = 6 - elif mutation == "control_scope": - control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py") - else: - control["schema_changes_authorized"] = True - control["schema_paths_authorized"].append("schemas/report.schema.json") - path.write_text(json.dumps(control), encoding="utf-8") - else: - raise AssertionError(f"Unknown active mutation case: {mutation}") - with pytest.raises(ValueError): - verify(tmp_path) - - -@pytest.mark.parametrize("relative,injection", [ - ("reports/json_report.py", b"\nimport socket\n"), - ("reports/json_report.py", b"\nfrom ..metrics.diversity import effective_state_diversity\n"), - ("reports/json_report.py", b"\nfrom ..io.validation import validate_bundle\n"), - ("reports/json_report.py", b"\nimport numpy\n"), - ("reports/json_report.py", b"\ndef invented_metric(values):\n return sum(value * value for value in values)\n"), - ("reports/json_report.py", b"\nrender_json = lambda value: '{}'\n"), - ("reports/json_report.py", b"\ndef dynamic_call(value):\n return eval(value)\n"), - ("reports/markdown_report.py", b"\nimport urllib.request\n"), - ("reports/markdown_report.py", b"\nfrom .assembly import assemble_report\n"), - ("reports/markdown_report.py", b"\nfrom ..metrics import coverage\n"), - ("reports/markdown_report.py", b"\nfrom ..io.readers import read_csv\n"), - ("reports/markdown_report.py", b"\nimport pandas\n"), - ("reports/markdown_report.py", b"\nopen('private.txt', 'w').write('raw content')\n"), - ("reports/markdown_report.py", b"\nexec('forged = True')\n"), - ("reports/markdown_report.py", b"\ndef effect_at_definition(value=globals().clear()):\n pass\n"), - ("reports/markdown_report.py", b"\n@print('raw content')\ndef decorated_effect():\n pass\n"), -]) -def test_phase4_step5_renderer_ast_checks_current_source_before_rejecting_scope_injection(repo_root, tmp_path, relative, injection): - checker = runpy.run_path(str(repo_root / "scripts/check_traceability.py"), run_name="phase4_step5_ast_tests") - verify = checker["phase4_step5_runtime_boundary"] - repository_path = "src/recursive_integrity_toolkit/" + relative - current = repo_root / repository_path - verify(current, repository_path) - copy = tmp_path / "source.py" - copy.write_bytes(current.read_bytes() + injection) - with pytest.raises(ValueError): - verify(copy, repository_path) - - -def test_phase4_step5_junit_dispatch_accepts_valid_subsets_and_rejects_bad_evidence(repo_root, tmp_path): - import json - - junit = tmp_path / "subset.xml" - command = [sys.executable, str(repo_root / "scripts/release_check.py"), - "--phase", "4", "--step", "5", "--junit", str(junit)] - valid = ('' - '' - '') - junit.write_text(valid, encoding="utf-8") - passed = subprocess.run(command, cwd=tmp_path, text=True, capture_output=True, check=False) - assert passed.returncode == 0, passed.stderr - assert json.loads(passed.stdout)["tests"] == 1 - assert json.loads(passed.stdout)["passed"] == 1 - too_small = subprocess.run(command + ["--minimum-tests", "3071"], cwd=tmp_path, - text=True, capture_output=True, check=False) - assert too_small.returncode != 0 - assert "Insufficient or duplicate test cases" in too_small.stderr - for outcome in ("failure", "error", "skipped"): - invalid = ('' - '' - f'<{outcome} message="independent rejection case" />' - '') - junit.write_text(invalid, encoding="utf-8") - rejected = subprocess.run(command, cwd=tmp_path, text=True, capture_output=True, check=False) - assert rejected.returncode != 0 - assert "Failed, errored or skipped test" in rejected.stderr - - -def test_phase4_step6_current_runtime_opens_only_output_helpers_and_freezes_schema(repo_root, phase4_step5_snapshot, phase4_step6_snapshot): - repo_root = phase4_step6_snapshot - package = "src/recursive_integrity_toolkit" - current = {path.relative_to(repo_root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (repo_root / package).rglob("*.py")} - frozen = {path.relative_to(phase4_step5_snapshot).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (phase4_step5_snapshot / package).rglob("*.py")} - assert len(current) == len(frozen) == 40 - assert current.keys() == frozen.keys() - opened = {package + "/" + relative for relative in ( - "utils/paths.py", "utils/logging.py", - )} - assert {path for path in current if current[path] != frozen[path]} == opened - assert {path: digest for path, digest in current.items() if path not in opened} == { - path: digest for path, digest in frozen.items() if path not in opened - } - schemas = {path.name: path.read_bytes() for path in (repo_root / "schemas").glob("*.json")} - assert len(schemas) == 5 - assert schemas == {path.name: path.read_bytes() for path in (phase4_step5_snapshot / "schemas").glob("*.json")} - - -def test_phase4_step6_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step6_snapshot): - repo_root = phase4_step6_snapshot - names = {"ci.yml", "golden.yml", "security.yml", "release.yml"} - root = repo_root / ".github/workflows" - expected_timeouts = {"ci.yml": [60, 20], "golden.yml": [15], "security.yml": [20], "release.yml": [25]} - assert {path.name for path in root.glob("*.yml")} == names - for name in sorted(names): - text = (root / name).read_text(encoding="utf-8") - assert [int(line.split(":", 1)[1]) for line in text.splitlines() - if line.strip().startswith("timeout-minutes:")] == expected_timeouts[name] - assert "Phase 4 Step 6" in text, name - assert "--phase 4 --step 6" in text, name - for older in ("--phase 4 --step 5", "--phase 4 --step 4", "--phase 4 --step 3", "--phase 4 --step 2", "--phase 4 --step 1", "--phase 3 --step 11"): - assert older not in text, (name, older) - assert "permissions:\n contents: read" in text - assert "persist-credentials: false" in text - assert "timeout-minutes:" in text and "set -euo pipefail" in text - assert "actions/upload-artifact@v4" in text and "if-no-files-found: error" in text - for line in text.splitlines(): - if "python scripts/release_check.py" in line: - assert "--phase 4 --step 6" in line, (name, line) - for forbidden in ("continue-on-error:", "|| true", "contents: write", "id-token: write", "twine upload", "git push"): - assert forbidden not in text, (name, forbidden) - ci = (root / "ci.yml").read_text(encoding="utf-8") - for required in ('os: [ubuntu-latest, windows-latest]', 'python-version: ["3.11", "3.12"]', - 'dependencies: [current, minimum]', '"numpy==2.0.0" "pandas==2.2.2"', - 'RIT_TEST_PARQUET: "0"', 'RIT_TEST_PARQUET: "1"', '--require-parquet', - '--baseline-evidence', 'python -m pip check', - '--minimum-tests 3215', '--minimum-tests 3218', - "find_spec('pyarrow') is None", 'import pyarrow'): - assert required in ci - assert ci.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in ci - golden = (root / "golden.yml").read_text(encoding="utf-8") - for required in ("tests/golden/test_phase3_math.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_phase3_metric_pipeline.py"): - assert required in golden - security = (root / "security.yml").read_text(encoding="utf-8") - for required in ('tests/integration/test_no_network.py', 'tests/integration/test_optional_dependency.py', - 'tests/unit/test_PR012_evidence_classes.py', 'tests/unit/test_PR013_report_schema.py', - 'tests/unit/test_PR014_unavailable.py', 'tests/unit/test_PR015_redaction.py', - 'tests/unit/test_PR016_determinism.py', 'tests/unit/test_PR018_language.py', - 'tests/integration/test_partial_provenance_report.py', - 'tests/integration/test_partial_lineage_report.py', - 'tests/unit/test_phase4_output_safety.py', 'tests/unit/test_phase4_contracts.py', 'tests/integration/test_phase4_gates.py'): - assert required in security - release = (root / "release.yml").read_text(encoding="utf-8") - assert "python -m build" in release and "python -m twine check --strict" in release - assert "--dist" in release and "--candidate" in release - assert "--delivery" not in release - assert "recursive-integrity-toolkit-phase4-step6-candidate" in release - assert "rit-phase4-step4" not in release - assert "--minimum-tests 3215" in release and "--minimum-tests 3218" in release - assert release.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - - -@pytest.mark.parametrize("mutation", [ - "unchanged", "formula", "math_oracle", "frozen_model", "frozen_schema", "hero", "dependency", "plan", - "cli", "html", "assembly", "result", "config", "hashing", "logging", "output_paths", "unknown_runtime", "missing_runtime", "premature_completion", "premature_report", - "control_step", "control_scope", "control_schema_scope", "historical_assertion", "historical_tooling", - "historical_binding", "historical_document", "input_hash_helper", "config_resolver_helper", -]) -def test_phase4_step6_current_snapshot_checks_valid_tree_before_mutations(repo_root, tmp_path, phase4_gate_tools, mutation, phase4_step6_snapshot): - import json - - paths = {path.relative_to(phase4_step6_snapshot).as_posix() - for path in phase4_step6_snapshot.rglob("*") if path.is_file()} - assert len(paths) == 228 - for relative in sorted(paths): - destination = tmp_path / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(phase4_step6_snapshot / relative, destination) - verify = phase4_gate_tools["verify_phase4_step6_snapshot"] - baseline = verify(tmp_path) - assert baseline["package_modules"] == 40 - assert baseline["frozen_runtime_modules"] == 38 - assert baseline["frozen_schemas"] == 5 - assert baseline["result_contracts_enabled"] is True - assert baseline["adapters_enabled"] is True - assert baseline["privacy_views_enabled"] is True - assert baseline["renderers_enabled"] is True - assert baseline["output_publication_enabled"] is True - assert baseline["cli_analysis_enabled"] is False - assert baseline["phase_complete"] is False - if mutation == "unchanged": - return - replacements = { - "formula": ("src/recursive_integrity_toolkit/metrics/diversity.py", - b"diversity = 1.0 - concentration", b"diversity = 0.5 - concentration"), - "math_oracle": ("tests/golden/phase3_math_cases.json", b'"v2_support":5', b'"v2_support":3'), - "hero": ("examples/hero/records_v2.csv", b"v2_08,v2,", b"v2_99,v2,"), - "dependency": ("pyproject.toml", b"numpy>=2.0", b"numpy>=3.0"), - "historical_assertion": ("tests/unit/test_phase4_contracts.py", - b' assert control["active_phase"] == 4 and control["active_step"] == 4\n', - b" assert True\n"), - "historical_tooling": ("scripts/release_check.py", - b"def verify_phase4_step4_control(control: dict, step: int = 4) -> None:", - b"def verify_phase4_step4_control(control: dict, step: int = 4) -> None:\n return"), - "historical_binding": ("tests/integration/test_phase4_gates.py", - b"def test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step4_snapshot):\n repo_root = phase4_step4_snapshot\n", - b"def test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step4_snapshot):\n"), - "historical_document": ("docs/cli.md", b"# CLI\n", - b"# Unauthorized historical document\n"), - "input_hash_helper": ("src/recursive_integrity_toolkit/utils/hashing.py", - b"return hashlib.sha256(data).hexdigest()", b"return hashlib.sha256(b'changed').hexdigest()"), - "config_resolver_helper": ("src/recursive_integrity_toolkit/config.py", - b"def resolve_config(", b"def resolve_config_disabled("), - } - appended = { - "frozen_model": "src/recursive_integrity_toolkit/models.py", - "frozen_schema": "schemas/report.schema.json", - "plan": "PHASE_4_PLAN.md", - "cli": "src/recursive_integrity_toolkit/cli.py", - "html": "src/recursive_integrity_toolkit/reports/html_report.py", - "assembly": "src/recursive_integrity_toolkit/reports/assembly.py", - "result": "src/recursive_integrity_toolkit/result.py", - "config": "src/recursive_integrity_toolkit/config.py", - "hashing": "src/recursive_integrity_toolkit/utils/hashing.py", - "logging": "src/recursive_integrity_toolkit/utils/logging.py", - "output_paths": "src/recursive_integrity_toolkit/utils/paths.py", - } - added = { - "unknown_runtime": "src/recursive_integrity_toolkit/reports/unauthorized.py", - "premature_completion": "PHASE_4_COMPLETION.md", "premature_report": "report.json", - } - if mutation in replacements: - relative, original, replacement = replacements[mutation] - path = tmp_path / relative - source = path.read_bytes() - assert source.count(original) == 1 - path.write_bytes(source.replace(original, replacement, 1)) - elif mutation in appended: - path = tmp_path / appended[mutation] - path.write_bytes(path.read_bytes() + b"\nUnauthorized Step 6 mutation\n") - elif mutation in added: - path = tmp_path / added[mutation] - assert not path.exists() - path.write_text("Unauthorized later-stage file\n", encoding="utf-8") - elif mutation == "missing_runtime": - (tmp_path / "src/recursive_integrity_toolkit/metrics/bounds.py").unlink() - elif mutation.startswith("control_"): - path = tmp_path / "PHASE_4_BASELINE.json" - control = json.loads(path.read_text(encoding="utf-8")) - if mutation == "control_step": - control["active_step"] = 7 - elif mutation == "control_scope": - control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py") - else: - control["schema_changes_authorized"] = True - control["schema_paths_authorized"].append("schemas/report.schema.json") - path.write_text(json.dumps(control), encoding="utf-8") - else: - raise AssertionError(f"Unknown active mutation case: {mutation}") - with pytest.raises(ValueError): - verify(tmp_path) - - -@pytest.mark.parametrize("relative,injection", [ - ("utils/paths.py", b"\nimport socket\n"), - ("utils/paths.py", b"\nfrom recursive_integrity_toolkit.metrics import diversity\n"), - ("utils/paths.py", b"\ndef new_reader(path):\n return path.read_bytes()\n"), - ("utils/paths.py", b"\nexec('forged=True')\n"), - ("utils/paths.py", b"\ndef effect(value=globals().clear()):\n pass\n"), - ("utils/logging.py", b"\nimport urllib.request\n"), - ("utils/logging.py", b"\ndef disclose(value):\n print(value)\n"), - ("utils/logging.py", b"\n@print('private')\ndef effect():\n pass\n"), -]) -def test_phase4_step6_current_output_ast_rejects_injections(repo_root, tmp_path, relative, injection): - checker = runpy.run_path(str(repo_root / "scripts/check_traceability.py"), run_name="phase4_step6_ast_tests") - verify = checker["phase4_step6_runtime_boundary"] - repository_path = "src/recursive_integrity_toolkit/" + relative - current = repo_root / repository_path - verify(current, repository_path) - target = tmp_path / "changed.py" - target.write_bytes(current.read_bytes() + injection) - with pytest.raises(ValueError): - verify(target, repository_path) - - -@pytest.mark.parametrize("relative,needle,replacement", [ - ("utils/paths.py", b"def resolve_content_reference", b"def disabled_content_reference"), - ("utils/logging.py", b"def format_diagnostic", b"def disabled_diagnostic"), -]) -def test_phase4_step6_preserves_existing_input_and_diagnostic_helpers(repo_root, phase4_step5_snapshot, phase4_gate_tools, relative, needle, replacement): - path = "src/recursive_integrity_toolkit/" + relative - before = (phase4_step5_snapshot / path).read_bytes() - after = (repo_root / path).read_bytes() - verify = phase4_gate_tools["_phase4_step6_preserve_runtime"] - verify(before, after, path) - assert needle in after - with pytest.raises(ValueError): - verify(before, after.replace(needle, replacement, 1), path) - - -@pytest.mark.parametrize("injection", [ - b"\nimport socket\n", b"\npytest.skip('inherited tests')\n", - b"\ndef test_unscoped():\n pass\n", - b"\ndef test_phase4_step6_unsafe(value=globals().clear()):\n pass\n", - b"\n@pytest.mark.parametrize('value', list(range(2)))\ndef test_phase4_step6_unsafe(value):\n pass\n", -]) -def test_phase4_step6_new_test_file_rejects_definition_effects(repo_root, phase4_gate_tools, injection): - path = "tests/unit/test_phase4_output_safety.py" - raw = (repo_root / path).read_bytes() - verify = phase4_gate_tools["_phase4_step6_new_test"] - verify(raw, path) - with pytest.raises(ValueError): - verify(raw + injection, path) - - -def test_phase4_step7_current_runtime_opens_only_cli_configuration_and_freezes_schema(repo_root, phase4_step6_snapshot, phase4_step7_snapshot): - repo_root = phase4_step7_snapshot - package = "src/recursive_integrity_toolkit" - current = {path.relative_to(repo_root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (repo_root / package).rglob("*.py")} - frozen = {path.relative_to(phase4_step6_snapshot).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (phase4_step6_snapshot / package).rglob("*.py")} - assert len(current) == len(frozen) == 40 - assert current.keys() == frozen.keys() - opened = {package + "/" + relative for relative in ( - "cli.py", "config.py", - )} - assert {path for path in current if current[path] != frozen[path]} == opened - assert {path: digest for path, digest in current.items() if path not in opened} == { - path: digest for path, digest in frozen.items() if path not in opened - } - schemas = {path.name: path.read_bytes() for path in (repo_root / "schemas").glob("*.json")} - assert len(schemas) == 5 - assert schemas == {path.name: path.read_bytes() for path in (phase4_step6_snapshot / "schemas").glob("*.json")} - - -def test_phase4_step7_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step7_snapshot): - repo_root = phase4_step7_snapshot - names = {"ci.yml", "golden.yml", "security.yml", "release.yml"} - root = repo_root / ".github/workflows" - expected_timeouts = {"ci.yml": [60, 20], "golden.yml": [15], "security.yml": [20], "release.yml": [40]} - assert {path.name for path in root.glob("*.yml")} == names - for name in sorted(names): - text = (root / name).read_text(encoding="utf-8") - assert [int(line.split(":", 1)[1]) for line in text.splitlines() - if line.strip().startswith("timeout-minutes:")] == expected_timeouts[name] - assert "Phase 4 Step 7" in text, name - assert "--phase 4 --step 7" in text, name - for older in ("--phase 4 --step 6", "--phase 4 --step 5", "--phase 4 --step 4", "--phase 4 --step 3", "--phase 4 --step 2", "--phase 4 --step 1", "--phase 3 --step 11"): - assert older not in text, (name, older) - assert "permissions:\n contents: read" in text - assert "persist-credentials: false" in text - assert "timeout-minutes:" in text and "set -euo pipefail" in text - assert "actions/upload-artifact@v4" in text and "if-no-files-found: error" in text - for line in text.splitlines(): - if "python scripts/release_check.py" in line: - assert "--phase 4 --step 7" in line, (name, line) - for forbidden in ("continue-on-error:", "|| true", "contents: write", "id-token: write", "twine upload", "git push"): - assert forbidden not in text, (name, forbidden) - ci = (root / "ci.yml").read_text(encoding="utf-8") - for required in ('os: [ubuntu-latest, windows-latest]', 'python-version: ["3.11", "3.12"]', - 'dependencies: [current, minimum]', '"numpy==2.0.0" "pandas==2.2.2"', - 'RIT_TEST_PARQUET: "0"', 'RIT_TEST_PARQUET: "1"', '--require-parquet', - '--baseline-evidence', 'python -m pip check', - '--minimum-tests 3420', '--minimum-tests 3423', - "find_spec('pyarrow') is None", 'import pyarrow'): - assert required in ci - assert ci.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in ci - golden = (root / "golden.yml").read_text(encoding="utf-8") - for required in ("tests/golden/test_phase3_math.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_phase3_metric_pipeline.py"): - assert required in golden - security = (root / "security.yml").read_text(encoding="utf-8") - for required in ('tests/integration/test_no_network.py', 'tests/integration/test_optional_dependency.py', - 'tests/unit/test_PR012_evidence_classes.py', 'tests/unit/test_PR013_report_schema.py', - 'tests/unit/test_PR014_unavailable.py', 'tests/unit/test_PR015_redaction.py', - 'tests/unit/test_PR016_determinism.py', 'tests/unit/test_PR018_language.py', - 'tests/integration/test_partial_provenance_report.py', - 'tests/integration/test_partial_lineage_report.py', - 'tests/integration/test_phase4_cli.py', 'tests/unit/test_phase4_output_safety.py', 'tests/unit/test_phase4_contracts.py', 'tests/integration/test_phase4_gates.py'): - assert required in security - release = (root / "release.yml").read_text(encoding="utf-8") - assert "python -m build" in release and "python -m twine check --strict" in release - assert "--dist" in release and "--candidate" in release - assert "--delivery" not in release - assert "recursive-integrity-toolkit-phase4-step7-candidate" in release - assert "rit-phase4-step4" not in release - assert "--minimum-tests 3420" in release and "--minimum-tests 3423" in release - assert release.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - - -@pytest.mark.parametrize("mutation", [ - "unchanged", "formula", "math_oracle", "frozen_model", "frozen_schema", "hero", "dependency", "plan", - "cli", "html", "assembly", "result", "config", "hashing", "logging", "output_paths", "unknown_runtime", "missing_runtime", "premature_completion", "premature_report", - "control_step", "control_scope", "control_schema_scope", "historical_assertion", "historical_tooling", - "historical_binding", "historical_document", "input_hash_helper", "config_resolver_helper", -]) -def test_phase4_step7_current_snapshot_checks_valid_tree_before_mutations(repo_root, tmp_path, phase4_gate_tools, mutation, phase4_step7_snapshot): - import json - - paths = {path.relative_to(phase4_step7_snapshot).as_posix() - for path in phase4_step7_snapshot.rglob("*") if path.is_file()} - assert len(paths) == 229 - for relative in sorted(paths): - destination = tmp_path / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(phase4_step7_snapshot / relative, destination) - verify = phase4_gate_tools["verify_phase4_step7_snapshot"] - baseline = verify(tmp_path) - assert baseline["package_modules"] == 40 - assert baseline["frozen_runtime_modules"] == 38 - assert baseline["frozen_schemas"] == 5 - assert baseline["result_contracts_enabled"] is True - assert baseline["adapters_enabled"] is True - assert baseline["privacy_views_enabled"] is True - assert baseline["renderers_enabled"] is True - assert baseline["output_publication_enabled"] is True - assert baseline["cli_analysis_enabled"] is True - assert baseline["phase_complete"] is False - if mutation == "unchanged": - return - replacements = { - "formula": ("src/recursive_integrity_toolkit/metrics/diversity.py", - b"diversity = 1.0 - concentration", b"diversity = 0.5 - concentration"), - "math_oracle": ("tests/golden/phase3_math_cases.json", b'"v2_support":5', b'"v2_support":3'), - "hero": ("examples/hero/records_v2.csv", b"v2_08,v2,", b"v2_99,v2,"), - "dependency": ("pyproject.toml", b"numpy>=2.0", b"numpy>=3.0"), - "historical_assertion": ("tests/unit/test_phase4_contracts.py", - b' assert control["active_phase"] == 4 and control["active_step"] == 4\n', - b" assert True\n"), - "historical_tooling": ("scripts/release_check.py", - b"def verify_phase4_step4_control(control: dict, step: int = 4) -> None:", - b"def verify_phase4_step4_control(control: dict, step: int = 4) -> None:\n return"), - "historical_binding": ("tests/integration/test_phase4_gates.py", - b"def test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step4_snapshot):\n repo_root = phase4_step4_snapshot\n", - b"def test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step4_snapshot):\n"), - "historical_document": ("docs/cli.md", b"# CLI\n", - b"# Unauthorized historical document\n"), - "input_hash_helper": ("src/recursive_integrity_toolkit/utils/hashing.py", - b"return hashlib.sha256(data).hexdigest()", b"return hashlib.sha256(b'changed').hexdigest()"), - "config_resolver_helper": ("src/recursive_integrity_toolkit/config.py", - b"def resolve_config(", b"def resolve_config_disabled("), - } - appended = { - "frozen_model": "src/recursive_integrity_toolkit/models.py", - "frozen_schema": "schemas/report.schema.json", - "plan": "PHASE_4_PLAN.md", - "cli": "src/recursive_integrity_toolkit/cli.py", - "html": "src/recursive_integrity_toolkit/reports/html_report.py", - "assembly": "src/recursive_integrity_toolkit/reports/assembly.py", - "result": "src/recursive_integrity_toolkit/result.py", - "config": "src/recursive_integrity_toolkit/config.py", - "hashing": "src/recursive_integrity_toolkit/utils/hashing.py", - "logging": "src/recursive_integrity_toolkit/utils/logging.py", - "output_paths": "src/recursive_integrity_toolkit/utils/paths.py", - } - added = { - "unknown_runtime": "src/recursive_integrity_toolkit/reports/unauthorized.py", - "premature_completion": "PHASE_4_COMPLETION.md", "premature_report": "report.json", - } - if mutation in replacements: - relative, original, replacement = replacements[mutation] - path = tmp_path / relative - source = path.read_bytes() - assert source.count(original) == 1 - path.write_bytes(source.replace(original, replacement, 1)) - elif mutation in appended: - path = tmp_path / appended[mutation] - path.write_bytes(path.read_bytes() + b"\nUnauthorized Step 7 mutation\n") - elif mutation in added: - path = tmp_path / added[mutation] - assert not path.exists() - path.write_text("Unauthorized later-stage file\n", encoding="utf-8") - elif mutation == "missing_runtime": - (tmp_path / "src/recursive_integrity_toolkit/metrics/bounds.py").unlink() - elif mutation.startswith("control_"): - path = tmp_path / "PHASE_4_BASELINE.json" - control = json.loads(path.read_text(encoding="utf-8")) - if mutation == "control_step": - control["active_step"] = 8 - elif mutation == "control_scope": - control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py") - else: - control["schema_changes_authorized"] = True - control["schema_paths_authorized"].append("schemas/report.schema.json") - path.write_text(json.dumps(control), encoding="utf-8") - else: - raise AssertionError(f"Unknown active mutation case: {mutation}") - with pytest.raises(ValueError): - verify(tmp_path) - - -@pytest.mark.parametrize("relative,injection", [ - ("cli.py", b"\nimport socket\n"), ("cli.py", b"\nmain()\n"), - ("cli.py", b"\ndef invented_metric():\n return 0.5\n"), - ("cli.py", b"\ndef unsafe(value=globals().clear()):\n pass\n"), - ("config.py", b"\nresolve_config = lambda *args: None\n"), - ("config.py", b"\nimport urllib.request\n"), -]) -def test_phase4_step7_current_cli_ast_rejects_injection(repo_root, tmp_path, relative, injection, phase4_step7_snapshot): - repo_root = phase4_step7_snapshot - checker = runpy.run_path(str(repo_root / "scripts/check_traceability.py"), run_name="phase4_step7_ast_tests") - verify = checker["phase4_step7_runtime_boundary"] - path = "src/recursive_integrity_toolkit/" + relative - verify(repo_root / path, path) - changed = tmp_path / "changed.py" - changed.write_bytes((repo_root / path).read_bytes() + injection) - with pytest.raises(ValueError): - verify(changed, path) - - -def test_phase4_step7_preserves_all_inherited_config_bytes(repo_root, phase4_step6_snapshot, phase4_gate_tools): - path = "src/recursive_integrity_toolkit/config.py" - before, after = (phase4_step6_snapshot / path).read_bytes(), (repo_root / path).read_bytes() - verify = phase4_gate_tools["_phase4_step7_preserve_runtime"] - verify(before, after, path) - for needle in (b"def resolve_config(", b"def resolve_phase4_options(", b"def phase4_config_hash("): - assert after.count(needle) == 1 - with pytest.raises(ValueError): - verify(before, after.replace(needle, b"def changed(", 1), path) - - -@pytest.mark.parametrize("injection", [ - b"\nimport socket\n", b"\npytest.skip('inherited tests')\n", - b"\ndef test_unscoped():\n pass\n", - b"\ndef test_phase4_step7_unsafe(value=globals().clear()):\n pass\n", -]) -def test_phase4_step7_new_cli_tests_reject_definition_effects(repo_root, phase4_gate_tools, injection, phase4_step7_snapshot): - repo_root = phase4_step7_snapshot - path = "tests/integration/test_phase4_cli.py" - raw = (repo_root / path).read_bytes() - verify = phase4_gate_tools["_phase4_step7_new_test"] - verify(raw, path) - with pytest.raises(ValueError): - verify(raw + injection, path) - - -def test_phase4_step8_current_runtime_opens_only_cli_configuration_and_freezes_schema(repo_root, phase4_step7_snapshot, phase4_step8_snapshot): - repo_root = phase4_step8_snapshot - package = "src/recursive_integrity_toolkit" - current = {path.relative_to(repo_root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (repo_root / package).rglob("*.py")} - frozen = {path.relative_to(phase4_step7_snapshot).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in (phase4_step7_snapshot / package).rglob("*.py")} - assert len(current) == len(frozen) == 40 - assert current.keys() == frozen.keys() - opened = {package + "/" + relative for relative in ( - "cli.py", "config.py", - )} - assert {path for path in current if current[path] != frozen[path]} == opened - assert {path: digest for path, digest in current.items() if path not in opened} == { - path: digest for path, digest in frozen.items() if path not in opened - } - schemas = {path.name: path.read_bytes() for path in (repo_root / "schemas").glob("*.json")} - assert len(schemas) == 5 - assert schemas == {path.name: path.read_bytes() for path in (phase4_step7_snapshot / "schemas").glob("*.json")} - - -def test_phase4_step8_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step8_snapshot): - repo_root = phase4_step8_snapshot - names = {"ci.yml", "golden.yml", "security.yml", "release.yml"} - root = repo_root / ".github/workflows" - expected_timeouts = {"ci.yml": [60, 20], "golden.yml": [15], "security.yml": [20], "release.yml": [40]} - assert {path.name for path in root.glob("*.yml")} == names - for name in sorted(names): - text = (root / name).read_text(encoding="utf-8") - assert [int(line.split(":", 1)[1]) for line in text.splitlines() - if line.strip().startswith("timeout-minutes:")] == expected_timeouts[name] - assert "Phase 4 Step 8" in text, name - assert "--phase 4 --step 8" in text, name - for older in ("--phase 4 --step 7", "--phase 4 --step 6", "--phase 4 --step 5", "--phase 4 --step 4", "--phase 4 --step 3", "--phase 4 --step 2", "--phase 4 --step 1", "--phase 3 --step 11"): - assert older not in text, (name, older) - assert "permissions:\n contents: read" in text - assert "persist-credentials: false" in text - assert "timeout-minutes:" in text and "set -euo pipefail" in text - assert "actions/upload-artifact@v4" in text and "if-no-files-found: error" in text - for line in text.splitlines(): - if "python scripts/release_check.py" in line: - assert "--phase 4 --step 8" in line, (name, line) - for forbidden in ("continue-on-error:", "|| true", "contents: write", "id-token: write", "twine upload", "git push"): - assert forbidden not in text, (name, forbidden) - ci = (root / "ci.yml").read_text(encoding="utf-8") - for required in ('os: [ubuntu-latest, windows-latest]', 'python-version: ["3.11", "3.12"]', - 'dependencies: [current, minimum]', '"numpy==2.0.0" "pandas==2.2.2"', - 'RIT_TEST_PARQUET: "0"', 'RIT_TEST_PARQUET: "1"', '--require-parquet', - '--baseline-evidence', 'python -m pip check', - '--minimum-tests 3620', '--minimum-tests 3623', - "find_spec('pyarrow') is None", 'import pyarrow'): - assert required in ci - assert ci.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in ci - golden = (root / "golden.yml").read_text(encoding="utf-8") - for required in ("tests/golden/test_phase3_math.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_phase3_metric_pipeline.py", "tests/integration/test_hero_end_to_end.py"): - assert required in golden - security = (root / "security.yml").read_text(encoding="utf-8") - for required in ('tests/integration/test_no_network.py', 'tests/integration/test_optional_dependency.py', - 'tests/unit/test_PR012_evidence_classes.py', 'tests/unit/test_PR013_report_schema.py', - 'tests/unit/test_PR014_unavailable.py', 'tests/unit/test_PR015_redaction.py', - 'tests/unit/test_PR016_determinism.py', 'tests/unit/test_PR018_language.py', - 'tests/integration/test_partial_provenance_report.py', - 'tests/integration/test_partial_lineage_report.py', - 'tests/integration/test_phase4_cli.py', 'tests/unit/test_phase4_output_safety.py', 'tests/unit/test_phase4_contracts.py', 'tests/integration/test_phase4_gates.py'): - assert required in security - release = (root / "release.yml").read_text(encoding="utf-8") - assert "python -m build" in release and "python -m twine check --strict" in release - assert "--dist" in release and "--candidate" in release - assert "--delivery" not in release - assert "recursive-integrity-toolkit-phase4-step8-candidate" in release - assert "rit-phase4-step4" not in release - assert "--minimum-tests 3620" in release and "--minimum-tests 3623" in release - assert release.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - - -@pytest.mark.parametrize("mutation", [ - "unchanged", "formula", "math_oracle", "frozen_model", "frozen_schema", "hero", "dependency", "plan", - "cli", "html", "assembly", "result", "config", "hashing", "logging", "output_paths", "unknown_runtime", "missing_runtime", "premature_completion", "premature_report", - "control_step", "control_scope", "control_schema_scope", "historical_assertion", "historical_tooling", - "historical_binding", "historical_document", "input_hash_helper", "config_resolver_helper", -]) -def test_phase4_step8_current_snapshot_checks_valid_tree_before_mutations(repo_root, tmp_path, phase4_gate_tools, mutation, phase4_step8_snapshot): - import json - - paths = {path.relative_to(phase4_step8_snapshot).as_posix() - for path in phase4_step8_snapshot.rglob("*") if path.is_file()} - assert len(paths) == 236 - for relative in sorted(paths): - destination = tmp_path / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(phase4_step8_snapshot / relative, destination) - verify = phase4_gate_tools["verify_phase4_step8_snapshot"] - baseline = verify(tmp_path) - assert baseline["package_modules"] == 40 - assert baseline["frozen_runtime_modules"] == 38 - assert baseline["frozen_schemas"] == 5 - assert baseline["result_contracts_enabled"] is True - assert baseline["adapters_enabled"] is True - assert baseline["privacy_views_enabled"] is True - assert baseline["renderers_enabled"] is True - assert baseline["output_publication_enabled"] is True - assert baseline["cli_analysis_enabled"] is True - assert baseline["phase_complete"] is False - if mutation == "unchanged": - return - replacements = { - "formula": ("src/recursive_integrity_toolkit/metrics/diversity.py", - b"diversity = 1.0 - concentration", b"diversity = 0.5 - concentration"), - "math_oracle": ("tests/golden/phase3_math_cases.json", b'"v2_support":5', b'"v2_support":3'), - "hero": ("examples/hero/records_v2.csv", b"v2_08,v2,", b"v2_99,v2,"), - "dependency": ("pyproject.toml", b"numpy>=2.0", b"numpy>=3.0"), - "historical_assertion": ("tests/unit/test_phase4_contracts.py", - b' assert control["active_phase"] == 4 and control["active_step"] == 4\n', - b" assert True\n"), - "historical_tooling": ("scripts/release_check.py", - b"def verify_phase4_step4_control(control: dict, step: int = 4) -> None:", - b"def verify_phase4_step4_control(control: dict, step: int = 4) -> None:\n return"), - "historical_binding": ("tests/integration/test_phase4_gates.py", - b"def test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step4_snapshot):\n repo_root = phase4_step4_snapshot\n", - b"def test_phase4_step4_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root, phase4_step4_snapshot):\n"), - "historical_document": ("docs/cli.md", b"# CLI\n", - b"# Unauthorized historical document\n"), - "input_hash_helper": ("src/recursive_integrity_toolkit/utils/hashing.py", - b"return hashlib.sha256(data).hexdigest()", b"return hashlib.sha256(b'changed').hexdigest()"), - "config_resolver_helper": ("src/recursive_integrity_toolkit/config.py", - b"def resolve_config(", b"def resolve_config_disabled("), - } - appended = { - "frozen_model": "src/recursive_integrity_toolkit/models.py", - "frozen_schema": "schemas/report.schema.json", - "plan": "PHASE_4_PLAN.md", - "cli": "src/recursive_integrity_toolkit/cli.py", - "html": "src/recursive_integrity_toolkit/reports/html_report.py", - "assembly": "src/recursive_integrity_toolkit/reports/assembly.py", - "result": "src/recursive_integrity_toolkit/result.py", - "config": "src/recursive_integrity_toolkit/config.py", - "hashing": "src/recursive_integrity_toolkit/utils/hashing.py", - "logging": "src/recursive_integrity_toolkit/utils/logging.py", - "output_paths": "src/recursive_integrity_toolkit/utils/paths.py", - } - added = { - "unknown_runtime": "src/recursive_integrity_toolkit/reports/unauthorized.py", - "premature_completion": "PHASE_4_COMPLETION.md", "premature_report": "report.json", - } - if mutation in replacements: - relative, original, replacement = replacements[mutation] - path = tmp_path / relative - source = path.read_bytes() - assert source.count(original) == 1 - path.write_bytes(source.replace(original, replacement, 1)) - elif mutation in appended: - path = tmp_path / appended[mutation] - path.write_bytes(path.read_bytes() + b"\nUnauthorized Step 8 mutation\n") - elif mutation in added: - path = tmp_path / added[mutation] - assert not path.exists() - path.write_text("Unauthorized later-stage file\n", encoding="utf-8") - elif mutation == "missing_runtime": - (tmp_path / "src/recursive_integrity_toolkit/metrics/bounds.py").unlink() - elif mutation.startswith("control_"): - path = tmp_path / "PHASE_4_BASELINE.json" - control = json.loads(path.read_text(encoding="utf-8")) - if mutation == "control_step": - control["active_step"] = 9 - elif mutation == "control_scope": - control["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/reports/assembly.py") - else: - control["schema_changes_authorized"] = True - control["schema_paths_authorized"].append("schemas/report.schema.json") - path.write_text(json.dumps(control), encoding="utf-8") - else: - raise AssertionError(f"Unknown active mutation case: {mutation}") - with pytest.raises(ValueError): - verify(tmp_path) - - -@pytest.mark.parametrize("relative,injection", [ - ("cli.py", b"\nimport socket\n"), ("cli.py", b"\nmain()\n"), - ("cli.py", b"\ndef invented_metric():\n return 0.5\n"), - ("cli.py", b"\ndef unsafe(value=globals().clear()):\n pass\n"), - ("config.py", b"\nresolve_config = lambda *args: None\n"), - ("config.py", b"\nimport urllib.request\n"), -]) -def test_phase4_step8_current_cli_ast_rejects_injection(repo_root, tmp_path, relative, injection): - checker = runpy.run_path(str(repo_root / "scripts/check_traceability.py"), run_name="phase4_step8_ast_tests") - verify = checker["phase4_step8_runtime_boundary"] - path = "src/recursive_integrity_toolkit/" + relative - verify(repo_root / path, path) - changed = tmp_path / "changed.py" - changed.write_bytes((repo_root / path).read_bytes() + injection) - with pytest.raises(ValueError): - verify(changed, path) - - -def test_phase4_step8_preserves_all_inherited_config_bytes(repo_root, phase4_step7_snapshot, phase4_gate_tools): - path = "src/recursive_integrity_toolkit/config.py" - before, after = (phase4_step7_snapshot / path).read_bytes(), (repo_root / path).read_bytes() - verify = phase4_gate_tools["_phase4_step8_preserve_runtime"] - verify(before, after, path) - for needle in (b"def resolve_config(", b"def resolve_phase4_options(", b"def phase4_config_hash("): - assert after.count(needle) == 1 - with pytest.raises(ValueError): - verify(before, after.replace(needle, b"def changed(", 1), path) - - -@pytest.mark.parametrize("mutation", ["copy", "schema", "extra", "glob", "version", "dependency", "backend"]) -def test_phase4_step8_resources_and_packaging_have_exact_boundaries(repo_root, tmp_path, phase4_gate_tools, phase4_step7_snapshot, mutation, phase4_step8_snapshot): - resources = phase4_gate_tools["PHASE4_STEP8_RESOURCES"] - for relative in {*resources, *resources.values(), "pyproject.toml"}: - target = tmp_path / relative - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes((phase4_step8_snapshot / relative).read_bytes()) - prior = {"pyproject.toml": (phase4_step7_snapshot / "pyproject.toml").read_bytes()} - verify = phase4_gate_tools["phase4_step8_resources"] - verify(tmp_path, prior) - if mutation in ("copy", "schema"): - path = "src/recursive_integrity_toolkit/data/hero/config.json" if mutation == "copy" else "src/recursive_integrity_toolkit/data/report.schema.json" - (tmp_path / path).write_bytes(b"{}") - elif mutation == "extra": - (tmp_path / "src/recursive_integrity_toolkit/data/extra.json").write_bytes(b"{}") - else: - path = tmp_path / "pyproject.toml" - raw = path.read_bytes() - a, b = {"glob": (b"data/hero/*", b"data/*"), "version": (b"0.1.0.dev2", b"0.1.0.dev3"), "dependency": (b"numpy>=2.0", b"numpy>=3.0"), "backend": (b"setuptools.build_meta", b"other.backend")}[mutation] - assert raw.count(a) == 1 - path.write_bytes(raw.replace(a, b)) - with pytest.raises(ValueError): verify(tmp_path, prior) - - -def test_phase4_step9_current_workflows_preserve_matrix_and_use_active_dispatch(repo_root): - import re - names = {"ci.yml", "golden.yml", "security.yml", "release.yml"} - root = repo_root / ".github/workflows" - expected_timeouts = {"ci.yml": [105, 75], "golden.yml": [15], "security.yml": [20], "release.yml": [40]} - assert {path.name for path in root.glob("*.yml")} == names - for name in sorted(names): - text = (root / name).read_text(encoding="utf-8") - assert [int(line.split(":", 1)[1]) for line in text.splitlines() - if line.strip().startswith("timeout-minutes:")] == expected_timeouts[name] - assert "Phase 4 Step 11" in text, name - assert "--phase 4 --step 11" in text, name - for older in ("--phase 4 --step 10", "--phase 4 --step 9", "--phase 4 --step 8", "--phase 4 --step 7", "--phase 4 --step 6", "--phase 4 --step 5", "--phase 4 --step 4", "--phase 4 --step 3", "--phase 4 --step 2", "--phase 4 --step 1", "--phase 3 --step 11"): - assert re.search(re.escape(older) + r"(?![0-9])", text) is None, (name, older) - assert "permissions:\n contents: read" in text - assert "persist-credentials: false" in text - assert "timeout-minutes:" in text and "set -euo pipefail" in text - assert "actions/upload-artifact@v4" in text and "if-no-files-found: error" in text - for line in text.splitlines(): - if "python scripts/release_check.py" in line: - assert "--phase 4 --step 11" in line, (name, line) - for forbidden in ("continue-on-error:", "|| true", "contents: write", "id-token: write", "twine upload", "git push"): - assert forbidden not in text, (name, forbidden) - ci = (root / "ci.yml").read_text(encoding="utf-8") - for required in ('os: [ubuntu-latest, windows-latest]', 'python-version: ["3.11", "3.12"]', - 'dependencies: [current, minimum]', '"numpy==2.0.0" "pandas==2.2.2"', - 'RIT_TEST_PARQUET: "0"', 'RIT_TEST_PARQUET: "1"', '--require-parquet', - '--baseline-evidence', 'python -m pip check', - '--minimum-tests 3768', '--minimum-tests 3771', - "find_spec('pyarrow') is None", 'import pyarrow'): - assert required in ci - assert ci.count("python -m pytest -p no:cacheprovider -q --junitxml=") == 2 - assert " -k " not in ci and "--ignore" not in ci - assert "needs: [core, parquet]" in ci - assert "uses: ./.github/workflows/release.yml" in ci - for name in ("ci.yml", "golden.yml", "security.yml"): - text = (root / name).read_text(encoding="utf-8") - assert " push:" not in text - assert " pull_request:" in text and " workflow_dispatch:" in text - golden = (root / "golden.yml").read_text(encoding="utf-8") - for required in ("tests/golden/test_phase3_math.py", "tests/integration/test_hero_structure.py", - "tests/integration/test_phase3_metric_pipeline.py", "tests/integration/test_hero_end_to_end.py", "tests/golden/test_phase4_reports.py", "--golden-junit"): - assert required in golden - security = (root / "security.yml").read_text(encoding="utf-8") - for required in ('tests/golden/test_phase4_reports.py', 'tests/integration/test_no_network.py', 'tests/integration/test_optional_dependency.py', - 'tests/unit/test_PR012_evidence_classes.py', 'tests/unit/test_PR013_report_schema.py', - 'tests/unit/test_PR014_unavailable.py', 'tests/unit/test_PR015_redaction.py', - 'tests/unit/test_PR016_determinism.py', 'tests/unit/test_PR018_language.py', - 'tests/integration/test_partial_provenance_report.py', - 'tests/integration/test_partial_lineage_report.py', - 'tests/integration/test_phase4_cli.py', 'tests/unit/test_phase4_output_safety.py', 'tests/unit/test_phase4_contracts.py', 'tests/integration/test_phase4_gates.py'): - assert required in security - release = (root / "release.yml").read_text(encoding="utf-8") - assert "python -m build" in release and "python -m twine check --strict" in release - assert "--candidate" in release - assert " workflow_call:" in release and " push:" not in release - assert release.count("uses: actions/download-artifact@v4") == 2 - assert "phase4-current-core-ubuntu-latest-3.12-current" in release - assert "phase4-current-real-parquet" in release - assert "core-evidence/source-commit.txt" in release and "parquet-evidence/source-commit.txt" in release - assert "git rev-parse HEAD" in release - assert "--delivery" not in release - assert "recursive-integrity-toolkit-phase4-current-candidate" in release - assert "rit-phase4-step4" not in release - assert "python -m pytest" not in release - assert "SOURCE_DATE_EPOCH" in release and "reproducibility.json" in release - - -def test_phase4_step9_runtime_schemas_resources_and_dependencies_equal_step8(repo_root, phase4_step8_snapshot, phase4_gate_tools): - prefixes = ("src/recursive_integrity_toolkit/", "schemas/", "examples/hero/") - current = {p.relative_to(repo_root).as_posix(): p.read_bytes() for directory in ("src/recursive_integrity_toolkit", "schemas", "examples/hero") for p in (repo_root / directory).rglob("*") if p.is_file() and "__pycache__" not in p.parts} - current["pyproject.toml"] = (repo_root / "pyproject.toml").read_bytes() - accepted = phase4_gate_tools["_phase4_current_files"]() - protected = {k: v for k, v in accepted.items() if k.startswith(prefixes) or k == "pyproject.toml"} - for path in phase4_gate_tools["PHASE4_CURRENT_VERSION_PATHS"]: - protected[path] = protected[path].replace(b"0.1.0.dev2", b"0.1.0.dev3", 1) - assert current == protected - assert sum(p.startswith("src/") and p.endswith(".py") for p in protected) == 40 - - -@pytest.mark.parametrize("relative", ["scripts/release_check.py", "scripts/check_traceability.py", "scripts/check_spec_consistency.py"]) -def test_phase4_step9_maintainer_definitions_preserved_except_exact_dispatch(repo_root, relative): - """Current entry points reject unapproved stages without preserving obsolete bodies.""" - result = subprocess.run([sys.executable, str(repo_root / relative), "--phase", "4", "--step", "12"], - cwd=repo_root, capture_output=True, text=True) - assert result.returncode != 0 - assert "Traceback" not in result.stdout - - -@pytest.mark.parametrize("mutation", [ - "unchanged", "formula", "cli", "assembly", "schema", "resource", "dependency", "version", "plan", "math_oracle", - "golden_json", "golden_markdown", "golden_rationale", "unknown_runtime", "unknown_resource", "missing_runtime", - "completion", "report", "control", -]) -def test_phase4_step9_snapshot_accepts_current_then_rejects_substantive_mutation(repo_root, phase4_step8_snapshot, phase4_gate_tools, tmp_path, mutation): - import json - prior = {p.relative_to(phase4_step8_snapshot).as_posix() for p in phase4_step8_snapshot.rglob("*") if p.is_file()} - paths = set(phase4_gate_tools["_phase4_current_files"]()) | phase4_gate_tools["PHASE4_CURRENT_NEW"] - assert len(prior) == 236 and len(paths) == 246 - for relative in sorted(paths): - target = tmp_path / relative - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes((repo_root / relative).read_bytes()) - verify = phase4_gate_tools["verify_phase4_current_snapshot"] - result = verify(tmp_path) - assert result["frozen_runtime_modules"] == 39 - assert result["package_modules"] == 40 and result["version_literal_changes"] == 2 - assert result["frozen_schemas"] == 5 and result["packaged_resources"] == 7 - assert result["frozen_report_oracle_files"] == 6 and result["phase_complete"] is False - if mutation == "unchanged": - return - paths_by_mutation = { - "formula": "src/recursive_integrity_toolkit/metrics/diversity.py", "cli": "src/recursive_integrity_toolkit/cli.py", - "assembly": "src/recursive_integrity_toolkit/reports/assembly.py", - "schema": "schemas/report.schema.json", "resource": "src/recursive_integrity_toolkit/data/hero/config.json", - "dependency": "pyproject.toml", "version": "src/recursive_integrity_toolkit/__init__.py", "plan": "PHASE_4_PLAN.md", - "math_oracle": "tests/golden/phase3_math_cases.json", "golden_json": "tests/golden/phase4_hero_report.json", - "golden_markdown": "tests/golden/phase4_hero_redacted.md", "golden_rationale": "tests/golden/phase4_report_cases.md", - } - if mutation in paths_by_mutation: - path = tmp_path / paths_by_mutation[mutation] - raw = path.read_bytes() - path.write_bytes(raw + b"\nchanged substantive evidence\n") - elif mutation == "completion": - (tmp_path / "PHASE_4_COMPLETION.md").unlink() - elif mutation in ("unknown_runtime", "unknown_resource", "report"): - relative = {"unknown_runtime": "src/recursive_integrity_toolkit/extra.py", "unknown_resource": "src/recursive_integrity_toolkit/data/extra.json", "report": "report.json"}[mutation] - (tmp_path / relative).write_text("{}", encoding="utf-8") - elif mutation == "missing_runtime": - (tmp_path / "src/recursive_integrity_toolkit/models.py").unlink() - elif mutation == "control": - path = tmp_path / "PHASE_4_BASELINE.json" - value = json.loads(path.read_text(encoding="utf-8")) - value["runtime_paths_authorized"].append("src/recursive_integrity_toolkit/cli.py") - path.write_text(json.dumps(value), encoding="utf-8") - with pytest.raises(ValueError): - verify(tmp_path) diff --git a/tests/integration/test_phase5_lineage_cli.py b/tests/integration/test_phase5_lineage_cli.py new file mode 100644 index 0000000..44a5680 --- /dev/null +++ b/tests/integration/test_phase5_lineage_cli.py @@ -0,0 +1,388 @@ +"""Explicit lineage orchestration, target scope and local input boundaries.""" + +from __future__ import annotations + +import csv +import importlib +import inspect +import json +import os +from pathlib import Path +import socket + +import pytest +from jsonschema import Draft202012Validator + + +def _write(path, rows): + if path.suffix == ".csv": + with path.open("w", encoding="utf-8", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + elif path.suffix == ".parquet": + import pyarrow as arrow + import pyarrow.parquet as parquet + parquet.write_table(arrow.Table.from_pylist(rows), path) + else: + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + + +def _inputs(directory, *, context_format="jsonl", content=False): + directory.mkdir(parents=True, exist_ok=True) + rows = [ + {"dataset_version": "v1", "record_id": "root1", "content": "PRIVATE_CONTENT_A", "topic": "A"}, + {"dataset_version": "v2", "record_id": "root2", "content": "PRIVATE_CONTENT_B", "topic": "B"}, + {"dataset_version": "v3", "record_id": "t1", "content": "PRIVATE_CONTENT_A", "topic": "A"}, + {"dataset_version": "v3", "record_id": "t2", "content": "PRIVATE_CONTENT_A", "topic": "A"}, + ] + primary = directory / "primary.jsonl" + contexts = [directory / f"context{i}.{context_format}" for i in (1, 2)] + _write(primary, rows[2:]) + for path, row in zip(contexts, rows[:2]): + _write(path, [row]) + provenance = directory / "provenance.jsonl" + manifests = [] + for index, row in enumerate(rows): + manifests.append({"dataset_version": row["dataset_version"], "record_id": row["record_id"], + "source_type": "human" if index < 2 else "synthetic", "provenance_confidence": "confirmed", + "external_grounding": "yes" if index < 2 else "no", "transformation": "generate", + "parent_ids": [] if index < 2 else [f"v{index - 1}::root{index - 1}"]}) + _write(provenance, manifests) + config = directory / "config.json" + representation = {"name": "exact" if content else "topic", "source": "content_hash" if content else "topic_field", + "field": "content" if content else "topic", "version": "1", "missing_value_policy": "error"} + if content: + representation["normalization_profile"] = "exact_utf8_v1" + config.write_text(json.dumps({"representation": representation, "version_order": ["v1", "v2", "v3"]}), encoding="utf-8") + return ["--records", str(primary), "--provenance", str(provenance), "--config", str(config)], primary, contexts, provenance, config + + +def _context_args(contexts): + return [value for path in contexts for value in ("--lineage-records", str(path))] + + +def _invoke(args, out, capsys, *, command="audit"): + from recursive_integrity_toolkit.cli import main + code = main([command, *args, "--out", str(out)]) + streams = capsys.readouterr() + directory = out / "reports" if command == "example" else out + report = json.loads((directory / "report.json").read_text(encoding="utf-8")) if (directory / "report.json").exists() else None + markdown = (directory / "report.md").read_text(encoding="utf-8") if (directory / "report.md").exists() else "" + if report is not None: + from recursive_integrity_toolkit.result import validate_report + validate_report(report) + assert report["run"]["report_schema_version"] == "1.1" + assert report["capabilities"] == report["observability"].get("capabilities", {}) + assert report["run"]["network_call_count"] == 0 + assert markdown.startswith("# Recursive Integrity Audit Report\n") + assert "PRIVATE_CONTENT_" not in json.dumps(report) + markdown + streams.out + streams.err + return code, report, markdown, streams + + +def _bounds(report, family): + value = report["derived_metrics"]["closure_exposure"][family] + return tuple(value[name]["value"] for name in ("lower_bound", "upper_bound", "interval_width")) + + +def _assert_primary_metrics(report, *, content=False): + metrics = report["derived_metrics"] + assert set(metrics["support"]["by_version"]) == {"v3"} + support = metrics["support"]["by_version"]["v3"]["support_size"] + assert support["value"] == 1 and support["scope"]["record_count"] == 2 + assert report["observed_facts"]["provenance"]["analyzed_record_count"]["value"] == 2 + shares = metrics["provenance"]["source_type_shares"] + assert shares["value"]["synthetic"] == 1.0 and shares["value"]["human"] == 0.0 + assert shares["denominator"] == 2 + assert _bounds(report, "direct") == (1.0, 1.0, 0.0) + if content: + duplicates = report["observed_facts"]["content"]["duplicate_record_count"] + assert duplicates["value"] == 1 and duplicates["scope"]["record_count"] == 2 + + +@pytest.mark.parametrize("context_format", ["csv", "jsonl"] + (["parquet"] if os.environ.get("RIT_TEST_PARQUET") == "1" else [])) +def test_context_loaders_and_repeated_inputs_keep_primary_only_denominators(tmp_path, capsys, repo_root, context_format): + args, primary, contexts, provenance, config = _inputs(tmp_path, context_format=context_format, content=True) + before = {path: path.read_bytes() for path in (primary, *contexts, provenance, config)} + code, report, markdown, streams = _invoke(args + ["--lineage"] + _context_args(contexts), tmp_path / "out", capsys) + assert code == 0 and report["run"]["run_status"] == "complete" + _assert_primary_metrics(report, content=True) + scope = report["observed_facts"]["lineage"]["graph_scope"]["value"] + assert scope == {"target_dataset_version": "v3", "target_record_count": 2, "loaded_record_count": 4, + "context_record_count": 2, "loaded_dataset_versions": ["v1", "v2", "v3"]} + assert report["derived_metrics"]["lineage"]["grounded_record_count"]["value"] == 2 + assert report["derived_metrics"]["lineage"]["ancestry_concentration_hhi"]["value"] == 0.5 + assert _bounds(report, "lineage") == (0.0, 0.0, 0.0) + assert report["capabilities"]["lineage"]["execution_status"] == "completed" + assert report["capabilities"]["dataset_longitudinal"]["execution_status"] == "not_requested" + assert "comparison_details" not in report["derived_metrics"]["support"] and report["simulations"] == {} + assert sum(item["role"] == "lineage_context" for item in report["inputs"]["artifacts"]) == 2 + Draft202012Validator(json.loads((repo_root / "schemas/report.schema.json").read_text())).validate(report) + assert all(path.read_bytes() == data for path, data in before.items()) + + +def test_plain_audit_does_not_dispatch_graph_even_when_provenance_has_parents(tmp_path, capsys, monkeypatch): + args, primary, contexts, provenance, config = _inputs(tmp_path) + _write(provenance, [json.loads(line) for line in provenance.read_text().splitlines() + if json.loads(line)["dataset_version"] == "v3"]) + graph = importlib.import_module("recursive_integrity_toolkit.lineage.graph") + ancestry = importlib.import_module("recursive_integrity_toolkit.lineage.ancestry") + def denied(*args, **kwargs): + pytest.fail("plain audit dispatched graph work") + monkeypatch.setattr(graph, "build_lineage_graph", denied) + monkeypatch.setattr(ancestry, "analyze_lineage", denied) + code, report, _, _ = _invoke(args, tmp_path / "out", capsys) + assert code == 0 + _assert_primary_metrics(report) + assert report["capabilities"]["lineage"]["execution_status"] == "not_requested" + assert report["observed_facts"]["lineage"]["cycle_status"]["value"] is None + + +def test_explicit_comparison_can_supply_ancestors_without_context_entering_pair(tmp_path, capsys): + args, primary, contexts, provenance, config = _inputs(tmp_path) + code, report, _, _ = _invoke(args + ["--lineage", "--compare", str(contexts[1]), + "--state-semantics", "literal shared topics"] + _context_args(contexts[:1]), tmp_path / "out", capsys) + assert code == 0 + assert set(report["derived_metrics"]["support"]["by_version"]) == {"v2", "v3"} + retention = report["derived_metrics"]["support"]["support_retention_ratio"] + assert retention["value"] == 0.0 and retention["scope"]["dataset_versions"] == ["v2", "v3"] + assert report["capabilities"]["dataset_longitudinal"]["execution_status"] == "partial" + assert report["capabilities"]["dataset_longitudinal"]["execution_scope"] == ["supplied_explicit_pair_support_and_diversity"] + assert report["capabilities"]["lineage"]["execution_status"] == "completed" + assert report["observed_facts"]["provenance"]["analyzed_record_count"]["value"] == 2 + assert report["observed_facts"]["lineage"]["graph_scope"]["value"]["target_record_count"] == 2 + assert _bounds(report, "direct") == (1.0, 1.0, 0.0) + assert _bounds(report, "lineage") == (0.0, 0.0, 0.0) + + +@pytest.mark.parametrize("configured", [False, True]) +def test_context_requires_explicit_audit_opt_in(tmp_path, capsys, configured): + args, primary, contexts, provenance, config = _inputs(tmp_path) + if configured: + raw = json.loads(config.read_text()); raw["inputs"] = {"lineage_context": [str(path) for path in contexts]} + config.write_text(json.dumps(raw)) + else: + args += _context_args(contexts) + code, report, _, streams = _invoke(args, tmp_path / "out", capsys) + assert code == 2 + assert "E_CONFIG_INVALID" in streams.err + assert report is None or report["derived_metrics"] == {} + + +def test_config_and_cli_context_paths_keep_their_own_relative_bases(tmp_path, capsys, monkeypatch): + directory = tmp_path / "settings" + args, primary, contexts, provenance, config = _inputs(directory) + raw = json.loads(config.read_text()) + raw.update(lineage=True, inputs={"lineage_context": contexts[0].name}) + config.write_text(json.dumps(raw)) + cli_context = tmp_path / "cli-context.jsonl" + cli_context.write_bytes(contexts[1].read_bytes()) + monkeypatch.chdir(tmp_path) + code, report, _, _ = _invoke(args + ["--lineage-records", cli_context.name], tmp_path / "out", capsys) + assert code == 0 + _assert_primary_metrics(report) + assert report["observed_facts"]["lineage"]["graph_scope"]["value"]["loaded_record_count"] == 4 + + +@pytest.mark.parametrize("case", ["primary_multiversion", "primary_overlap", "comparison_overlap", "duplicate_context", "empty_primary"]) +def test_invalid_role_scope_cannot_produce_completed_lineage(tmp_path, capsys, case): + args, primary, contexts, provenance, config = _inputs(tmp_path) + extra = ["--lineage"] + _context_args(contexts) + if case == "primary_multiversion": + rows = [json.loads(line) for line in primary.read_text().splitlines()] + rows[0]["dataset_version"] = "v4" + _write(primary, rows) + manifests = [json.loads(line) for line in provenance.read_text().splitlines()] + manifests[2]["dataset_version"] = "v4" + _write(provenance, manifests) + raw = json.loads(config.read_text()); raw["version_order"].append("v4") + config.write_text(json.dumps(raw)) + elif case == "primary_overlap": + _write(contexts[0], [{"dataset_version": "v3", "record_id": "separate", "content": "separate", "topic": "C"}]) + elif case == "comparison_overlap": + comparison = tmp_path / "compare.jsonl" + _write(comparison, [{"dataset_version": "v2", "record_id": "separate", "content": "separate", "topic": "C"}]) + extra += ["--compare", str(comparison), "--state-semantics", "literal shared topics"] + elif case == "duplicate_context": + extra += _context_args(contexts[:1]) + else: + primary.write_bytes(b"") + code, report, _, _ = _invoke(args + extra, tmp_path / "out", capsys) + assert code != 0 and report is not None and report["errors"] + assert report["capabilities"].get("lineage", {}).get("execution_status") != "completed" + if case == "empty_primary": + assert report["observed_facts"] == report["derived_metrics"] == {} + + +def test_validate_context_remains_input_only(tmp_path, capsys, monkeypatch): + from recursive_integrity_toolkit import cli + args, primary, contexts, provenance, config = _inputs(tmp_path) + names = ("metrics.diversity", "metrics.provenance", "metrics.bounds", "metrics.duplicates", "metrics.tail", + "metrics.resampling", "representations.base", "representations.field", "representations.content_hash", + "representations.compatibility", "lineage.graph", "lineage.cycles", "lineage.ancestry") + modules = [importlib.import_module("recursive_integrity_toolkit." + name) for name in names] + touched = [] + def denied(*args, **kwargs): + touched.append(True) + raise AssertionError("validate dispatched calculation") + for module in modules: + for name, value in vars(module).copy().items(): + if inspect.isfunction(value) and value.__module__ == module.__name__: + monkeypatch.setattr(module, name, denied) + monkeypatch.setattr(cli, "_calculations", denied) + code, report, _, _ = _invoke(args + _context_args(contexts), tmp_path / "out", capsys, command="validate") + assert code == 0 and touched == [] + assert report["derived_metrics"] == report["proxy_signals"] == report["simulations"] == {} + assert report["capabilities"]["lineage"]["execution_status"] == "not_requested" + assert set(report["observed_facts"]["record_counts"]) == {"v1", "v2", "v3"} + + +@pytest.mark.parametrize("configured", [False, True]) +def test_validate_rejects_lineage_execution_request(tmp_path, capsys, configured): + args, primary, contexts, provenance, config = _inputs(tmp_path) + if configured: + raw = json.loads(config.read_text()); raw["lineage"] = True + config.write_text(json.dumps(raw)) + else: + args += ["--lineage"] + code, report, _, streams = _invoke(args + _context_args(contexts), tmp_path / "out", capsys, command="validate") + assert code == 2 and "E_CONFIG_INVALID" in streams.err + assert report is None or report["derived_metrics"] == {} + + +@pytest.mark.parametrize("case,expected_exit,execution", [ + ("cycle", 3, "partial"), ("missing", 0, "partial"), + ("nodes", 1, "failed"), ("edges", 1, "failed"), ("roots", 1, "failed"), ("visits", 1, "failed"), +]) +def test_partial_and_resource_failures_preserve_independent_primary_values(tmp_path, capsys, case, expected_exit, execution): + args, primary, contexts, provenance, config = _inputs(tmp_path) + rows = [json.loads(line) for line in provenance.read_text().splitlines()] + if case == "cycle": + rows[2]["parent_ids"] = ["v3::t1"] + elif case == "missing": + rows[2]["parent_ids"] = ["v1::missing"] + else: + raw = json.loads(config.read_text()) + limit, value = {"nodes": ("max_lineage_nodes", 3), "edges": ("max_lineage_edges", 1), + "roots": ("max_lineage_root_memberships", 2), + "visits": ("max_lineage_root_union_visits", 1)}[case] + raw["resource_limits"] = {limit: value} + config.write_text(json.dumps(raw)) + _write(provenance, rows) + code, report, _, _ = _invoke(args + ["--lineage"] + _context_args(contexts), tmp_path / "out", capsys) + assert code == expected_exit + _assert_primary_metrics(report) + assert report["capabilities"]["lineage"]["execution_status"] == execution + expected_run = "complete" if case == "missing" else "failed" if case in ("nodes", "edges") else "partial" + assert report["run"]["run_status"] == expected_run + if case in ("cycle", "missing"): + assert report["derived_metrics"]["lineage"]["unresolved_record_count"]["value"] == 1 + assert _bounds(report, "lineage") == (0.0, 0.5, 0.5) + else: + if case in ("nodes", "edges"): + assert "lineage" not in report["derived_metrics"]["closure_exposure"] + else: + assert _bounds(report, "lineage") == (None, None, None) + assert any(row["code"] == "E_LINEAGE_RESOURCE_LIMIT_EXCEEDED" for row in report["errors"]) + facts = report["observed_facts"]["lineage"] + usage = facts["resource_usage"]["value"] + exhausted, admitted_field, admitted, attempted = { + "nodes": ("max_nodes", "admitted_node_count", 3, 4), + "edges": ("max_edges", "admitted_edge_count", 1, 2), + "roots": ("max_root_memberships", "stored_root_membership_count", 2, 3), + "visits": ("max_root_union_visits", "root_union_visit_count", 1, 2), + }[case] + assert usage["exhausted_limit"] == exhausted and usage["attempted_value"] == attempted + assert usage["limits"][exhausted] == admitted and usage[admitted_field] == admitted + if case in ("roots", "visits"): + assert facts["cycle_status"]["value"] == "acyclic" + assert facts["depth_summary"]["value"]["maximum_resolved_target_depth"] == 1 + else: + assert facts["cycle_status"]["value"] is None + assert "graph_scope" not in facts + + +def test_conflicting_context_chronology_preserves_primary_metrics_without_lineage_claim(tmp_path, capsys): + args, primary, contexts, provenance, config = _inputs(tmp_path) + raw = json.loads(config.read_text()); del raw["version_order"] + config.write_text(json.dumps(raw)) + order = tmp_path / "conflicting-order.json" + order.write_text(json.dumps({"version_order": ["v1", "v2", "v3"], + "version_rank": {"v1": 1, "v2": 0, "v3": 2}}), encoding="utf-8") + code, report, _, _ = _invoke(args + ["--lineage", "--version-order", str(order)] + + _context_args(contexts), tmp_path / "out", capsys) + assert code != 0 and report["errors"] + assert any(item["code"] == "E_VERSION_ORDER_CONFLICT" for item in report["errors"]) + _assert_primary_metrics(report) + assert report["capabilities"]["lineage"]["execution_status"] == "failed" + assert report["observed_facts"]["lineage"]["cycle_status"]["value"] is None + assert "comparison_details" not in report["derived_metrics"]["support"] + assert any(item["role"] == "version_order" for item in report["inputs"]["artifacts"]) + + +@pytest.mark.parametrize("record_ids", ["hash", "omit"]) +def test_lineage_context_redaction_protects_every_sink_and_never_fetches_sources(tmp_path, capsys, monkeypatch, record_ids): + directory = tmp_path / "PRIVATE_PATH" + args, primary, contexts, provenance, config = _inputs(directory) + for path in (primary, *contexts, provenance): + rows = [json.loads(line) for line in path.read_text().splitlines()] + for row in rows: + row["dataset_version"] = "PRIVATE_VERSION_" + row["dataset_version"] + row["record_id"] = "PRIVATE_ID_" + row["record_id"] + if "parent_ids" in row: + row["parent_ids"] = ["PRIVATE_VERSION_" + ref.replace("::", "::PRIVATE_ID_") for ref in row["parent_ids"]] + row["source_uri"] = "https://invalid.example/PRIVATE_URI?token=secret" + _write(path, rows) + raw = json.loads(config.read_text()); raw["version_order"] = ["PRIVATE_VERSION_" + item for item in raw["version_order"]] + config.write_text(json.dumps(raw)) + touched = [] + def denied(*args, **kwargs): + touched.append(True) + raise AssertionError("network attempted") + monkeypatch.setattr(socket, "create_connection", denied) + monkeypatch.setattr(socket.socket, "connect", denied) + code, report, markdown, streams = _invoke(args + ["--lineage", "--redacted", "--record-ids", record_ids] + + _context_args(contexts), directory / "PRIVATE_OUTPUT", capsys) + assert code == 0 and touched == [] + rendered = json.dumps(report) + markdown + streams.out + streams.err + assert not any(value in rendered for value in ("PRIVATE_PATH", "PRIVATE_VERSION", "PRIVATE_ID", "PRIVATE_URI", "PRIVATE_OUTPUT")) + roots = report["derived_metrics"]["lineage"]["top_shared_ancestors"]["value"] + assert roots["total_count"] == 2 + if record_ids == "omit": + assert roots["items"] is None and roots["detail_status"] == "omitted" + else: + assert roots["returned_count"] == 2 + assert all(row["record_key"]["record_id"].startswith("hmac-sha256:") for row in roots["items"]) + + +def test_invalid_context_input_cannot_be_overwritten_by_failure_report(tmp_path, capsys): + args, primary, contexts, provenance, config = _inputs(tmp_path) + out = tmp_path / "out" + out.mkdir() + collision = out / "report.json" + collision.write_text("{PRIVATE_MALFORMED_CONTEXT", encoding="utf-8") + before = collision.read_bytes() + from recursive_integrity_toolkit.cli import main + assert main(["audit", *args, "--lineage", "--lineage-records", str(collision), "--out", str(out)]) != 0 + streams = capsys.readouterr() + assert collision.read_bytes() == before and not (out / "report.md").exists() + assert "PRIVATE_MALFORMED_CONTEXT" not in streams.out + streams.err + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_literal_hero_has_exact_requested_lineage_and_plain_default(tmp_path, capsys, enabled): + code, report, _, streams = _invoke(["--lineage"] if enabled else [], tmp_path / "hero", capsys, command="example") + assert code == 0 + assert report["derived_metrics"]["support"]["by_version"]["v1"]["support_size"]["value"] == 8 + assert report["derived_metrics"]["support"]["by_version"]["v2"]["support_size"]["value"] == 5 + assert _bounds(report, "direct") == (0.5, 0.5, 0.0) + assert report["capabilities"]["lineage"]["execution_status"] == ("completed" if enabled else "not_requested") + if enabled: + lineage = report["derived_metrics"]["lineage"] + assert lineage["distinct_external_root_count"]["value"] == 5 + assert lineage["ancestry_concentration_hhi"]["value"] == 0.25 + assert lineage["effective_external_root_count"]["value"] == 4.0 + assert lineage["top_shared_ancestors"]["value"]["items"][0]["incidence_share"] == 3 / 8 + assert _bounds(report, "lineage") == (0.0, 0.0, 0.0) + assert report["proxy_signals"]["shared_ancestry_dependence"]["level"] == "present" + assert "lineage execution is deferred" not in streams.err diff --git a/tests/integration/test_phase5_lineage_privacy.py b/tests/integration/test_phase5_lineage_privacy.py new file mode 100644 index 0000000..22a8ebc --- /dev/null +++ b/tests/integration/test_phase5_lineage_privacy.py @@ -0,0 +1,301 @@ +"""Lineage identities cross the existing privacy boundary without data loss. + +Small real inputs exercise roots, cycle witnesses, unresolved rows and public +scope labels. One 202-record case reaches the existing 100-row presentation cap. +""" + +from copy import deepcopy +import json + +import pytest +from jsonschema import Draft202012Validator + +from recursive_integrity_toolkit.io.validation import validate_bundle +from recursive_integrity_toolkit.lineage.ancestry import analyze_lineage, shared_ancestry_dependence +from recursive_integrity_toolkit.metrics.bounds import lineage_closure_exposure +from recursive_integrity_toolkit.models import AuditBundle, FileRole, InputSource +from recursive_integrity_toolkit.reports.assembly import assemble_report, privacy_view +from recursive_integrity_toolkit.reports.json_report import render_json +from recursive_integrity_toolkit.reports.markdown_report import render_markdown +from recursive_integrity_toolkit.result import CanonicalReport +from recursive_integrity_toolkit.utils.hashing import IdentifierProtection + + +_VERSION_OLD = "VERSION_SENTINEL_old" +_VERSION_TARGET = "VERSION_SENTINEL_target" +_ROOT_ID = "RECORD_SENTINEL_root|`" +_SECRET = b"lineage-privacy-test-secret-only!!" +_RAW_PAYLOAD = ("CONTENT_SENTINEL", "NOTES_SENTINEL", "URI_SENTINEL") + + +def _run(): + nullable = ("started_at", "completed_at", "duration_seconds", "python_version", + "platform", "command", "config_hash", "random_seed") + return { + "run_id": "RUN_SENTINEL", "toolkit_version": "0.1.0.dev4", + "report_schema_version": "1.1", **dict.fromkeys(nullable), + "strict_mode": False, "redacted_mode": False, "network_call_count": 0, + "deterministic": True, "privacy_mode": "standard", "run_status": "complete", + "null_reasons": {key: "No execution metadata supplied." for key in nullable}, + } + + +def _report(tmp_path, *, capped=False): + tmp_path.mkdir(parents=True, exist_ok=True) + # A matching record_id in different versions is two distinct identities. + if capped: + declarations = { + **{(_VERSION_TARGET, f"RECORD_SENTINEL_root_{i:03}"): ([], "yes") for i in range(101)}, + **{(_VERSION_TARGET, f"RECORD_SENTINEL_cycle_{i:03}"): + ([f"{_VERSION_TARGET}::RECORD_SENTINEL_cycle_{i:03}"], "no") for i in range(101)}, + } + else: + declarations = { + (_VERSION_OLD, _ROOT_ID): ([], "yes"), + (_VERSION_OLD, "RECORD_SENTINEL_cycle_a"): + ([f"{_VERSION_OLD}::RECORD_SENTINEL_cycle_b"], "no"), + (_VERSION_OLD, "RECORD_SENTINEL_cycle_b"): + ([f"{_VERSION_OLD}::RECORD_SENTINEL_cycle_a"], "no"), + (_VERSION_TARGET, _ROOT_ID): ([], "yes"), + (_VERSION_TARGET, "RECORD_SENTINEL_child"): + ([f"{_VERSION_OLD}::{_ROOT_ID}"], "no"), + (_VERSION_TARGET, "RECORD_SENTINEL_child_two"): + ([f"{_VERSION_OLD}::{_ROOT_ID}"], "no"), + (_VERSION_TARGET, "RECORD_SENTINEL_closed"): ([], "no"), + (_VERSION_TARGET, "RECORD_SENTINEL_unknown"): ([], "unknown"), + (_VERSION_TARGET, "RECORD_SENTINEL_affected"): + ([f"{_VERSION_OLD}::RECORD_SENTINEL_cycle_a"], "no"), + } + records, provenance = [], [] + for (version, record_id), (parents, grounding) in declarations.items(): + identity = {"dataset_version": version, "record_id": record_id} + records.append({**identity, "content": "CONTENT_SENTINEL", + "notes": "NOTES_SENTINEL"}) + provenance.append({**identity, "source_type": "human" if grounding == "yes" else "synthetic", + "provenance_confidence": "confirmed", "external_grounding": grounding, + "parent_ids": parents, "transformation": "generate", + "source_uri": "https://invalid.example/URI_SENTINEL", + "notes": "NOTES_SENTINEL"}) + sources = [] + for name, role, rows in ( + ("primary", FileRole.RECORDS_PRIMARY, [row for row in records if row["dataset_version"] == _VERSION_TARGET]), + ("context", FileRole.RECORDS_COMPARE, [row for row in records if row["dataset_version"] != _VERSION_TARGET]), + ("provenance", FileRole.PROVENANCE_MANIFEST, provenance), + ): + if rows: + path = tmp_path / ("PATH_SENTINEL_" + name + ".jsonl") + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + sources.append(InputSource(role, path)) + versions = [_VERSION_TARGET] if capped else [_VERSION_OLD, _VERSION_TARGET] + bundle = validate_bundle(AuditBundle(tuple(sources)), configuration={"version_order": versions}) + lineage = analyze_lineage(bundle, target_dataset_version=_VERSION_TARGET) + return assemble_report(bundle, run=_run(), lineage=lineage, + lineage_bounds=lineage_closure_exposure(lineage), + shared_ancestry=shared_ancestry_dependence(lineage)) + + +def _view(report, mode="redacted", record_id_mode="hash", *, secret=_SECRET): + return privacy_view(report, mode=mode, record_id_mode=record_id_mode, + protection=IdentifierProtection.create(secret=secret)) + + +def _validate(view, repo_root): + value = view.to_dict() + schema = json.loads((repo_root / "schemas/report.schema.json").read_text(encoding="utf-8")) + Draft202012Validator(schema).validate(value) + CanonicalReport.from_dict(value) + assert json.loads(render_json(view)) == value + return value + + +def _details(value): + observed = value["observed_facts"]["lineage"] + cycle = observed["cycle_analysis"]["value"] + return { + **{name: cycle[name] for name in ("components", "cycle_member_record_keys", "affected_record_keys")}, + "unresolved_record_details": observed["unresolved_record_details"]["value"], + "top_shared_ancestors": value["derived_metrics"]["lineage"]["top_shared_ancestors"]["value"], + } + + +def _identities(value, path=()): + if isinstance(value, dict): + if set(value) == {"dataset_version", "record_id"}: + yield path, value + else: + for name, item in value.items(): + yield from _identities(item, (*path, name)) + elif isinstance(value, list): + for index, item in enumerate(value): + yield from _identities(item, (*path, index)) + + +def _lineage_identity_evidence(value): + return { + "observations": value["observed_facts"]["lineage"], + "metrics": value["derived_metrics"]["lineage"], + "bounds": value["derived_metrics"]["closure_exposure"]["lineage"], + "proxy": value["proxy_signals"]["shared_ancestry_dependence"], + "errors": value["errors"], "warnings": value["warnings"], + } + + +def _quantitative_evidence(value): + """Exclude presentation identities while comparing exact analytical values.""" + observed = value["observed_facts"]["lineage"] + cycle = observed["cycle_analysis"]["value"] + metrics = value["derived_metrics"]["lineage"] + graph_scope = observed["graph_scope"]["value"] + return { + "run_status": value["run"]["run_status"], + "capability": value["capabilities"]["lineage"], + "cycle_counts": {name: item for name, item in cycle.items() if not isinstance(item, dict)}, + "graph_counts": {name: item for name, item in graph_scope.items() if type(item) is int}, + "depth": observed["depth_summary"]["value"], + "resources": observed["resource_usage"]["value"], + "metrics": {name: {field: item for field, item in envelope.items() + if field in {"value", "status", "coverage", "denominator", "evidence_class", "unit"}} + for name, envelope in metrics.items() if name != "top_shared_ancestors"}, + "closure": {name: {field: item for field, item in envelope.items() + if field in {"value", "status", "coverage", "denominator", "evidence_class", "unit"}} + for name, envelope in value["derived_metrics"]["closure_exposure"]["lineage"].items()}, + "proxy": {name: item for name, item in value["proxy_signals"]["shared_ancestry_dependence"].items() + if name in {"status", "level", "coverage", "denominator", "evidence_class"}}, + "diagnostics": [(name, row["severity"], row.get("effect_on_run"), row["effect_on_capabilities"]) + for name in ("errors", "warnings") for row in value[name]], + } + + +@pytest.mark.parametrize("mode,record_id_mode", [ + ("standard", "preserve"), ("redacted", "hash"), + ("redacted", "omit"), ("redacted", "preserve"), +]) +def test_lineage_privacy_modes_preserve_analytical_values_and_safe_rendering( + tmp_path, repo_root, mode, record_id_mode): + report = _report(tmp_path) + original = deepcopy(report.to_dict()) + view = _view(report, mode, record_id_mode) + actual = _validate(view, repo_root) + assert _quantitative_evidence(actual) == _quantitative_evidence(original) + assert report.to_dict() == original + for rendered in (render_json(view), render_markdown(view)): + assert all(sentinel not in rendered for sentinel in _RAW_PAYLOAD) + assert ("VERSION_SENTINEL" in rendered) is (mode == "standard") + assert ("PATH_SENTINEL" in rendered) is (mode == "standard") + assert ("RECORD_SENTINEL" in rendered) is (record_id_mode == "preserve") + markdown = render_markdown(view) + assert "