Phase 5: complete lineage and ancestry dev4 milestone

.github/workflows/ci.yml

@@ -1,15 +1,31 @@
-name: Phase 4 Step 11 CI
+name: Current verification
 
 on:
   pull_request:
+    types: [opened, synchronize, reopened, labeled]
     paths-ignore:
       - README.md
       - CHANGELOG.md
-      - "docs/**"
+      - docs/architecture.md
+      - docs/cli.md
+      - docs/data_schema.md
+      - docs/privacy.md
+      - docs/release_process.md
+      - docs/report_schema.md
+      - docs/theory_traceability.md
       - PHASE_4_COMPLETION.md
       - PHASE_4_VALIDATION_REPORT.md
       - PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md
+      - PHASE_5_STEP_*.md
+      - PHASE_5_COMPLETION.md
   workflow_dispatch:
+    inputs:
+      gate:
+        description: Focused development checks or complete candidate verification
+        required: true
+        default: focused
+        type: choice
+        options: [focused, candidate]
 
 permissions:
   contents: read
@@ -19,10 +35,37 @@ defaults:
     shell: bash
 
 jobs:
+  focused:
+    if: (github.event_name != 'workflow_dispatch' || inputs.gate != 'candidate') && github.event.action != 'labeled'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    env:
+      PYTHONDONTWRITEBYTECODE: "1"
+      RIT_TEST_PARQUET: "0"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: python -m pip install ".[test]"
+      - name: Check current lineage, reports and affected boundaries
+        run: |
+          set -euo pipefail
+          python scripts/check_spec_consistency.py
+          python scripts/check_traceability.py
+          python scripts/release_check.py
+          python -m pytest -p no:cacheprovider -q tests/unit/test_PR009_generation.py tests/unit/test_PR008_parent_resolution.py tests/unit/test_PR007_version_order.py tests/unit/test_parent_batch.py tests/unit/test_PR001_identity.py tests/unit/test_PR002_loaders.py tests/unit/test_PR003_schema_mapping.py tests/unit/test_PR004_coverage.py tests/unit/test_PR010_observability.py tests/unit/test_lineage_context_inputs.py tests/unit/test_lineage_graph.py tests/unit/test_T6_cycles.py tests/unit/test_T4_ancestry.py tests/unit/test_lineage_root_resources.py tests/unit/test_lineage_concentration.py tests/unit/test_lineage_bounds_proxy.py tests/unit/test_lineage_fixture_inputs.py tests/unit/test_phase3_contracts.py tests/unit/test_PR016_determinism.py
+          python -m pytest -p no:cacheprovider -q tests/integration/test_phase3_metric_pipeline.py tests/integration/test_phase5_lineage_cli.py tests/integration/test_phase5_lineage_reports.py tests/integration/test_phase5_lineage_privacy.py tests/integration/test_phase4_cli.py tests/integration/test_hero_end_to_end.py tests/integration/test_partial_lineage_report.py tests/golden/test_phase4_reports.py tests/unit/test_PR013_report_schema.py tests/unit/test_PR015_redaction.py tests/integration/test_no_network.py tests/integration/test_current_verification.py tests/integration/test_ci_workflows.py tests/integration/test_no_algorithms.py tests/integration/test_repository_structure.py
+
   core:
+    if: (github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate') || (github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == 'verification:candidate')
     name: Core (${{ matrix.dependencies }}) on ${{ matrix.os }} with Python ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 105
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
@@ -66,32 +109,32 @@ jobs:
           fi
       - name: Verify actual optional dependency absence
         run: python -c "import importlib.util; assert importlib.util.find_spec('pyarrow') is None"
-      - name: Run the complete core suite
+      - name: Run complete canonical regression
         run: |
           set -euo pipefail
           mkdir -p "$RUNNER_TEMP/rit-core"
-          python -m pytest -p no:cacheprovider -q --junitxml="$RUNNER_TEMP/rit-core/core.xml" --maxfail=1 | tee "$RUNNER_TEMP/rit-core/phase4_test_results.log"
-          python scripts/release_check.py --phase 4 --step 11 --junit "$RUNNER_TEMP/rit-core/core.xml" --minimum-tests 3768
-      - name: Check current authorized scope and canonical suite
+          python -m pytest -p no:cacheprovider -q --ignore=tests/performance --junitxml="$RUNNER_TEMP/rit-core/core.xml" --maxfail=1 | tee "$RUNNER_TEMP/rit-core/tests.log"
+          python scripts/release_check.py --junit "$RUNNER_TEMP/rit-core/core.xml"
+      - name: Check current source, specifications and traceability
         run: |
           set -euo pipefail
-          python scripts/check_spec_consistency.py --phase 4 --step 11
-          python scripts/check_traceability.py --phase 4 --step 11
-          python scripts/release_check.py --phase 4 --step 11 --diff
-          python scripts/release_check.py --phase 4 --step 11 --baseline-evidence "$RUNNER_TEMP/rit-core"
+          python scripts/check_spec_consistency.py
+          python scripts/check_traceability.py
+          python scripts/release_check.py
       - name: Retain core evidence without masking failure
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: phase4-current-core-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.dependencies }}
+          name: current-core-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.dependencies }}
           path: ${{ runner.temp }}/rit-core/
           if-no-files-found: error
           retention-days: 30
 
   parquet:
+    if: (github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate') || (github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == 'verification:candidate')
     name: Real optional Parquet on Ubuntu with Python 3.12
     runs-on: ubuntu-latest
-    timeout-minutes: 75
+    timeout-minutes: 30
     env:
       PYTHONDONTWRITEBYTECODE: "1"
       RIT_TEST_PARQUET: "1"
@@ -108,7 +151,7 @@ jobs:
           python-version: "3.12"
       - name: Install approved real Parquet extra
         run: python -m pip install ".[test,parquet]"
-      - name: Run the complete extra-enabled suite
+      - name: Run complete canonical regression with Parquet
         run: |
           set -euo pipefail
           mkdir -p "$RUNNER_TEMP/rit-parquet"
@@ -117,19 +160,60 @@ jobs:
           python -c "import pyarrow; print('PyArrow', pyarrow.__version__)"
           git rev-parse HEAD > "$RUNNER_TEMP/rit-parquet/source-commit.txt"
           free --bytes | tee "$RUNNER_TEMP/rit-parquet/host-memory-bytes.txt"
-          python -m pytest -p no:cacheprovider -q --junitxml="$RUNNER_TEMP/rit-parquet/parquet.xml" --maxfail=1 | tee "$RUNNER_TEMP/rit-parquet/phase4_parquet_results.log"
-          python scripts/release_check.py --phase 4 --step 11 --junit "$RUNNER_TEMP/rit-parquet/parquet.xml" --require-parquet --minimum-tests 3771
-          python scripts/release_check.py --phase 4 --step 11 --baseline-evidence "$RUNNER_TEMP/rit-parquet"
+          python -m pytest -p no:cacheprovider -q --ignore=tests/performance --junitxml="$RUNNER_TEMP/rit-parquet/parquet.xml" --maxfail=1 | tee "$RUNNER_TEMP/rit-parquet/tests.log"
+          python scripts/release_check.py --junit "$RUNNER_TEMP/rit-parquet/parquet.xml" --require-parquet
       - name: Retain Parquet evidence without masking failure
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: phase4-current-real-parquet
+          name: current-real-parquet
           path: ${{ runner.temp }}/rit-parquet/
           if-no-files-found: error
           retention-days: 30
 
+  performance:
+    if: (github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate') || (github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == 'verification:candidate')
+    runs-on: ubuntu-latest
+    timeout-minutes: 75
+    env:
+      PYTHONDONTWRITEBYTECODE: "1"
+      RIT_TEST_PARQUET: "0"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: python -m pip install ".[test]"
+      - name: Measure reference profile including required 100k workload
+        run: |
+          set -euo pipefail
+          mkdir -p "$RUNNER_TEMP/rit-performance"
+          python -m pip freeze > "$RUNNER_TEMP/rit-performance/dependencies.txt"
+          python -c "import platform; print(platform.platform(), platform.python_version())"
+          free --bytes
+          python -m pytest -p no:cacheprovider -q tests/performance --junitxml="$RUNNER_TEMP/rit-performance/performance.xml" | tee "$RUNNER_TEMP/rit-performance/tests.log"
+          python scripts/release_check.py --junit "$RUNNER_TEMP/rit-performance/performance.xml"
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: current-reference-performance
+          path: ${{ runner.temp }}/rit-performance/
+          if-no-files-found: error
+          retention-days: 30
+
+  hero:
+    if: (github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate') || (github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == 'verification:candidate')
+    uses: ./.github/workflows/golden.yml
+
+  security:
+    if: (github.event_name == 'workflow_dispatch' && inputs.gate == 'candidate') || (github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == 'verification:candidate')
+    uses: ./.github/workflows/security.yml
+
   delivery:
     name: Build verified candidate
-    needs: [core, parquet]
+    needs: [core, parquet, performance, hero, security]
     uses: ./.github/workflows/release.yml

.github/workflows/golden.yml

@@ -1,14 +1,7 @@
-name: Phase 4 Step 11 Hero and Mathematical Contract
+name: Hero and mathematical contract
 
 on:
-  pull_request:
-    paths-ignore:
-      - README.md
-      - CHANGELOG.md
-      - "docs/**"
-      - PHASE_4_COMPLETION.md
-      - PHASE_4_VALIDATION_REPORT.md
-      - PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md
+  workflow_call:
   workflow_dispatch:
 
 permissions:
@@ -41,16 +34,15 @@ jobs:
         run: |
           set -euo pipefail
           mkdir -p "$RUNNER_TEMP/rit-math"
-          python scripts/check_spec_consistency.py --phase 4 --step 11
-          python -m pytest -p no:cacheprovider -q tests/integration/test_hero_structure.py tests/integration/test_hero_end_to_end.py tests/integration/test_phase3_metric_pipeline.py tests/golden/test_phase3_math.py tests/golden/test_phase4_reports.py --junitxml="$RUNNER_TEMP/rit-math/math.xml" | tee "$RUNNER_TEMP/rit-math/phase4_math_validation.log"
-          python scripts/release_check.py --phase 4 --step 11 --golden-junit "$RUNNER_TEMP/rit-math/math.xml"
-          python scripts/release_check.py --phase 4 --step 11 --diff
+          python scripts/check_spec_consistency.py
+          python -m pytest -p no:cacheprovider -q tests/integration/test_hero_structure.py tests/integration/test_hero_end_to_end.py tests/integration/test_phase3_metric_pipeline.py tests/golden/test_phase3_math.py tests/golden/test_phase4_reports.py tests/unit/test_T4_ancestry.py tests/unit/test_lineage_concentration.py tests/unit/test_lineage_bounds_proxy.py tests/integration/test_phase5_lineage_reports.py --junitxml="$RUNNER_TEMP/rit-math/math.xml" | tee "$RUNNER_TEMP/rit-math/tests.log"
+          python scripts/release_check.py --junit "$RUNNER_TEMP/rit-math/math.xml"
+          python scripts/release_check.py
       - name: Retain actual Hero and mathematical evidence
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: phase4-current-hero-mathematical
+          name: current-hero-mathematical
           path: ${{ runner.temp }}/rit-math/
           if-no-files-found: error
           retention-days: 30
-# Frozen mathematical expectations are unchanged; Step 11 Hero reports use independent assertions.

.github/workflows/release.yml

@@ -1,4 +1,4 @@
-name: Phase 4 Step 11 Build and Candidate Evidence
+name: Package and candidate verification
 
 on:
   workflow_call:
@@ -33,37 +33,37 @@ jobs:
       - name: Reuse complete core results from this candidate run
         uses: actions/download-artifact@v4
         with:
-          name: phase4-current-core-ubuntu-latest-3.12-current
-          path: ${{ runner.temp }}/rit-phase4-current/core-evidence/
+          name: current-core-ubuntu-latest-3.12-current
+          path: ${{ runner.temp }}/rit-candidate/core-evidence/
       - name: Reuse complete real-Parquet results from this candidate run
         uses: actions/download-artifact@v4
         with:
-          name: phase4-current-real-parquet
-          path: ${{ runner.temp }}/rit-phase4-current/parquet-evidence/
+          name: current-real-parquet
+          path: ${{ runner.temp }}/rit-candidate/parquet-evidence/
       - name: Install approved build and test tools
         run: python -m pip install ".[release,test,parquet]" "setuptools>=82"
       - name: Check authority, declarations and current-step scope
         run: |
           set -euo pipefail
-          evidence="$RUNNER_TEMP/rit-phase4-current"
+          evidence="$RUNNER_TEMP/rit-candidate"
           test "$(cat "$evidence/core-evidence/source-commit.txt")" = "$(git rev-parse HEAD)"
           test "$(cat "$evidence/parquet-evidence/source-commit.txt")" = "$(git rev-parse HEAD)"
           cp "$evidence/core-evidence/core.xml" "$evidence/core.xml"
           cp "$evidence/parquet-evidence/parquet.xml" "$evidence/parquet.xml"
           python -m pip freeze > "$evidence/build-dependencies.txt"
           python -m pip check
           python -c "import pyarrow; print('PyArrow', pyarrow.__version__)"
-          python scripts/check_spec_consistency.py --phase 4 --step 11
-          python scripts/check_traceability.py --phase 4 --step 11
-          python scripts/release_check.py --phase 4 --step 11 --diff
+          python scripts/check_spec_consistency.py
+          python scripts/check_traceability.py
+          python scripts/release_check.py
           rit version
-      - name: Build and repeat the dev3 artifacts from the same committed source
+      - name: Build and repeat the development artifacts from the same committed source
         run: |
           set -euo pipefail
-          evidence="$RUNNER_TEMP/rit-phase4-current"
+          evidence="$RUNNER_TEMP/rit-candidate"
           export SOURCE_DATE_EPOCH="$(git show -s --format=%ct HEAD)"
           printf '%s\n' "$SOURCE_DATE_EPOCH" > "$evidence/source-date-epoch.txt"
-          python -m build --no-isolation --outdir "$evidence/dist" | tee "$evidence/phase4_build_results.log"
+          python -m build --no-isolation --outdir "$evidence/dist" | tee "$evidence/build.log"
           python -m twine check --strict "$evidence/dist/"*
           mkdir -p "$evidence/rebuild-source"
           git archive HEAD | tar -x -C "$evidence/rebuild-source"
@@ -79,7 +79,7 @@ jobs:
           from pathlib import Path
           import tarfile
 
-          evidence = Path(os.environ["RUNNER_TEMP"]) / "rit-phase4-current"
+          evidence = Path(os.environ["RUNNER_TEMP"]) / "rit-candidate"
           first = {p.name: p for p in (evidence / "dist").iterdir() if p.is_file()}
           second = {p.name: p for p in (evidence / "rebuild-dist").iterdir() if p.is_file()}
           assert first.keys() == second.keys() and len(first) == 2
@@ -109,17 +109,17 @@ jobs:
           (evidence / "reproducibility.json").write_text(json.dumps(result, indent=2) + "\n")
           print(json.dumps(result, indent=2))
           PY
-      - name: Verify installed packages and produce the byte-verified Step 11 candidate
+      - name: Verify installed packages and produce the byte-verified current candidate
         run: |
           set -euo pipefail
-          python scripts/release_check.py --phase 4 --step 11 --candidate "$RUNNER_TEMP/rit-phase4-current" | tee "$RUNNER_TEMP/rit-phase4-current/installed-candidate.log"
+          python scripts/release_check.py --candidate "$RUNNER_TEMP/rit-candidate" | tee "$RUNNER_TEMP/rit-candidate/installed-candidate.log"
       - name: Retain candidate or failure evidence without masking failure
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: recursive-integrity-toolkit-phase4-current-candidate
+          name: recursive-integrity-toolkit-current-candidate
           path: |
-            ${{ runner.temp }}/rit-phase4-current/
-            !${{ runner.temp }}/rit-phase4-current/rebuild-source/
+            ${{ runner.temp }}/rit-candidate/
+            !${{ runner.temp }}/rit-candidate/rebuild-source/
           if-no-files-found: error
           retention-days: 90

.github/workflows/security.yml

@@ -1,14 +1,7 @@
-name: Phase 4 Step 11 Security Boundary
+name: Security boundaries
 
 on:
-  pull_request:
-    paths-ignore:
-      - README.md
-      - CHANGELOG.md
-      - "docs/**"
-      - PHASE_4_COMPLETION.md
-      - PHASE_4_VALIDATION_REPORT.md
-      - PHASE_4_ARCHITECTURE_COMPLIANCE_REPORT.md
+  workflow_call:
   workflow_dispatch:
 
 permissions:
@@ -52,6 +45,7 @@ jobs:
             tests/unit/test_PR017_content_refs.py \
             tests/integration/test_hero_end_to_end.py tests/golden/test_phase4_reports.py \
             tests/integration/test_partial_lineage_report.py \
+            tests/integration/test_phase5_lineage_privacy.py tests/unit/test_lineage_root_resources.py \
             tests/unit/test_T1_representation.py \
             tests/unit/test_PR006_duplicates.py \
             tests/unit/test_T1_support.py \
@@ -73,17 +67,15 @@ jobs:
             tests/integration/test_partial_provenance_report.py \
             tests/unit/test_phase4_output_safety.py \
             tests/integration/test_phase4_cli.py \
-            tests/unit/test_phase4_contracts.py \
-            tests/integration/test_phase4_gates.py \
-            --junitxml="$RUNNER_TEMP/rit-security/security.xml" | tee "$RUNNER_TEMP/rit-security/phase4_current_security_results.log"
-          python scripts/release_check.py --phase 4 --step 11 --junit "$RUNNER_TEMP/rit-security/security.xml"
-          python scripts/check_traceability.py --phase 4 --step 11
-          python scripts/release_check.py --phase 4 --step 11
+            --junitxml="$RUNNER_TEMP/rit-security/security.xml" | tee "$RUNNER_TEMP/rit-security/tests.log"
+          python scripts/release_check.py --junit "$RUNNER_TEMP/rit-security/security.xml"
+          python scripts/check_traceability.py
+          python scripts/release_check.py
       - name: Retain actual security test evidence
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: phase4-current-security-boundary
+          name: current-security-boundary
           path: ${{ runner.temp }}/rit-security/
           if-no-files-found: error
           retention-days: 30