diff --git a/.github/actions/build-cli/action.yml b/.github/actions/build-cli/action.yml index e8790bd152..ec0ac245fe 100644 --- a/.github/actions/build-cli/action.yml +++ b/.github/actions/build-cli/action.yml @@ -37,6 +37,7 @@ runs: id: bazel if: inputs.prepared-build != 'true' with: + remote-download-outputs: toplevel cache-key: ${{ inputs.cache-key }} tailscale-client-id: ${{ inputs.tailscale-client-id }} diff --git a/.github/actions/setup-bazel/action.yml b/.github/actions/setup-bazel/action.yml index c40e430704..8b9d5c4b60 100644 --- a/.github/actions/setup-bazel/action.yml +++ b/.github/actions/setup-bazel/action.yml @@ -1,7 +1,8 @@ name: Set up Bazel description: > Installs Bazel through the pinned bazelisk with its repository cache, and a - disk cache named by `cache-key` that only master saves automatically: PRs + disk cache for hosted-only execution named by `cache-key`. Only master saves + automatically: PRs restore master's entry; explicit native Windows dispatches may also prime their branch's cache. Jobs with the same configuration and runner image share a restore prefix; master callers save fresh @@ -18,6 +19,9 @@ inputs: prefix misses. Use when two jobs share rustc flags and runner image so Clippy can reuse Test Linux action outputs. default: "" + remote-download-outputs: + description: Remote outputs to materialize; packaging jobs require toplevel. + default: "minimal" cache-save: description: Whether this setup may save Bazelisk and repository caches. default: "true" @@ -42,6 +46,53 @@ outputs: runs: using: composite steps: + - name: Free runner disk space + if: runner.os == 'Linux' && runner.environment == 'github-hosted' + shell: bash + run: | + set -euo pipefail + df -h / + status=0 + sudo rm -rf /usr/local/lib/android & + p1=$! + sudo rm -rf /usr/share/dotnet & + p2=$! + sudo rm -rf /opt/ghc & + p3=$! + sudo rm -rf /opt/hostedtoolcache/CodeQL & + p4=$! + for pid in "$p1" "$p2" "$p3" "$p4"; do + wait "$pid" || status=1 + done + df -h / + exit "$status" + + - name: Remove inactive Xcode installations + if: runner.os == 'macOS' && runner.environment == 'github-hosted' + shell: bash + run: | + df -h / + python3 - <<'PYTHON' + import pathlib + import subprocess + + applications = pathlib.Path("/Applications") + selected = pathlib.Path(subprocess.check_output(["xcode-select", "-p"], text=True).strip()).resolve(strict=True) + active = next((path for path in selected.parents if path.suffix == ".app"), None) + if active is None: + raise SystemExit(f"Selected developer directory is not an Xcode application: {selected}") + print(f"Selected Xcode before cleanup: {active}", flush=True) + inactive = [path for path in applications.glob("Xcode*.app") if path.resolve() != active] + for path in inactive: + print(f"Removing inactive Xcode: {path}", flush=True) + subprocess.run(["sudo", "rm", "-rf", "--", str(path)], check=True) + after = pathlib.Path(subprocess.check_output(["xcode-select", "-p"], text=True).strip()).resolve(strict=True) + if after != selected: + raise SystemExit(f"Selected developer directory changed: {selected} -> {after}") + print(f"Selected Xcode after cleanup: {active}", flush=True) + PYTHON + df -h / + - name: Verify Windows build runner configuration if: runner.os == 'Windows' && runner.environment == 'github-hosted' shell: pwsh @@ -104,18 +155,6 @@ runs: ); JS - - uses: bazel-contrib/setup-bazel@104e3eea9447cdaa4b00d0fb46191a3985d26ab3 # 0.19.1 - with: - bazelisk-cache: true - repository-cache: true - disk-cache: false - bazelrc: >- - build --disk_cache=${{ steps.disk-cache-config.outputs.path }} - cache-save: ${{ github.ref == 'refs/heads/master' && inputs.cache-save == 'true' }} - # Windows build-script runfiles nest each vendored crate's sources - # under its repository name twice; keep the output base short. - output-base: ${{ steps.disk-cache-config.outputs.output-base }} - - name: Preserve the runner network environment for Tailscale if: runner.os == 'Linux' && inputs.tailscale-client-id != '' shell: bash @@ -153,6 +192,7 @@ runs: shell: bash env: TAILSCALE_OUTCOME: ${{ steps.tailscale.outcome }} + REMOTE_DOWNLOAD_OUTPUTS: ${{ inputs.remote-download-outputs }} run: | if [[ "$TAILSCALE_OUTCOME" == success ]]; then # Resolve the peer through the local API without changing runner DNS. @@ -168,6 +208,18 @@ runs: fi python3 "$GITHUB_ACTION_PATH/configure_fleet.py" + - uses: bazel-contrib/setup-bazel@104e3eea9447cdaa4b00d0fb46191a3985d26ab3 # 0.19.1 + with: + bazelisk-cache: true + repository-cache: true + disk-cache: false + bazelrc: >- + build --disk_cache=${{ steps.fleet.outputs.enabled != 'true' && steps.disk-cache-config.outputs.path || '' }} + cache-save: ${{ github.ref == 'refs/heads/master' && inputs.cache-save == 'true' }} + # Windows build-script runfiles nest each vendored crate's sources + # under its repository name twice; keep the output base short. + output-base: ${{ steps.disk-cache-config.outputs.output-base }} + - name: Map the shared Windows compiler path if: steps.fleet.outputs.enabled == 'true' && runner.os == 'Windows' shell: pwsh @@ -207,3 +259,8 @@ runs: path: ${{ steps.disk-cache-config.outputs.path }} key: ${{ steps.disk-cache-config.outputs.prefix }}run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} restore-keys: ${{ steps.disk-cache-config.outputs.restore-keys }} + + - name: Report available build storage + continue-on-error: true + shell: bash + run: bash "$GITHUB_ACTION_PATH/report_disk.sh" diff --git a/.github/actions/setup-bazel/configure_fleet.py b/.github/actions/setup-bazel/configure_fleet.py index 101f9b825c..87e71930d7 100644 --- a/.github/actions/setup-bazel/configure_fleet.py +++ b/.github/actions/setup-bazel/configure_fleet.py @@ -16,7 +16,10 @@ def configure_fleet( runner_arch: str, host: str = "ubuntu-main", port: int = 50051, + remote_download_outputs: str = "minimal", ) -> bool: + if remote_download_outputs not in {"minimal", "toplevel", "all"}: + raise ValueError(f"Invalid remote output mode: {remote_download_outputs}") profile = { ("Linux", "X64"): "rbe-linux", ("macOS", "ARM64"): "rbe-macos", @@ -77,7 +80,7 @@ def configure_fleet( build --remote_timeout=600 # Survive brief coordinator restarts; five retries expire in about 3s. build --remote_retries=10 -build --remote_download_outputs=toplevel +build --remote_download_outputs={remote_download_outputs} build --jobs={16 if windows else 40} # Leave room for Bazel analysis and downloads while large Rust actions compile. build --local_resources=cpu=2 @@ -108,4 +111,5 @@ def configure_fleet( os.environ["RUNNER_OS"], os.environ["RUNNER_ARCH"], host=os.environ.get("BAZEL_FLEET_HOST", "ubuntu-main"), + remote_download_outputs=os.environ.get("REMOTE_DOWNLOAD_OUTPUTS", "minimal"), ) diff --git a/.github/actions/setup-bazel/report_disk.sh b/.github/actions/setup-bazel/report_disk.sh new file mode 100644 index 0000000000..3c30f98ad7 --- /dev/null +++ b/.github/actions/setup-bazel/report_disk.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo '::group::Build filesystem capacity' +df -h "$GITHUB_WORKSPACE" "$HOME" "$RUNNER_TEMP" +df -i "$GITHUB_WORKSPACE" +echo '::endgroup::' +echo '::group::Build outputs, caches, and temporary files' +for path in "$GITHUB_WORKSPACE/bazel-out/" "$HOME/.cache" "$RUNNER_TEMP"; do + if [[ -d "$path" ]]; then + du -sh "$path" + fi +done +echo '::endgroup::' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89502441d4..09352a4958 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,6 +161,13 @@ jobs: if-no-files-found: error retention-days: 1 + - name: Report final build disk usage + if: ${{ always() && hashFiles('.github/actions/setup-bazel/report_disk.sh') != '' }} + continue-on-error: true + timeout-minutes: 2 + shell: bash + run: bash .github/actions/setup-bazel/report_disk.sh + benchmark-harness: name: Benchmark harness self-tests runs-on: ubuntu-latest @@ -324,6 +331,13 @@ jobs: - name: Check rusqlite stays inside storage owners run: python3 scripts/check-rusqlite-storage-boundary.py + - name: Report final build disk usage + if: ${{ always() && hashFiles('.github/actions/setup-bazel/report_disk.sh') != '' }} + continue-on-error: true + timeout-minutes: 2 + shell: bash + run: bash .github/actions/setup-bazel/report_disk.sh + # The macOS and Windows lanes run the same Bazel workspace the Linux # authority does, on demand: they exercise target-specific behavior the # Linux lane cannot prove. @@ -343,31 +357,6 @@ jobs: with: fetch-depth: 0 - - name: Remove inactive Xcode installations - shell: bash - run: | - df -h / - python3 - <<'PYTHON' - import pathlib - import subprocess - - applications = pathlib.Path("/Applications") - selected = pathlib.Path(subprocess.check_output(["xcode-select", "-p"], text=True).strip()).resolve(strict=True) - active = next((path for path in selected.parents if path.suffix == ".app"), None) - if active is None: - raise SystemExit(f"Selected developer directory is not an Xcode application: {selected}") - print(f"Selected Xcode before cleanup: {active}", flush=True) - inactive = [path for path in applications.glob("Xcode*.app") if path.resolve() != active] - for path in inactive: - print(f"Removing inactive Xcode: {path}", flush=True) - subprocess.run(["sudo", "rm", "-rf", "--", str(path)], check=True) - after = pathlib.Path(subprocess.check_output(["xcode-select", "-p"], text=True).strip()).resolve(strict=True) - if after != selected: - raise SystemExit(f"Selected developer directory changed: {selected} -> {after}") - print(f"Selected Xcode after cleanup: {active}", flush=True) - PYTHON - df -h / - - uses: ./.github/actions/setup-bazel id: bazel with: @@ -418,6 +407,13 @@ jobs: path: ${{ steps.bazel.outputs.disk-cache-path }} key: ${{ steps.bazel.outputs.disk-cache-key }} + - name: Report final build disk usage + if: ${{ always() && hashFiles('.github/actions/setup-bazel/report_disk.sh') != '' }} + continue-on-error: true + timeout-minutes: 2 + shell: bash + run: bash .github/actions/setup-bazel/report_disk.sh + # A trusted fleet run submits the whole workspace from one hosted runner. # Hosted-only runs retain their shards so local CPU and disk limits do not # serialize the entire workspace. Both routes feed the Test Linux check. @@ -461,29 +457,6 @@ jobs: || (github.actor_id == '158243242' && vars.TS_BAZEL_BOT_CLIENT_ID) || (github.actor_id == '29139614' && vars.TS_BAZEL_RENOVATE_CLIENT_ID)) || '' }} - # Fleet measurements leave about 46 GiB free without deleting SDKs. - # Keep cleanup for unmeasured hosted shards and fleet fallback, where - # local builds have previously exhausted the runner disk. - - name: Free runner disk space - if: matrix.shard != 'fleet' || steps.bazel.outputs.fleet-enabled != 'true' - run: | - set -euo pipefail - df -h / - status=0 - sudo rm -rf /usr/local/lib/android & - p1=$! - sudo rm -rf /usr/share/dotnet & - p2=$! - sudo rm -rf /opt/ghc & - p3=$! - sudo rm -rf /opt/hostedtoolcache/CodeQL & - p4=$! - for pid in "$p1" "$p2" "$p3" "$p4"; do - wait "$pid" || status=1 - done - df -h / - exit "$status" - - name: Select Linux test targets id: targets env: @@ -605,11 +578,6 @@ jobs: prebuild=false fi fi - if [[ $SHARD == fleet ]]; then - # Keep test binaries and intermediate outputs in the remote CAS. - # Bazel still downloads test logs and XML for failure diagnostics. - echo 'build --remote_download_outputs=minimal' >> .bazelrc.local - fi { echo "patterns=${targets[*]}" echo "prebuild=$prebuild" @@ -647,6 +615,13 @@ jobs: path: ${{ steps.bazel.outputs.disk-cache-path }} key: ${{ steps.bazel.outputs.disk-cache-key }} + - name: Report final build disk usage + if: ${{ always() && hashFiles('.github/actions/setup-bazel/report_disk.sh') != '' }} + continue-on-error: true + timeout-minutes: 2 + shell: bash + run: bash .github/actions/setup-bazel/report_disk.sh + # Preserves the "Test Linux" check name for branch protection. linux-test-result: name: Test Linux @@ -795,6 +770,13 @@ jobs: exit 1 fi + - name: Report final build disk usage + if: ${{ always() && hashFiles('.github/actions/setup-bazel/report_disk.sh') != '' }} + continue-on-error: true + timeout-minutes: 2 + shell: bash + run: bash .github/actions/setup-bazel/report_disk.sh + # Blocking Clippy policy over every Bazel Rust target, the # `cargo clippy --workspace --all-targets -D warnings` equivalent. clippy: @@ -838,6 +820,13 @@ jobs: path: ${{ steps.bazel.outputs.disk-cache-path }} key: ${{ steps.bazel.outputs.disk-cache-key }} + - name: Report final build disk usage + if: ${{ always() && hashFiles('.github/actions/setup-bazel/report_disk.sh') != '' }} + continue-on-error: true + timeout-minutes: 2 + shell: bash + run: bash .github/actions/setup-bazel/report_disk.sh + # Branch protection requires this check name. The generator feature policy # test (`//scripts:gen_builds_test`) runs inside Repository gates, whose # Bazel analysis it shares; on its own runner it paid 52 s of Bazel setup @@ -914,6 +903,13 @@ jobs: TRACEDECAY_BIN="$binary" bazel run //scripts:mcp-conformance-smoke.sh bazel run //scripts:packaged_lsp_bridge -- "$binary" "$smoke_work/lsp" + - name: Report final build disk usage + if: ${{ always() && hashFiles('.github/actions/setup-bazel/report_disk.sh') != '' }} + continue-on-error: true + timeout-minutes: 2 + shell: bash + run: bash .github/actions/setup-bazel/report_disk.sh + # Frontend checks require no Rust compilation and must also validate drafts. # Do not serialize them behind repository-wide Rust formatting and gates. dashboard: @@ -985,6 +981,13 @@ jobs: path: ${{ steps.bazel.outputs.disk-cache-path }} key: ${{ steps.bazel.outputs.disk-cache-key }} + - name: Report final build disk usage + if: ${{ always() && hashFiles('.github/actions/setup-bazel/report_disk.sh') != '' }} + continue-on-error: true + timeout-minutes: 2 + shell: bash + run: bash .github/actions/setup-bazel/report_disk.sh + hermes-integration: name: Hermes integration (stock) needs: debug-cli @@ -1081,4 +1084,3 @@ jobs: env: TRACEDECAY_BIN: ${{ github.workspace }}/target/debug/tracedecay run: scripts/opencode_stock_integration.sh - diff --git a/.github/workflows/hermetic-evals.yml b/.github/workflows/hermetic-evals.yml index 18f369725f..c505f2f873 100644 --- a/.github/workflows/hermetic-evals.yml +++ b/.github/workflows/hermetic-evals.yml @@ -56,3 +56,10 @@ jobs: --test_output=errors - name: Execute stock-edit oracle regressions run: python3 evals/agent_adoption/stock_check_selftest.py + + - name: Report final build disk usage + if: ${{ always() && hashFiles('.github/actions/setup-bazel/report_disk.sh') != '' }} + continue-on-error: true + timeout-minutes: 2 + shell: bash + run: bash .github/actions/setup-bazel/report_disk.sh diff --git a/.github/workflows/release-beta.yml b/.github/workflows/release-beta.yml index f79a818409..49799a15fc 100644 --- a/.github/workflows/release-beta.yml +++ b/.github/workflows/release-beta.yml @@ -163,6 +163,7 @@ jobs: id: bazel uses: ./.release-automation/.github/actions/setup-bazel with: + remote-download-outputs: toplevel cache-key: release-${{ matrix.target }} tailscale-client-id: >- ${{ (matrix.name == 'x86_64-linux' || matrix.name == 'aarch64-macos' || matrix.name == 'x86_64-windows') @@ -240,6 +241,13 @@ jobs: path: ${{ steps.bazel.outputs.disk-cache-path }} key: ${{ steps.bazel.outputs.disk-cache-key }} + - name: Report final build disk usage + if: ${{ always() && hashFiles('.release-automation/.github/actions/setup-bazel/report_disk.sh') != '' }} + continue-on-error: true + timeout-minutes: 2 + shell: bash + run: bash .release-automation/.github/actions/setup-bazel/report_disk.sh + publish: needs: [validate, build] # Publish only a whole release. beta.86 shipped without macOS because a diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 4f6d32be19..78b27f0b70 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -177,3 +177,10 @@ jobs: run: scripts/check-release-drift.sh env: GITHUB_TOKEN: ${{ secrets.RELEASE_PLZ_TOKEN }} + + - name: Report final build disk usage + if: ${{ always() && hashFiles('.github/actions/setup-bazel/report_disk.sh') != '' }} + continue-on-error: true + timeout-minutes: 2 + shell: bash + run: bash .github/actions/setup-bazel/report_disk.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9ec1c91de0..658e0700c0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -164,6 +164,7 @@ jobs: id: bazel uses: ./.release-automation/.github/actions/setup-bazel with: + remote-download-outputs: toplevel cache-key: release-${{ matrix.target }} tailscale-client-id: >- ${{ (matrix.name == 'x86_64-linux' || matrix.name == 'aarch64-macos' || matrix.name == 'x86_64-windows') @@ -261,6 +262,13 @@ jobs: path: ${{ steps.bazel.outputs.disk-cache-path }} key: ${{ steps.bazel.outputs.disk-cache-key }} + - name: Report final build disk usage + if: ${{ always() && hashFiles('.release-automation/.github/actions/setup-bazel/report_disk.sh') != '' }} + continue-on-error: true + timeout-minutes: 2 + shell: bash + run: bash .release-automation/.github/actions/setup-bazel/report_disk.sh + verify-release: name: Verify immutable GitHub release needs: [validate-release, build] @@ -765,6 +773,13 @@ jobs: if-no-files-found: error retention-days: 7 + - name: Report final build disk usage + if: ${{ always() && hashFiles('.github/actions/setup-bazel/report_disk.sh') != '' }} + continue-on-error: true + timeout-minutes: 2 + shell: bash + run: bash .github/actions/setup-bazel/report_disk.sh + publish-typescript: name: Publish @tracedecay/sdk to npm needs: [validate-release, build-typescript, verify-release] diff --git a/.github/workflows/renovate-embedded-bundles.yml b/.github/workflows/renovate-embedded-bundles.yml index eee374115f..b455108e5f 100644 --- a/.github/workflows/renovate-embedded-bundles.yml +++ b/.github/workflows/renovate-embedded-bundles.yml @@ -63,6 +63,13 @@ jobs: if-no-files-found: error retention-days: 1 + - name: Report final build disk usage + if: ${{ always() && hashFiles('.github/actions/setup-bazel/report_disk.sh') != '' }} + continue-on-error: true + timeout-minutes: 2 + shell: bash + run: bash .github/actions/setup-bazel/report_disk.sh + push: name: Push regenerated bundles needs: build diff --git a/.github/workflows/sdk-conformance.yml b/.github/workflows/sdk-conformance.yml index 84e60bd7ec..f574316c9b 100644 --- a/.github/workflows/sdk-conformance.yml +++ b/.github/workflows/sdk-conformance.yml @@ -110,6 +110,13 @@ jobs: path: ${{ steps.bazel.outputs.disk-cache-path }} key: ${{ steps.bazel.outputs.disk-cache-key }} + - name: Report final build disk usage + if: ${{ always() && hashFiles('.github/actions/setup-bazel/report_disk.sh') != '' }} + continue-on-error: true + timeout-minutes: 2 + shell: bash + run: bash .github/actions/setup-bazel/report_disk.sh + production-router: permissions: contents: read @@ -143,3 +150,10 @@ jobs: --test_arg=--include-ignored - run: pnpm run test:installed working-directory: sdks/typescript + + - name: Report final build disk usage + if: ${{ always() && hashFiles('.github/actions/setup-bazel/report_disk.sh') != '' }} + continue-on-error: true + timeout-minutes: 2 + shell: bash + run: bash .github/actions/setup-bazel/report_disk.sh diff --git a/.github/workflows/upstream-compat.yml b/.github/workflows/upstream-compat.yml index 5a3d485cec..d960e74e07 100644 --- a/.github/workflows/upstream-compat.yml +++ b/.github/workflows/upstream-compat.yml @@ -47,3 +47,10 @@ jobs: --incompatible_no_implicit_file_export \ --rewind_lost_inputs \ //crates/... + + - name: Report final build disk usage + if: ${{ always() && hashFiles('.github/actions/setup-bazel/report_disk.sh') != '' }} + continue-on-error: true + timeout-minutes: 2 + shell: bash + run: bash .github/actions/setup-bazel/report_disk.sh diff --git a/Cargo.lock b/Cargo.lock index 12682e8a5f..f4394e5755 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2885,7 +2885,7 @@ dependencies = [ [[package]] name = "grafeo-adapters" version = "0.5.43" -source = "git+https://github.com/ScriptedAlchemy/grafeo.git?rev=be4713a5c2ee9df7e4639671c1e1244b7680f103#be4713a5c2ee9df7e4639671c1e1244b7680f103" +source = "git+https://github.com/ScriptedAlchemy/grafeo.git?branch=main#5116be3c7b9392006ef4d229634963e1306a7b6b" dependencies = [ "bincode", "grafeo-common", @@ -2900,7 +2900,7 @@ dependencies = [ [[package]] name = "grafeo-common" version = "0.5.43" -source = "git+https://github.com/ScriptedAlchemy/grafeo.git?rev=be4713a5c2ee9df7e4639671c1e1244b7680f103#be4713a5c2ee9df7e4639671c1e1244b7680f103" +source = "git+https://github.com/ScriptedAlchemy/grafeo.git?branch=main#5116be3c7b9392006ef4d229634963e1306a7b6b" dependencies = [ "arcstr", "bincode", @@ -2920,7 +2920,7 @@ dependencies = [ [[package]] name = "grafeo-core" version = "0.5.43" -source = "git+https://github.com/ScriptedAlchemy/grafeo.git?rev=be4713a5c2ee9df7e4639671c1e1244b7680f103#be4713a5c2ee9df7e4639671c1e1244b7680f103" +source = "git+https://github.com/ScriptedAlchemy/grafeo.git?branch=main#5116be3c7b9392006ef4d229634963e1306a7b6b" dependencies = [ "arc-swap", "arcstr", @@ -2943,7 +2943,7 @@ dependencies = [ [[package]] name = "grafeo-engine" version = "0.5.43" -source = "git+https://github.com/ScriptedAlchemy/grafeo.git?rev=be4713a5c2ee9df7e4639671c1e1244b7680f103#be4713a5c2ee9df7e4639671c1e1244b7680f103" +source = "git+https://github.com/ScriptedAlchemy/grafeo.git?branch=main#5116be3c7b9392006ef4d229634963e1306a7b6b" dependencies = [ "arcstr", "bincode", @@ -2965,7 +2965,7 @@ dependencies = [ [[package]] name = "grafeo-storage" version = "0.5.43" -source = "git+https://github.com/ScriptedAlchemy/grafeo.git?rev=be4713a5c2ee9df7e4639671c1e1244b7680f103#be4713a5c2ee9df7e4639671c1e1244b7680f103" +source = "git+https://github.com/ScriptedAlchemy/grafeo.git?branch=main#5116be3c7b9392006ef4d229634963e1306a7b6b" dependencies = [ "bincode", "byteorder", diff --git a/Cargo.toml b/Cargo.toml index f181a8429d..8d448c3b59 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,14 +81,16 @@ asupersync = { git = "https://github.com/Dicklesworthstone/asupersync", rev = "e # and in-place, page-verified sealed sections that sealed generation stores # are built and reopened with. Its `memory_usage()` also counts the catalog, # empty property-index shards and plan-cache tables, which the serving graph -# heap charge reads. The fork's workspace resolves sibling crates -# by path, so all five crates must name the same `rev`; mixed revisions build -# type-incompatible instances. -grafeo-adapters = { git = "https://github.com/ScriptedAlchemy/grafeo.git", rev = "be4713a5c2ee9df7e4639671c1e1244b7680f103", default-features = false, features = ["algos"] } -grafeo-common = { git = "https://github.com/ScriptedAlchemy/grafeo.git", rev = "be4713a5c2ee9df7e4639671c1e1244b7680f103", default-features = false } -grafeo-core = { git = "https://github.com/ScriptedAlchemy/grafeo.git", rev = "be4713a5c2ee9df7e4639671c1e1244b7680f103", default-features = false, features = ["lpg"] } -grafeo-engine = { git = "https://github.com/ScriptedAlchemy/grafeo.git", rev = "be4713a5c2ee9df7e4639671c1e1244b7680f103", default-features = false, features = ["lpg", "gql", "algos", "wal", "grafeo-file"] } -grafeo-storage = { git = "https://github.com/ScriptedAlchemy/grafeo.git", rev = "be4713a5c2ee9df7e4639671c1e1244b7680f103", default-features = false, features = ["wal"] } +# heap charge reads, and reports native read allocation demand before +# materializing rows. These APIs live in the fork so every build uses the +# same implementation. The fork resolves sibling crates by path: keep all +# five dependencies on main together and lock their exact commit in Cargo.lock +# to avoid type-incompatible instances from mixed sources. +grafeo-adapters = { git = "https://github.com/ScriptedAlchemy/grafeo.git", branch = "main", default-features = false, features = ["algos"] } +grafeo-common = { git = "https://github.com/ScriptedAlchemy/grafeo.git", branch = "main", default-features = false } +grafeo-core = { git = "https://github.com/ScriptedAlchemy/grafeo.git", branch = "main", default-features = false, features = ["lpg"] } +grafeo-engine = { git = "https://github.com/ScriptedAlchemy/grafeo.git", branch = "main", default-features = false, features = ["lpg", "gql", "algos", "wal", "grafeo-file"] } +grafeo-storage = { git = "https://github.com/ScriptedAlchemy/grafeo.git", branch = "main", default-features = false, features = ["wal"] } # Pinned once: the MCP git handlers and the root's edit/lifecycle readers must # resolve the same object database and status implementation, so a feature # added for one is never missing for the other. diff --git a/MODULE.bazel b/MODULE.bazel index c4f38a74ae..4cb5ca79a8 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -103,13 +103,6 @@ crate.annotation( version = "0.46.0", ) -crate.annotation( - crate = "grafeo-core", - patch_args = ["-p1"], - patches = ["//scripts:bazel/patches/grafeo-core-read-allocation.patch"], - version = "0.5.43", -) - # arborium grammar build scripts read `env!("CARGO_MANIFEST_DIR")`, which bakes # the compiling output base's absolute path into the build-script binary. The # action key does not include that path, so the disk cache served one output diff --git a/crates/tracedecay-agent-hosts/src/agents/hermes/lifecycle.rs b/crates/tracedecay-agent-hosts/src/agents/hermes/lifecycle.rs index d83c5f1bd5..8d2c85a565 100644 --- a/crates/tracedecay-agent-hosts/src/agents/hermes/lifecycle.rs +++ b/crates/tracedecay-agent-hosts/src/agents/hermes/lifecycle.rs @@ -110,8 +110,8 @@ mod tests { "config should select tracedecay memory provider:\n{config}" ); assert!( - config.contains("engine: tracedecay"), - "config should select tracedecay context engine:\n{config}" + !config.contains("engine: tracedecay"), + "config must leave Hermes' native context engine selected:\n{config}" ); } @@ -144,7 +144,7 @@ mod tests { let named_config = text(&home.path().join(".hermes/profiles/work/config.yaml")); assert!(named_config.contains("- tracedecay")); assert!(named_config.contains("provider: tracedecay")); - assert!(named_config.contains("engine: tracedecay")); + assert!(!named_config.contains("engine: tracedecay")); assert_eq!( super::super::hermes_registration_state( home.path(), diff --git a/crates/tracedecay-agent-hosts/src/agents/hermes/profile_config.rs b/crates/tracedecay-agent-hosts/src/agents/hermes/profile_config.rs index 2b3b83ec72..f238136ae1 100644 --- a/crates/tracedecay-agent-hosts/src/agents/hermes/profile_config.rs +++ b/crates/tracedecay-agent-hosts/src/agents/hermes/profile_config.rs @@ -85,12 +85,12 @@ pub(super) fn registration_state( .and_then(|memory| string_value(&memory, "provider")) .as_deref() == Some("tracedecay"); - let context = root + let replaces_compressor = root .get_mapping("context") .and_then(|context| string_value(&context, "engine")) .as_deref() == Some("tracedecay"); - if enabled && memory && context { + if enabled && memory && !replaces_compressor { State::Current } else { State::Repairable @@ -145,7 +145,6 @@ pub(super) fn disable_plugin(config_path: &Path) -> Result<()> { // install/update surfaces stay stable. const PLUGINS_ERR: &str = "unsupported Hermes plugins config"; const MEMORY_ERR: &str = "unsupported Hermes memory config"; -const CONTEXT_ERR: &str = "unsupported Hermes context config"; // The mutation path is deliberately text/span oriented rather than CST oriented. // `yaml-edit` (as of 0.2.3) cannot reliably synthesize nested block or flow @@ -159,7 +158,7 @@ const CONTEXT_ERR: &str = "unsupported Hermes context config"; /// The containers enabling TraceDecay may add to a profile, as JSON pointers /// into the YAML document. -const OWNED_CONTAINERS: [&str; 4] = ["/plugins", "/plugins/enabled", "/memory", "/context"]; +const OWNED_CONTAINERS: [&str; 3] = ["/plugins", "/plugins/enabled", "/memory"]; /// The enabled config and the [`OWNED_CONTAINERS`] enabling added to it. fn enable_plugin_config( @@ -216,20 +215,13 @@ fn enable_normalized(existing: &str) -> std::result::Result { "memory", "provider", "tracedecay", - &[], MEMORY_ERR, "Hermes memory provider already configured; refusing to overwrite it", )?; - let text = ensure_scalar( - &text, - "context", - "engine", - "tracedecay", - &["compressor"], - CONTEXT_ERR, - "Hermes context engine already configured; refusing to overwrite it", - )?; - Ok(text) + // TraceDecay owns retrieval and memory, not Hermes' native compressor. + // Repair profiles activated by the former no-op engine while preserving + // every user-selected engine and the rest of the context configuration. + disable_scalar(&text, "context", "engine", false) } fn disable_normalized( @@ -394,7 +386,6 @@ fn ensure_scalar( container: &str, key: &str, value: &str, - allow_overwrite: &[&str], unsupported: &str, conflict: &str, ) -> std::result::Result { @@ -424,21 +415,6 @@ fn ensure_scalar( } match string_value(mapping, key).as_deref() { Some(current) if current == value => Ok(text.to_string()), - Some(current) if allow_overwrite.contains(¤t) => { - let entry = mapping - .find_entry_by_key(key) - .ok_or_else(|| unsupported.to_string())?; - let range = entry - .value_node() - .and_then(|node| node.as_scalar().map(yaml_edit::Scalar::byte_range)) - .ok_or_else(|| unsupported.to_string())?; - Ok(format!( - "{}{}{}", - &text[..range.start as usize], - value, - &text[range.end as usize..] - )) - } Some(_) => Err(conflict.to_string()), None => Err(unsupported.to_string()), } @@ -833,7 +809,7 @@ mod tests { .as_deref(), Some("tracedecay") ); - assert_eq!( + assert_ne!( root.get_mapping("context") .and_then(|context| string_value(&context, "engine")) .as_deref(), @@ -1004,11 +980,11 @@ mod tests { for (original, appended) in [ ( "theme: dark\nplugins:\n enabled:\n - foreign\n", - &["/memory", "/context"][..], + &["/memory"][..], ), ( "model: x\nmemory: {}\n", - &["/plugins", "/plugins/enabled", "/context"][..], + &["/plugins", "/plugins/enabled"][..], ), ] { let (enabled, created) = enable_plugin_config(original).unwrap(); @@ -1020,6 +996,48 @@ mod tests { } } + #[test] + fn activation_preserves_native_and_user_selected_context_engines() { + for context in [ + "", + "context: {engine: compressor, budget: 42}\n", + "context: {engine: custom-engine, budget: 42}\n", + ] { + let original = format!( + "plugins: {{enabled: [tracedecay]}}\nmemory: {{provider: tracedecay}}\n\ + {context}compression: {{enabled: false, threshold: 0.6}}\n" + ); + let (updated, created) = enable_plugin_config(&original).unwrap(); + assert_eq!(updated, original); + assert!(created.is_empty()); + } + } + + #[test] + fn activation_repairs_the_unavailable_context_engine_without_losing_settings() { + use crate::agents::host_bundle::HostBundleRegistrationStateV1 as State; + + let dir = TempDir::new().unwrap(); + let config = dir.path().join("config.yaml"); + let original = concat!( + "plugins: {enabled: [tracedecay]}\n", + "memory: {provider: tracedecay}\n", + "context: {engine: tracedecay, budget: 42}\n", + "compression: {enabled: true, threshold: 0.6}\n", + ); + std::fs::write(&config, original).unwrap(); + assert_eq!(registration_state(&config), State::Repairable); + + enable_plugin(&config).unwrap(); + + assert_eq!(registration_state(&config), State::Current); + let updated = read(&config); + assert_enabled(&updated); + assert!(updated.contains("budget: 42")); + assert!(updated.contains("compression: {enabled: true, threshold: 0.6}")); + assert_eq!(enable_plugin_config(&updated).unwrap().0, updated); + } + #[test] fn disable_empty_mapping_preserves_immediately_following_line() { let input = concat!( diff --git a/crates/tracedecay-agent-hosts/src/agents/hermes/templates.rs b/crates/tracedecay-agent-hosts/src/agents/hermes/templates.rs index aba852798e..b00e619a46 100644 --- a/crates/tracedecay-agent-hosts/src/agents/hermes/templates.rs +++ b/crates/tracedecay-agent-hosts/src/agents/hermes/templates.rs @@ -22,7 +22,7 @@ pub(super) fn plugin_manifest( kind: standalone\n\ version: {version}\n\ generator_commit: {commit}\n\ - description: TraceDecay code-graph tools, memory provider, and LCM context engine for Hermes\n\ + description: TraceDecay code-graph tools, memory provider, and LCM retrieval for Hermes\n\ author: tracedecay (generated by `tracedecay install --agent hermes`)\n\ provides_tools:\n{tools}\n\ provides_hooks:\n\ diff --git a/crates/tracedecay-agent-hosts/src/agents/hermes/templates/plugin_init.py b/crates/tracedecay-agent-hosts/src/agents/hermes/templates/plugin_init.py index 85d64c2ac4..087a312dd7 100644 --- a/crates/tracedecay-agent-hosts/src/agents/hermes/templates/plugin_init.py +++ b/crates/tracedecay-agent-hosts/src/agents/hermes/templates/plugin_init.py @@ -12,6 +12,7 @@ import threading import time from collections import deque +from functools import partial from pathlib import Path from . import schemas, tools @@ -38,12 +39,6 @@ class MemoryProvider: pass -try: - from agent.context_engine import ContextEngine -except Exception: - class ContextEngine: - pass - # Hermes' centralized auxiliary LLM facade is the MODULE-LEVEL # agent.auxiliary_client.call_llm(task=..., messages=..., ...). AIAgent # instances carry no ``auxiliary_client`` attribute and no host call site @@ -229,7 +224,8 @@ def _resolve_auxiliary_client(agent=None): "type": "string", "description": "Optional session id override (for example, expand a cross-session grep hit in its owning session).", }, - "max_tokens": {"type": "integer", "description": "Token budget for returned content.", "default": 4000}, + "max_tokens": {"type": "integer", "description": "Token budget for returned content.", + "default": 4000}, "source_limit": { "type": "integer", "minimum": 1, @@ -320,21 +316,10 @@ def _resolve_auxiliary_client(agent=None): LCM_TOOL_ALIASES = { schema["name"]: "tracedecay_" + schema["name"] for schema in LCM_NATIVE_SCHEMAS } -LCM_DIRECT_TOOL_NAMES = frozenset(LCM_TOOL_ALIASES.values()) LCM_DIRECT_TO_NATIVE = {tracedecay_name: native_name for native_name, tracedecay_name in LCM_TOOL_ALIASES.items()} -# Public LCM readers never depend on forwarding the host's live message list. -MESSAGE_DEPENDENT_TOOLS = frozenset() - STANDARD_HERMES_LCM_PROVIDER = "hermes" -LCM_PROVIDER_LOCAL_TOOL_NAMES = frozenset(( - "tracedecay_lcm_describe", - "tracedecay_lcm_doctor", - "tracedecay_lcm_expand", - "tracedecay_lcm_expand_query", -)) - # Direct duplicates of the memory provider's own tool surface # (fact_store / fact_feedback / memory_status). Skipped at register() time # when tracedecay is the active memory.provider so the same store is not @@ -349,8 +334,6 @@ def _resolve_auxiliary_client(agent=None): # first-turn guidance nudge so it never advertises tools that are not # actually registered. _REGISTERED_TOOL_NAMES = set() -_CONTEXT_TOOL_NAMES = set() -_HOST_FORWARDS_MESSAGES = None def _active_memory_provider(ctx=None): """The memory.provider configured for this profile, if any.""" @@ -371,19 +354,12 @@ def _active_memory_provider(ctx=None): pass return None -def _make_wrapped_lcm_handler(tool_name: str, engine): - def _wrapped(args: dict, **kwargs) -> str: - return engine.handle_tool_call(tool_name, args, **kwargs) - return _wrapped - -def _host_forwards_registered_tool_messages(ctx) -> bool: - capability = getattr(ctx, "context_engine_tool_handlers_receive_messages", False) - if callable(capability): - try: - capability = capability() - except Exception: - return False - return bool(capability) +def _make_wrapped_lcm_handler(tool_name: str, config=None, hermes_home=None): + return _make_project_safe_handler( + LCM_TOOL_ALIASES[tool_name], + partial(_handle_lcm_tool_call, tool_name, config=config), + hermes_home, + ) _NUDGE_CODE_REQUEST_RE = re.compile( r"\b(" @@ -1094,9 +1070,6 @@ def safe_handler(args, **kwargs): "nudge": (True, "Inject a first-turn tracedecay guidance nudge only for codebase/navigation requests."), "sync_turn": (True, "Mirror each completed turn into the LCM raw store."), "prefetch": (True, "Background fact recall injected at turn start."), - "context_threshold": ("", "Compression trigger as a fraction of the context window (default: hermes compression.threshold)."), - "threshold_tokens": ("", "Absolute compression trigger in tokens (overrides context_threshold)."), - "context_length": ("", "Context window override when the host does not report one."), "expansion_model": ("", "Model used for lcm_expand_query synthesis."), "expansion_context_tokens": ("", "Expanded-context budget for lcm_expand_query (default 32000)."), "expansion_timeout_ms": ("", "lcm_expand_query synthesis timeout in milliseconds."), @@ -1148,7 +1121,7 @@ def _with_plugin_block(config, hermes_home=None): """Layer the profile's plugins.tracedecay config block under a host config. Host-provided values always win; the block fills the gaps so profile - config.yaml settings reach the engine/provider without bespoke env vars. + config.yaml settings reach tools and the provider without bespoke env vars. """ block = { key: value @@ -1297,116 +1270,6 @@ def _lcm_int_setting(config, env_key, *names, default=None): pass return _configured_int(config, *names, default=default) -def _config_bool_disabled(value): - if isinstance(value, bool): - return value is False - if isinstance(value, (int, float)): - return value == 0 - if isinstance(value, str): - normalized = value.strip().lower() - if normalized in ("0", "false", "no", "off"): - return True - try: - return float(normalized) == 0 - except ValueError: - return False - return False - -def _hermes_yaml_compression_threshold(default, hermes_home=None): - # Port of hermes-lcm config._hermes_compression_threshold: read the main - # Hermes compression.threshold from the resolved host config when no LCM - # override exists. Disabled Hermes compression must not leak its threshold. - home = ( - hermes_home - or os.path.join(os.path.expanduser("~"), ".hermes") - ) - cfg_path = Path(home) / "config.yaml" - try: - text = cfg_path.read_text() - except Exception: - return default - try: - import yaml - except Exception: - yaml = None - try: - if yaml is not None: - cfg = yaml.safe_load(text) or {} - compression = cfg.get("compression") or {} - if _config_bool_disabled(compression.get("enabled")): - return default - value = compression.get("threshold") - if value is None: - return default - return float(value) - - in_compression = False - direct_indent = None - compression_disabled = False - threshold_value = None - for raw_line in text.splitlines(): - line = raw_line.split('#', 1)[0].rstrip() - if not line.strip(): - continue - if not line.startswith((" ", "\t")): - in_compression = line.strip() == "compression:" - direct_indent = None - continue - if not in_compression: - continue - indent = len(line) - len(line.lstrip(" \t")) - if direct_indent is None: - direct_indent = indent - if indent != direct_indent or ":" not in line: - continue - key, raw_value = line.strip().split(":", 1) - value = raw_value.strip().strip("'\"") - if key == "enabled" and _config_bool_disabled(value): - compression_disabled = True - elif key == "threshold": - threshold_value = value - if compression_disabled or threshold_value is None: - return default - return float(threshold_value) - except Exception: - return default - -def _lcm_context_threshold(config, hermes_home=None): - raw = os.environ.get("LCM_CONTEXT_THRESHOLD") - if raw is not None: - try: - return float(raw) - except (TypeError, ValueError): - pass - configured = _configured_value(config, "context_threshold") - if configured is not None: - try: - return float(configured) - except (TypeError, ValueError): - pass - return _hermes_yaml_compression_threshold(0.75, hermes_home=hermes_home) - -def _configured_threshold_tokens(config, hermes_home=None, context_length_override=None): - explicit = _configured_int(config, "threshold_tokens") - if explicit is not None: - return explicit - context_length = context_length_override - if context_length is None: - context_length = _configured_int( - config, - "context_length", - "max_context_tokens", - "model_context_tokens", - ) - if context_length is None: - return None - try: - return int(int(context_length) * float(_lcm_context_threshold(config, hermes_home=hermes_home))) - except (TypeError, ValueError): - return None - - - def _lcm_expansion_model(config): value = _lcm_str_setting(config, "LCM_EXPANSION_MODEL", "expansion_model", default="") return str(value or "").strip() @@ -1588,9 +1451,20 @@ def _synthesize_expand_query_payload(retrieval, agent=None, **kwargs): def _handle_lcm_expand_query(args, **kwargs) -> str: kwargs = dict(kwargs) agent = kwargs.pop("agent", None) + config = _with_plugin_block(kwargs.pop("config", None), kwargs.pop("hermes_home", None)) args = dict(args or {}) args.setdefault("provider", STANDARD_HERMES_LCM_PROVIDER) - retrieval = call_tracedecay_json("tracedecay_lcm_expand_query", args, **kwargs) + args.setdefault("context_max_tokens", _lcm_expansion_context_tokens(config)) + retrieval = call_tracedecay_json( + "tracedecay_lcm_expand_query", args, + **_project_call_kwargs(kwargs.get("project_root")), + ) + if kwargs.get("model") is None: + model = _lcm_expansion_model(config) + if model: + kwargs["model"] = model + if kwargs.get("timeout") is None and kwargs.get("expansion_timeout") is None: + kwargs["expansion_timeout"] = _lcm_expansion_timeout_ms(config) / 1000 payload = _synthesize_expand_query_payload(retrieval, agent=agent, **kwargs) return json.dumps(payload) @@ -1681,503 +1555,41 @@ def _translate_lcm_args(native_name: str, args: dict) -> dict: return translated return translated -_ENGINE_DEFAULT_SESSION = "__default__" - -class _EngineSessionState: - """Mutable per-conversation engine state. - - The host registers ONE engine instance and shares it across every - AIAgent in the process (gateway sessions, parallel delegate_task - children), so conversation-scoped fields must never live directly on - the engine. - """ - - _FIELDS = ( - "project_root", - "agent", - "model", - "last_prompt_tokens", - "last_real_prompt_tokens", - "last_completion_tokens", - "last_total_tokens", - "context_length", - "threshold_tokens", - "compression_count", - "last_compress_result", - "_last_compress_aborted", - "_last_summary_error", - "_runtime_context_length", - "_session_start_context_length", - ) - - def __init__(self): - self.project_root = None - self.agent = None - self.model = "" - self.last_prompt_tokens = 0 - self.last_real_prompt_tokens = 0 - self.last_completion_tokens = 0 - self.last_total_tokens = 0 - # Host-contract token state (run_agent.py reads these directly; the - # minimum-context guard in agent/agent_init.py checks context_length). - self.context_length = 0 - self.threshold_tokens = 0 - self.compression_count = 0 - # Compression abort/diagnostic state read by - # agent/conversation_compression.py after each compress() call. - self.last_compress_result = None - self._last_compress_aborted = False - self._last_summary_error = None - self._runtime_context_length = None - self._session_start_context_length = None - - def adopt(self, other): - """Carry conversation state across a compression-rotation rebind.""" - for field in self._FIELDS: - setattr(self, field, getattr(other, field)) - -def _engine_session_property(field): - """Engine attribute view onto the calling context's session state. - - Keeps the host contract intact: run_agent.py reads (and - conversation_compression.py occasionally writes) these as plain - attributes on the shared engine instance, and each access resolves to - the session bound to the calling thread. - """ - - def _get(self): - return getattr(self._state(), field) - - def _set(self, value): - setattr(self._state(), field, value) - - return property(_get, _set) - -class TraceDecayContextEngine(ContextEngine): - def __init__(self, config=None, hermes_home=None): - # Per-session mutable state, guarded by an RLock. Calls that carry - # no session id (compress, update_from_response, should_compress, - # ...) resolve the session bound to the calling thread, falling - # back to the most recently bound session. That keeps concurrent - # gateway sessions / delegate_task children from clobbering each - # other whenever the host gives us a binding signal, and at minimum - # serializes state access when it does not. - self._state_lock = threading.RLock() - self._session_states = {} - self._thread_sessions = threading.local() - self.active_session_id = None - self._host_config = config - self.hermes_home = _resolve_hermes_home(config, hermes_home) - self.config = _with_plugin_block(config, self.hermes_home) - self.project_root = _resolved_project_scope( - _configured_project_root(self.config), - self.hermes_home, - ) - - def __deepcopy__(self, memo): - """Create an agent-local engine without copying locks or live agents.""" - clone = type(self)( - config=copy.deepcopy(self._host_config, memo), - hermes_home=self.hermes_home, - ) - memo[id(self)] = clone - clone.config = copy.deepcopy(self.config, memo) - clone.project_root = self.project_root - with self._state_lock: - clone.active_session_id = self.active_session_id - for key, state in self._session_states.items(): - copied_state = _EngineSessionState() - for field in _EngineSessionState._FIELDS: - if field == "agent": - continue - setattr(copied_state, field, copy.deepcopy(getattr(state, field), memo)) - clone._session_states[key] = copied_state - return clone - - project_root = _engine_session_property("project_root") - agent = _engine_session_property("agent") - model = _engine_session_property("model") - last_prompt_tokens = _engine_session_property("last_prompt_tokens") - last_real_prompt_tokens = _engine_session_property("last_real_prompt_tokens") - last_completion_tokens = _engine_session_property("last_completion_tokens") - last_total_tokens = _engine_session_property("last_total_tokens") - context_length = _engine_session_property("context_length") - threshold_tokens = _engine_session_property("threshold_tokens") - compression_count = _engine_session_property("compression_count") - last_compress_result = _engine_session_property("last_compress_result") - _last_compress_aborted = _engine_session_property("_last_compress_aborted") - _last_summary_error = _engine_session_property("_last_summary_error") - _runtime_context_length = _engine_session_property("_runtime_context_length") - _session_start_context_length = _engine_session_property("_session_start_context_length") - - def _session_key(self, session_id=None): - if session_id: - return str(session_id) - bound = getattr(self._thread_sessions, "session_id", None) - if bound: - return bound - return self.active_session_id or _ENGINE_DEFAULT_SESSION - - def _state(self, session_id=None): - key = self._session_key(session_id) - with self._state_lock: - state = self._session_states.get(key) - if state is None: - state = _EngineSessionState() - self._session_states[key] = state - return state - - @property - def name(self) -> str: - return "tracedecay" - - def _bind_session(self, session_id=None, hermes_home=None, project_root=None, **kwargs): - if session_id is not None: - session_key = str(session_id) - with self._state_lock: - if session_key not in self._session_states: - state = _EngineSessionState() - # Compression rotation continues the same conversation: - # carry model/window/counters over from the predecessor - # session's state (old_session_id on boundary starts, - # else whatever this thread was bound to). - source_key = str(kwargs.get("old_session_id") or "") - source = self._session_states.get(source_key) if source_key else None - if source is None and kwargs.get("boundary_reason") == "compression": - source = self._session_states.get(self._session_key()) - if source is None: - source = self._session_states.get(_ENGINE_DEFAULT_SESSION) - if source is not None: - state.adopt(source) - self._session_states[session_key] = state - # Bind the calling thread so session-less calls (compress, - # update_from_response, should_compress) resolve this session. - self._thread_sessions.session_id = session_key - self.active_session_id = session_key - if kwargs.get("config") is not None: - self._host_config = kwargs.get("config") - if "context_length" in kwargs: - applied_context_length = self._apply_context_length(kwargs.get("context_length")) - if applied_context_length is not None: - self._session_start_context_length = applied_context_length - next_agent = kwargs.get("agent") - if next_agent is not None: - self.agent = next_agent - explicit_hermes_home = hermes_home or kwargs.get("hermes_home") - if explicit_hermes_home or kwargs.get("config") is not None or self.hermes_home is None: - next_hermes_home = _resolve_hermes_home( - kwargs.get("config", self._host_config), - explicit_hermes_home, - ) - if next_hermes_home: - self.hermes_home = next_hermes_home - # Re-layer the profile's plugins.tracedecay block now that the host - # config and hermes_home are settled for this session. - self.config = _with_plugin_block(self._host_config, self.hermes_home) - explicit_project_root = project_root or kwargs.get("project_root") - runtime_cwd = kwargs.get("cwd") - configured_project_root = _configured_project_root(self.config) - routing_supplied = bool(explicit_project_root or runtime_cwd) - if explicit_project_root: - route_state, next_project_root = _project_scope_resolution( - explicit_project_root, - self.hermes_home, - ) - if route_state == "unregistered": - next_project_root = _code_project_root( - explicit=explicit_project_root, - hermes_home=self.hermes_home, - ) - elif runtime_cwd: - next_project_root = _resolved_project_scope(runtime_cwd, self.hermes_home) - elif self.project_root: - route_state, next_project_root = _project_scope_resolution( - self.project_root, - self.hermes_home, - ) - if route_state == "unregistered": - next_project_root = _code_project_root( - explicit=self.project_root, - hermes_home=self.hermes_home, - ) - routing_supplied = routing_supplied or next_project_root is None - elif configured_project_root: - next_project_root = _resolved_project_scope( - configured_project_root, - self.hermes_home, - ) - routing_supplied = True - elif session_id is not None and not self.project_root: - next_project_root = _resolved_project_scope( - _runtime_working_directory(), - self.hermes_home, - ) - else: - next_project_root = None - if next_project_root: - self.project_root = next_project_root - elif routing_supplied: - self.project_root = None - - def initialize(self, session_id=None, hermes_home=None, project_root=None, **kwargs): - self._bind_session(session_id, hermes_home, project_root, **kwargs) - - def on_session_start(self, session_id=None, hermes_home=None, project_root=None, **kwargs): - self._bind_session(session_id, hermes_home, project_root, **kwargs) - - def on_session_end(self, session_id=None, messages=None, **kwargs): - _join_host_receipts() - # Real session boundary: drop the per-session record so long-lived - # gateway processes do not accumulate dead conversation state. - key = self._session_key(session_id) - with self._state_lock: - self._session_states.pop(key, None) - if getattr(self._thread_sessions, "session_id", None) == key: - self._thread_sessions.session_id = None - if self.active_session_id == key: - self.active_session_id = None - - def _apply_context_length(self, context_length): - """Track a context length on the host-contract attrs. - - run_agent.py logs ``context_length``/``threshold_tokens`` directly and - the minimum-context guard in agent/agent_init.py reads - ``context_length``, leaving them 0 logged a bogus 0-token window and - silently bypassed that guard. - """ - try: - parsed = int(context_length) - except (TypeError, ValueError): - return None - if parsed <= 0: - return None - self.context_length = parsed - threshold_tokens = _configured_threshold_tokens( - self.config, - hermes_home=self.hermes_home, - context_length_override=parsed, - ) - if threshold_tokens is None: - threshold_tokens = int( - parsed * _lcm_context_threshold(self.config, hermes_home=self.hermes_home) - ) - self.threshold_tokens = threshold_tokens - return parsed - - def update_model(self, model, context_length, base_url="", api_key="", provider="", api_mode=""): - self.model = str(model or "") - self._runtime_context_length = self._apply_context_length(context_length) - - def update_from_response(self, usage): - # Required by newer Hermes ContextEngine ABCs (abstract method). - # Tracks normalized token usage; run_agent.py reads these attrs. - usage = usage or {} - - def _as_int(value): - try: - return int(value) - except (TypeError, ValueError): - return 0 - - self.last_prompt_tokens = _as_int( - usage.get("prompt_tokens") or usage.get("input_tokens") - ) - self.last_real_prompt_tokens = self.last_prompt_tokens - self.last_completion_tokens = _as_int( - usage.get("completion_tokens") or usage.get("output_tokens") - ) - self.last_total_tokens = _as_int(usage.get("total_tokens")) or ( - self.last_prompt_tokens + self.last_completion_tokens - ) - - def _effective_context_length(self): - if self._runtime_context_length is not None: - return self._runtime_context_length - if self._session_start_context_length is not None: - return self._session_start_context_length - return None - - def _tool_args(self, session_id=None): - return { - "session_id": session_id if session_id is not None else self.active_session_id - } - - def should_compress_preflight(self, messages, current_tokens=None, **kwargs): - del messages, current_tokens, kwargs - # Hermes does not expose an authentic raw-compression protocol. Its - # transcript still ingests through the daemon, but compaction is a - # typed unavailable capability. - return False - - def _preflight_probe(self, messages, current_tokens=None, **kwargs): - del messages, current_tokens, kwargs - return { - "status": "unavailable", - "reason": "host_raw_compression_unavailable", - "should_compress": False, - } - - def should_compress(self, prompt_tokens=None, **kwargs): - del prompt_tokens, kwargs - return False - - def has_content_to_compress(self, messages, current_tokens=None, **kwargs): - del current_tokens, kwargs - non_empty = [ - message - for message in (messages or []) - if str((message or {}).get("content") or "").strip() - ] - return len(non_empty) >= 2 - - def should_defer_preflight_to_real_usage(self, rough_tokens=None): - del rough_tokens - return False - - def carry_over_new_session_context(self, old_session_id, new_session_id): - old_session_id = str(old_session_id or "") - new_session_id = str(new_session_id or "") - if not new_session_id: - return - self._bind_session(new_session_id, old_session_id=old_session_id) - - def status(self, session_id=None, **kwargs): - args = self._tool_args(session_id) - args["provider"] = STANDARD_HERMES_LCM_PROVIDER - args = _lcm_store_args(args, kwargs.get("project_root") or self.project_root) - return call_tracedecay_json( - "tracedecay_lcm_status", - args, - **_project_call_kwargs(kwargs.get("project_root") or self.project_root), - ) - - def get_tool_schemas(self): - return list(LCM_NATIVE_SCHEMAS) - - def get_status(self): - last_result = self.last_compress_result - if not isinstance(last_result, dict): - last_result = {"status": "never_ran"} - return { - "engine": self.name, - "session_id": self.active_session_id, - "active_session_id": self.active_session_id, - "project_root": self.project_root, - "tracedecay_binary_path": tools.TRACEDECAY_BIN, - "tracedecay_binary_available": _tracedecay_binary_available(), - "context_engine_tool_names": sorted( - schema["name"] for schema in self.get_tool_schemas() - ), - "last_compress_result": last_result, - "awaiting_real_usage_after_compression": False, - "live_ingest": { - "registered_tool_names": sorted(_REGISTERED_TOOL_NAMES), - "context_tool_names": sorted(_CONTEXT_TOOL_NAMES), - "host_forwards_messages": _HOST_FORWARDS_MESSAGES, - "message_dependent_tools_registered": bool( - MESSAGE_DEPENDENT_TOOLS.intersection(_REGISTERED_TOOL_NAMES) - ), - "gate_reason": ( - "not_registered" - if not _REGISTERED_TOOL_NAMES and not _CONTEXT_TOOL_NAMES - else ( - "host_does_not_forward_messages" - if _HOST_FORWARDS_MESSAGES is False - else "registered" - ) - ), - }, - } - - def handle_tool_call(self, name, arguments=None, **kwargs) -> str: - tool_name, tool_args = _normalize_memory_tool_call(name, arguments) - native_name = tool_name - tracedecay_name = LCM_TOOL_ALIASES.get(native_name) - if tracedecay_name is None and native_name in LCM_DIRECT_TOOL_NAMES: - tracedecay_name = native_name - native_name = LCM_DIRECT_TO_NATIVE.get(native_name, native_name) - if tracedecay_name is None: - return tools.error_payload(f"unknown LCM tool: {tool_name}") - - preflight_kwargs = dict(kwargs) - preflight_kwargs.pop("messages", None) - - tool_args = _translate_lcm_args(native_name, dict(tool_args)) - if tool_args.get("error"): - return json.dumps({"error": tool_args["error"]}) - if tracedecay_name in LCM_PROVIDER_LOCAL_TOOL_NAMES: - tool_args.setdefault("provider", STANDARD_HERMES_LCM_PROVIDER) - if self.active_session_id: - tool_args.setdefault("session_id", self.active_session_id) - tool_args = _lcm_store_args( - tool_args, - preflight_kwargs.get("project_root") or self.project_root, - ) - - if tracedecay_name == "tracedecay_lcm_expand_query": - expand_kwargs = dict(preflight_kwargs) - agent = expand_kwargs.pop("agent", None) or self.agent - return _handle_lcm_expand_query( - tool_args, - agent=agent, - **_project_call_kwargs( - expand_kwargs.get("project_root") or self.project_root - ), - ) - return tools.call_tracedecay_tool( - tracedecay_name, +def _handle_lcm_tool_call(name, arguments=None, config=None, **kwargs) -> str: + """Dispatch retrieval using the calling Hermes session, without owning compaction.""" + tool_name, tool_args = _normalize_memory_tool_call(name, arguments) + native_name = LCM_DIRECT_TO_NATIVE.get(tool_name, tool_name) + tracedecay_name = LCM_TOOL_ALIASES.get(native_name) + if tracedecay_name is None: + return tools.error_payload(f"unknown LCM tool: {tool_name}") + + tool_args = _translate_lcm_args(native_name, dict(tool_args)) + if tool_args.get("error"): + return json.dumps({"error": tool_args["error"]}) + tool_args.setdefault("provider", STANDARD_HERMES_LCM_PROVIDER) + if kwargs.get("session_id"): + tool_args.setdefault("session_id", kwargs["session_id"]) + if ( + native_name == "lcm_grep" + and tool_args.get("scope") == "current" + and not tool_args.get("session_id") + ): + return tools.error_payload("lcm_grep current scope requires a Hermes session_id") + project_root = kwargs.get("project_root") + tool_args = _lcm_store_args(tool_args, project_root) + if tracedecay_name == "tracedecay_lcm_expand_query": + return _handle_lcm_expand_query( tool_args, - **_project_call_kwargs( - preflight_kwargs.get("project_root") or self.project_root - ), - ) - - def expand_query(self, prompt, query=None, node_ids=None, **kwargs): - kwargs = dict(kwargs) - args = self._tool_args(kwargs.pop("session_id", None)) - args["provider"] = STANDARD_HERMES_LCM_PROVIDER - args["prompt"] = prompt - if query is not None: - args["query"] = query - if node_ids is not None: - args["node_ids"] = node_ids - for key in ("max_results", "max_tokens", "context_max_tokens"): - if key in kwargs and kwargs[key] is not None: - args[key] = kwargs[key] - if "context_max_tokens" not in args: - args["context_max_tokens"] = _lcm_expansion_context_tokens(self.config) - args = _lcm_store_args(args, kwargs.get("project_root") or self.project_root) - retrieval = call_tracedecay_json( - "tracedecay_lcm_expand_query", - args, - **_project_call_kwargs(kwargs.get("project_root") or self.project_root), + config=config, + hermes_home=kwargs.get("hermes_home"), + agent=kwargs.get("agent"), + **_project_call_kwargs(project_root), ) - synthesis_kwargs = dict(kwargs) - synthesis_agent = synthesis_kwargs.pop("agent", None) or self.agent - if synthesis_kwargs.get("model") is None: - expansion_model = _lcm_expansion_model(self.config) - if expansion_model: - synthesis_kwargs["model"] = expansion_model - if ( - synthesis_kwargs.get("timeout") is None - and synthesis_kwargs.get("expansion_timeout") is None - ): - synthesis_kwargs["expansion_timeout"] = _lcm_expansion_timeout_ms(self.config) / 1000 - return _synthesize_expand_query_payload(retrieval, agent=synthesis_agent, **synthesis_kwargs) - - def compress(self, messages, current_tokens=None, focus_topic=None, **kwargs): - """Return the unchanged transcript when host compaction is unavailable.""" - del current_tokens, focus_topic, kwargs - original = list(messages or []) - reason = "host_raw_compression_unavailable" - self.last_compress_result = { - "status": "unavailable", - "reason": reason, - "semantic_error": True, - } - self._last_compress_aborted = True - self._last_summary_error = reason - return original + # Retrieval never sends the host's live messages or other execution kwargs + # to the CLI. Completed turns still ingest through the memory provider. + return tools.call_tracedecay_tool( + tracedecay_name, tool_args, **_project_call_kwargs(project_root) + ) class TracedecayMemoryProvider(MemoryProvider): provider_id = "tracedecay" @@ -2585,7 +1997,6 @@ def handle_tool_call(self, name, arguments=None, **kwargs) -> str: ) def register(ctx): - global _HOST_FORWARDS_MESSAGES context_config = getattr(ctx, "config", None) explicit_context_home = ( getattr(ctx, "hermes_home", None) or getattr(ctx, "_hermes_home", None) @@ -2604,6 +2015,7 @@ def bound(*args, **kwargs): return bound ctx.register_hook("pre_llm_call", _pre_llm_call) + ctx.register_hook("on_session_end", lambda **_kwargs: _join_host_receipts()) try: ctx.register_hook("post_tool_call", bind_hermes_home(_post_tool_call)) except Exception as exc: @@ -2635,32 +2047,14 @@ def bound(*args, **kwargs): memory_provider.hermes_home = context_hermes_home ctx.register_memory_provider(memory_provider) - context_engine = TraceDecayContextEngine( - config=context_config, - hermes_home=context_hermes_home, - ) - ctx.register_context_engine(context_engine) - - # Direct tool registration is split by capability: - # - Code-graph / memory / transcript tools register UNCONDITIONALLY. - # They work without host message forwarding, and hermes defers - # registered non-core tools through its tool-search bridge - # (tools/tool_search.py), so schema footprint is not a blocker. - # - Only the live-ingest LCM verbs whose schemas take the in-memory - # ``messages`` list (MESSAGE_DEPENDENT_TOOLS) and the context-engine - # native tool mirrors stay gated behind the message-forwarding - # capability flag, without forwarding their ingest piggyback can - # never fire (the host still mounts the native LCM tools itself via - # context_engine.get_tool_schemas()). + # TraceDecay supplies retrieval and memory. Hermes keeps its configured + # context engine (the built-in compressor by default), which owns native + # context-window pressure, compression, and conversation boundaries. register_tool = getattr(ctx, "register_tool", None) - host_forwards_messages = _host_forwards_registered_tool_messages(ctx) - _HOST_FORWARDS_MESSAGES = host_forwards_messages if callable(register_tool) else None tracedecay_is_memory_provider = _active_memory_provider(ctx) == "tracedecay" if callable(register_tool): for schema in schemas.TOOL_SCHEMAS: name = schema["name"] - if name in MESSAGE_DEPENDENT_TOOLS and not host_forwards_messages: - continue if name in MEMORY_PROVIDER_TOOLS and tracedecay_is_memory_provider: # The active memory provider already exposes this store as # fact_store/fact_feedback/memory_status, registering the @@ -2692,34 +2086,30 @@ def bound(*args, **kwargs): ) else: _REGISTERED_TOOL_NAMES.add(name) - if host_forwards_messages: - for schema in context_engine.get_tool_schemas(): - name = schema["name"] - visible_schema = _agent_visible_schema(schema) - try: - register_tool( - name=name, - toolset="context_engine", - schema=visible_schema, - handler=_make_wrapped_lcm_handler(name, context_engine), - description=schema.get("description", ""), - ) - except Exception as exc: - logger.warning( - "tracedecay LCM tool registration failed for %s; continuing with context-engine schemas: %s", - name, - exc, - ) - else: - _CONTEXT_TOOL_NAMES.add(name) - else: - logger.info( - "tracedecay LCM registered tools skipped: this Hermes host does not forward messages to registered tool handlers; transcript sync remains on the memory-provider hook" - ) + # LCM readers use stored transcripts and forwarded session identity; + # they do not require access to the live in-memory message list. + for schema in LCM_NATIVE_SCHEMAS: + name = schema["name"] + try: + register_tool( + name=name, + toolset="tracedecay", + schema=_agent_visible_schema(schema), + handler=_make_wrapped_lcm_handler( + name, config=context_config, hermes_home=context_hermes_home + ), + description=schema.get("description", ""), + ) + except Exception as exc: + logger.warning( + "tracedecay LCM tool registration failed for %s; continuing: %s", + name, + exc, + ) + else: + _REGISTERED_TOOL_NAMES.add(name) else: - logger.info( - "tracedecay direct tool registration unavailable on this Hermes host; continuing with context-engine schemas" - ) + logger.info("tracedecay direct tool registration unavailable on this Hermes host") register_skill = getattr(ctx, "register_skill", None) skills_dir = Path(__file__).parent / "skills" diff --git a/crates/tracedecay-agent-hosts/src/agents/hermes/templates/skill.md b/crates/tracedecay-agent-hosts/src/agents/hermes/templates/skill.md index 68f1b0fb79..f41570b5c2 100644 --- a/crates/tracedecay-agent-hosts/src/agents/hermes/templates/skill.md +++ b/crates/tracedecay-agent-hosts/src/agents/hermes/templates/skill.md @@ -46,7 +46,7 @@ compresses a session. 2. Hermes exposes native aliases `lcm_grep`, `lcm_load_session`, `lcm_describe`, `lcm_expand`, `lcm_expand_query`, `lcm_status`, and `lcm_doctor`. They dispatch to their matching `tracedecay_lcm_*` commands; - use the Hermes alias and its schema when the context engine offers it, or the + use the Hermes alias and its schema when the plugin offers it, or the canonical command and its schema elsewhere. Do not invent fields by mixing the two surfaces. 3. Narrow temporal evidence with `lcm_grep` / `tracedecay_lcm_grep` (default @@ -87,9 +87,10 @@ and root identity stay inside the mounted daemon authority. Never redirect a profile refresh through whichever project happens to be active, and pass the same `scope`, `session`, `source`, and `target` selectors to `status` and `cancel` as to `begin`. -Leave host context-window -preflight, compression, and boundaries to the Hermes context engine rather -than triggering them during recall. +Hermes keeps its configured context engine, including its built-in compressor +by default. That engine handles context-window preflight, compression, and +boundaries. TraceDecay supplies retrieval tools and memory-provider transcript +ingest without replacing it or triggering compression during recall. ## Storage and project identity diff --git a/crates/tracedecay-automation-runtime/src/automation/skill_usage/recommendations.rs b/crates/tracedecay-automation-runtime/src/automation/skill_usage/recommendations.rs index 2534382215..d4b396af80 100644 --- a/crates/tracedecay-automation-runtime/src/automation/skill_usage/recommendations.rs +++ b/crates/tracedecay-automation-runtime/src/automation/skill_usage/recommendations.rs @@ -82,6 +82,17 @@ fn skill_improvement_recommendation(summary: &SkillUsageRecord) -> SkillImprovem ); } + // A missing use observation is not a measured low-use rate. Keep patch + // counts available as evidence without turning absent telemetry into a + // repair diagnosis. This does not establish that the skill needs no work. + if summary.use_count == 0 || summary.last_used_at.is_none() { + return no_improvement_recommendation( + summary, + "skill use evidence is unavailable; patch counts alone do not establish a repair need", + evidence, + ); + } + if summary.patch_count >= 2 && summary.use_count <= summary.patch_count { return SkillImprovementRecommendation { skill_id: summary.skill_id.clone(), @@ -264,7 +275,7 @@ mod tests { } #[test] - fn only_repeated_patches_recommend_skill_repair() { + fn repeated_patches_without_use_evidence_do_not_recommend_repair() { let now = tracedecay_runtime_core::tracedecay::current_timestamp(); let mut viewed_unused = activated_long_ago("viewed-unused-skill", now); viewed_unused.view_count = 3; @@ -286,14 +297,116 @@ mod tests { let id: fn(&SkillImprovementRecommendation) -> &str = |recommendation| &recommendation.skill_id; - for skill_id in ["viewed-unused-skill", "patched-once-skill", "unused-skill"] { + for skill_id in [ + "viewed-unused-skill", + "patched-once-skill", + "unused-skill", + "patched-twice-skill", + ] { let recommendation = by_id(&recommendations, skill_id, id); assert!(!recommendation.improvement, "{recommendation:?}"); assert_eq!(recommendation.recommendation, "none"); + assert_eq!(recommendation.priority, "none"); + assert!( + recommendation + .reason + .contains("use evidence is unavailable") + ); + } + let patched = by_id(&recommendations, "patched-twice-skill", id); + for expected in ["patches=2", "uses=0", "use_telemetry=unobserved"] { + assert!(patched.evidence.iter().any(|entry| entry == expected)); + } + } + + #[test] + fn incomplete_use_evidence_cannot_authorize_a_patch_review() { + for (use_count, last_used_at) in [(1, None), (0, Some(100))] { + let mut summary = automation_skill("incomplete-use"); + summary.patch_count = 2; + summary.use_count = use_count; + summary.last_used_at = last_used_at; + let recommendations = skill_improvement_recommendations(&[summary]); + assert_eq!(recommendations.len(), 1); + let recommendation = &recommendations[0]; + assert!(!recommendation.improvement, "{recommendation:?}"); + assert_eq!(recommendation.recommendation, "none"); + assert!( + recommendation + .reason + .contains("use evidence is unavailable") + ); + } + } + + #[test] + fn observed_use_preserves_the_existing_patch_review_policy() { + for (use_count, patch_count, expected) in [(1, 2, true), (3, 2, false), (1, 1, false)] { + let mut summary = automation_skill("observed-use"); + summary.use_count = use_count; + summary.patch_count = patch_count; + summary.last_used_at = Some(100); + let recommendations = skill_improvement_recommendations(&[summary]); + assert_eq!(recommendations.len(), 1); + let recommendation = &recommendations[0]; + assert_eq!(recommendation.improvement, expected, "{recommendation:?}"); + assert_eq!( + recommendation.recommendation, + if expected { "repair_candidate" } else { "none" } + ); + assert_eq!( + recommendation.priority, + if expected { "medium" } else { "none" } + ); + assert!( + recommendation + .evidence + .contains(&"use_telemetry=observed".to_string()) + ); + } + } + + #[test] + fn observed_usage_does_not_override_protected_skill_exclusions() { + for (pinned, source, state, reason) in [ + ( + true, + ManagedSkillSource::AutomationRun, + ManagedSkillState::Active, + "pinned", + ), + ( + false, + ManagedSkillSource::User, + ManagedSkillState::Active, + "user-authored", + ), + ( + false, + ManagedSkillSource::AutomationRun, + ManagedSkillState::Disabled, + "disabled", + ), + ( + false, + ManagedSkillSource::AutomationRun, + ManagedSkillState::Archived, + "archived", + ), + ] { + let mut summary = automation_skill("protected-skill"); + summary.pinned = pinned; + summary.provenance_source = Some(source); + summary.state = Some(state); + summary.use_count = 1; + summary.last_used_at = Some(100); + summary.patch_count = 2; + let recommendations = skill_improvement_recommendations(&[summary]); + assert_eq!(recommendations.len(), 1); + let recommendation = &recommendations[0]; + assert!(!recommendation.improvement, "{recommendation:?}"); + assert_eq!(recommendation.recommendation, "none"); + assert!(recommendation.reason.contains(reason), "{recommendation:?}"); } - let repair = by_id(&recommendations, "patched-twice-skill", id); - assert!(repair.improvement); - assert_eq!(repair.recommendation, "repair_candidate"); - assert_eq!(repair.priority, "medium"); } } diff --git a/crates/tracedecay-code-index-retention/src/code_index_generations/generation_scan.rs b/crates/tracedecay-code-index-retention/src/code_index_generations/generation_scan.rs index ba1908dc2f..94e144df53 100644 --- a/crates/tracedecay-code-index-retention/src/code_index_generations/generation_scan.rs +++ b/crates/tracedecay-code-index-retention/src/code_index_generations/generation_scan.rs @@ -12,18 +12,23 @@ use super::{ }; const MAX_FORMAT_REVISION_PREFIX_BYTES: usize = 4 * 1024; +const GENERATION_METADATA_READ_CHUNK_BYTES: usize = 4 * 1024; pub(super) fn read_generation_format_revision( path: &Path, is_cancelled: &dyn Fn() -> bool, ) -> Result { - let mut file = File::open(path).map_err(deferred_if_absent)?; - let mut prefix = vec![0_u8; MAX_FORMAT_REVISION_PREFIX_BYTES]; - let bytes_read = file.read(&mut prefix).map_err(storage)?; if is_cancelled() { return Err(CodeGenerationRetentionErrorV1::Cancelled); } - prefix.truncate(bytes_read); + let file = File::open(path).map_err(deferred_if_absent)?; + let mut prefix = Vec::with_capacity(MAX_FORMAT_REVISION_PREFIX_BYTES); + file.take(MAX_FORMAT_REVISION_PREFIX_BYTES as u64) + .read_to_end(&mut prefix) + .map_err(storage)?; + if is_cancelled() { + return Err(CodeGenerationRetentionErrorV1::Cancelled); + } parse_json_u32_field(&prefix, b"format_revision").ok_or_else(|| { CodeGenerationRetentionErrorV1::UnsafeState(format!( "generation file '{}' has no readable format revision in its bounded prefix", @@ -39,28 +44,80 @@ pub(super) fn read_generation_metadata( is_cancelled: &dyn Fn() -> bool, ) -> Result<(u32, SealedGenerationManifestMetadataV1, String, u64), CodeGenerationRetentionErrorV1> { + if is_cancelled() { + return Err(CodeGenerationRetentionErrorV1::Cancelled); + } let mut file = File::open(path).map_err(deferred_if_absent)?; let size_bytes = file.metadata().map_err(storage)?.len(); let mut hasher = Sha256::new(); - let mut prefix = Vec::with_capacity(MAX_GENERATION_METADATA_PREFIX_BYTES); - let mut buffer = vec![0u8; 1024 * 1024]; - loop { - let bytes_read = file.read(&mut buffer).map_err(storage)?; + let (format_revision, manifest) = + read_generation_metadata_prefix(&mut file, path, verification, &mut hasher, is_cancelled)?; + let state_digest = match verification { + GenerationDigestVerificationV1::Full => { + // The prefix reader hashed exactly the bytes it consumed from this + // handle. Continue at that position: a small manifest must never + // turn a full integrity check into a prefix-only digest. + let mut buffer = vec![0u8; 1024 * 1024]; + loop { + if is_cancelled() { + return Err(CodeGenerationRetentionErrorV1::Cancelled); + } + let bytes_read = file.read(&mut buffer).map_err(storage)?; + if bytes_read == 0 { + break; + } + hasher.update(&buffer[..bytes_read]); + if is_cancelled() { + return Err(CodeGenerationRetentionErrorV1::Cancelled); + } + } + encode_tagged_lowercase_hex("sha256:", &hasher.finalize()) + } + GenerationDigestVerificationV1::MetadataOnly => named_state_digest(path)?, + }; + Ok((format_revision, manifest, state_digest, size_bytes)) +} + +/// Read only as far as complete metadata requires, up to the existing bound. +/// Geometrically spaced probes keep repeated prefix parsing linear in bytes +/// even when a manifest is large. Reads stay small and cancellable between +/// probes; the prefix allocation is released before full-payload hashing. +fn read_generation_metadata_prefix( + file: &mut File, + path: &Path, + verification: GenerationDigestVerificationV1, + hasher: &mut Sha256, + is_cancelled: &dyn Fn() -> bool, +) -> Result<(u32, SealedGenerationManifestMetadataV1), CodeGenerationRetentionErrorV1> { + let mut prefix = Vec::with_capacity(GENERATION_METADATA_READ_CHUNK_BYTES); + let mut buffer = [0u8; GENERATION_METADATA_READ_CHUNK_BYTES]; + let mut next_probe = GENERATION_METADATA_READ_CHUNK_BYTES; + while prefix.len() < MAX_GENERATION_METADATA_PREFIX_BYTES { + if is_cancelled() { + return Err(CodeGenerationRetentionErrorV1::Cancelled); + } + let remaining = MAX_GENERATION_METADATA_PREFIX_BYTES - prefix.len(); + let read_limit = buffer.len().min(remaining).min(next_probe - prefix.len()); + let bytes_read = file.read(&mut buffer[..read_limit]).map_err(storage)?; if bytes_read == 0 { break; } if verification == GenerationDigestVerificationV1::Full { hasher.update(&buffer[..bytes_read]); } - let remaining = MAX_GENERATION_METADATA_PREFIX_BYTES.saturating_sub(prefix.len()); - prefix.extend_from_slice(&buffer[..bytes_read.min(remaining)]); + prefix.extend_from_slice(&buffer[..bytes_read]); if is_cancelled() { return Err(CodeGenerationRetentionErrorV1::Cancelled); } - if verification == GenerationDigestVerificationV1::MetadataOnly - && prefix.len() >= MAX_GENERATION_METADATA_PREFIX_BYTES - { - break; + if prefix.len() >= next_probe { + if parse_json_u32_field(&prefix, b"format_revision").is_some() + && extract_json_object_field(&prefix, b"manifest").is_some() + { + break; + } + next_probe = next_probe + .saturating_mul(2) + .min(MAX_GENERATION_METADATA_PREFIX_BYTES); } } let format_revision = parse_json_u32_field(&prefix, b"format_revision").ok_or_else(|| { @@ -81,13 +138,7 @@ pub(super) fn read_generation_metadata( path.display() )) })?; - let state_digest = match verification { - GenerationDigestVerificationV1::Full => { - encode_tagged_lowercase_hex("sha256:", &hasher.finalize()) - } - GenerationDigestVerificationV1::MetadataOnly => named_state_digest(path)?, - }; - Ok((format_revision, manifest, state_digest, size_bytes)) + Ok((format_revision, manifest)) } fn named_state_digest(path: &Path) -> Result { @@ -112,7 +163,16 @@ fn parse_json_u32_field(prefix: &[u8], field: &[u8]) -> Option { .iter() .position(|byte| !byte.is_ascii_digit()) .map_or(prefix.len(), |offset| start + offset); - std::str::from_utf8(&prefix[start..end]).ok()?.parse().ok() + // A prefix ending in digits is not yet a complete number. In particular, + // do not accept the integer prefix of a fractional or exponent value. + let terminator = prefix[end..] + .iter() + .copied() + .find(|byte| !byte.is_ascii_whitespace())?; + if !matches!(terminator, b',' | b'}') { + return None; + } + serde_json::from_slice(&prefix[start..end]).ok() } fn extract_json_object_field<'a>(prefix: &'a [u8], field: &[u8]) -> Option<&'a [u8]> { @@ -167,3 +227,231 @@ fn json_field_value_start(prefix: &[u8], field: &[u8]) -> Option { } Some(cursor) } + +#[cfg(test)] +mod tests { + use std::cell::Cell; + use std::fs::{self, File, OpenOptions}; + use std::io::{Seek, SeekFrom, Write}; + use std::path::{Path, PathBuf}; + + use sha2::{Digest, Sha256}; + use tracedecay_code_index::production::SEALED_GENERATION_FORMAT_REVISION_V1; + use tracedecay_domain::canonical_text::encode_tagged_lowercase_hex; + + use super::{ + CodeGenerationRetentionErrorV1, GENERATION_METADATA_READ_CHUNK_BYTES, + GenerationDigestVerificationV1, MAX_GENERATION_METADATA_PREFIX_BYTES, parse_json_u32_field, + read_generation_format_revision, read_generation_metadata, read_generation_metadata_prefix, + }; + + fn generation_bytes(manifest_padding: usize, payload_bytes: usize) -> Vec { + serde_json::to_vec(&serde_json::json!({ + "format_revision": SEALED_GENERATION_FORMAT_REVISION_V1, + "manifest": { + "generation_id": "generation.v1.scan", + "padding": "x".repeat(manifest_padding), + "seal": { "sealed_at": 7 }, + }, + "payload": "x".repeat(payload_bytes), + })) + .unwrap() + } + + fn write_generation(root: &Path, bytes: &[u8]) -> (PathBuf, String) { + let digest = encode_tagged_lowercase_hex("sha256:", &Sha256::digest(bytes)); + let path = root.join(format!("generation-{}.json", &digest[7..])); + fs::write(&path, bytes).unwrap(); + (path, digest) + } + + #[test] + fn metadata_only_stops_at_the_complete_manifest() { + let root = tempfile::tempdir().unwrap(); + let bytes = generation_bytes(0, 256 * 1024); + let (path, expected_digest) = write_generation(root.path(), &bytes); + assert_eq!( + read_generation_format_revision(&path, &|| false).unwrap(), + SEALED_GENERATION_FORMAT_REVISION_V1 + ); + let mut file = File::open(&path).unwrap(); + let (revision, manifest) = read_generation_metadata_prefix( + &mut file, + &path, + GenerationDigestVerificationV1::MetadataOnly, + &mut Sha256::new(), + &|| false, + ) + .unwrap(); + assert_eq!(revision, SEALED_GENERATION_FORMAT_REVISION_V1); + assert_eq!(manifest.generation_id.as_str(), "generation.v1.scan"); + assert_eq!(manifest.seal.sealed_at.0, 7); + assert_eq!( + file.stream_position().unwrap(), + GENERATION_METADATA_READ_CHUNK_BYTES as u64 + ); + let (_, _, digest, size) = + read_generation_metadata(&path, GenerationDigestVerificationV1::MetadataOnly, &|| { + false + }) + .unwrap(); + assert_eq!(digest, expected_digest); + assert_eq!(size, bytes.len() as u64); + } + + #[test] + fn full_verification_still_hashes_the_payload_after_the_manifest() { + let root = tempfile::tempdir().unwrap(); + let bytes = generation_bytes(0, 2 * 1024 * 1024); + let (path, named_digest) = write_generation(root.path(), &bytes); + let (_, _, digest, size) = + read_generation_metadata(&path, GenerationDigestVerificationV1::Full, &|| false) + .unwrap(); + assert_eq!(digest, named_digest); + assert_eq!(size, bytes.len() as u64); + + let mut file = OpenOptions::new().write(true).open(&path).unwrap(); + file.seek(SeekFrom::End(-3)).unwrap(); + file.write_all(b"y").unwrap(); + drop(file); + let (_, _, changed_digest, _) = + read_generation_metadata(&path, GenerationDigestVerificationV1::Full, &|| false) + .unwrap(); + assert_ne!(changed_digest, named_digest); + assert_eq!( + changed_digest, + encode_tagged_lowercase_hex("sha256:", &Sha256::digest(fs::read(&path).unwrap())) + ); + } + + #[test] + fn a_manifest_spanning_multiple_chunks_is_read_completely() { + let root = tempfile::tempdir().unwrap(); + let bytes = generation_bytes(12 * 1024, 64 * 1024); + let (path, _) = write_generation(root.path(), &bytes); + let mut file = File::open(&path).unwrap(); + let (_, manifest) = read_generation_metadata_prefix( + &mut file, + &path, + GenerationDigestVerificationV1::MetadataOnly, + &mut Sha256::new(), + &|| false, + ) + .unwrap(); + assert_eq!(manifest.generation_id.as_str(), "generation.v1.scan"); + assert_eq!(manifest.seal.sealed_at.0, 7); + assert_eq!(file.stream_position().unwrap(), 16 * 1024); + } + + #[test] + fn metadata_beyond_the_existing_prefix_bound_is_refused() { + let root = tempfile::tempdir().unwrap(); + let mut bytes = vec![b' '; MAX_GENERATION_METADATA_PREFIX_BYTES]; + bytes.extend(generation_bytes(0, 0)); + let (path, _) = write_generation(root.path(), &bytes); + let mut file = File::open(&path).unwrap(); + let result = read_generation_metadata_prefix( + &mut file, + &path, + GenerationDigestVerificationV1::MetadataOnly, + &mut Sha256::new(), + &|| false, + ); + assert!(matches!( + result, + Err(CodeGenerationRetentionErrorV1::UnsafeState(_)) + )); + assert_eq!( + file.stream_position().unwrap(), + MAX_GENERATION_METADATA_PREFIX_BYTES as u64 + ); + } + + #[test] + fn malformed_manifest_metadata_is_refused_in_both_modes() { + let root = tempfile::tempdir().unwrap(); + let bytes = serde_json::to_vec(&serde_json::json!({ + "format_revision": SEALED_GENERATION_FORMAT_REVISION_V1, + "manifest": { "generation_id": "generation.v1.scan" }, + "payload": "x".repeat(64 * 1024), + })) + .unwrap(); + let (path, _) = write_generation(root.path(), &bytes); + for verification in [ + GenerationDigestVerificationV1::MetadataOnly, + GenerationDigestVerificationV1::Full, + ] { + assert!(matches!( + read_generation_metadata(&path, verification, &|| false), + Err(CodeGenerationRetentionErrorV1::UnsafeState(_)) + )); + } + } + + #[test] + fn metadata_reads_observe_cancellation_before_and_after_io() { + let root = tempfile::tempdir().unwrap(); + let (path, _) = write_generation(root.path(), &generation_bytes(0, 64 * 1024)); + let mut file = File::open(&path).unwrap(); + let result = read_generation_metadata_prefix( + &mut file, + &path, + GenerationDigestVerificationV1::MetadataOnly, + &mut Sha256::new(), + &|| true, + ); + assert!(matches!( + result, + Err(CodeGenerationRetentionErrorV1::Cancelled) + )); + assert_eq!(file.stream_position().unwrap(), 0); + + let checks = Cell::new(0); + let result = read_generation_metadata_prefix( + &mut file, + &path, + GenerationDigestVerificationV1::MetadataOnly, + &mut Sha256::new(), + &|| { + let previous = checks.get(); + checks.set(previous + 1); + previous > 0 + }, + ); + assert!(matches!( + result, + Err(CodeGenerationRetentionErrorV1::Cancelled) + )); + assert_eq!( + file.stream_position().unwrap(), + GENERATION_METADATA_READ_CHUNK_BYTES as u64 + ); + } + + #[test] + fn format_revision_requires_a_complete_unsigned_json_integer() { + for input in [ + r#"{"format_revision":21,"manifest":{}}"#, + r#"{"format_revision":21 }"#, + ] { + assert_eq!( + parse_json_u32_field(input.as_bytes(), b"format_revision"), + Some(21) + ); + } + for input in [ + r#"{"format_revision":21"#, + r#"{"format_revision":21.5}"#, + r#"{"format_revision":21e1}"#, + r#"{"format_revision":021}"#, + r#"{"format_revision":-1}"#, + r#"{"format_revision":"21"}"#, + r#"{"format_revision":4294967296}"#, + ] { + assert_eq!( + parse_json_u32_field(input.as_bytes(), b"format_revision"), + None + ); + } + } +} diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/queries.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/queries.rs index 1a0a5ff773..a71306187a 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/queries.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/queries.rs @@ -1936,35 +1936,19 @@ impl CodeIndexSchedulerRegistryV1 { // request that spent its budget mid-warm, was cancelled, or lost // the wait task answers the typed unavailable rather than opening // a reader on a store it never waited out. - let mut catalog_retain = None; - if let Some(budget) = remaining_generation_resolution_wait(context.request) { - let requirement = match seed { - Some(seed) if !seed_is_served(&store, &latest, seed, budget).await? => { - CodeGraphReadinessRequirement::Engine - } - _ => CodeGraphReadinessRequirement::Catalog, - }; - // The seed read spent part of the request; the wait gets the rest. - let budget = - remaining_generation_resolution_wait(context.request).unwrap_or(Duration::ZERO); - let waiting = Arc::clone(&store); - match native_runtime::spawn_blocking(move || { - waiting.await_rewarm_and_retain_for(budget, requirement) - }) - .await - { - Ok(Ok(retain)) => catalog_retain = Some(retain), - Ok(Err(_pending)) => return Err(CallableCodeCursorError::Unavailable), - Err(join_error) => { - tracing::warn!( - event = "code_graph_rewarm_wait_failed", - error = %join_error, - "a callable's graph re-warm wait did not finish" - ); - return Err(CallableCodeCursorError::Unavailable); - } + let seed_budget = remaining_generation_resolution_wait(context.request) + .ok_or(CallableCodeCursorError::Unavailable)?; + let requirement = match seed { + Some(seed) if !seed_is_served(&store, &latest, seed, seed_budget).await? => { + CodeGraphReadinessRequirement::Engine } - } + _ => CodeGraphReadinessRequirement::Catalog, + }; + // The seed point read may have spent part of the budget. A released + // catalog must not consume the rest of the enclosing MCP deadline. + let budget = remaining_generation_resolution_wait(context.request) + .ok_or(CallableCodeCursorError::Unavailable)?; + let catalog_retain = Some(await_callable_rewarm(&store, requirement, budget).await?); let cost = CodeGraphReadCostMeter::start(); let reader = store .interactive_reader_with_cancellation( @@ -1987,6 +1971,47 @@ impl CodeIndexSchedulerRegistryV1 { } } +/// A callable's deadline includes route settlement and response delivery. +/// Probe without waiting to start a released graph's background re-warm, then +/// wait only if measured remaining work fits inside the reserved budget. +async fn await_callable_rewarm( + store: &Arc, + requirement: CodeGraphReadinessRequirement, + remaining: Duration, +) -> Result, CallableCodeCursorError> { + // Even a zero-budget probe can wait for the synchronous serving-engine + // mutex while a background engine open holds it. Keep both the probe and + // the measured wait off Tokio workers, and bound the entire admission. + let waiting = Arc::clone(store); + let admission = native_runtime::timeout( + remaining, + native_runtime::spawn_blocking(move || { + let pending = match waiting.await_rewarm_and_retain_for(Duration::ZERO, requirement) { + Ok(retain) => return Ok(retain), + Err(pending) => pending, + }; + let budget = crate::project_reads::rewarm_wait_budget(remaining, pending.retry_after) + .ok_or(CallableCodeCursorError::Unavailable)?; + waiting + .await_rewarm_and_retain_for(budget, requirement) + .map_err(|_| CallableCodeCursorError::Unavailable) + }), + ) + .await; + match admission { + Ok(Ok(result)) => result, + Ok(Err(error)) => { + tracing::warn!( + event = "code_graph_rewarm_wait_failed", + %error, + "a callable's graph re-warm wait did not finish" + ); + Err(CallableCodeCursorError::Unavailable) + } + Err(_) => Err(CallableCodeCursorError::Unavailable), + } +} + /// Whether `seed` names a symbol `latest` serves, read once the engine is /// resident. A malformed seed is not served; the query refuses it as invalid. async fn seed_is_served( @@ -1998,12 +2023,12 @@ async fn seed_is_served( let Ok(seed) = typed::(seed) else { return Ok(false); }; + let engine_retain = + await_callable_rewarm(store, CodeGraphReadinessRequirement::Engine, budget).await?; let probing = Arc::clone(store); let generation = latest.metadata().manifest().generation_id.clone(); native_runtime::spawn_blocking(move || { - let _engine = probing - .await_rewarm_and_retain_for(budget, CodeGraphReadinessRequirement::Engine) - .map_err(|_| CallableCodeCursorError::Unavailable)?; + let _engine_retain = engine_retain; let cancellation: Arc = Arc::new(tracedecay_graph_db::NeverCancelled); probing @@ -4503,6 +4528,22 @@ mod tests { ); } + #[test] + fn slow_callable_rewarm_reserves_time_for_seed_and_catalog_reads() { + let available = generation_resolution_wait_from_remaining(Duration::from_secs(10)); + assert_eq!(available, Duration::from_secs(9)); + assert_eq!( + crate::project_reads::rewarm_wait_budget(available, Duration::from_secs(17)), + None, + "a measured 17-second warm must not occupy a 10-second MCP dispatch" + ); + assert_eq!( + crate::project_reads::rewarm_wait_budget(available, Duration::from_secs(2)), + Some(Duration::from_millis(4_500)), + "a short warm retains a bounded wait and leaves response time" + ); + } + #[test] fn rejected_unpinned_query_does_not_claim_the_request_sentinel_as_source() { let outcome = rejected_cursor::>( diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs index dc1bf10119..3a5e4f37ec 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs @@ -210,6 +210,15 @@ impl CodeIndexBuildProgressSlotStateV1 { if self.generation_id.as_ref() != Some(generation_id) || self.owner_epoch != owner_epoch { return false; } + // A progress producer can timestamp its snapshot before another + // producer publishes. Preserve the most recent observed timestamp + // even when the later lock acquisition delivers an older sample. + // Replacing the generation clears the snapshot and starts a new owner. + if let Some(current) = self.snapshot.as_ref() { + snapshot.last_progress_micros = snapshot + .last_progress_micros + .max(current.last_progress_micros); + } self.progress_epoch = self.progress_epoch.saturating_add(1).max(1); snapshot.progress_epoch = self.progress_epoch; diff --git a/crates/tracedecay-code-index-runtime/src/project_reads.rs b/crates/tracedecay-code-index-runtime/src/project_reads.rs index ab66aa22ae..c87ef9c98d 100644 --- a/crates/tracedecay-code-index-runtime/src/project_reads.rs +++ b/crates/tracedecay-code-index-runtime/src/project_reads.rs @@ -66,7 +66,7 @@ fn refuse_projection_wait(request: &CodeGraphReadRequest<'_>) -> Result<(), Code /// also covers response delivery, so spending all of it waiting on the warm /// would turn a retryable warming state into a generic transport timeout. /// Wait only while the measured warm is likely to fit with time to answer. -fn rewarm_wait_budget(remaining: Duration, predicted: Duration) -> Option { +pub(crate) fn rewarm_wait_budget(remaining: Duration, predicted: Duration) -> Option { let wait = remaining / 2; (wait > Duration::ZERO && predicted < wait).then_some(wait) } @@ -87,9 +87,19 @@ async fn await_rewarm( // The zero-budget probe starts the background re-warm and returns its // measured remaining time without parking the request. let requirement = request.readiness; - let pending = match store.await_rewarm_and_retain_for(Duration::ZERO, requirement) { - Ok(retain) => return Ok(retain), - Err(pending) => pending, + let probing = Arc::clone(store); + let pending = match native_runtime::spawn_blocking(move || { + probing.await_rewarm_and_retain_for(Duration::ZERO, requirement) + }) + .await + { + Ok(Ok(retain)) => return Ok(retain), + Ok(Err(pending)) => pending, + Err(error) => { + return Err(CodeGraphReadError::Unavailable { + detail: format!("the code graph re-warm probe did not finish: {error}"), + }); + } }; let now = now_micros(); let expires_at = diff --git a/crates/tracedecay-code-index-runtime/src/project_reads/ignored_dependency_admission_tests.rs b/crates/tracedecay-code-index-runtime/src/project_reads/ignored_dependency_admission_tests.rs index 35ab833191..9135cf4f90 100644 --- a/crates/tracedecay-code-index-runtime/src/project_reads/ignored_dependency_admission_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/project_reads/ignored_dependency_admission_tests.rs @@ -21,7 +21,6 @@ use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; use crate::code_index_scheduler::{CodeIndexSchedulerRegistryV1, LatestCompleteCodeIndexV1}; use crate::project_reads::project_code_index_ignored_dependency_admission_port; -use tracedecay_runtime_core::path_safety::canonical_existing_identity; const PROJECT_ID: &str = "project.project-open-ignored-dependency"; @@ -70,7 +69,6 @@ export function generationAnchor() { return 1; } .mount_worktree(project_id(), root.path(), store.path().to_path_buf()) .await .expect("mount code-index scheduler"); - wait_for_initial_generation(®istry, root.path()).await; let baseline = latest(®istry, root.path()).await; let generation = baseline.generation(); let snapshot = generation.snapshot(); @@ -242,31 +240,11 @@ fn request_context(scope: ResolvedScope, suffix: &str) -> RequestContext { .expect("request context") } -async fn wait_for_initial_generation(registry: &CodeIndexSchedulerRegistryV1, project_root: &Path) { - if registry.latest_generation_id(project_root).await.is_some() { - return; - } - let canonical_root = canonical_existing_identity(project_root).expect("canonical fixture root"); - let mut publications = registry.subscribe_generation_publications(); - tokio::time::timeout(Duration::from_secs(5), async { - loop { - let event = publications.recv().await.expect("generation publication"); - if event.project_root == canonical_root { - break; - } - } - }) - .await - .expect("initial generation publication"); -} - /// Wait for the complete serving generation, not merely its publication. /// -/// The generation-publication broadcast and `latest_generation_id` both fire -/// on the reconcile pass, ahead of the bind and the serving swap that fill -/// `serving_generation`. Reading the complete arm the instant -/// `wait_for_initial_generation` returns therefore raced the seat and reported -/// a missing generation for a store that had one. +/// Publication and `latest_generation_id` precede the serving swap. Wait +/// for the actual fresh serving state before using imports as admission +/// evidence; no earlier broadcast establishes readiness. async fn latest( registry: &CodeIndexSchedulerRegistryV1, project_root: &Path, diff --git a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs index 1f816a111f..6a48bb325e 100644 --- a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs @@ -3003,6 +3003,44 @@ fn text_progress_rate_and_eta_require_two_monotonic_committed_samples() { ); } +#[test] +fn progress_timestamp_never_regresses_within_one_owner() { + let generation = CodeGenerationId::new("generation.progress-clock").unwrap(); + let mut slot = super::super::CodeIndexBuildProgressSlotStateV1::default(); + let owner = slot.replace_generation(generation.clone()); + + let mut first = progress_snapshot_for_generation(&generation, 1); + first.last_progress_micros = 500; + assert!(slot.publish(&generation, owner, first)); + let first_epoch = slot.snapshot().unwrap().progress_epoch; + + // This later producer committed more pages but assembled its snapshot + // before the earlier publisher took the slot lock. + let mut delayed = progress_snapshot_for_generation(&generation, 2); + delayed.last_progress_micros = 400; + assert!(slot.publish(&generation, owner, delayed)); + let current = slot.snapshot().unwrap(); + assert_eq!(current.committed_pages, 2); + assert!(current.progress_epoch > first_epoch); + assert_eq!( + current.last_progress_micros, 500, + "last observed progress cannot move backwards within one owner" + ); + + let mut later = progress_snapshot_for_generation(&generation, 3); + later.last_progress_micros = 600; + assert!(slot.publish(&generation, owner, later)); + assert_eq!(slot.snapshot().unwrap().last_progress_micros, 600); + + // Ownership replacement is a new observation sequence, even when the + // generation identity is reused. It must not inherit the old timestamp. + let replacement_owner = slot.replace_generation(generation.clone()); + let mut replacement = progress_snapshot_for_generation(&generation, 1); + replacement.last_progress_micros = 42; + assert!(slot.publish(&generation, replacement_owner, replacement)); + assert_eq!(slot.snapshot().unwrap().last_progress_micros, 42); +} + #[test] fn progress_owner_epoch_rejects_original_a_after_a_b_a_replacement() { let generation_a = CodeGenerationId::new("generation.progress-a").expect("generation A"); diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index add7683673..63d4be6e40 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -165,12 +165,11 @@ pub fn source_walk( if !is_dir { return !path_policy.excludes(&relative); } - // A directory with its own Git authority is another project, not - // source owned by this one. This covers linked worktrees (`.git` - // file), nested clones/submodules (`.git` directory), and keeps a - // primary checkout containing agent worktrees from indexing many - // copies of itself. The project root is deliberately exempt at - // depth zero above. + // Includes select paths, not repository authority. A parent + // gitlink does not capture the child's files or prove their + // freshness, so even registered submodules remain separate + // sources. Linked worktrees and nested clones share this boundary. + // The project root is deliberately exempt at depth zero above. if std::fs::symlink_metadata(entry.path().join(".git")).is_ok() { return false; } @@ -226,7 +225,7 @@ mod tests { fn walked_files(root: &std::path::Path, path_policy: &IndexPathPolicyV1) -> Vec { let mut files = source_walk(root, None, path_policy) .expect("source walk") - .filter_map(Result::ok) + .map(|entry| entry.expect("source entry")) .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) .map(|entry| { entry @@ -375,6 +374,109 @@ mod tests { ); } + fn fixture_git(root: &std::path::Path, args: &[&str]) -> std::process::Output { + let program = tracedecay_runtime_core::git::try_git_program().expect("Git program"); + let mut command = std::process::Command::new(program); + command.current_dir(root); + for (key, _) in std::env::vars_os() { + if key.to_string_lossy().starts_with("GIT_") { + command.env_remove(key); + } + } + let output = command + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", root.join(".fixture-gitconfig")) + .args([ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .expect("run fixture Git"); + assert!( + output.status.success(), + "Git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + output + } + + #[test] + fn includes_cannot_admit_registered_submodule_files_as_parent_source() { + let fixture = TempDir::new().expect("repository fixture"); + let root = fixture.path().join("parent"); + let child = fixture.path().join("child"); + fs::create_dir_all(root.join("src")).unwrap(); + fs::create_dir_all(child.join("src")).unwrap(); + fixture_git(&root, &["init", "-q"]); + fixture_git(&child, &["init", "-q"]); + fs::write(root.join("src/parent.rs"), "pub fn parent() {}\n").unwrap(); + fs::write(child.join("src/feature.rs"), "pub fn feature() {}\n").unwrap(); + fixture_git(&child, &["add", "."]); + fixture_git(&child, &["commit", "-qm", "child source"]); + fixture_git( + &root, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + "../child", + "app", + ], + ); + let index = fixture_git(&root, &["ls-files", "--stage", "-z", "--", "app"]); + assert!(index.stdout.starts_with(b"160000 ")); + assert!(index.stdout.ends_with(b" 0\tapp\0")); + assert!(root.join("app/src/feature.rs").is_file()); + + for rule in ["app/**", "app/src/feature.rs", "app/src/fe*.rs"] { + let policy = IndexPathPolicyV1::new(vec![], vec![rule.into()]).unwrap(); + let files = walked_files(&root, &policy); + assert!(files.contains(&PathBuf::from("src/parent.rs")), "{files:?}"); + assert!( + files.iter().all(|path| !path.starts_with("app")), + "{rule}: a parent gitlink is not child source authority: {files:?}" + ); + let scoped = source_walk(&root, Some(rule), &policy) + .expect("explicit submodule scope") + .map(|entry| entry.expect("source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .collect::>(); + assert!( + scoped.is_empty(), + "{rule}: an explicit scope cannot cross repositories" + ); + } + } + + #[test] + fn submodule_metadata_cannot_break_an_unrelated_source_include() { + let root = TempDir::new().expect("project root"); + fs::create_dir_all(root.path().join(".git")).unwrap(); + fs::create_dir_all(root.path().join("src")).unwrap(); + fs::create_dir_all(root.path().join("app/.git")).unwrap(); + fs::write(root.path().join("src/keep.rs"), "pub fn keep() {}\n").unwrap(); + fs::write(root.path().join("app/foreign.rs"), "pub fn foreign() {}\n").unwrap(); + fs::write( + root.path().join(".gitmodules"), + "[invalid submodule metadata\n", + ) + .unwrap(); + let policy = IndexPathPolicyV1::new(vec![], vec!["src/**".into()]).unwrap(); + let files = walked_files(root.path(), &policy); + assert!(files.contains(&PathBuf::from("src/keep.rs"))); + assert!( + files.iter().all(|path| !path.starts_with("app")), + "{files:?}" + ); + } + #[test] fn explicit_glob_cannot_read_excluded_paths_without_policy_include() { let root = TempDir::new().expect("project root"); diff --git a/crates/tracedecay-global-db/src/observation/canonical_bodies.rs b/crates/tracedecay-global-db/src/observation/canonical_bodies.rs index 0ab7f8d5e8..e4d039f490 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_bodies.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_bodies.rs @@ -1,8 +1,8 @@ use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, params}; +use tracedecay_store::canonical_body::LOAD_CANONICAL_BODIES_SQL; use tracedecay_store::{ - CANONICAL_BODIES_TABLE_SQL, CanonicalBodyError, INLINE_BODY_BYTES, LOAD_CANONICAL_BODY_SQL, - StoredCanonicalBody, UPSERT_CANONICAL_BODY_SQL, collect_body_refs, parse_stored_observation, - unpack_body, + CANONICAL_BODIES_TABLE_SQL, CanonicalBodyError, INLINE_BODY_BYTES, StoredCanonicalBody, + UPSERT_CANONICAL_BODY_SQL, collect_body_refs, parse_stored_observation, unpack_body, }; use super::super::global_db_operation_error; @@ -112,8 +112,36 @@ pub async fn decode_observation_json( ) -> tracedecay_store::ObservationStoreResult { let hashes = collect_body_refs(observation_json).map_err(|error| storage(operation, error))?; let mut bodies = std::collections::HashMap::new(); - for hash in hashes { - bodies.insert(hash.clone(), load_canonical_body(conn, &hash).await?); + if !hashes.is_empty() { + const READ_OPERATION: &str = "read canonical observation body"; + let encoded = serde_json::to_string(&hashes).map_err(|error| storage(operation, error))?; + let mut rows = conn + .query(LOAD_CANONICAL_BODIES_SQL, params![encoded]) + .await + .map_err(|error| storage(READ_OPERATION, error))?; + while let Some(row) = rows + .next() + .await + .map_err(|error| storage(READ_OPERATION, error))? + { + let hash: String = row.get(0).map_err(|error| storage(READ_OPERATION, error))?; + let encoding = row + .get::>(1) + .map_err(|error| storage(READ_OPERATION, error))? + .ok_or_else(|| { + storage( + READ_OPERATION, + CanonicalBodyError::Missing { + content_hash: hash.clone(), + }, + ) + })?; + let blob: Vec = row.get(2).map_err(|error| storage(READ_OPERATION, error))?; + let uncompressed = row.get(3).map_err(|error| storage(READ_OPERATION, error))?; + let bytes = unpack_body(&hash, &encoding, &blob, uncompressed) + .map_err(|error| storage(READ_OPERATION, error))?; + bodies.insert(hash, bytes); + } } parse_stored_observation(observation_json, |hash| { bodies @@ -126,38 +154,6 @@ pub async fn decode_observation_json( .map_err(|error| storage(operation, error)) } -pub(crate) async fn load_canonical_body( - conn: &impl QueryExecutor, - content_hash: &str, -) -> tracedecay_store::ObservationStoreResult> { - const OPERATION: &str = "read canonical observation body"; - let mut rows = conn - .query(LOAD_CANONICAL_BODY_SQL, params![content_hash]) - .await - .map_err(|error| storage(OPERATION, error))?; - let row = rows - .next() - .await - .map_err(|error| storage(OPERATION, error))? - .ok_or_else(|| { - storage( - OPERATION, - CanonicalBodyError::Missing { - content_hash: content_hash.to_owned(), - }, - ) - })?; - let encoding: String = row.get(0).map_err(|error| storage(OPERATION, error))?; - let blob: Vec = row.get(1).map_err(|error| storage(OPERATION, error))?; - unpack_body( - content_hash, - &encoding, - &blob, - row.get(2).map_err(|error| storage(OPERATION, error))?, - ) - .map_err(|error| storage(OPERATION, error)) -} - async fn persist_bodies( conn: &impl Executor, bodies: &[StoredCanonicalBody], @@ -197,6 +193,18 @@ async fn table_exists( #[cfg(test)] mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use tracedecay_domain::{ + ComponentVersion, DurableObservationV1, ObservationId, ObservationIdentityMaterialV1, + ObservationOrderingDomainV1, ObservationScopeV1, ObservationSourceGenerationV1, + ObservationSourceIdentityV1, ObservationSourceRangeV1, PayloadReferenceV1, RetentionClass, + SanitizationReceiptId, SanitizationReceiptRefV1, SanitizationReceiptV1, + SanitizerDispositionV1, SensitivityV1, SessionId, + }; + use tracedecay_runtime_core::db::engine::{IntoParams, ReadConnection, Rows}; + use tracedecay_store::slim_stored_json; + use super::*; use tempfile::TempDir; use tracedecay_runtime_core::db::engine::TestConnection; @@ -205,13 +213,21 @@ mod tests { async fn body_reads_distinguish_storage_failures_from_missing_rows() { let tmp = TempDir::new().unwrap(); let conn = TestConnection::open(&tmp.path().join("body-read-errors.db")); - let error = load_canonical_body(&conn, "missing").await.unwrap_err(); + let encoded = serde_json::json!({ + (tracedecay_store::canonical_body::BODY_REF_KEY): {"/payload/text": "missing"} + }) + .to_string(); + let error = decode_observation_json(&conn, &encoded, "hydrate fixture") + .await + .unwrap_err(); let tracedecay_store::ObservationStoreError::Storage { source, .. } = error else { panic!("expected a storage error"); }; assert!(source.is::()); ensure_canonical_bodies_table(&conn).await.unwrap(); - let error = load_canonical_body(&conn, "missing").await.unwrap_err(); + let error = decode_observation_json(&conn, &encoded, "hydrate fixture") + .await + .unwrap_err(); let tracedecay_store::ObservationStoreError::Storage { source, .. } = error else { panic!("expected a missing-body error"); }; @@ -284,4 +300,139 @@ mod tests { "CAS rows keep neither the body nor a full placeholder" ); } + + fn body_observation(payload: serde_json::Value) -> DurableObservationV1 { + let identity = ObservationIdentityMaterialV1::for_native_record( + ObservationSourceIdentityV1::new(SessionId::new("session.bodies").unwrap()).unwrap(), + ObservationScopeV1::Profile, + ObservationSourceGenerationV1::new(1).unwrap(), + ObservationSourceRangeV1::new(12, 34).unwrap(), + ObservationOrderingDomainV1::FileBytes, + ObservationId::new("record.bodies").unwrap(), + ) + .unwrap(); + let receipt = SanitizationReceiptV1::new( + SanitizationReceiptRefV1::new( + SanitizationReceiptId::new("receipt.bodies").unwrap(), + ComponentVersion::new("sanitizer.fixture.v1").unwrap(), + ) + .unwrap(), + SanitizerDispositionV1::Accepted, + SensitivityV1::NonSensitive, + Some(PayloadReferenceV1::for_payload(&payload).unwrap()), + ) + .unwrap(); + DurableObservationV1::new( + identity, + receipt, + RetentionClass::new("transcript.fixture").unwrap(), + payload, + ) + .unwrap() + } + + struct CountedRead { + connection: ReadConnection, + queries: AtomicUsize, + } + + impl QueryExecutor for CountedRead { + async fn query

( + &self, + sql: &str, + params: P, + ) -> tracedecay_runtime_core::db::engine::Result + where + P: IntoParams, + { + self.queries.fetch_add(1, Ordering::Relaxed); + self.connection.query(sql, params).await + } + } + + #[asupersync::test] + async fn observation_body_hydration_batches_reads_and_preserves_verification() { + let tmp = TempDir::new().unwrap(); + let conn = TestConnection::open(&tmp.path().join("observation-bodies.db")); + ensure_canonical_bodies_table(&conn).await.unwrap(); + let observation = body_observation(serde_json::json!({ + "first": "alpha ".repeat(INLINE_BODY_BYTES), + "second": "beta ".repeat(INLINE_BODY_BYTES), + "same_as_first": "alpha ".repeat(INLINE_BODY_BYTES), + })); + let expected = serde_json::to_value(&observation).unwrap(); + let (encoded, bodies) = slim_stored_json(&expected.to_string()).unwrap(); + persist_bodies(&conn, &bodies).await.unwrap(); + let hashes = collect_body_refs(&encoded).unwrap(); + assert_eq!(hashes.len(), 2, "repeated payloads share one body"); + // An unrelated corrupt row must not be read by the bounded lookup. + conn.execute( + UPSERT_CANONICAL_BODY_SQL, + params!["unrelated", "identity", b"wrong".as_slice(), 5_i64], + ) + .await + .unwrap(); + let counted = CountedRead { + connection: conn.read_only(), + queries: AtomicUsize::new(0), + }; + let decoded = decode_observation_json(&counted, &encoded, "hydrate fixture") + .await + .unwrap(); + assert_eq!(serde_json::to_value(decoded).unwrap(), expected); + assert_eq!(counted.queries.load(Ordering::Relaxed), 1); + + let inline = body_observation(serde_json::json!({"text": "small"})); + let encoded_inline = serde_json::to_string(&inline).unwrap(); + let decoded = decode_observation_json(&counted, &encoded_inline, "hydrate fixture") + .await + .unwrap(); + assert_eq!( + serde_json::to_value(decoded).unwrap(), + serde_json::to_value(inline).unwrap() + ); + assert_eq!( + counted.queries.load(Ordering::Relaxed), + 1, + "inline rows issue no body query" + ); + + conn.execute( + "DELETE FROM session_canonical_bodies WHERE content_hash = ?1", + params![hashes[0].as_str()], + ) + .await + .unwrap(); + let error = decode_observation_json(&counted, &encoded, "hydrate fixture") + .await + .unwrap_err(); + let tracedecay_store::ObservationStoreError::Storage { source, .. } = error else { + panic!("expected missing body refusal"); + }; + assert!(matches!( + source.downcast_ref::(), + Some(CanonicalBodyError::Missing { content_hash }) if content_hash == &hashes[0] + )); + assert_eq!(counted.queries.load(Ordering::Relaxed), 2); + + persist_bodies(&conn, &bodies).await.unwrap(); + conn.execute( + "UPDATE session_canonical_bodies SET uncompressed_bytes = uncompressed_bytes + 1 + WHERE content_hash = ?1", + params![hashes[1].as_str()], + ) + .await + .unwrap(); + let error = decode_observation_json(&counted, &encoded, "hydrate fixture") + .await + .unwrap_err(); + let tracedecay_store::ObservationStoreError::Storage { source, .. } = error else { + panic!("expected corrupt body refusal"); + }; + assert!(matches!( + source.downcast_ref::(), + Some(CanonicalBodyError::Corrupt { content_hash, .. }) if content_hash == &hashes[1] + )); + assert_eq!(counted.queries.load(Ordering::Relaxed), 3); + } } diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs new file mode 100644 index 0000000000..6c5af50fea --- /dev/null +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs @@ -0,0 +1,342 @@ +//! Bounded reclamation of bodies shared by raw messages and observations. +//! +//! Writer triggers maintain a derived reference index in the same transaction +//! as each owner. Existing owners and body candidates are enrolled incrementally +//! after those triggers are installed. No body is collected before all three +//! bootstrap frontiers are complete. A candidate and its indexed references are +//! checked under one guarded writer, so another owner cannot race collection. + +use serde::{Deserialize, Serialize}; +use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_runtime_core::db::Database; +use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, params}; +use tracedecay_store::collect_body_refs; + +pub(crate) mod schema; +#[cfg(test)] +mod tests; + +const OPERATION: &str = "collect unreferenced session canonical bodies"; +// Observation records may approach 1 MiB. This matches the observation audit's +// payload page bound and leaves headroom below the SQL channel's 64 MiB cap. +pub const MAX_CANONICAL_BODY_GC_ROWS: usize = 48; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct CanonicalBodyGcReport { + pub roots_scanned: u64, + pub bodies_scanned: u64, + pub bodies_deleted: u64, + /// Stored BLOB bytes deleted, including the effect of their existing + /// compression. This is not database-file or filesystem space reclaimed. + pub stored_bytes_deleted: u64, + pub bootstrap_complete: bool, + /// Resume on the short maintenance cadence while bounded work remains. + pub has_more: bool, +} + +struct Progress { + raw_through: i64, + raw_until: i64, + observations_through: i64, + observations_until: i64, + bodies_after: String, + bodies_until: String, +} + +impl Progress { + fn complete(&self) -> bool { + self.raw_through == self.raw_until + && self.observations_through == self.observations_until + && self.bodies_after == self.bodies_until + } +} + +fn db_error(error: impl std::fmt::Display) -> TraceDecayError { + integrity_error(error.to_string()) +} + +fn integrity_error(message: impl Into) -> TraceDecayError { + TraceDecayError::Database { + operation: OPERATION.to_owned(), + message: message.into(), + } +} + +async fn read_progress(conn: &(impl QueryExecutor + ?Sized)) -> Result { + let mut rows = conn + .query( + "SELECT raw_through, raw_until, observations_through, observations_until, + bodies_after, bodies_until + FROM canonical_body_gc_progress WHERE singleton = 1", + (), + ) + .await + .map_err(db_error)?; + let row = + rows.next().await.map_err(db_error)?.ok_or_else(|| { + integrity_error("canonical body ownership bootstrap state is missing") + })?; + let progress = Progress { + raw_through: row.get(0).map_err(db_error)?, + raw_until: row.get(1).map_err(db_error)?, + observations_through: row.get(2).map_err(db_error)?, + observations_until: row.get(3).map_err(db_error)?, + bodies_after: row.get(4).map_err(db_error)?, + bodies_until: row.get(5).map_err(db_error)?, + }; + if progress.raw_through < 0 + || progress.raw_through > progress.raw_until + || progress.observations_through < 0 + || progress.observations_through > progress.observations_until + || progress.bodies_after > progress.bodies_until + { + return Err(integrity_error( + "canonical body ownership bootstrap state is invalid", + )); + } + Ok(progress) +} + +async fn save_progress(conn: &(impl Executor + ?Sized), progress: &Progress) -> Result<()> { + let changed = conn + .execute( + "UPDATE canonical_body_gc_progress + SET raw_through = ?1, observations_through = ?2, bodies_after = ?3 + WHERE singleton = 1 AND raw_until = ?4 AND observations_until = ?5 + AND bodies_until = ?6", + params![ + progress.raw_through, + progress.observations_through, + progress.bodies_after.as_str(), + progress.raw_until, + progress.observations_until, + progress.bodies_until.as_str() + ], + ) + .await + .map_err(db_error)?; + if changed != 1 { + return Err(integrity_error("canonical body ownership frontier changed")); + } + Ok(()) +} + +#[tracing::instrument(name = "global_db.canonical_bodies.gc", level = "trace", skip_all)] +pub(crate) async fn run(database: &Database, max_rows: usize) -> Result { + let transaction = database.begin_write_transaction(OPERATION).await?; + match step(&transaction, max_rows).await { + Ok(report) => { + transaction.commit().await?; + Ok(report) + } + Err(error) => match transaction.rollback().await { + Ok(()) => Err(error), + Err(rollback) => Err(integrity_error(format!( + "{error}; rollback failed: {rollback}" + ))), + }, + } +} + +async fn step(conn: &(impl Executor + ?Sized), max_rows: usize) -> Result { + schema::validate(conn).await?; + let limit = max_rows.clamp(1, MAX_CANONICAL_BODY_GC_ROWS) as i64; + let mut progress = read_progress(conn).await?; + let mut report = CanonicalBodyGcReport::default(); + if !progress.complete() { + if progress.raw_through < progress.raw_until { + report.roots_scanned = bootstrap_raw(conn, &mut progress, limit).await?; + } else if progress.observations_through < progress.observations_until { + report.roots_scanned = bootstrap_observations(conn, &mut progress, limit).await?; + } else { + report.bodies_scanned = bootstrap_bodies(conn, &mut progress, limit).await?; + } + save_progress(conn, &progress).await?; + } else { + collect_candidates(conn, limit, &mut report).await?; + } + report.bootstrap_complete = progress.complete(); + report.has_more = !report.bootstrap_complete || has_candidates(conn).await?; + Ok(report) +} + +async fn bootstrap_raw( + conn: &(impl Executor + ?Sized), + progress: &mut Progress, + limit: i64, +) -> Result { + let mut rows = conn + .query( + "SELECT store_id, CASE WHEN storage_kind = 'inline' AND content IS NULL + THEN content_hash END + FROM lcm_raw_messages WHERE store_id > ?1 AND store_id <= ?2 + ORDER BY store_id LIMIT ?3", + params![progress.raw_through, progress.raw_until, limit], + ) + .await + .map_err(db_error)?; + let mut page = Vec::new(); + while let Some(row) = rows.next().await.map_err(db_error)? { + page.push(( + row.get::(0).map_err(db_error)?, + row.get::>(1).map_err(db_error)?, + )); + } + drop(rows); + for (owner, hash) in &page { + if let Some(hash) = hash { + conn.execute( + "INSERT OR IGNORE INTO canonical_body_gc_refs(owner_kind, owner_id, content_hash) + VALUES ('raw', ?1, ?2)", + params![*owner, hash.as_str()], + ) + .await + .map_err(db_error)?; + } + progress.raw_through = *owner; + } + if page.len() < limit as usize { + progress.raw_through = progress.raw_until; + } + Ok(page.len() as u64) +} + +async fn bootstrap_observations( + conn: &(impl Executor + ?Sized), + progress: &mut Progress, + limit: i64, +) -> Result { + let mut rows = conn + .query( + "SELECT sequence, observation_json FROM observations + WHERE sequence > ?1 AND sequence <= ?2 ORDER BY sequence LIMIT ?3", + params![ + progress.observations_through, + progress.observations_until, + limit + ], + ) + .await + .map_err(db_error)?; + let mut page = Vec::new(); + while let Some(row) = rows.next().await.map_err(db_error)? { + page.push(( + row.get::(0).map_err(db_error)?, + row.get::(1).map_err(db_error)?, + )); + } + drop(rows); + for (owner, json) in &page { + for hash in collect_body_refs(json).map_err(db_error)? { + conn.execute( + "INSERT OR IGNORE INTO canonical_body_gc_refs(owner_kind, owner_id, content_hash) + VALUES ('observation', ?1, ?2)", + params![*owner, hash], + ) + .await + .map_err(db_error)?; + } + progress.observations_through = *owner; + } + if page.len() < limit as usize { + progress.observations_through = progress.observations_until; + } + Ok(page.len() as u64) +} + +async fn bootstrap_bodies( + conn: &(impl Executor + ?Sized), + progress: &mut Progress, + limit: i64, +) -> Result { + let mut rows = conn + .query( + "SELECT content_hash FROM session_canonical_bodies + WHERE content_hash > ?1 AND content_hash <= ?2 ORDER BY content_hash LIMIT ?3", + params![ + progress.bodies_after.as_str(), + progress.bodies_until.as_str(), + limit + ], + ) + .await + .map_err(db_error)?; + let mut page = Vec::new(); + while let Some(row) = rows.next().await.map_err(db_error)? { + page.push(row.get::(0).map_err(db_error)?); + } + drop(rows); + for hash in &page { + conn.execute( + "INSERT OR IGNORE INTO canonical_body_gc_candidates(content_hash) VALUES (?1)", + params![hash.as_str()], + ) + .await + .map_err(db_error)?; + progress.bodies_after.clone_from(hash); + } + if page.len() < limit as usize { + progress.bodies_after.clone_from(&progress.bodies_until); + } + Ok(page.len() as u64) +} + +async fn collect_candidates( + conn: &(impl Executor + ?Sized), + limit: i64, + report: &mut CanonicalBodyGcReport, +) -> Result<()> { + let mut rows = conn + .query( + "SELECT content_hash FROM canonical_body_gc_candidates ORDER BY content_hash LIMIT ?1", + params![limit], + ) + .await + .map_err(db_error)?; + let mut hashes = Vec::new(); + while let Some(row) = rows.next().await.map_err(db_error)? { + hashes.push(row.get::(0).map_err(db_error)?); + } + drop(rows); + report.bodies_scanned = hashes.len() as u64; + if hashes.is_empty() { + return Ok(()); + } + let hashes = serde_json::to_string(&hashes).map_err(db_error)?; + let mut deleted = conn + .query( + "DELETE FROM session_canonical_bodies + WHERE content_hash IN (SELECT value FROM json_each(?1)) + AND NOT EXISTS (SELECT 1 FROM canonical_body_gc_refs r + WHERE r.content_hash = session_canonical_bodies.content_hash) + RETURNING length(body)", + params![hashes.as_str()], + ) + .await + .map_err(db_error)?; + while let Some(row) = deleted.next().await.map_err(db_error)? { + let bytes = u64::try_from(row.get::(0).map_err(db_error)?) + .map_err(|_| integrity_error("canonical body stored byte count is negative"))?; + report.bodies_deleted += 1; + report.stored_bytes_deleted = report.stored_bytes_deleted.saturating_add(bytes); + } + drop(deleted); + // Referenced candidates are discharged too. The final owner removal + // records them again; idle ticks never revisit every retained body. + conn.execute( + "DELETE FROM canonical_body_gc_candidates + WHERE content_hash IN (SELECT value FROM json_each(?1))", + params![hashes], + ) + .await + .map_err(db_error)?; + Ok(()) +} + +async fn has_candidates(conn: &(impl QueryExecutor + ?Sized)) -> Result { + let mut rows = conn + .query("SELECT 1 FROM canonical_body_gc_candidates LIMIT 1", ()) + .await + .map_err(db_error)?; + rows.next().await.map(|row| row.is_some()).map_err(db_error) +} diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/schema.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/schema.rs new file mode 100644 index 0000000000..3d80ad0bd7 --- /dev/null +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/schema.rs @@ -0,0 +1,249 @@ +use std::collections::BTreeMap; + +use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, params}; + +use super::{db_error, integrity_error}; + +/// Derived ownership, never another copy of a body. The bootstrap progress is +/// committed with installation of every writer trigger, before deletion is +/// permitted. Admission can install this component without rewriting evidence. +pub(crate) const OBJECTS: &[(&str, &str)] = &[ + ( + "canonical_body_gc_refs", + "CREATE TABLE canonical_body_gc_refs ( + owner_kind TEXT NOT NULL CHECK(owner_kind IN ('raw', 'observation')), + owner_id INTEGER NOT NULL, + content_hash TEXT NOT NULL, + PRIMARY KEY(owner_kind, owner_id, content_hash), + FOREIGN KEY(content_hash) REFERENCES session_canonical_bodies(content_hash) + ON DELETE RESTRICT + )", + ), + ( + "canonical_body_gc_by_hash", + "CREATE INDEX canonical_body_gc_by_hash ON canonical_body_gc_refs(content_hash)", + ), + ( + "canonical_body_gc_candidates", + "CREATE TABLE canonical_body_gc_candidates ( + content_hash TEXT NOT NULL PRIMARY KEY, + FOREIGN KEY(content_hash) REFERENCES session_canonical_bodies(content_hash) + ON DELETE CASCADE + )", + ), + ( + "canonical_body_gc_progress", + "CREATE TABLE canonical_body_gc_progress ( + singleton INTEGER PRIMARY KEY CHECK(singleton = 1), + raw_through INTEGER NOT NULL CHECK(raw_through >= 0), + raw_until INTEGER NOT NULL CHECK(raw_until >= raw_through), + observations_through INTEGER NOT NULL CHECK(observations_through >= 0), + observations_until INTEGER NOT NULL CHECK(observations_until >= observations_through), + bodies_after TEXT NOT NULL, + bodies_until TEXT NOT NULL CHECK(bodies_until >= bodies_after) + )", + ), + ( + "canonical_body_gc_body_insert", + "CREATE TRIGGER canonical_body_gc_body_insert + AFTER INSERT ON session_canonical_bodies BEGIN + INSERT OR IGNORE INTO canonical_body_gc_candidates(content_hash) + VALUES (NEW.content_hash); + END", + ), + ( + "canonical_body_gc_ref_delete", + "CREATE TRIGGER canonical_body_gc_ref_delete + AFTER DELETE ON canonical_body_gc_refs BEGIN + INSERT OR IGNORE INTO canonical_body_gc_candidates(content_hash) + VALUES (OLD.content_hash); + END", + ), + ( + "canonical_body_gc_raw_insert", + "CREATE TRIGGER canonical_body_gc_raw_insert + AFTER INSERT ON lcm_raw_messages + WHEN NEW.storage_kind = 'inline' AND NEW.content IS NULL BEGIN + INSERT OR IGNORE INTO canonical_body_gc_refs(owner_kind, owner_id, content_hash) + VALUES ('raw', NEW.store_id, NEW.content_hash); + END", + ), + ( + "canonical_body_gc_raw_delete", + "CREATE TRIGGER canonical_body_gc_raw_delete + AFTER DELETE ON lcm_raw_messages BEGIN + DELETE FROM canonical_body_gc_refs + WHERE owner_kind = 'raw' AND owner_id = OLD.store_id; + END", + ), + ( + "canonical_body_gc_raw_update", + "CREATE TRIGGER canonical_body_gc_raw_update + AFTER UPDATE OF store_id, storage_kind, content, content_hash ON lcm_raw_messages + WHEN OLD.store_id IS NOT NEW.store_id + OR OLD.content_hash IS NOT NEW.content_hash + OR (OLD.storage_kind = 'inline' AND OLD.content IS NULL) + IS NOT (NEW.storage_kind = 'inline' AND NEW.content IS NULL) + BEGIN + DELETE FROM canonical_body_gc_refs + WHERE owner_kind = 'raw' AND owner_id = OLD.store_id; + INSERT OR IGNORE INTO canonical_body_gc_refs(owner_kind, owner_id, content_hash) + SELECT 'raw', NEW.store_id, NEW.content_hash + WHERE NEW.storage_kind = 'inline' AND NEW.content IS NULL; + END", + ), + ( + "canonical_body_gc_observation_insert", + "CREATE TRIGGER canonical_body_gc_observation_insert + AFTER INSERT ON observations BEGIN + SELECT CASE + WHEN NOT json_valid(NEW.observation_json) + THEN RAISE(ABORT, 'invalid canonical body reference document') + WHEN json_type(NEW.observation_json, '$._tracedecay_canonical_body_refs') + IS NOT NULL AND json_type(NEW.observation_json, + '$._tracedecay_canonical_body_refs') <> 'object' + THEN RAISE(ABORT, 'invalid canonical body reference map') + WHEN EXISTS (SELECT 1 FROM json_each(NEW.observation_json, + '$._tracedecay_canonical_body_refs') WHERE type <> 'text') + THEN RAISE(ABORT, 'invalid canonical body reference hash') + END; + INSERT OR IGNORE INTO canonical_body_gc_refs(owner_kind, owner_id, content_hash) + SELECT 'observation', NEW.sequence, value + FROM json_each(NEW.observation_json, '$._tracedecay_canonical_body_refs'); + END", + ), + ( + "canonical_body_gc_observation_delete", + "CREATE TRIGGER canonical_body_gc_observation_delete + AFTER DELETE ON observations BEGIN + DELETE FROM canonical_body_gc_refs + WHERE owner_kind = 'observation' AND owner_id = OLD.sequence; + END", + ), + ( + "canonical_body_gc_observation_update", + "CREATE TRIGGER canonical_body_gc_observation_update + AFTER UPDATE OF sequence, observation_json ON observations + WHEN OLD.sequence IS NOT NEW.sequence OR OLD.observation_json IS NOT NEW.observation_json + BEGIN + SELECT CASE + WHEN NOT json_valid(NEW.observation_json) + THEN RAISE(ABORT, 'invalid canonical body reference document') + WHEN json_type(NEW.observation_json, '$._tracedecay_canonical_body_refs') + IS NOT NULL AND json_type(NEW.observation_json, + '$._tracedecay_canonical_body_refs') <> 'object' + THEN RAISE(ABORT, 'invalid canonical body reference map') + WHEN EXISTS (SELECT 1 FROM json_each(NEW.observation_json, + '$._tracedecay_canonical_body_refs') WHERE type <> 'text') + THEN RAISE(ABORT, 'invalid canonical body reference hash') + END; + DELETE FROM canonical_body_gc_refs + WHERE owner_kind = 'observation' AND owner_id = OLD.sequence; + INSERT OR IGNORE INTO canonical_body_gc_refs(owner_kind, owner_id, content_hash) + SELECT 'observation', NEW.sequence, value + FROM json_each(NEW.observation_json, '$._tracedecay_canonical_body_refs'); + END", + ), +]; + +async fn inventory( + conn: &(impl QueryExecutor + ?Sized), +) -> tracedecay_domain::errors::Result> { + let mut rows = conn + .query( + "SELECT name, sql FROM sqlite_master + WHERE name GLOB 'canonical_body_gc_*' ORDER BY name LIMIT ?1", + params![OBJECTS.len() as i64 + 1], + ) + .await + .map_err(db_error)?; + let mut objects = BTreeMap::new(); + while let Some(row) = rows.next().await.map_err(db_error)? { + objects.insert( + row.get::(0).map_err(db_error)?, + row.get::(1).map_err(db_error)?, + ); + } + Ok(objects) +} + +fn require_inventory(objects: &BTreeMap) -> tracedecay_domain::errors::Result<()> { + if objects.len() != OBJECTS.len() { + return Err(integrity_error( + "canonical body ownership schema is incomplete", + )); + } + for &(name, expected) in OBJECTS { + // These objects are installed from these exact statements. Preserve + // case inside JSON paths and owner literals: case-folding a changed + // trigger could hide a reference-tracking failure. + if objects + .get(name) + .is_none_or(|actual| actual.trim().trim_end_matches(';') != expected) + { + return Err(integrity_error(format!( + "canonical body ownership object {name} does not match its contract" + ))); + } + } + Ok(()) +} + +pub(super) async fn validate( + conn: &(impl QueryExecutor + ?Sized), +) -> tracedecay_domain::errors::Result<()> { + require_inventory(&inventory(conn).await?) +} + +pub(crate) async fn install( + conn: &(impl Executor + ?Sized), +) -> tracedecay_domain::errors::Result<()> { + let objects = inventory(conn).await?; + if !objects.is_empty() { + require_inventory(&objects)?; + super::read_progress(conn).await?; + return Ok(()); + } + // The keyset bootstrap begins at zero. INTEGER PRIMARY KEY alone does + // not forbid hand-written nonpositive identities, so prove the frontier + // cannot skip an old owner before installing its collection authority. + let mut minima = conn + .query( + "SELECT (SELECT MIN(store_id) FROM lcm_raw_messages), + (SELECT MIN(sequence) FROM observations)", + (), + ) + .await + .map_err(db_error)?; + let minima = minima + .next() + .await + .map_err(db_error)? + .ok_or_else(|| integrity_error("canonical body ownership minima are missing"))?; + for index in 0..2 { + if minima + .get::>(index) + .map_err(db_error)? + .is_some_and(|id| id <= 0) + { + return Err(integrity_error( + "canonical body ownership requires positive source identities", + )); + } + } + for &(_, sql) in OBJECTS { + conn.execute_batch(sql).await.map_err(db_error)?; + } + conn.execute( + "INSERT INTO canonical_body_gc_progress ( + singleton, raw_through, raw_until, observations_through, observations_until, + bodies_after, bodies_until + ) SELECT 1, 0, COALESCE((SELECT MAX(store_id) FROM lcm_raw_messages), 0), + 0, COALESCE((SELECT MAX(sequence) FROM observations), 0), + '', COALESCE((SELECT MAX(content_hash) FROM session_canonical_bodies), '')", + (), + ) + .await + .map_err(db_error)?; + Ok(()) +} diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs new file mode 100644 index 0000000000..9da4f65432 --- /dev/null +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs @@ -0,0 +1,508 @@ +use tracedecay_runtime_core::db::engine::{ + Connection, Executor, QueryExecutor, TestConnection, TransactionBehavior, params, +}; +use tracedecay_store::{ + BODY_REF_KEY, CANONICAL_BODIES_TABLE_SQL, StoredCanonicalBody, UPSERT_CANONICAL_BODY_SQL, +}; + +use super::{CanonicalBodyGcReport, has_candidates, read_progress, schema, step}; + +struct Store { + runtime: TestConnection, + _directory: tempfile::TempDir, +} + +impl Store { + async fn open() -> Self { + let directory = tempfile::tempdir().unwrap(); + let runtime = TestConnection::open(&directory.path().join("canonical-bodies.db")); + runtime + .execute_batch(CANONICAL_BODIES_TABLE_SQL) + .await + .unwrap(); + runtime + .execute_batch( + "CREATE TABLE lcm_raw_messages ( + store_id INTEGER PRIMARY KEY AUTOINCREMENT, + storage_kind TEXT NOT NULL, content TEXT, content_hash TEXT NOT NULL + ); + CREATE TABLE observations ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + observation_json TEXT NOT NULL + );", + ) + .await + .unwrap(); + Self { + runtime, + _directory: directory, + } + } + + fn conn(&self) -> &Connection { + &self.runtime + } + + async fn install(&self) { + let transaction = self + .conn() + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + schema::install(&transaction).await.unwrap(); + transaction.commit().await.unwrap(); + } + + async fn tick(&self, limit: usize) -> CanonicalBodyGcReport { + let transaction = self + .conn() + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let report = step(&transaction, limit).await.unwrap(); + transaction.commit().await.unwrap(); + report + } +} + +async fn insert_body(conn: &(impl Executor + ?Sized), body: &StoredCanonicalBody) { + conn.execute( + UPSERT_CANONICAL_BODY_SQL, + params![ + body.content_hash.as_str(), + body.encoding, + body.blob.as_slice(), + body.uncompressed_bytes + ], + ) + .await + .unwrap(); +} + +async fn body_exists(conn: &(impl QueryExecutor + ?Sized), hash: &str) -> bool { + conn.query( + "SELECT 1 FROM session_canonical_bodies WHERE content_hash = ?1", + params![hash], + ) + .await + .unwrap() + .next() + .await + .unwrap() + .is_some() +} + +fn observation(hash: &str) -> String { + serde_json::json!({ + "payload": {"text": null}, + (BODY_REF_KEY): {"/payload/text": hash} + }) + .to_string() +} + +async fn drain(store: &Store, limit: usize) -> (u64, u64) { + let mut deleted = 0; + let mut bytes = 0; + for _ in 0..32 { + let report = store.tick(limit).await; + assert!(report.roots_scanned <= limit as u64); + assert!(report.bodies_scanned <= limit as u64); + if !report.bootstrap_complete { + assert_eq!(report.bodies_deleted, 0, "bootstrap never collects"); + } + deleted += report.bodies_deleted; + bytes += report.stored_bytes_deleted; + if !report.has_more { + return (deleted, bytes); + } + } + panic!("bounded fixture must converge"); +} + +#[asupersync::test] +async fn bootstrap_and_owner_release_preserve_shared_bodies_until_the_final_reference() { + let store = Store::open().await; + let shared = + StoredCanonicalBody::pack("shared complete text ".repeat(1024).as_bytes()).unwrap(); + let orphan = + StoredCanonicalBody::pack("unreferenced old text ".repeat(1024).as_bytes()).unwrap(); + insert_body(store.conn(), &shared).await; + insert_body(store.conn(), &orphan).await; + store.conn().execute( + "INSERT INTO lcm_raw_messages(storage_kind, content, content_hash) VALUES ('inline', NULL, ?1)", + params![shared.content_hash.as_str()], + ).await.unwrap(); + store + .conn() + .execute( + "INSERT INTO observations(observation_json) VALUES (?1)", + params![observation(&shared.content_hash)], + ) + .await + .unwrap(); + store.install().await; + + let first = store.tick(1).await; + assert_eq!(first.roots_scanned, 1); + assert!( + !first.bootstrap_complete, + "observation and body seed are still pending" + ); + assert_eq!(first.bodies_deleted, 0); + assert!(body_exists(store.conn(), &orphan.content_hash).await); + let (deleted, bytes) = drain(&store, 1).await; + assert_eq!(deleted, 1); + assert_eq!(bytes, orphan.blob.len() as u64); + assert!(body_exists(store.conn(), &shared.content_hash).await); + assert!( + store + .conn() + .execute( + "DELETE FROM session_canonical_bodies WHERE content_hash = ?1", + params![shared.content_hash.as_str()] + ) + .await + .is_err(), + "the reference index also enforces its body foreign key" + ); + + store + .conn() + .execute("DELETE FROM lcm_raw_messages", ()) + .await + .unwrap(); + assert_eq!( + store.tick(1).await.bodies_deleted, + 0, + "observation still owns the body" + ); + store.conn().execute("UPDATE observations SET observation_json = '{\"__retention_released\":\"observation\"}'", ()).await.unwrap(); + let released = store.tick(1).await; + assert_eq!(released.bodies_deleted, 1); + assert_eq!(released.stored_bytes_deleted, shared.blob.len() as u64); + assert!(released.stored_bytes_deleted < shared.uncompressed_bytes as u64); + assert!(!released.has_more); +} + +#[asupersync::test] +async fn bootstrap_survives_interruption_and_tracks_owners_changed_between_pages() { + let store = Store::open().await; + let old = StoredCanonicalBody::pack("old complete body ".repeat(1024).as_bytes()).unwrap(); + let new = StoredCanonicalBody::pack("new complete body ".repeat(1024).as_bytes()).unwrap(); + insert_body(store.conn(), &old).await; + store + .conn() + .execute( + "INSERT INTO lcm_raw_messages(storage_kind, content_hash) VALUES ('inline', ?1)", + params![old.content_hash.as_str()], + ) + .await + .unwrap(); + store + .conn() + .execute( + "INSERT INTO observations(observation_json) VALUES (?1)", + params![observation(&old.content_hash)], + ) + .await + .unwrap(); + store.install().await; + + let transaction = store + .conn() + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + assert_eq!(step(&transaction, 1).await.unwrap().roots_scanned, 1); + transaction.rollback().await.unwrap(); + assert_eq!( + read_progress(store.conn()).await.unwrap().raw_through, + 0, + "interruption rolls back the ownership page and frontier together" + ); + assert!(body_exists(store.conn(), &old.content_hash).await); + assert_eq!(store.tick(1).await.roots_scanned, 1); + + let transaction = store + .conn() + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + insert_body(&transaction, &new).await; + transaction + .execute( + "INSERT INTO lcm_raw_messages(storage_kind, content_hash) VALUES ('inline', ?1)", + params![new.content_hash.as_str()], + ) + .await + .unwrap(); + transaction + .execute("DELETE FROM lcm_raw_messages WHERE store_id = 1", ()) + .await + .unwrap(); + transaction + .execute( + "UPDATE observations SET observation_json = ?1", + params![observation(&new.content_hash)], + ) + .await + .unwrap(); + transaction.commit().await.unwrap(); + + let (deleted, _) = drain(&store, 1).await; + assert_eq!(deleted, 1); + assert!(!body_exists(store.conn(), &old.content_hash).await); + assert!( + body_exists(store.conn(), &new.content_hash).await, + "owners above the initial frontier are tracked by writer triggers" + ); + store + .conn() + .execute("DELETE FROM observations", ()) + .await + .unwrap(); + assert_eq!( + store.tick(1).await.bodies_deleted, + 0, + "the raw owner added during bootstrap survives" + ); +} + +#[asupersync::test] +async fn damaged_guards_and_unresolvable_old_references_block_collection() { + let store = Store::open().await; + let orphan = StoredCanonicalBody::pack(b"unreferenced body").unwrap(); + insert_body(store.conn(), &orphan).await; + store + .conn() + .execute( + "INSERT INTO observations(observation_json) VALUES (?1)", + params![observation("missing-body")], + ) + .await + .unwrap(); + store.install().await; + let transaction = store + .conn() + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + assert!( + step(&transaction, 1).await.is_err(), + "missing referenced body must not earn bootstrap completeness" + ); + transaction.rollback().await.unwrap(); + assert_eq!( + read_progress(store.conn()) + .await + .unwrap() + .observations_through, + 0 + ); + assert!(body_exists(store.conn(), &orphan.content_hash).await); + store + .conn() + .execute_batch("DROP TRIGGER canonical_body_gc_observation_insert") + .await + .unwrap(); + let transaction = store + .conn() + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let error = step(&transaction, 1).await.unwrap_err(); + assert!(error.to_string().contains("schema is incomplete")); + transaction.rollback().await.unwrap(); + assert!(body_exists(store.conn(), &orphan.content_hash).await); +} + +#[asupersync::test] +async fn registered_collection_commits_and_lost_authority_rolls_back_deleted_bodies() { + let mut harness = + crate::tests::harness::RegisteredGlobalDbHarness::open("canonical-body-gc").await; + let database = harness.registered.clone(); + let first = StoredCanonicalBody::pack(b"first unowned canonical body").unwrap(); + let retained = StoredCanonicalBody::pack(b"body whose collection is revoked").unwrap(); + let transaction = database.begin_write_transaction().await.unwrap(); + insert_body(&transaction, &first).await; + transaction.commit().await.unwrap(); + let report = database.run_canonical_body_gc(1).await.unwrap(); + assert!(report.bootstrap_complete); + assert_eq!(report.bodies_deleted, 1); + assert_eq!(report.stored_bytes_deleted, first.blob.len() as u64); + + let transaction = database.begin_write_transaction().await.unwrap(); + insert_body(&transaction, &retained).await; + transaction.commit().await.unwrap(); + let transaction = database.begin_write_transaction().await.unwrap(); + assert_eq!(step(&transaction, 1).await.unwrap().bodies_deleted, 1); + harness.revoke(); + assert!( + transaction.commit().await.is_err(), + "revoked writer cannot publish collection" + ); + assert!(body_exists(&database.read_connection(), &retained.content_hash).await); + assert!(database.run_canonical_body_gc(1).await.is_err()); +} + +#[asupersync::test] +async fn registered_legacy_bootstrap_and_live_references_survive_restarts() { + let mut harness = + crate::tests::harness::RegisteredGlobalDbHarness::open("canonical-body-gc-restart").await; + let live = StoredCanonicalBody::pack(b"retained by the registered raw owner").unwrap(); + let orphan = StoredCanonicalBody::pack(b"unowned before additive installation").unwrap(); + let transaction = harness.registered.begin_write_transaction().await.unwrap(); + insert_body(&transaction, &live).await; + insert_body(&transaction, &orphan).await; + transaction + .execute_batch( + "INSERT INTO sessions(provider, session_id, project_key, project_path) + VALUES ('codex', 'gc-session', '/gc-project', '/gc-project');", + ) + .await + .unwrap(); + transaction + .execute( + "INSERT INTO lcm_raw_messages(provider, message_id, session_id, role, ordinal, + content, content_hash, storage_kind) + VALUES ('codex', 'gc-message', 'gc-session', 'assistant', 1, NULL, ?1, 'inline')", + params![live.content_hash.as_str()], + ) + .await + .unwrap(); + // Model an older admitted store: remove only derived GC objects while + // retaining canonical bodies and both source-root tables. + for &(name, sql) in schema::OBJECTS.iter().rev() { + let kind = if sql.starts_with("CREATE TRIGGER") { + "TRIGGER" + } else if sql.starts_with("CREATE INDEX") { + "INDEX" + } else { + "TABLE" + }; + transaction + .execute_batch(&format!("DROP {kind} {name}")) + .await + .unwrap(); + } + transaction.commit().await.unwrap(); + harness = harness.restart().await; + let report = harness.registered.run_canonical_body_gc(1).await.unwrap(); + assert_eq!(report.roots_scanned, 1); + assert!(!report.bootstrap_complete); + assert_eq!(report.bodies_deleted, 0); + let before = read_progress(&harness.registered.read_connection()) + .await + .unwrap(); + harness = harness.restart().await; + let after = read_progress(&harness.registered.read_connection()) + .await + .unwrap(); + assert_eq!(after.raw_through, before.raw_through); + assert_eq!(after.bodies_after, before.bodies_after); + schema::validate(&harness.registered.read_connection()) + .await + .unwrap(); + let transaction = harness.registered.begin_write_transaction().await.unwrap(); + assert!( + transaction + .execute( + "DELETE FROM session_canonical_bodies WHERE content_hash = ?1", + params![live.content_hash.as_str()], + ) + .await + .is_err(), + "the registered production writer enforces the owner foreign key" + ); + transaction.rollback().await.unwrap(); + let seed = harness.registered.run_canonical_body_gc(1).await.unwrap(); + assert_eq!(seed.bodies_scanned, 1); + harness = harness.restart().await; + assert!( + has_candidates(&harness.registered.read_connection()) + .await + .unwrap(), + "candidate seed persists across reopen" + ); + let mut deleted = 0; + for _ in 0..8 { + let report = harness.registered.run_canonical_body_gc(1).await.unwrap(); + deleted += report.bodies_deleted; + if !report.has_more { + break; + } + } + assert_eq!(deleted, 1); + assert!(body_exists(&harness.registered.read_connection(), &live.content_hash).await); + assert!(!body_exists(&harness.registered.read_connection(), &orphan.content_hash).await); + let transaction = harness.registered.begin_write_transaction().await.unwrap(); + transaction + .execute( + "DELETE FROM lcm_raw_messages WHERE provider = 'codex' AND message_id = 'gc-message'", + (), + ) + .await + .unwrap(); + transaction.commit().await.unwrap(); + assert_eq!( + harness + .registered + .run_canonical_body_gc(1) + .await + .unwrap() + .bodies_deleted, + 1 + ); +} + +#[asupersync::test] +async fn nonpositive_legacy_owner_ids_cannot_be_skipped_by_bootstrap() { + for raw_owner in [true, false] { + let store = Store::open().await; + let body = StoredCanonicalBody::pack(b"retained legacy body").unwrap(); + insert_body(store.conn(), &body).await; + if raw_owner { + store + .conn() + .execute( + "INSERT INTO lcm_raw_messages(store_id, storage_kind, content, content_hash) + VALUES (0, 'inline', NULL, ?1)", + params![body.content_hash.as_str()], + ) + .await + .unwrap(); + } else { + store + .conn() + .execute( + "INSERT INTO observations(sequence, observation_json) VALUES (-1, ?1)", + params![observation(&body.content_hash)], + ) + .await + .unwrap(); + } + let transaction = store + .conn() + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let error = schema::install(&transaction).await.unwrap_err(); + assert!(error.to_string().contains("positive source identities")); + transaction.rollback().await.unwrap(); + assert!(body_exists(store.conn(), &body.content_hash).await); + assert!( + store + .conn() + .query( + "SELECT 1 FROM sqlite_master WHERE name GLOB 'canonical_body_gc_*' LIMIT 1", + () + ) + .await + .unwrap() + .next() + .await + .unwrap() + .is_none() + ); + } +} diff --git a/crates/tracedecay-global-db/src/observation/mod.rs b/crates/tracedecay-global-db/src/observation/mod.rs index 9b7d176b21..a7b4c667ab 100644 --- a/crates/tracedecay-global-db/src/observation/mod.rs +++ b/crates/tracedecay-global-db/src/observation/mod.rs @@ -1,4 +1,5 @@ mod canonical_bodies; +pub(crate) mod canonical_body_gc; mod codec; mod persist; mod refusal_census; @@ -7,6 +8,7 @@ mod schema; pub(crate) use canonical_bodies::compact_lcm_bodies; pub use canonical_bodies::decode_observation_json; +pub use canonical_body_gc::{CanonicalBodyGcReport, MAX_CANONICAL_BODY_GC_ROWS}; pub use refusal_census::ingest_refusal_read_from_censuses; pub(super) use schema::{OBSERVATION_ADMISSION_MARKERS, ensure_observation_schema}; diff --git a/crates/tracedecay-global-db/src/observation/retention.rs b/crates/tracedecay-global-db/src/observation/retention.rs index 36c2154bcd..33efbdae93 100644 --- a/crates/tracedecay-global-db/src/observation/retention.rs +++ b/crates/tracedecay-global-db/src/observation/retention.rs @@ -240,7 +240,8 @@ pub struct ObservationRetentionPhaseReport { pub eligible: u64, /// Rows actually released (`0` in a dry run). pub acted: u64, - /// Bytes of payload reclaimed from the database by this pass. + /// Logical UTF-8 payload bytes removed, excluding replacement markers. + /// Page and file reclamation are measured separately. pub bytes_reclaimed: u64, /// Oldest governing disposition timestamp among the bounded eligible rows. #[serde(default)] @@ -606,11 +607,9 @@ async fn execute_required(executor: &(impl Executor + ?Sized), sql: &str) -> Res .map_err(db_error) } -/// Reclaimed bytes for one released column: the original length minus the -/// compact marker that replaces it (saturating so a payload already smaller -/// than the marker never underflows). -fn reclaimed_bytes(original_len: u64, marker: &str) -> u64 { - original_len.saturating_sub(marker.len() as u64) +/// Logical payload reduction across the columns replaced with a marker. +fn reclaimed_bytes(original_len: u64, marker: &str, marker_columns: u64) -> u64 { + original_len.saturating_sub((marker.len() as u64).saturating_mul(marker_columns)) } /// One row whose payload a pass selected for release. @@ -626,6 +625,7 @@ struct PayloadRelease { drop_trigger: &'static str, create_trigger: &'static str, marker: &'static str, + marker_columns: u64, /// `UPDATE … RETURNING id` over `{ids}` that re-checks each row's /// eligibility (`?1` is the marker, `?2` the disposition cutoff). update: String, @@ -673,7 +673,9 @@ async fn run_release_pass( if !mode.is_apply() { report.bytes_reclaimed = targets .iter() - .map(|target| reclaimed_bytes(target.original_len, release.marker)) + .map(|target| { + reclaimed_bytes(target.original_len, release.marker, release.marker_columns) + }) .sum(); return Ok(report); } @@ -733,7 +735,9 @@ async fn release_payload_batch( chunk .iter() .filter(|target| released.contains(&target.id)) - .map(|target| reclaimed_bytes(target.original_len, release.marker)) + .map(|target| { + reclaimed_bytes(target.original_len, release.marker, release.marker_columns) + }) .sum(), ); } @@ -819,7 +823,7 @@ async fn run_anchor_pass( }; let cutoff = release_cutoff(window, now); let sql = format!( - "SELECT a.anchor_id, LENGTH(a.anchor_json) AS len, + "SELECT a.anchor_id, LENGTH(CAST(a.anchor_json AS BLOB)) AS len, ( SELECT d.effective_at FROM retrieval_anchor_dispositions d @@ -840,6 +844,7 @@ async fn run_anchor_pass( drop_trigger: DROP_ANCHOR_UPDATE_TRIGGER, create_trigger: CREATE_ANCHOR_UPDATE_TRIGGER, marker: ANCHOR_RELEASED_MARKER, + marker_columns: 1, update: format!( "UPDATE retrieval_anchors AS a SET anchor_json = ?1 WHERE a.anchor_id IN ({{ids}}) @@ -900,7 +905,7 @@ async fn run_observation_pass( // rewrites the payload, and restores the exact canonical trigger before // commit. let sql = format!( - "SELECT o.observation_id, LENGTH(o.observation_json) AS len, + "SELECT o.observation_id, LENGTH(CAST(o.observation_json AS BLOB)) AS len, released.effective_at FROM observations o JOIN ( @@ -932,6 +937,7 @@ async fn run_observation_pass( drop_trigger: DROP_OBSERVATION_UPDATE_TRIGGER, create_trigger: CREATE_OBSERVATION_UPDATE_TRIGGER, marker: OBSERVATION_RELEASED_MARKER, + marker_columns: 1, update: format!( "UPDATE observations AS o SET observation_json = ?1 WHERE o.observation_id IN ({{ids}}) @@ -967,7 +973,8 @@ async fn run_provenance_pass( // IS NULL)` satisfied. let sql = format!( "SELECT p.observation_id, - LENGTH(p.availability_json) + LENGTH(COALESCE(p.capture_json, '')) AS len, + LENGTH(CAST(p.availability_json AS BLOB)) + + LENGTH(CAST(COALESCE(p.capture_json, '') AS BLOB)) AS len, ( SELECT d.effective_at FROM retrieval_anchor_dispositions d @@ -990,6 +997,7 @@ async fn run_provenance_pass( drop_trigger: DROP_PROVENANCE_UPDATE_TRIGGER, create_trigger: CREATE_PROVENANCE_UPDATE_TRIGGER, marker: PROVENANCE_RELEASED_MARKER, + marker_columns: 2, update: format!( "UPDATE observation_repository_provenance AS p SET availability_json = ?1, capture_json = ?1 diff --git a/crates/tracedecay-global-db/src/observation/retention/tests.rs b/crates/tracedecay-global-db/src/observation/retention/tests.rs index 91add69739..c3afed8fdc 100644 --- a/crates/tracedecay-global-db/src/observation/retention/tests.rs +++ b/crates/tracedecay-global-db/src/observation/retention/tests.rs @@ -101,6 +101,16 @@ async fn seed_evidence( generation: &str, size: usize, ) -> Result<(), String> { + seed_evidence_with_text(conn, anchor_id, generation, &"x".repeat(size)).await +} + +async fn seed_evidence_with_text( + conn: &RetentionTestStore, + anchor_id: &str, + generation: &str, + text: &str, +) -> Result<(), String> { + let payload = |field: &str| serde_json::json!({ (field): text }).to_string(); let observation_id = format!("obs-{anchor_id}"); let receipt_id = format!("receipt-{anchor_id}"); conn.execute( @@ -117,12 +127,18 @@ async fn seed_evidence( params![ observation_id.as_str(), receipt_id.as_str(), - payload("body", size) + payload("body") ], ) .await .map_err(|err| format!("insert observation: {err}"))?; - insert_anchor(conn, anchor_id, generation, size).await?; + conn.execute( + "INSERT INTO retrieval_anchors(anchor_id, anchor_json, owner_json, projection_generation) + VALUES (?1, ?2, ?3, ?4)", + params![anchor_id, payload("target"), OWNER, generation], + ) + .await + .map_err(|err| format!("insert anchor: {err}"))?; conn.execute( "INSERT INTO observation_retrieval_anchors(observation_id, anchor_id) VALUES (?1, ?2)", @@ -136,8 +152,8 @@ async fn seed_evidence( VALUES (?1, ?2, ?3, ?4, ?5)", params![ observation_id.as_str(), - payload("avail", size), - payload("capture", size), + payload("avail"), + payload("capture"), anchor_id, OWNER ], @@ -702,6 +718,90 @@ async fn reports_measurable_reclaim_metrics() -> Result<(), String> { Ok(()) } +async fn evidence_payloads(conn: &RetentionTestStore) -> Result<[String; 4], String> { + let mut rows = conn + .query( + "SELECT a.anchor_json, o.observation_json, p.availability_json, p.capture_json + FROM retrieval_anchors a + JOIN observation_retrieval_anchors b ON b.anchor_id = a.anchor_id + JOIN observations o ON o.observation_id = b.observation_id + JOIN observation_repository_provenance p ON p.observation_id = o.observation_id + WHERE a.anchor_id = 'measured'", + (), + ) + .await + .map_err(|error| error.to_string())?; + let row = rows + .next() + .await + .map_err(|error| error.to_string())? + .ok_or("evidence payload row")?; + Ok([ + row.get(0).map_err(|error| error.to_string())?, + row.get(1).map_err(|error| error.to_string())?, + row.get(2).map_err(|error| error.to_string())?, + row.get(3).map_err(|error| error.to_string())?, + ]) +} + +#[asupersync::test] +async fn reports_utf8_payload_reduction_after_every_replacement_marker() -> Result<(), String> { + for text in ["evidence".repeat(64), "é漢🙂".repeat(64), String::new()] { + let conn = test_store().await; + seed_evidence_with_text(&conn, "measured", GEN, &text).await?; + set_disposition(&conn, "measured", "deleted", NOW - 90 * DAY, None).await?; + let before = evidence_payloads(&conn).await?; + let dry = conn + .database() + .run_observation_retention( + None, + &released_config(), + RetentionMode::DryRun, + UtcMicros(NOW), + ) + .await + .map_err(|error| error.to_string())?; + assert_eq!(evidence_payloads(&conn).await?, before); + + let applied = run_apply(&conn, None, &released_config()).await?; + let after = evidence_payloads(&conn).await?; + assert_eq!(after[0], ANCHOR_RELEASED_MARKER); + assert_eq!(after[1], OBSERVATION_RELEASED_MARKER); + assert_eq!(after[2], PROVENANCE_RELEASED_MARKER); + assert_eq!(after[3], PROVENANCE_RELEASED_MARKER); + let lengths = |payloads: &[String; 4]| { + [ + payloads[0].len(), + payloads[1].len(), + payloads[2].len() + payloads[3].len(), + ] + }; + let expected = lengths(&before) + .into_iter() + .zip(lengths(&after)) + .map(|(before, after)| before.saturating_sub(after) as u64) + .collect::>(); + for report in [&dry, &applied] { + assert_eq!( + [ + report.anchors_released.bytes_reclaimed, + report.observations_released.bytes_reclaimed, + report.provenance_released.bytes_reclaimed, + ], + expected.as_slice(), + ); + assert_eq!(report.bytes_reclaimed(), expected.iter().sum::()); + } + assert_eq!(dry.anchors_released.acted, 0); + assert_eq!(dry.observations_released.acted, 0); + assert_eq!(dry.provenance_released.acted, 0); + assert_eq!(applied.anchors_released.acted, 1); + assert_eq!(applied.observations_released.acted, 1); + assert_eq!(applied.provenance_released.acted, 1); + } + Ok(()) +} + // Retention is generation-scoped: only the targeted generation is released. #[asupersync::test] async fn retention_is_generation_scoped() -> Result<(), String> { diff --git a/crates/tracedecay-global-db/src/registered.rs b/crates/tracedecay-global-db/src/registered.rs index b95c02f80a..64dfb5ef25 100644 --- a/crates/tracedecay-global-db/src/registered.rs +++ b/crates/tracedecay-global-db/src/registered.rs @@ -781,6 +781,27 @@ impl RegisteredGlobalDb { .map_err(|error| registered_error("run registered session retention", error)) } + /// Advances one bounded ownership-bootstrap or orphan-collection page. + /// All body and reference mutations share the guarded database writer. + pub async fn run_canonical_body_gc( + &self, + max_rows: usize, + ) -> tracedecay_domain::errors::Result { + crate::observation::canonical_body_gc::run(&self.database, max_rows).await + } + + /// Removes old exact replay-string duplicates using verified raw bodies, + /// without changing summary evidence or the effective replay message. + pub async fn compact_replay_metadata( + &self, + max_rows: usize, + ) -> tracedecay_domain::errors::Result + { + tracedecay_lcm::retention::compact_replay_metadata(&self.database, max_rows) + .await + .map_err(|error| registered_error("compact duplicate LCM replay metadata", error)) + } + #[tracing::instrument( name = "global_db.registered.retention.observations", level = "trace", diff --git a/crates/tracedecay-global-db/src/registered_accounting.rs b/crates/tracedecay-global-db/src/registered_accounting.rs index ad42cce109..f86b063337 100644 --- a/crates/tracedecay-global-db/src/registered_accounting.rs +++ b/crates/tracedecay-global-db/src/registered_accounting.rs @@ -87,6 +87,14 @@ impl RegisteredGlobalDb { after_tokens: u64, timestamp: i64, ) -> tracedecay_domain::errors::Result<()> { + // SQLite's signed INTEGER cannot represent every u64. Refuse before + // writing rather than wrapping a valid token count into a negative. + let before_tokens = i64::try_from(before_tokens).map_err(|error| { + global_db_operation_error("encode savings ledger before tokens", error) + })?; + let after_tokens = i64::try_from(after_tokens).map_err(|error| { + global_db_operation_error("encode savings ledger after tokens", error) + })?; let project_path = RegisteredGlobalDb::canonical_project_key(Path::new(project_path)); let tool_name = tool_name.to_owned(); self.transact(async move |transaction| { @@ -99,8 +107,8 @@ impl RegisteredGlobalDb { timestamp, project_path.as_str(), tool_name.as_str(), - before_tokens as i64, - after_tokens as i64 + before_tokens, + after_tokens ], ) .await diff --git a/crates/tracedecay-global-db/src/schema_contract/mod.rs b/crates/tracedecay-global-db/src/schema_contract/mod.rs index 7e6d8a9d39..1aed76024b 100644 --- a/crates/tracedecay-global-db/src/schema_contract/mod.rs +++ b/crates/tracedecay-global-db/src/schema_contract/mod.rs @@ -130,6 +130,13 @@ fn hash_admitted_schema(omit: Option<(&str, &str)>) -> (String, bool) { )); } } + for (name, sql) in crate::observation::canonical_body_gc::schema::OBJECTS { + parts.push(format!("canonical_body_gc:{name}:{sql}")); + } + parts.push(format!( + "lcm_fts_update:{}", + tracedecay_lcm::schema::RAW_FTS_UPDATE_TRIGGER_SQL + )); for (table, columns) in TEMPORAL_TABLE_COLUMNS { parts.push(format!("temporal:{table}")); for column in *columns { @@ -172,6 +179,7 @@ pub use invariants::{ pub(crate) use invariants::{ensure_authority_invariants, ensure_fresh_authority_invariants}; pub use validation::validate_registry_schema_contract; pub(crate) use validation::{ - validate_authority_schema_contract, validate_remote_deletion_schema_contract, - validate_session_graph_publication_schema_contract, validate_session_temporal_schema_contract, + validate_authority_schema_contract, validate_existing_observation_table, + validate_remote_deletion_schema_contract, validate_session_graph_publication_schema_contract, + validate_session_temporal_schema_contract, }; diff --git a/crates/tracedecay-global-db/src/schema_contract/validation.rs b/crates/tracedecay-global-db/src/schema_contract/validation.rs index fb3f0b593e..ae0a81e2d6 100644 --- a/crates/tracedecay-global-db/src/schema_contract/validation.rs +++ b/crates/tracedecay-global-db/src/schema_contract/validation.rs @@ -378,6 +378,28 @@ async fn validate_named_tables_and_indexes( validate_contracts(conn, &contracts).await } +/// Reuse the final authority table contract before recovery work can install +/// maintenance triggers on an existing incompatible observation table. A +/// missing table remains installable; its schema has no rows to protect. +pub(crate) async fn validate_existing_observation_table( + conn: &impl QueryExecutor, +) -> tracedecay_domain::errors::Result<()> { + let metadata = read_table_metadata(conn, &["observations"]).await?; + let actual = metadata.get("observations").ok_or_else(|| { + global_db_operation_message(OPERATION, "observation table inventory is unavailable") + })?; + if actual.columns.is_empty() { + return Ok(()); + } + let contract = TABLES + .iter() + .find(|table| table.name == "observations") + .ok_or_else(|| { + global_db_operation_message(OPERATION, "observation table contract is missing") + })?; + validate_table(contract, actual) +} + async fn validate_contracts( conn: &impl QueryExecutor, contracts: &[&Table], diff --git a/crates/tracedecay-global-db/src/schema_stages.rs b/crates/tracedecay-global-db/src/schema_stages.rs index 525f78dd41..d6dee305a8 100644 --- a/crates/tracedecay-global-db/src/schema_stages.rs +++ b/crates/tracedecay-global-db/src/schema_stages.rs @@ -9,7 +9,8 @@ use super::schema_contract::{ ensure_authority_invariant_schema, ensure_authority_invariants, ensure_fresh_authority_invariants, require_foreign_key_audit, validate_authority_rows_exhaustive, validate_authority_schema_contract, - validate_registry_schema_contract, validate_remote_deletion_schema_contract, + validate_existing_observation_table, validate_registry_schema_contract, + validate_remote_deletion_schema_contract, }; use super::{ configuration, git_index_transactions, global_db_operation_error, global_db_operation_message, @@ -702,6 +703,26 @@ async fn git_correlation_schema_refusal( /// storage classification. const AUTHORITY_SCHEMA_AUTHORITY: &str = "authority schema"; +/// Refuse a drifted observation table before maintenance installs triggers. +/// Both initial admission and subsequent attachments use the same read-only +/// check so a missing table's triggers never mask its typed reset. +async fn validate_existing_observation_for_admission( + conn: &impl QueryExecutor, + is_fresh: bool, +) -> tracedecay_domain::errors::Result<()> { + if is_fresh { + return Ok(()); + } + validate_existing_observation_table(conn) + .await + .map_err(|error| { + tracedecay_domain::errors::TraceDecayError::reset_required( + AUTHORITY_SCHEMA_AUTHORITY, + error.to_string(), + ) + }) +} + async fn validate_admitted_authority_schema( conn: &RegisteredSchemaInstallationV1, is_fresh: bool, @@ -744,6 +765,7 @@ pub async fn ensure_registered_schema_for_admission( } }; let is_fresh = configuration_fresh.is_some(); + validate_existing_observation_for_admission(installation, is_fresh).await?; let triggers_intact: Pin< Box> + Send + '_>, > = Box::pin(authority_invariant_triggers_intact(installation)); @@ -1140,6 +1162,7 @@ async fn install_registered_schema_stage_sequence( }, error => global_db_operation_error("initialize LCM schema", error), })?; + observation::canonical_body_gc::schema::install(transaction).await?; } // `force_exhaustive` means admission observed damaged or missing guard // triggers (for example a dropped guarded table takes its triggers with @@ -1313,6 +1336,8 @@ pub(crate) async fn ensure_attached_registered_schema( return Ok(RegisteredSchemaAttachmentV1::SessionsRefused(refused)); } }; + validate_existing_observation_for_admission(&read_connection, configuration_fresh.is_some()) + .await?; let triggers_intact: Pin< Box> + Send + '_>, > = Box::pin(authority_invariant_triggers_intact(&read_connection)); @@ -1409,6 +1434,12 @@ pub(crate) async fn attached_registered_schema_reset_refusal( } Err(error) => return reset_or_error(error), }; + if let Err(error) = + validate_existing_observation_for_admission(&read_connection, configuration_fresh.is_some()) + .await + { + return reset_or_error(error); + } let triggers_intact: Pin< Box> + Send + '_>, > = Box::pin(authority_invariant_triggers_intact(&read_connection)); diff --git a/crates/tracedecay-global-db/src/tests.rs b/crates/tracedecay-global-db/src/tests.rs index 19cb23afe0..4b100c5093 100644 --- a/crates/tracedecay-global-db/src/tests.rs +++ b/crates/tracedecay-global-db/src/tests.rs @@ -1644,6 +1644,45 @@ async fn registered_runtime_does_not_recreate_a_missing_database() { ); } +#[asupersync::test] +async fn savings_ledger_refuses_unrepresentable_token_counts() { + let harness = RegisteredGlobalDbHarness::open("savings-ledger-token-encoding").await; + let database = &harness.registered; + let out_of_range = i64::MAX as u64 + 1; + + for (before, after) in [ + (out_of_range, 0), + (0, out_of_range), + (u64::MAX, 3), + (3, u64::MAX), + ] { + let error = database + .try_record_savings("/project", "tool", before, after, 1) + .await + .expect_err("unrepresentable token count must be refused"); + assert!( + error.to_string().contains("encode savings ledger"), + "unexpected refusal: {error}" + ); + } + assert_eq!( + row_count(database, "savings_ledger").await, + 0, + "a rejected count must not append a corrupted or partial row" + ); + + database + .try_record_savings("/project", "tool", i64::MAX as u64, 0, 1) + .await + .expect("the largest representable count must be accepted"); + let total = database + .sum_savings(Some("/project"), 0) + .await + .expect("read the committed savings row"); + assert_eq!(total.saved_tokens, i64::MAX as u64); + assert_eq!(total.calls, 1); +} + #[asupersync::test] async fn project_token_projection_rejects_unrepresentable_totals() { let harness = RegisteredGlobalDbHarness::open("project-tokens-overflow").await; diff --git a/crates/tracedecay-graph-db/src/corrupt_store.rs b/crates/tracedecay-graph-db/src/corrupt_store.rs index f11c913eb7..594535180f 100644 --- a/crates/tracedecay-graph-db/src/corrupt_store.rs +++ b/crates/tracedecay-graph-db/src/corrupt_store.rs @@ -90,6 +90,8 @@ pub(crate) fn recover_deterministically_corrupt_container_with( tracing::error!( event = "store_corrupt_not_deterministic", container = %container.display(), + first_fault_fingerprint = %format!("sha256:{}", sha256_hex(first_fault.as_bytes())), + second_fault_fingerprint = %format!("sha256:{}", sha256_hex(second_fault.as_bytes())), first_fault = %first_fault, second_fault = %second_fault, "graph container corruption is not deterministic; deletion refused. \ @@ -376,6 +378,14 @@ mod tests { ); for field in [ format!("container={}", container.display()), + format!( + "first_fault_fingerprint=sha256:{}", + sha256_hex(b"block 18 CRC mismatch") + ), + format!( + "second_fault_fingerprint=sha256:{}", + sha256_hex(b"block 7 CRC mismatch") + ), "first_fault=block 18 CRC mismatch".to_owned(), "second_fault=block 7 CRC mismatch".to_owned(), ] { diff --git a/crates/tracedecay-graph-db/src/generation_runtime.rs b/crates/tracedecay-graph-db/src/generation_runtime.rs index c8accc16b4..d2fdd35dd5 100644 --- a/crates/tracedecay-graph-db/src/generation_runtime.rs +++ b/crates/tracedecay-graph-db/src/generation_runtime.rs @@ -1837,7 +1837,7 @@ impl GraphDb { // and nothing serves it, so its directory leaves the disk now // instead of waiting, possibly forever, on a project that never // publishes again, behind the deferred row delete. - self.retire_sealed_generation_store(locator); + self.retire_sealed_generation_store(locator)?; return Ok(GenerationContentsDeletion::RetentionPending); } let mut state = self.wait_verified_generations_write()?; @@ -1858,7 +1858,7 @@ impl GraphDb { state.heads.remove(&locator.projection); } drop(state); - self.retire_sealed_generation_store(locator); + self.retire_sealed_generation_store(locator)?; Ok(GenerationContentsDeletion::Deleted) } @@ -2270,7 +2270,7 @@ impl GraphDb { drop(state); // A quarantined generation's rows are in doubt; its derived sealed // artifact must not outlive that doubt. - self.retire_sealed_generation_store(&locator); + self.retire_sealed_generation_store(&locator)?; Ok(()) } } @@ -2799,8 +2799,8 @@ mod tests { }; use super::{ - GenerationLocator, GenerationStageOutcome, StagedGenerationRowsDeletion, - generation_stage_pages, + GenerationContentsDeletion, GenerationLocator, GenerationStageOutcome, + StagedGenerationRowsDeletion, generation_stage_pages, }; fn manifest(source: &str, watermark: &str) -> GraphGenerationManifest { @@ -3977,4 +3977,49 @@ mod tests { ); owner.close().unwrap(); } + + #[test] + fn sealed_generation_retirement_keeps_unlink_errors_retryable() { + let temp = TempDir::new().unwrap(); + let (owner, database) = persistent_database(&temp); + let generation = manifest("source:one", "watermark:one"); + let locator = + GenerationLocator::new(generation.projection.clone(), generation.generation.clone()); + let namespace = generation.identity().physical_namespace().unwrap(); + let directory = temp.path().join("commit-metadata.sealed").join( + namespace + .as_str() + .strip_prefix("generation:") + .unwrap_or(namespace.as_str()), + ); + std::fs::create_dir_all(directory.parent().unwrap()).unwrap(); + // A regular file in the exact artifact slot deterministically refuses + // remove_dir_all even on hosts that run the test with full privileges. + std::fs::write(&directory, b"not a directory").unwrap(); + + let error = database + .delete_generation_contents(&locator, &|| Ok(())) + .expect_err("a failed sealed unlink must not complete retention"); + assert!( + error + .to_string() + .contains("sealed directory removal failed") + ); + assert_eq!(std::fs::read(&directory).unwrap(), b"not a directory"); + + std::fs::remove_file(&directory).unwrap(); + std::fs::create_dir(&directory).unwrap(); + std::fs::write(directory.join("derived-rows"), b"retired").unwrap(); + assert!(matches!( + database.delete_generation_contents(&locator, &|| Ok(())), + Ok(GenerationContentsDeletion::Deleted) + )); + assert!(!directory.exists()); + // Idempotent replay after the directory is gone still succeeds. + assert!(matches!( + database.delete_generation_contents(&locator, &|| Ok(())), + Ok(GenerationContentsDeletion::Deleted) + )); + owner.close().unwrap(); + } } diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index a6e140d089..e644148982 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -472,12 +472,12 @@ impl SealedGenerationStore { ) } - /// Best-effort teardown used only when the generation is quarantined or - /// retired, or before staging rows become releasable. A sealed-only head - /// is never discarded through this path while it remains serveable. - fn discard(&self) { + /// Retirement tears down the reader and requires a successful artifact + /// unlink. A failed unlink remains a typed error for the durable cleanup + /// owner to retry; a sealed-only live head never enters this path. + fn discard(&self) -> Result<(), GraphDbError> { let _ = self.close(); - remove_sealed_directory(&self.directory); + remove_sealed_directory_checked(&self.directory) } } @@ -751,15 +751,28 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { } } -fn remove_sealed_directory(directory: &Path) { +fn remove_sealed_directory_checked(directory: &Path) -> Result<(), GraphDbError> { match std::fs::remove_dir_all(directory) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(_) => { - // Retirement re-runs are idempotent; a leaked directory is - // re-collected on the next retirement or rebuild of this - // generation and never serves reads without a matching receipt. - } + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(sealed_store_io_failure( + "sealed directory removal failed", + error, + )), + } +} + +/// Staging and invalid-artifact cleanup are best effort. Retiring an accepted +/// replay instead uses the checked path, so a failed unlink cannot be reported +/// as a completed retirement and lose its durable retry ownership. +fn remove_sealed_directory(directory: &Path) { + if let Err(error) = remove_sealed_directory_checked(directory) { + tracing::warn!( + event = "sealed_artifact_cleanup_deferred", + path = %directory.display(), + %error, + "sealed artifact cleanup did not complete" + ); } } @@ -1657,30 +1670,31 @@ impl GraphDb { /// Retires the sealed artifact for `locator`: uninstalls the reader and /// deletes its directory. Idempotent, and never touches staging rows. #[tracing::instrument(name = "graph_db.sealed_store.retire", level = "trace", skip_all)] - pub(crate) fn retire_sealed_generation_store(&self, locator: &GenerationLocator) { + pub(crate) fn retire_sealed_generation_store( + &self, + locator: &GenerationLocator, + ) -> Result<(), GraphDbError> { let removed = self .inner .sealed_generations .write() - .ok() - .and_then(|mut sealed| sealed.remove(locator)); + .map_err(|_| GraphDbError::unavailable("sealed generation registry lock poisoned"))? + .remove(locator); if let Some(store) = removed { - store.discard(); - return; + return store.discard(); } - // No installed reader: still delete any on-disk artifact so a - // retired generation leaves nothing behind. + // A removed in-memory reader is not necessary for the artifact to + // exist on disk. Unlink by exact locator even after a prior failed + // retirement; a durable tombstone can then retry without the reader. let Some(reopen) = self.inner.reopen.as_ref() else { - return; + return Ok(()); }; let Some(database_path) = reopen.config.path.as_ref() else { - return; - }; - let Ok(physical_namespace) = locator.physical_namespace() else { - return; + return Ok(()); }; + let physical_namespace = locator.physical_namespace()?; let root = sealed_store_root(database_path); - remove_sealed_directory(&sealed_generation_directory(&root, &physical_namespace)); + remove_sealed_directory_checked(&sealed_generation_directory(&root, &physical_namespace)) } } diff --git a/crates/tracedecay-lcm/src/compression.rs b/crates/tracedecay-lcm/src/compression.rs index 41d77e8fda..9c33f83aaf 100644 --- a/crates/tracedecay-lcm/src/compression.rs +++ b/crates/tracedecay-lcm/src/compression.rs @@ -1420,17 +1420,17 @@ fn assemble_replay_messages( .flat_map(|unit| unit.messages) .collect(), summaries.iter().collect::>(), - latest_user_context_anchor(anchor_source, raws), + latest_user_context_anchor(anchor_source, anchors, raws), ), Some(cap) => { let used = anchors .iter() - .map(|message| crate::lcm_budget_tokens(&message.content)) + .map(|message| replay_transactions::replay_message_tokens(message)) .sum::(); let (selected_raws, tail_tokens) = select_budget_tail(raws, used, cap); let mut summary_budget = (cap - used - tail_tokens).max(0); let preserved_objective_anchor = - latest_user_context_anchor(anchor_source, &selected_raws).and_then( + latest_user_context_anchor(anchor_source, anchors, &selected_raws).and_then( |(store_id, part, already_preserved)| { if already_preserved { return Some((store_id, part, already_preserved)); @@ -1548,11 +1548,12 @@ fn is_budget_droppable_tail_message(message: &LcmRawMessage) -> bool { fn latest_user_context_anchor( raws: &[LcmRawMessage], + anchors: &[&LcmRawMessage], selected_tail: &[&LcmRawMessage], ) -> Option<(i64, String, bool)> { for message in raws.iter().rev() { if let Some(preserved) = preserved_objective_context_content(&message.content) { - if selected_tail.iter().any(|selected| { + if selected_tail.iter().chain(anchors).any(|selected| { preserved_objective_context_content(&selected.content) == Some(preserved) }) { return None; @@ -1714,7 +1715,10 @@ fn context_recovery_hint(summary_nodes: &[LcmSummaryNode]) -> Option { } fn replay_token_estimate(messages: &[Value]) -> i64 { - messages.iter().map(crate::lcm_message_budget_tokens).sum() + messages + .iter() + .map(crate::lcm_replay_message_budget_tokens) + .sum() } fn replay_exceeds_budget(replay_token_estimate: i64, max_assembly_tokens: Option) -> bool { @@ -2098,7 +2102,8 @@ async fn ingest_active_messages( let mut replay = message.clone(); replay["role"] = Value::String(role.clone()); replay["content"] = original_content.clone(); - let initial_metadata_json = active_message_metadata(message, &replay); + let initial_metadata_json = + active_message_metadata(message, &replay, &storage_text); let expected_content_hash = projected_content_hash(&storage_text); if let Some(existing) = existing_state { let matches_stored_row = existing.ordinal == ordinal @@ -2167,6 +2172,7 @@ async fn ingest_active_messages( let metadata_json = active_replay_metadata_json( upsert.projection_metadata_json.as_deref(), &replay, + &raw, ); if metadata_json != initial_metadata_json { update_active_replay_metadata(conn, provider, &message_id, &metadata_json) @@ -2354,7 +2360,7 @@ fn default_message_kind(role: &str) -> String { } } -fn active_message_metadata(message: &Value, replay: &Value) -> String { +fn active_message_metadata(message: &Value, replay: &Value, storage_text: &str) -> String { let mut metadata = Map::new(); metadata.insert( replay_transactions::ACTIVE_REPLAY_METADATA_KEY.to_string(), @@ -2362,7 +2368,7 @@ fn active_message_metadata(message: &Value, replay: &Value) -> String { ); metadata.insert( replay_transactions::ACTIVE_REPLAY_MESSAGE_KEY.to_string(), - active_replay_for_metadata(replay), + active_replay_for_metadata(replay, Some(storage_text)), ); if let Some(lcm_ingest) = message.get("lcm_ingest") { metadata.insert("lcm_ingest".to_string(), lcm_ingest.clone()); @@ -2384,7 +2390,11 @@ fn replay_content_value( serde_json::from_str(&raw.content).unwrap_or_else(|_| Value::String(raw.content.clone())) } -fn active_replay_metadata_json(existing_metadata_json: Option<&str>, replay: &Value) -> String { +fn active_replay_metadata_json( + existing_metadata_json: Option<&str>, + replay: &Value, + raw: &LcmRawMessage, +) -> String { let mut metadata = existing_metadata_json .and_then(|text| serde_json::from_str::(text).ok()) .and_then(|value| value.as_object().cloned()) @@ -2395,18 +2405,29 @@ fn active_replay_metadata_json(existing_metadata_json: Option<&str>, replay: &Va ); metadata.insert( replay_transactions::ACTIVE_REPLAY_MESSAGE_KEY.to_string(), - active_replay_for_metadata(replay), + active_replay_for_metadata( + replay, + (raw.storage_kind == LcmStorageKind::Inline).then_some(raw.content.as_str()), + ), ); Value::Object(metadata).to_string() } -fn active_replay_for_metadata(replay: &Value) -> Value { +fn active_replay_for_metadata(replay: &Value, canonical_content: Option<&str>) -> Value { let mut replay = replay.clone(); if let Some(object) = replay.as_object_mut() { // This locator is regenerated from the canonical raw row at replay. // Persisting it makes our own replay look like a source metadata edit // and invalidates otherwise unchanged summary lineage on re-ingest. object.remove("store_id"); + // The canonical raw body already owns this exact string. The replay + // reader restores omitted content from it; structured content and + // external payload placeholders retain their separate representation. + if canonical_content + .is_some_and(|content| object.get("content").and_then(Value::as_str) == Some(content)) + { + object.remove("content"); + } replay_transactions::strip_disposable_assistant_replay_sidecars(object, ""); } replay diff --git a/crates/tracedecay-lcm/src/compression_policy.rs b/crates/tracedecay-lcm/src/compression_policy.rs index de320bb9d4..57e4464708 100644 --- a/crates/tracedecay-lcm/src/compression_policy.rs +++ b/crates/tracedecay-lcm/src/compression_policy.rs @@ -103,7 +103,7 @@ pub fn overflow_recovery_assembly_cap(input: OverflowRecoveryCapInput<'_>) -> Op let message_tokens = input .messages .iter() - .map(crate::lcm_message_budget_tokens) + .map(crate::lcm_replay_message_budget_tokens) .sum::(); let overhead_tokens = (current_tokens - message_tokens).max(0); Some((assembly_cap - overhead_tokens).max(1)) diff --git a/crates/tracedecay-lcm/src/lib.rs b/crates/tracedecay-lcm/src/lib.rs index f61a0f6663..40041bc34e 100644 --- a/crates/tracedecay-lcm/src/lib.rs +++ b/crates/tracedecay-lcm/src/lib.rs @@ -142,6 +142,32 @@ pub(crate) fn lcm_message_budget_tokens(message: &Value) -> i64 { } } +/// Tokens replayed by a message, including tool calls and other fields +/// outside `content`. Replay selection and reporting must use the same +/// tokenizer for this envelope as for visible text. Role framing belongs to +/// the host's prompt overhead; store and summary IDs are local provenance. +pub(crate) fn lcm_replay_message_budget_tokens(message: &Value) -> i64 { + let content_tokens = lcm_message_budget_tokens(message); + let Some(object) = message.as_object() else { + return content_tokens; + }; + let envelope = object + .iter() + .filter(|(key, _)| { + !matches!( + key.as_str(), + "content" | "role" | "store_id" | "lcm_summary_node_id" + ) + }) + .map(|(key, value)| (key.clone(), value.clone())) + .collect::>(); + if envelope.is_empty() { + content_tokens + } else { + content_tokens.saturating_add(lcm_budget_tokens(&Value::Object(envelope).to_string())) + } +} + enum VisibleContent<'a> { Empty, Text(&'a str), @@ -278,7 +304,10 @@ impl GitScopeFilter { #[cfg(test)] mod budget_tests { - use super::{lcm_budget_tokens, lcm_message_budget_tokens, lcm_message_visible_text}; + use super::{ + lcm_budget_tokens, lcm_message_budget_tokens, lcm_message_visible_text, + lcm_replay_message_budget_tokens, + }; use serde_json::json; /// The borrowed counter must agree with counting the materialized visible @@ -343,6 +372,49 @@ mod budget_tests { ); } + #[test] + fn replay_budget_ignores_local_provenance_and_role_framing() { + let message = json!({ + "role": "system", + "content": "compact summary", + "store_id": 123, + "lcm_summary_node_id": "summary-local-id", + }); + assert_eq!( + lcm_replay_message_budget_tokens(&message), + lcm_budget_tokens("compact summary") + ); + } + + #[test] + fn overflow_recovery_does_not_charge_tool_arguments_as_fixed_overhead() { + use crate::compression_policy::{OverflowRecoveryCapInput, overflow_recovery_assembly_cap}; + + let message = json!({ + "role": "assistant", + "content": "", + "tool_calls": [{ + "id": "call-a", + "type": "function", + "function": { + "name": "execute", + "arguments": json!({"source": "f(x:u32)->u32{x+1};".repeat(32)}).to_string(), + }, + }], + }); + let message_tokens = lcm_replay_message_budget_tokens(&message); + let prompt_overhead = 128; + assert_eq!( + overflow_recovery_assembly_cap(OverflowRecoveryCapInput { + current_tokens: Some(message_tokens + prompt_overhead), + max_assembly_tokens: Some(message_tokens + prompt_overhead), + messages: &[message], + }), + Some(message_tokens), + "the replay budget must reserve fixed prompt overhead exactly once" + ); + } + /// The cap is inclusive: a chunk of messages totalling exactly the cap /// fits, one token under it does not. #[test] diff --git a/crates/tracedecay-lcm/src/raw.rs b/crates/tracedecay-lcm/src/raw.rs index dc99667e9b..7a307488b7 100644 --- a/crates/tracedecay-lcm/src/raw.rs +++ b/crates/tracedecay-lcm/src/raw.rs @@ -28,12 +28,15 @@ pub const RAW_MESSAGE_SELECT_COLUMNS: &str = "provider, message_id, session_id, store_id, role, ordinal, timestamp, content, content_hash, storage_kind, payload_ref, snippet_text, metadata_json, - (SELECT encoding FROM session_canonical_bodies - WHERE content_hash = lcm_raw_messages.content_hash), - (SELECT body FROM session_canonical_bodies - WHERE content_hash = lcm_raw_messages.content_hash), - (SELECT uncompressed_bytes FROM session_canonical_bodies - WHERE content_hash = lcm_raw_messages.content_hash)"; + CASE WHEN storage_kind = 'inline' AND content IS NULL THEN + (SELECT encoding FROM session_canonical_bodies + WHERE content_hash = lcm_raw_messages.content_hash) END, + CASE WHEN storage_kind = 'inline' AND content IS NULL THEN + (SELECT body FROM session_canonical_bodies + WHERE content_hash = lcm_raw_messages.content_hash) END, + CASE WHEN storage_kind = 'inline' AND content IS NULL THEN + (SELECT uncompressed_bytes FROM session_canonical_bodies + WHERE content_hash = lcm_raw_messages.content_hash) END"; fn record_select_columns(alias: &str, text: &str, metadata: &str) -> String { format!( "{alias}.provider, {alias}.message_id, {alias}.session_id, {alias}.role, @@ -231,6 +234,14 @@ async fn persist_canonical_text( ) -> Result<(), LcmError> { let body = StoredCanonicalBody::pack(text.as_bytes()) .map_err(|error| LcmError::Db(format!("canonical body pack failed: {error}")))?; + persist_canonical_body(conn, &body).await +} + +/// Persists an already packed body inside the caller's raw-owner transaction. +pub(crate) async fn persist_canonical_body( + conn: &(impl Executor + ?Sized), + body: &StoredCanonicalBody, +) -> Result<(), LcmError> { conn.execute( UPSERT_CANONICAL_BODY_SQL, params![ diff --git a/crates/tracedecay-lcm/src/replay_transactions.rs b/crates/tracedecay-lcm/src/replay_transactions.rs index ba4cd4cbf3..487f704686 100644 --- a/crates/tracedecay-lcm/src/replay_transactions.rs +++ b/crates/tracedecay-lcm/src/replay_transactions.rs @@ -153,18 +153,11 @@ fn active_replay_value(message: &LcmRawMessage) -> Option { active_replay_message_from_metadata(message) } -fn replay_message_tokens(message: &LcmRawMessage) -> i64 { - let mut tokens = crate::lcm_budget_tokens(&message.content); - if let Some(mut replay) = active_replay_value(message) { - if let Some(object) = replay.as_object_mut() { - object.remove("content"); - object.remove("role"); - object.remove("store_id"); - } - let serialized = replay.to_string(); - tokens += serialized.len().div_ceil(4) as i64; +pub(crate) fn replay_message_tokens(message: &LcmRawMessage) -> i64 { + match active_replay_value(message) { + Some(replay) => crate::lcm_replay_message_budget_tokens(&replay), + None => crate::lcm_budget_tokens(&message.content), } - tokens } #[tracing::instrument(name = "sessions.lcm.normalize_pairs", level = "trace", skip_all)] diff --git a/crates/tracedecay-lcm/src/retention.rs b/crates/tracedecay-lcm/src/retention.rs index 3bf7b09734..fc7bfcd935 100644 --- a/crates/tracedecay-lcm/src/retention.rs +++ b/crates/tracedecay-lcm/src/retention.rs @@ -20,22 +20,28 @@ //! //! # Passes //! -//! Both passes reuse the store's content-addressed external payload lifecycle -//! (`lcm_external_payloads` keyed by a content-hash-derived `payload_ref`, -//! with a full reaping GC in [`super::gc`]). In reclaim order: +//! External payloads use the store's owner-scoped lifecycle +//! (`lcm_external_payloads`, with a reaping GC in [`super::gc`]). In order: //! //! 1. **Drop** (terminal, longest window): projection-durable raw rows past //! `drop_after_days` are deleted. Any now unreferenced external payload is //! reaped by the existing payload GC. -//! 2. **Offload** (recoverable, shorter window): projection-durable *inline* -//! raw rows past `offload_after_days` have their bulky `content` -//! externalized to the content-addressed store (deduplicated by hash) and -//! replaced with a recoverable placeholder, reclaiming the inline column and -//! its FTS shadow. +//! 2. **Inline** (lossless): bodies previously externalized by retention return +//! to their raw rows or the existing shared canonical-body store when its +//! packed body, retrieval snippet and record allowance are smaller. Released +//! files follow the existing GC grace period; this pass does not delete files. +//! 3. **Offload** (lossless, shorter window): projection-durable direct-inline +//! bodies past `offload_after_days` move to shared canonical storage only +//! when the same storage comparison succeeds. Non-beneficial bodies stay +//! intact; no new uncompressed payload files are created. Generated search +//! text follows the retained body or canonical snippet, and verified reads +//! continue to return the complete original text. //! -//! Every pass is bounded (`max_batch_size`) and incremental so the daemon can -//! schedule it off the hot path without competing with foreground writes, and a -//! dry run counts what would be reclaimed without mutating anything. +//! Every pass is incremental. The lossless storage passes examine at most 500 +//! candidates and 8 MiB of body/metadata bytes per pass, with a 4 MiB cap on each +//! body and row metadata envelope before body hydration. Larger records remain +//! intact. Dry runs count candidates without hydrating, compressing or changing +//! them, so their candidate count does not predict how many bodies will shrink. use std::collections::BTreeSet; use std::path::Path; @@ -52,11 +58,25 @@ use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, Row, params}; use tracedecay_runtime_core::db::{ Database, DatabaseEngineReadConnection, DatabaseMemoryTransaction, }; +use tracedecay_store::{INLINE_BODY_BYTES, StoredCanonicalBody}; +#[cfg(test)] use super::payload::{ExternalPayloadWrite, PayloadFileRollback}; -use super::{LcmError, payload, schema, util}; +use super::{LcmError, payload, raw, schema, summary_convergence, util}; + +mod replay_metadata; +pub use replay_metadata::{ + MAX_REPLAY_METADATA_COMPACTION_ROWS, ReplayMetadataCompactionReport, compact_replay_metadata, +}; const SECONDS_PER_DAY: i64 = 24 * 60 * 60; +const MAX_REPACK_BODY_BYTES: usize = 4 * 1024 * 1024; +const MAX_STORAGE_PASS_BYTES: usize = 8 * 1024 * 1024; +const MAX_STORAGE_PASS_ROWS: i64 = 500; +const RESTORE_CURSOR: &str = "restore_offloaded_v2"; +// Covers the canonical record, hash indexes and owner/candidate references. +// This is a conservative logical storage allowance, not a filesystem estimate. +const CANONICAL_RECORD_ALLOWANCE: usize = 1024; /// SQL predicate (over an aliased `lcm_raw_messages` row `r`) that is true when /// the raw row's `store_id` is covered by a durable summary node's lineage. @@ -83,17 +103,18 @@ const OFFLOAD_KIND: &str = "retention_offload"; /// Per-table/per-store retention windows for the session store. Defaults keep /// a six-month recovery horizon for projection-durable raw evidence and -/// offload its bulky inline payload after 30 days. Rows without durable -/// summary lineage remain untouched. +/// compact its bulky inline payload after 30 days when canonical storage is +/// smaller. Rows without durable summary lineage remain untouched. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct LcmRetentionConfig { /// Master switch. When `false`, [`run_session_retention`] is a /// no-op even in [`RetentionMode::Apply`]. #[serde(default = "default_retention_enabled")] pub enabled: bool, - /// Window after which a projection-durable, still-inline raw row has its - /// content offloaded to the content-addressed store. `None` disables the - /// offload pass. + /// Window after which projection-durable bodies may be compacted into the + /// existing canonical store. Retention-offloaded files past this window may + /// return inline or become canonical when smaller; non-beneficial and + /// oversized bodies remain intact. `None` disables both lossless passes. #[serde(default = "default_offload_after_days")] pub offload_after_days: Option, /// Window after which a projection-durable raw row is dropped. `None` @@ -171,7 +192,9 @@ pub struct LcmRetentionPhaseReport { pub eligible: u64, /// Rows actually acted on (`0` in a dry run). pub acted: u64, - /// Bytes of message content reclaimed from the database by this pass. + /// Logical inline-content bytes removed from the database by this pass. + /// This excludes metadata, indexes, canonical bodies and payload files, and + /// does not measure physical disk reclamation. Inlining reports zero. pub bytes_reclaimed: u64, /// Oldest timestamp among the bounded eligible rows, when any. #[serde(default)] @@ -195,6 +218,19 @@ pub struct LcmRetentionReport { pub ended_at: i64, pub dropped: LcmRetentionPhaseReport, pub offloaded: LcmRetentionPhaseReport, + /// Retention-offloaded bodies restored to normal inline/canonical storage. + /// Released files remain available until existing payload GC verifies them + /// after its grace period. + #[serde(default)] + pub inlined: LcmRetentionPhaseReport, + /// Subset of restored/offloaded rows compacted into shared canonical bodies. + /// This counts representation changes, not filesystem bytes reclaimed. + #[serde(default)] + pub canonicalized_rows: u64, + /// A lossless pass reached its row/byte budget before draining its cursor. + /// The daemon may continue this work without rerunning age-based deletion. + #[serde(default)] + pub storage_has_more: bool, /// Database `PRAGMA freelist_count` before/after (freed pages are the /// measurable, VACUUM-free signal that space was reclaimed). pub freelist_before: u64, @@ -245,7 +281,7 @@ pub async fn read_session_retention_backlog( .unwrap_or(now); let sql = format!( "SELECT MIN(r.timestamp), - COALESCE(SUM(LENGTH(COALESCE(r.content, ''))), 0) + COALESCE(SUM(LENGTH(CAST(COALESCE(r.content, '') AS BLOB))), 0) FROM lcm_raw_messages r WHERE {DURABLE_ROW_IDS} AND r.timestamp IS NOT NULL @@ -254,7 +290,7 @@ pub async fn read_session_retention_backlog( OR (?3 = 1 AND +r.timestamp < ?4 AND r.storage_kind = 'inline' AND r.content IS NOT NULL - AND LENGTH(r.content) > 0) + AND LENGTH(CAST(r.content AS BLOB)) >= {INLINE_BODY_BYTES}) )" ); let mut rows = conn @@ -383,6 +419,9 @@ async fn run_session_retention_inner( ended_at: now, dropped: LcmRetentionPhaseReport::disabled(), offloaded: LcmRetentionPhaseReport::disabled(), + inlined: LcmRetentionPhaseReport::disabled(), + canonicalized_rows: 0, + storage_has_more: false, freelist_before, freelist_after: freelist_before, page_count_before, @@ -393,6 +432,7 @@ async fn run_session_retention_inner( if !config.enabled || !config.any_window() { report.dropped.window_days = config.drop_after_days; report.offloaded.window_days = config.offload_after_days; + report.inlined.window_days = config.offload_after_days; return Ok(report); } @@ -400,8 +440,8 @@ async fn run_session_retention_inner( provider, session_id, }; - // Drop first (terminal, longest window) so offload never externalizes a row - // that is about to be deleted. + // Drop first (terminal, longest window) so the lossless passes never compact + // a row that is about to be deleted. let (dropped, drop_cursor) = run_drop_pass( store, scope, @@ -413,7 +453,7 @@ async fn run_session_retention_inner( ) .await?; report.dropped = dropped; - let (offloaded, offload_cursor) = run_offload_pass( + let (inlined, inline_cursor, inline_has_more) = run_inline_offloaded_pass( store, storage_root, scope, @@ -421,14 +461,32 @@ async fn run_session_retention_inner( mode, now, &mut report.errors, + &mut report.canonicalized_rows, + authorize, + ) + .await?; + report.inlined = inlined; + let (offloaded, offload_cursor, offload_has_more) = run_offload_pass( + store, + scope, + config, + mode, + now, + &mut report.errors, + &mut report.canonicalized_rows, authorize, ) .await?; report.offloaded = offloaded; + report.storage_has_more = inline_has_more || offload_has_more; if mode.is_apply() { // Consume the staged GC/reporting meta cards: record the last run so a // scheduler and Doctor can report retention backlog without a rescan. - let acted = report.dropped.acted.saturating_add(report.offloaded.acted); + let acted = report + .dropped + .acted + .saturating_add(report.offloaded.acted) + .saturating_add(report.inlined.acted); let transaction = store .begin_memory_write_transaction("begin session retention metadata", authorize) .await?; @@ -436,7 +494,11 @@ async fn run_session_retention_inner( // A scoped run examined only its own rows, so only an unscoped run may // move the cursors past what it examined. if scope.is_unscoped() { - for (pass, cursor) in [("drop", drop_cursor), ("offload", offload_cursor)] { + for (pass, cursor) in [ + ("drop", drop_cursor), + ("offload", offload_cursor), + (RESTORE_CURSOR, inline_cursor), + ] { if let Some(cursor) = cursor { cursor.persist(&transaction, pass).await?; } @@ -863,7 +925,7 @@ async fn run_drop_pass( cursor, cutoff, config.batch_limit(), - "r.store_id, r.timestamp, LENGTH(COALESCE(r.content, ''))", + "r.store_id, r.timestamp, LENGTH(CAST(COALESCE(r.content, '') AS BLOB))", "1 = 1", |row| { Ok(DropRow { @@ -922,6 +984,280 @@ async fn run_drop_pass( Ok((report, (errors.len() == errors_before).then_some(next))) } +struct InlineOffloadedRow { + store_id: i64, + timestamp: i64, + provider: String, + session_id: String, + message_id: String, + payload_ref: String, + content_hash: String, + placeholder_text: Option, + metadata_json: Option, + byte_count: usize, +} + +fn restorable_offload_predicate() -> String { + format!( + "r.storage_kind = 'external' AND r.content IS NULL + AND COALESCE(LENGTH(CAST(r.metadata_json AS BLOB)), 0) + + COALESCE(LENGTH(CAST(r.placeholder_text AS BLOB)), 0) <= {MAX_REPACK_BODY_BYTES} + AND EXISTS ( + SELECT 1 FROM lcm_external_payloads e + WHERE e.payload_ref = r.payload_ref + AND e.provider = r.provider AND e.session_id = r.session_id + AND e.message_id = r.message_id AND e.content_hash = r.content_hash + AND e.kind = '{OFFLOAD_KIND}' + AND e.byte_count > 0 AND e.byte_count <= {MAX_REPACK_BODY_BYTES} + )" + ) +} + +#[allow(clippy::too_many_arguments)] +async fn run_inline_offloaded_pass( + store: RetentionStore<'_>, + storage_root: &Path, + scope: RetentionScope<'_>, + config: &LcmRetentionConfig, + mode: RetentionMode, + now: i64, + errors: &mut Vec, + canonicalized: &mut u64, + authorize: Option<&RetentionAuthorization<'_>>, +) -> Result<(LcmRetentionPhaseReport, Option, bool), LcmError> { + let mut report = LcmRetentionPhaseReport { + window_days: config.offload_after_days, + ..LcmRetentionPhaseReport::default() + }; + let Some(window) = config.offload_after_days else { + return Ok((report, None, false)); + }; + let read = store.read_connection(); + // The v2 cursor revisits files skipped by the prior tiny-file-only pass. + let mut cursor = RetentionCursor::read(&read, RESTORE_CURSOR).await?; + let mut remaining = MAX_STORAGE_PASS_BYTES; + let mut has_more = true; + for _ in 0..config.batch_limit().min(MAX_STORAGE_PASS_ROWS) { + let (mut targets, next) = select_retention_candidates( + &read, + scope, + cursor, + cutoff_secs(window, now), + 1, + "r.store_id, r.timestamp, r.provider, r.session_id, r.message_id, + r.payload_ref, r.content_hash, r.placeholder_text, r.metadata_json, + (SELECT byte_count FROM lcm_external_payloads WHERE payload_ref = r.payload_ref)", + &restorable_offload_predicate(), + |row| { + Ok(InlineOffloadedRow { + store_id: row.get(0)?, + timestamp: row.get(1)?, + provider: row.get(2)?, + session_id: row.get(3)?, + message_id: row.get(4)?, + payload_ref: row.get(5)?, + content_hash: row.get(6)?, + placeholder_text: row.get(7)?, + metadata_json: row.get(8)?, + byte_count: usize::try_from(row.get::(9)?) + .map_err(|_| LcmError::PayloadIntegrityMismatch)?, + }) + }, + ) + .await?; + if let Some(target) = targets.pop() { + let cost = target + .byte_count + .saturating_add(target.metadata_json.as_ref().map_or(0, String::len)) + .saturating_add(target.placeholder_text.as_ref().map_or(0, String::len)); + if cost > remaining { + break; + } + remaining -= cost; + report.eligible += 1; + report.oldest_eligible_at = Some( + report + .oldest_eligible_at + .map_or(target.timestamp, |old| old.min(target.timestamp)), + ); + if mode.is_apply() { + match inline_offloaded_one(store, storage_root, &target, authorize).await { + Ok(Some(compacted)) => { + report.acted += 1; + *canonicalized += u64::from(compacted); + } + Ok(None) => {} + Err(error) => { + errors.push(format!("inline raw row {}: {error}", target.store_id)); + has_more = false; + break; + } + } + } + } else if next.durable_through == cursor.durable_through { + // No eligible aged row and no later summary lineage to examine. + // Reaching the cutoff itself is not pending work, even when a + // scoped pass cannot persist the shared cursor. + has_more = false; + break; + } + if next == cursor { + has_more = false; + break; + } + cursor = next; + } + // Inlining releases a file reference, not physical bytes. Existing owner + // GC proves it is unreferenced after grace and reports its own reclamation. + Ok((report, mode.is_apply().then_some(cursor), has_more)) +} + +async fn inline_offloaded_one( + store: RetentionStore<'_>, + storage_root: &Path, + target: &InlineOffloadedRow, + authorize: Option<&RetentionAuthorization<'_>>, +) -> Result, LcmError> { + let read = store.read_connection(); + let metadata = payload::load_payload_metadata(&read, &target.payload_ref).await?; + if metadata.provider != target.provider + || metadata.session_id != target.session_id + || metadata.message_id != target.message_id + { + return Err(LcmError::PayloadNotOwnedBySession); + } + let bytes = + usize::try_from(metadata.byte_count).map_err(|_| LcmError::PayloadIntegrityMismatch)?; + let chars = + usize::try_from(metadata.char_count).map_err(|_| LcmError::PayloadIntegrityMismatch)?; + if metadata.kind != OFFLOAD_KIND + || metadata.content_hash != target.content_hash + || bytes == 0 + || bytes != target.byte_count + || bytes > MAX_REPACK_BODY_BYTES + || chars > bytes + { + return Err(LcmError::PayloadIntegrityMismatch); + } + let content = payload::read_verified_payload_content_with_checkpoint( + storage_root, + &target.payload_ref, + &target.content_hash, + bytes, + chars, + &mut || { + if let Some(authorize) = authorize { + authorize("read session retention inline payload")?; + } + Ok(()) + }, + )?; + let packed = (bytes >= INLINE_BODY_BYTES) + .then(|| pack_canonical(&content)) + .transpose()?; + let placeholder = raw::derived_text_for_snippet(&content); + let txn = store + .begin_memory_write_transaction("begin session retention inline pass", authorize) + .await?; + if let Some(body) = &packed { + if !canonical_storage_saves_bytes(&txn, body, &placeholder).await? { + txn.rollback().await?; + return Ok(None); + } + raw::persist_canonical_body(&txn, body).await?; + } + if let Some(authorize) = authorize { + authorize("compare and swap session retention inline row")?; + } + summary_convergence::suspend_raw_update_guard(&txn).await?; + let update_sql = format!( + "UPDATE lcm_raw_messages AS r + SET content = ?2, storage_kind = 'inline', payload_ref = NULL, placeholder_text = ?11 + WHERE r.store_id = ?1 AND r.content_hash = ?3 AND r.payload_ref = ?4 + AND r.provider = ?5 AND r.session_id = ?6 AND r.message_id = ?7 + AND r.timestamp = ?8 AND r.placeholder_text IS ?9 AND r.metadata_json IS ?10 + AND {PROJECTION_DURABLE} AND {}", + restorable_offload_predicate() + ); + let changed = txn + .execute( + &update_sql, + params![ + target.store_id, + if packed.is_none() { + Some(content.as_str()) + } else { + None + }, + target.content_hash.as_str(), + target.payload_ref.as_str(), + target.provider.as_str(), + target.session_id.as_str(), + target.message_id.as_str(), + target.timestamp, + util::opt_text(target.placeholder_text.as_deref()), + util::opt_text(target.metadata_json.as_deref()), + packed.as_ref().map(|_| placeholder.as_str()) + ], + ) + .await?; + if changed != 1 { + txn.rollback().await?; + return Err(LcmError::Db(format!( + "inline compare-and-swap rejected changed row {}", + target.store_id + ))); + } + // The normal raw read authority verifies the original sanitizer receipt + // against the restored full text. A corrupt receipt never becomes visible + // as inline text, and its external file remains intact on rollback. + if let Err(error) = raw::load_raw_message_by_store_id(&txn, target.store_id).await { + txn.rollback().await?; + return Err(error); + } + if let Err(error) = summary_convergence::restore_raw_update_guard(&txn).await { + txn.rollback().await?; + return Err(error); + } + commit_authorized(txn, authorize, "commit session retention inline pass").await?; + Ok(Some(packed.is_some())) +} + +fn pack_canonical(content: &str) -> Result { + StoredCanonicalBody::pack(content.as_bytes()) + .map_err(|error| LcmError::Db(format!("retention canonical body pack failed: {error}"))) +} + +async fn canonical_storage_saves_bytes( + conn: &(impl QueryExecutor + ?Sized), + body: &StoredCanonicalBody, + snippet: &str, +) -> Result { + let original_bytes = + usize::try_from(body.uncompressed_bytes).map_err(|_| LcmError::PayloadIntegrityMismatch)?; + let mut rows = conn.query( + "SELECT LENGTH(body), uncompressed_bytes FROM session_canonical_bodies WHERE content_hash = ?1", + params![body.content_hash.as_str()], + ).await?; + let marginal_body_bytes = if let Some(existing) = rows.next().await? { + // Validate sizes before the normal raw reader hydrates an existing body. + // It subsequently proves its decoded bytes, hash and sanitizer receipt + // under this same transaction, before releasing the original source. + let stored_bytes = usize::try_from(existing.get::(0)?) + .map_err(|_| LcmError::PayloadIntegrityMismatch)?; + if existing.get::(1)? != body.uncompressed_bytes || stored_bytes > original_bytes { + return Err(LcmError::PayloadIntegrityMismatch); + } + 0 + } else { + body.blob.len() + }; + Ok(marginal_body_bytes + .saturating_add(snippet.len()) + .saturating_add(CANONICAL_RECORD_ALLOWANCE) + < original_bytes) +} + struct OffloadRow { store_id: i64, provider: String, @@ -929,74 +1265,202 @@ struct OffloadRow { message_id: String, timestamp: i64, content: String, + metadata_json: Option, +} + +struct OffloadHeader { + store_id: i64, + timestamp: i64, + provider: String, + session_id: String, + message_id: String, + content_hash: String, + metadata_json: Option, + byte_count: usize, +} + +async fn load_offload_row( + read: &(impl QueryExecutor + ?Sized), + target: OffloadHeader, +) -> Result { + let mut rows = read + .query( + &format!( + "SELECT {} FROM lcm_raw_messages + WHERE store_id = ?1 AND timestamp = ?2 AND provider = ?3 AND session_id = ?4 + AND message_id = ?5 AND content_hash = ?6 AND metadata_json IS ?7 + AND storage_kind = 'inline' AND content IS NOT NULL + AND LENGTH(CAST(content AS BLOB)) = ?8", + raw::RAW_MESSAGE_SELECT_COLUMNS + ), + params![ + target.store_id, + target.timestamp, + target.provider.as_str(), + target.session_id.as_str(), + target.message_id.as_str(), + target.content_hash.as_str(), + target.metadata_json.as_deref(), + target.byte_count as i64 + ], + ) + .await?; + let row = rows.next().await?.ok_or_else(|| { + LcmError::Db(format!( + "offload compare-and-swap rejected changed row {}", + target.store_id + )) + })?; + let message = raw::verified_raw_message_from_row(&row)?; + Ok(OffloadRow { + store_id: target.store_id, + timestamp: target.timestamp, + provider: target.provider, + session_id: target.session_id, + message_id: target.message_id, + content: message.content, + metadata_json: target.metadata_json, + }) } #[allow(clippy::too_many_arguments)] async fn run_offload_pass( store: RetentionStore<'_>, - storage_root: &Path, scope: RetentionScope<'_>, config: &LcmRetentionConfig, mode: RetentionMode, now: i64, errors: &mut Vec, + canonicalized: &mut u64, authorize: Option<&RetentionAuthorization<'_>>, -) -> Result<(LcmRetentionPhaseReport, Option), LcmError> { +) -> Result<(LcmRetentionPhaseReport, Option, bool), LcmError> { let mut report = LcmRetentionPhaseReport { window_days: config.offload_after_days, ..LcmRetentionPhaseReport::default() }; let Some(window) = config.offload_after_days else { - return Ok((report, None)); + return Ok((report, None, false)); }; let cutoff = cutoff_secs(window, now); let read = store.read_connection(); - let cursor = RetentionCursor::read(&read, "offload").await?; - let (targets, next) = select_retention_candidates( - &read, - scope, - cursor, - cutoff, - config.batch_limit(), - "r.store_id, r.timestamp, r.provider, r.session_id, r.message_id, r.content", - "r.storage_kind = 'inline' AND r.content IS NOT NULL AND LENGTH(r.content) > 0", - |row| { - Ok(OffloadRow { - store_id: row.get(0)?, - timestamp: row.get(1)?, - provider: row.get(2)?, - session_id: row.get(3)?, - message_id: row.get(4)?, - content: row.get(5)?, - }) - }, - ) - .await?; - report.eligible = targets.len() as u64; - report.oldest_eligible_at = targets.iter().map(|target| target.timestamp).min(); - if !mode.is_apply() { - report.bytes_reclaimed = targets.iter().map(|t| t.content.len() as u64).sum(); - return Ok((report, None)); + let mut cursor = RetentionCursor::read(&read, "offload").await?; + let mut remaining = MAX_STORAGE_PASS_BYTES; + let mut has_more = true; + for _ in 0..config.batch_limit().min(MAX_STORAGE_PASS_ROWS) { + let (mut targets, next) = select_retention_candidates( + &read, scope, cursor, cutoff, 1, + "r.store_id, r.timestamp, r.provider, r.session_id, r.message_id, r.content_hash, + r.metadata_json, LENGTH(CAST(r.content AS BLOB))", + &format!("r.storage_kind = 'inline' AND r.content IS NOT NULL + AND COALESCE(LENGTH(CAST(r.metadata_json AS BLOB)), 0) <= {MAX_REPACK_BODY_BYTES} + AND LENGTH(CAST(r.content AS BLOB)) BETWEEN {INLINE_BODY_BYTES} AND {MAX_REPACK_BODY_BYTES}"), + |row| Ok(OffloadHeader { + store_id: row.get(0)?, timestamp: row.get(1)?, provider: row.get(2)?, + session_id: row.get(3)?, message_id: row.get(4)?, content_hash: row.get(5)?, + metadata_json: row.get(6)?, + byte_count: usize::try_from(row.get::(7)?).map_err(|_| LcmError::PayloadIntegrityMismatch)?, + }), + ).await?; + if let Some(header) = targets.pop() { + let cost = header + .byte_count + .saturating_add(header.metadata_json.as_ref().map_or(0, String::len)); + if cost > remaining { + break; + } + remaining -= cost; + report.eligible += 1; + report.oldest_eligible_at = Some( + report + .oldest_eligible_at + .map_or(header.timestamp, |old| old.min(header.timestamp)), + ); + if mode.is_apply() { + let store_id = header.store_id; + let result = async { + let target = load_offload_row(&read, header).await?; + offload_one(store, &target, authorize).await + } + .await; + match result { + Ok(Some(bytes)) => { + report.acted += 1; + report.bytes_reclaimed = report.bytes_reclaimed.saturating_add(bytes); + *canonicalized += 1; + } + Ok(None) => {} + Err(error) => { + errors.push(format!("offload raw row {store_id}: {error}")); + has_more = false; + break; + } + } + } + } else if next.durable_through == cursor.durable_through { + // No eligible aged row and no later summary lineage to examine. + // Reaching the cutoff itself is not pending work, even when a + // scoped pass cannot persist the shared cursor. + has_more = false; + break; + } + if next == cursor { + has_more = false; + break; + } + cursor = next; } + Ok((report, mode.is_apply().then_some(cursor), has_more)) +} - // Each row is offloaded atomically: write the content-addressed file, then - // flip the row to external + placeholder in its own transaction. A crash - // between file write and commit is cleaned up by the rollback guard. - let errors_before = errors.len(); - for target in targets { - match offload_one(store, storage_root, &target, authorize).await { - Ok(bytes) => { - report.acted += 1; - report.bytes_reclaimed = report.bytes_reclaimed.saturating_add(bytes); +async fn offload_one( + store: RetentionStore<'_>, + target: &OffloadRow, + authorize: Option<&RetentionAuthorization<'_>>, +) -> Result, LcmError> { + if target.content.len() > MAX_REPACK_BODY_BYTES { + return Err(LcmError::Db( + "retention body exceeds the 4 MiB repack limit".to_owned(), + )); + } + if target.content.len() >= INLINE_BODY_BYTES { + let packed = pack_canonical(&target.content)?; + let placeholder = raw::derived_text_for_snippet(&target.content); + let txn = store + .begin_memory_write_transaction("begin session retention canonical pass", authorize) + .await?; + if canonical_storage_saves_bytes(&txn, &packed, &placeholder).await? { + raw::persist_canonical_body(&txn, &packed).await?; + let changed = txn.execute(&format!( + "UPDATE lcm_raw_messages AS r SET content = NULL, placeholder_text = ?2 + WHERE r.store_id = ?1 AND r.content_hash = ?3 AND r.provider = ?4 + AND r.session_id = ?5 AND r.message_id = ?6 AND r.timestamp = ?7 + AND r.content = ?8 AND r.metadata_json IS ?9 + AND r.storage_kind = 'inline' AND r.payload_ref IS NULL AND {PROJECTION_DURABLE}"), + params![target.store_id, placeholder.as_str(), packed.content_hash.as_str(), + target.provider.as_str(), target.session_id.as_str(), target.message_id.as_str(), + target.timestamp, target.content.as_str(), target.metadata_json.as_deref()] + ).await?; + if changed != 1 { + txn.rollback().await?; + return Err(LcmError::Db(format!( + "offload compare-and-swap rejected changed row {}", + target.store_id + ))); + } + if let Err(error) = raw::load_raw_message_by_store_id(&txn, target.store_id).await { + txn.rollback().await?; + return Err(error); } - Err(err) => errors.push(format!("offload raw row {}: {err}", target.store_id)), + commit_authorized(txn, authorize, "commit session retention canonical pass").await?; + return Ok(Some(target.content.len() as u64)); } + txn.rollback().await?; } - // A failed row stays behind the cursor so the next pass retries it. - Ok((report, (errors.len() == errors_before).then_some(next))) + Ok(None) } -async fn offload_one( +#[cfg(test)] +async fn offload_external_one( store: RetentionStore<'_>, storage_root: &Path, target: &OffloadRow, @@ -1051,6 +1515,7 @@ async fn offload_one( AND r.message_id = ?7 AND r.timestamp = ?8 AND r.content = ?9 + AND r.metadata_json IS ?10 AND r.storage_kind = 'inline' AND r.payload_ref IS NULL AND {PROJECTION_DURABLE}" @@ -1067,7 +1532,8 @@ async fn offload_one( target.session_id.as_str(), target.message_id.as_str(), target.timestamp, - target.content.as_str() + target.content.as_str(), + target.metadata_json.as_deref() ], ) .await?; diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata.rs b/crates/tracedecay-lcm/src/retention/replay_metadata.rs new file mode 100644 index 0000000000..b7be38726d --- /dev/null +++ b/crates/tracedecay-lcm/src/retention/replay_metadata.rs @@ -0,0 +1,291 @@ +//! Bounded, lossless removal of old replay strings duplicated by raw bodies. +//! +//! The raw reader verifies canonical bytes and their sanitizer receipt before +//! a duplicate is removed. Structured replay content and external/media +//! representations keep their own content. A durable rowid frontier and a byte +//! budget keep this repair off the foreground ingest path. + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tracedecay_runtime_core::db::Database; +use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, params}; + +use crate::{LcmError, LcmRawMessage, gc, payload, raw, schema, security, summary_convergence}; + +#[cfg(test)] +mod tests; + +const FRONTIER_KEY: &str = "replay_metadata_compaction_store_id_v1"; +const OPERATION: &str = "compact duplicate LCM replay metadata"; +pub const MAX_REPLAY_METADATA_COMPACTION_ROWS: usize = 48; +// Bound both receipt verification and metadata sanitization while a writer is +// held. The raw metadata envelope itself is capped at 1 MiB by ingestion. +const MAX_BATCH_BYTES: u64 = 4 * 1024 * 1024; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ReplayMetadataCompactionReport { + pub rows_scanned: u64, + pub rows_compacted: u64, + /// Eligible rows retained because their representation, sanitizer result, + /// or byte bound could not prove this narrow lossless transformation. + pub rows_skipped: u64, + /// Subset of skipped rows whose metadata plus canonical/stored body + /// representations exceed this pass's 4 MiB read and verification budget. + pub rows_skipped_oversized: u64, + /// UTF-8 metadata bytes removed. This does not measure file shrinkage. + pub metadata_bytes_removed: u64, + pub has_more: bool, +} + +fn db_error(error: impl std::fmt::Display) -> LcmError { + LcmError::Db(format!("{OPERATION}: {error}")) +} + +async fn frontier(conn: &(impl QueryExecutor + ?Sized)) -> Result { + match schema::get_gc_meta(conn, FRONTIER_KEY).await? { + None => Ok(0), + Some(value) => value + .parse::() + .ok() + .filter(|value| *value >= 0) + .ok_or_else(|| db_error("invalid replay metadata compaction frontier")), + } +} + +async fn has_work(conn: &(impl QueryExecutor + ?Sized), after: i64) -> Result { + Ok(conn + .query( + "SELECT 1 FROM lcm_raw_messages WHERE store_id > ?1 LIMIT 1", + params![after], + ) + .await? + .next() + .await? + .is_some()) +} + +/// Advances one bounded repair page under the registered database writer. +/// Body bytes, lineage and effective replay content remain unchanged. +pub async fn compact_replay_metadata( + database: &Database, + max_rows: usize, +) -> Result { + let read = database.read_connection(); + if !has_work(&read, frontier(&read).await?).await? { + return Ok(ReplayMetadataCompactionReport::default()); + } + let transaction = database + .begin_write_transaction(OPERATION) + .await + .map_err(db_error)?; + match step(&transaction, max_rows).await { + Ok(report) => { + transaction.commit().await.map_err(db_error)?; + Ok(report) + } + Err(error) => match transaction.rollback().await { + Ok(()) => Err(error), + Err(rollback) => Err(db_error(format!("{error}; rollback failed: {rollback}"))), + }, + } +} + +struct Candidate { + store_id: i64, + eligible: bool, + read_bytes: Option, +} + +/// Read only sizes and identities until each candidate earns the byte budget. +async fn candidates( + conn: &(impl QueryExecutor + ?Sized), + after: i64, + limit: usize, +) -> Result, LcmError> { + let mut rows = conn + .query( + "SELECT r.store_id, + CASE WHEN r.storage_kind = 'inline' AND json_valid(r.metadata_json) + THEN json_type(r.metadata_json, '$.lcm_active_replay') = 'true' + AND json_type(r.metadata_json, '$.active_replay.content') = 'text' + ELSE 0 END, + LENGTH(CAST(r.metadata_json AS BLOB)), + COALESCE(LENGTH(CAST(r.content AS BLOB)), b.uncompressed_bytes), + COALESCE(LENGTH(b.body), 0) + FROM lcm_raw_messages r + LEFT JOIN session_canonical_bodies b ON b.content_hash = r.content_hash + WHERE r.store_id > ?1 ORDER BY r.store_id LIMIT ?2", + params![ + after, + limit.clamp(1, MAX_REPLAY_METADATA_COMPACTION_ROWS) as i64 + ], + ) + .await?; + let mut result = Vec::new(); + while let Some(row) = rows.next().await? { + let metadata: Option = row.get(2)?; + let body: Option = row.get(3)?; + let stored: i64 = row.get(4)?; + let read_bytes = metadata + .zip(body) + .and_then(|(metadata, body)| { + Some(( + u64::try_from(metadata).ok()?, + u64::try_from(body).ok()?, + u64::try_from(stored).ok()?, + )) + }) + .and_then(|(metadata, body, stored)| metadata.checked_add(body)?.checked_add(stored)); + result.push(Candidate { + store_id: row.get(0)?, + eligible: row.get::>(1)?.unwrap_or(0) != 0, + read_bytes, + }); + } + Ok(result) +} + +async fn step( + conn: &(impl Executor + ?Sized), + max_rows: usize, +) -> Result { + let mut after = frontier(conn).await?; + let candidates = candidates(conn, after, max_rows).await?; + let mut report = ReplayMetadataCompactionReport::default(); + let mut remaining = MAX_BATCH_BYTES; + let mut guard_suspended = false; + for candidate in candidates { + if candidate.eligible { + let bytes = candidate.read_bytes.ok_or_else(|| { + db_error(format!( + "raw row {} has no bounded canonical body", + candidate.store_id + )) + })?; + if bytes > MAX_BATCH_BYTES { + report.rows_skipped += 1; + report.rows_skipped_oversized += 1; + } else if bytes > remaining { + break; + } else { + remaining -= bytes; + let message = raw::load_raw_message_by_store_id(conn, candidate.store_id).await?; + if let Some(metadata) = compacted_metadata(&message)? { + if !guard_suspended { + summary_convergence::suspend_raw_update_guard(conn).await?; + guard_suspended = true; + } + let changed = + apply_metadata(conn, candidate.store_id, &message, &metadata).await?; + if changed { + report.rows_compacted += 1; + report.metadata_bytes_removed += message + .metadata_json + .as_deref() + .map_or(0, str::len) + .saturating_sub(metadata.len()) + as u64; + } else { + report.rows_skipped += 1; + } + } else { + report.rows_skipped += 1; + } + } + } + report.rows_scanned += 1; + after = candidate.store_id; + } + if guard_suspended { + summary_convergence::restore_raw_update_guard(conn).await?; + } + schema::set_gc_meta(conn, FRONTIER_KEY, &after.to_string()).await?; + report.has_more = has_work(conn, after).await?; + Ok(report) +} + +fn compacted_metadata(message: &LcmRawMessage) -> Result, LcmError> { + if message.storage_kind != crate::LcmStorageKind::Inline { + return Ok(None); + } + let Some(stored) = message.metadata_json.as_deref() else { + return Ok(None); + }; + let mut metadata: Value = serde_json::from_str(stored).map_err(db_error)?; + if metadata.get("lcm_active_replay").and_then(Value::as_bool) != Some(true) { + return Ok(None); + } + let Some(replay) = metadata + .get_mut("active_replay") + .and_then(Value::as_object_mut) + else { + return Ok(None); + }; + let Some(content) = replay.get("content").and_then(Value::as_str) else { + return Ok(None); + }; + if content != message.content + || gc::is_known_payload_placeholder_prefix(content.trim_start()) + || !payload::extract_payload_refs_from_text(content).is_empty() + || security::contains_media_payload(content) + // Ingest's large-media detector intentionally requires substantial + // base64 data. Compaction is stricter: even a tiny data: URI carries + // a provider media representation that must remain in replay metadata. + || content + .as_bytes() + .windows(5) + .any(|bytes| bytes.eq_ignore_ascii_case(b"data:")) + { + return Ok(None); + } + replay.remove("content"); + let rebound = raw::resanitize_stored_provider_metadata_json(&metadata.to_string())?; + let mut expected = metadata; + let mut actual: Value = serde_json::from_str(&rebound).map_err(db_error)?; + // Only the metadata receipt may change in addition to the proven duplicate. + // New sanitizer policy changes belong to privacy remediation, not this pass. + remove_metadata_receipt(&mut expected); + remove_metadata_receipt(&mut actual); + if expected != actual || rebound.len() >= stored.len() { + return Ok(None); + } + Ok(Some(rebound)) +} + +fn remove_metadata_receipt(metadata: &mut Value) { + if let Some(protection) = metadata + .get_mut("ingest_protection") + .and_then(Value::as_object_mut) + { + protection.remove("metadata_sanitization_receipt"); + } +} + +async fn apply_metadata( + conn: &(impl Executor + ?Sized), + store_id: i64, + message: &LcmRawMessage, + metadata: &str, +) -> Result { + let changed = conn + .execute( + "UPDATE lcm_raw_messages SET metadata_json = ?1 + WHERE store_id = ?2 AND provider = ?3 AND session_id = ?4 AND message_id = ?5 + AND content_hash = ?6 AND metadata_json IS ?7 AND storage_kind = 'inline' + AND role = ?8 AND ordinal = ?9 AND timestamp IS ?10 AND payload_ref IS NULL", + params![ + metadata, + store_id, + message.provider.as_str(), + message.session_id.as_str(), + message.message_id.as_str(), + message.content_hash.as_str(), + message.metadata_json.as_deref(), + message.role.as_str(), + message.ordinal, + message.timestamp + ], + ) + .await?; + Ok(changed == 1) +} diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs new file mode 100644 index 0000000000..1bc50de1ce --- /dev/null +++ b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs @@ -0,0 +1,371 @@ +use tracedecay_domain::{ComponentVersion, SanitizationReceiptV1}; +use tracedecay_privacy::{ + LCM_PAYLOAD_SANITIZER_VERSION_V1, sanitize_lcm_payload_text, verify_sanitized_json_payload, +}; +use tracedecay_runtime_core::db::engine::{TestConnection, TransactionBehavior}; +use tracedecay_store::{StoredCanonicalBody, UPSERT_CANONICAL_BODY_SQL}; + +use super::*; + +struct Store { + conn: TestConnection, + _directory: tempfile::TempDir, +} + +impl Store { + async fn open() -> Self { + let directory = tempfile::tempdir().unwrap(); + let conn = TestConnection::open(&directory.path().join("replay-metadata.db")); + conn.execute_batch( + "CREATE TABLE sessions ( + provider TEXT NOT NULL, session_id TEXT NOT NULL, + PRIMARY KEY(provider, session_id) + ); INSERT INTO sessions VALUES ('codex', 'session');", + ) + .await + .unwrap(); + schema::ensure_lcm_schema(&conn).await.unwrap(); + Self { + conn, + _directory: directory, + } + } + + async fn insert(&self, ordinal: i64, text: &str, replay: Value, canonical: bool) -> i64 { + let sanitized = sanitize_lcm_payload_text(text).unwrap(); + assert_eq!(sanitized.sanitized_text(), text); + let body = StoredCanonicalBody::pack(text.as_bytes()).unwrap(); + let metadata = serde_json::json!({ + "lcm_active_replay": true, + "active_replay": {"role": "assistant", "content": replay, "provider_field": "kept"}, + "ingest_protection": {"sanitization_receipt": sanitized.receipt()} + }); + let metadata = + raw::resanitize_stored_provider_metadata_json(&metadata.to_string()).unwrap(); + let transaction = self + .conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + if canonical { + transaction + .execute( + UPSERT_CANONICAL_BODY_SQL, + params![ + body.content_hash.as_str(), + body.encoding, + body.blob.as_slice(), + body.uncompressed_bytes + ], + ) + .await + .unwrap(); + } + transaction + .execute( + "INSERT INTO lcm_raw_messages(provider, message_id, session_id, role, ordinal, + timestamp, content, content_hash, storage_kind, metadata_json) + VALUES ('codex', ?1, 'session', 'assistant', ?2, NULL, ?3, ?4, 'inline', ?5)", + params![ + format!("message-{ordinal}"), + ordinal, + (!canonical).then_some(text), + body.content_hash.as_str(), + metadata.as_str() + ], + ) + .await + .unwrap(); + let mut rows = transaction + .query( + "SELECT store_id FROM lcm_raw_messages WHERE ordinal = ?1", + params![ordinal], + ) + .await + .unwrap(); + let id = rows.next().await.unwrap().unwrap().get(0).unwrap(); + transaction.commit().await.unwrap(); + id + } + + async fn tick(&self, rows: usize) -> ReplayMetadataCompactionReport { + let transaction = self + .conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let report = step(&transaction, rows).await.unwrap(); + transaction.commit().await.unwrap(); + report + } +} + +async fn scalar(conn: &(impl QueryExecutor + ?Sized), sql: &str) -> i64 { + conn.query(sql, ()) + .await + .unwrap() + .next() + .await + .unwrap() + .unwrap() + .get(0) + .unwrap() +} + +fn metadata(message: &LcmRawMessage) -> Value { + serde_json::from_str(message.metadata_json.as_deref().unwrap()).unwrap() +} + +#[asupersync::test] +async fn removes_only_the_duplicate_and_preserves_receipts_replay_and_summary_state() { + let store = Store::open().await; + let text = "retained verified reasoning and tool evidence ".repeat(180); + let id = store + .insert(1, &text, Value::String(text.clone()), true) + .await; + store + .conn + .execute_batch( + "INSERT INTO session_summary_nodes(summary_id, provider, conversation_id, session_id, + depth, summary_text, summary_hash, summary_token_count, source_token_count) + VALUES ('summary', 'codex', 'conversation', 'session', 0, 'summary', 'hash', 1, 1); + INSERT INTO session_summary_sources(summary_id, source_kind, source_id, ordinal) + VALUES ('summary', 'raw_message', '1', 0); + UPDATE lcm_summary_convergence_queue SET state = 'current', attempted_raw_store_id = 1;", + ) + .await + .unwrap(); + let before = raw::load_raw_message_by_store_id(&store.conn, id) + .await + .unwrap(); + let before_replay = crate::replay_transactions::raw_replay_message(&before); + let revision = scalar( + &store.conn, + "SELECT raw_revision_generation FROM lcm_summary_convergence_queue", + ) + .await; + let report = store.tick(48).await; + assert_eq!(report.rows_compacted, 1); + assert!(report.metadata_bytes_removed >= text.len() as u64); + assert!(!report.has_more); + let after = raw::load_raw_message_by_store_id(&store.conn, id) + .await + .unwrap(); + assert_eq!(after.content, before.content); + assert_eq!(after.content_hash, before.content_hash); + assert_eq!( + crate::replay_transactions::raw_replay_message(&after), + before_replay + ); + let mut after_metadata = metadata(&after); + assert!(after_metadata["active_replay"].get("content").is_none()); + assert_eq!( + after_metadata["ingest_protection"]["sanitization_receipt"], + metadata(&before)["ingest_protection"]["sanitization_receipt"] + ); + let protection = after_metadata + .as_object_mut() + .unwrap() + .remove("ingest_protection") + .unwrap(); + let receipt: SanitizationReceiptV1 = + serde_json::from_value(protection["metadata_sanitization_receipt"].clone()).unwrap(); + verify_sanitized_json_payload( + &Value::String(after_metadata.to_string()), + &receipt, + &ComponentVersion::new(LCM_PAYLOAD_SANITIZER_VERSION_V1).unwrap(), + ) + .unwrap(); + assert_eq!( + scalar( + &store.conn, + "SELECT raw_revision_generation FROM lcm_summary_convergence_queue" + ) + .await, + revision + ); + assert_eq!( + scalar( + &store.conn, + "SELECT COUNT(*) FROM lcm_summary_convergence_dirty_raw" + ) + .await, + 0 + ); + assert_eq!( + scalar(&store.conn, "SELECT COUNT(*) FROM session_summary_sources").await, + 1 + ); + assert_eq!( + scalar(&store.conn, "SELECT COUNT(*) FROM session_canonical_bodies").await, + 1 + ); + summary_convergence::validate_raw_update_guard(&store.conn) + .await + .unwrap(); + store.conn.execute("UPDATE lcm_raw_messages SET metadata_json = json_set(metadata_json, '$.provider_note', 'changed') WHERE store_id = ?1", params![id]).await.unwrap(); + assert_eq!( + scalar( + &store.conn, + "SELECT COUNT(*) FROM lcm_summary_convergence_dirty_raw" + ) + .await, + 1, + "ordinary updates still invalidate" + ); +} + +#[asupersync::test] +async fn structured_different_media_and_external_representations_are_retained() { + let store = Store::open().await; + let text = "ordinary original body ".repeat(30); + store + .insert( + 1, + &text, + serde_json::json!([{"type": "text", "text": text}]), + false, + ) + .await; + store + .insert( + 2, + &text, + Value::String("provider-specific replay representation".repeat(30)), + false, + ) + .await; + let placeholder = "[externalized tool output: retained-payload-placeholder]".repeat(20); + store + .insert(3, &placeholder, Value::String(placeholder.clone()), false) + .await; + let id = store + .insert(4, &text, Value::String(text.clone()), false) + .await; + store.conn.execute("UPDATE lcm_raw_messages SET storage_kind = 'external', payload_ref = 'payload-ref', placeholder_text = 'external' WHERE store_id = ?1", params![id]).await.unwrap(); + let media = "data:image/png;base64,aGVsbG8="; + store + .insert(5, media, Value::String(media.to_owned()), false) + .await; + let report = store.tick(48).await; + assert_eq!(report.rows_scanned, 5); + assert_eq!(report.rows_compacted, 0); + assert_eq!(report.metadata_bytes_removed, 0); + // All five provider representations still carry their original content. + assert_eq!(scalar(&store.conn, "SELECT COUNT(*) FROM lcm_raw_messages WHERE json_type(metadata_json, '$.active_replay.content') IS NOT NULL").await, 5); +} + +#[asupersync::test] +async fn corrupt_body_rolls_back_prior_changes_frontier_and_guard_suspension() { + let store = Store::open().await; + let first = "first verified duplicated string ".repeat(120); + let second = "second verified duplicated string ".repeat(160); + let id = store + .insert(1, &first, Value::String(first.clone()), false) + .await; + store + .insert(2, &second, Value::String(second.clone()), true) + .await; + let original = raw::load_raw_message_by_store_id(&store.conn, id) + .await + .unwrap(); + store + .conn + .execute_batch("UPDATE session_canonical_bodies SET body = X'00'") + .await + .unwrap(); + let transaction = store + .conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + assert!(step(&transaction, 48).await.is_err()); + transaction.rollback().await.unwrap(); + assert_eq!(frontier(&store.conn).await.unwrap(), 0); + assert_eq!( + raw::load_raw_message_by_store_id(&store.conn, id) + .await + .unwrap(), + original + ); + summary_convergence::validate_raw_update_guard(&store.conn) + .await + .unwrap(); + store.conn.execute("UPDATE lcm_raw_messages SET metadata_json = json_set(metadata_json, '$.provider_note', 'changed') WHERE store_id = ?1", params![id]).await.unwrap(); + assert_eq!( + scalar( + &store.conn, + "SELECT COUNT(*) FROM lcm_summary_convergence_dirty_raw" + ) + .await, + 1, + "rollback restores ordinary invalidation" + ); +} + +#[asupersync::test] +async fn stale_metadata_cannot_be_overwritten_and_missing_guard_refuses_repair() { + let store = Store::open().await; + let text = "verified duplicate content ".repeat(80); + let id = store + .insert(1, &text, Value::String(text.clone()), false) + .await; + let snapshot = raw::load_raw_message_by_store_id(&store.conn, id) + .await + .unwrap(); + let compacted = compacted_metadata(&snapshot).unwrap().unwrap(); + store.conn.execute("UPDATE lcm_raw_messages SET metadata_json = json_set(metadata_json, '$.provider_note', 'concurrent') WHERE store_id = ?1", params![id]).await.unwrap(); + assert!( + !apply_metadata(&store.conn, id, &snapshot, &compacted) + .await + .unwrap() + ); + assert_eq!( + metadata( + &raw::load_raw_message_by_store_id(&store.conn, id) + .await + .unwrap() + )["provider_note"], + "concurrent" + ); + store + .conn + .execute_batch("DROP TRIGGER lcm_summary_convergence_raw_unprotected_update") + .await + .unwrap(); + let transaction = store + .conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let error = step(&transaction, 48).await.unwrap_err(); + assert!(error.to_string().contains("guard is missing")); + transaction.rollback().await.unwrap(); + assert_eq!(frontier(&store.conn).await.unwrap(), 0); +} + +#[asupersync::test] +async fn row_and_byte_budgets_advance_a_durable_frontier_and_finished_pages_are_noops() { + let store = Store::open().await; + let text = "bounded replay text ".repeat(35_000); + for ordinal in 1..=4 { + store + .insert(ordinal, &text, Value::String(text.clone()), true) + .await; + } + let first = store.tick(1).await; + assert_eq!(first.rows_scanned, 1); + assert_eq!(first.rows_compacted, 1); + assert!(first.has_more); + assert_eq!(frontier(&store.conn).await.unwrap(), 1); + let second = store.tick(48).await; + assert!( + second.rows_scanned > 0 && second.rows_scanned < 3, + "byte cap stops before the third large row" + ); + assert!(second.has_more); + let last = store.tick(48).await; + assert!(!last.has_more); + let idle = store.tick(48).await; + assert_eq!(idle, ReplayMetadataCompactionReport::default()); +} diff --git a/crates/tracedecay-lcm/src/retention/tests.rs b/crates/tracedecay-lcm/src/retention/tests.rs index 6978cf853b..cfbedbf733 100644 --- a/crates/tracedecay-lcm/src/retention/tests.rs +++ b/crates/tracedecay-lcm/src/retention/tests.rs @@ -402,7 +402,8 @@ async fn dry_run_counts_without_mutating() -> Result<(), String> { #[asupersync::test] async fn backlog_read_reports_real_eligible_bytes_and_watermark() -> Result<(), String> { let store = test_store().await?; - let durable = insert_message(&store.conn, 1, 90, "retention backlog bytes").await?; + let content = "retention backlog café ☕"; + let durable = insert_message(&store.conn, 1, 90, content).await?; make_projection_durable(&store.conn, durable).await?; let records = read_session_retention_backlog( @@ -417,7 +418,7 @@ async fn backlog_read_reports_real_eligible_bytes_and_watermark() -> Result<(), assert_eq!(records.len(), 1); assert_eq!(records[0].table.as_str(), "lcm_raw_messages"); - assert!(records[0].past_window_bytes.get() > 0); + assert_eq!(records[0].past_window_bytes.get(), content.len() as u64); assert_eq!( records[0].oldest_past_window_at, tracedecay_domain::UtcMicros((NOW - 90 * DAY) * 1_000_000) @@ -487,16 +488,15 @@ async fn drop_removes_the_single_content_copy_and_its_index() -> Result<(), Stri Ok(()) } -// (a) offload only externalizes projection-durable rows AFTER durability; the -// bulky inline content leaves the raw column, replaced by a recoverable -// content-addressed placeholder (§4 one content copy). +// Compaction only changes projection-durable rows, and retains complete text +// through the canonical verified reader without creating a plain payload file. #[asupersync::test] -async fn offload_externalizes_durable_content_after_durability() -> Result<(), String> { +async fn offload_compacts_durable_content_without_creating_payload_files() -> Result<(), String> { let store = test_store().await?; let conn = &store.conn; - let content = "x".repeat(4096); + let content = "x".repeat(8192); let durable = insert_message(conn, 1, 90, &content).await?; - let _live = insert_message(conn, 2, 90, &content).await?; + let live = insert_message(conn, 2, 90, &content).await?; make_projection_durable(conn, durable).await?; let config = LcmRetentionConfig { @@ -511,34 +511,840 @@ async fn offload_externalizes_durable_content_after_durability() -> Result<(), S "only the durable row is offloaded" ); assert!(report.offloaded.bytes_reclaimed >= 4096); + assert_eq!(report.canonicalized_rows, 1); + assert!(report.errors.is_empty(), "{:?}", report.errors); - // Durable row: inline content cleared, now external with a payload_ref. + // The ordinary inline reader hydrates the shared canonical body. let kind: String = fetch_str( conn, "SELECT storage_kind FROM lcm_raw_messages WHERE store_id = ?1", params![durable], ) .await?; - assert_eq!(kind, "external"); + assert_eq!(kind, "inline"); let payload_present = fetch_i64(conn, "SELECT COUNT(*) FROM lcm_external_payloads", ()).await?; - assert_eq!(payload_present, 1, "content stored once, addressed by hash"); - // The un-projected live row is untouched (still inline). - let live_kind: i64 = fetch_i64( + assert_eq!(payload_present, 0); + assert_eq!(count(conn, "session_canonical_bodies").await?, 1); + assert!(!payload::payload_dir(&store.storage_root).exists()); + let restored = raw::load_raw_message_by_store_id(conn, durable) + .await + .map_err(|error| error.to_string())?; + assert_eq!(restored.content, content); + assert!(restored.payload_ref.is_none()); + let live_content = fetch_str( conn, - "SELECT COUNT(*) FROM lcm_raw_messages WHERE storage_kind = 'inline'", + "SELECT content FROM lcm_raw_messages WHERE store_id = ?1", + params![live], + ) + .await?; + assert_eq!(live_content, content, "live un-projected row stays intact"); + let repeated = run_apply(conn, &store.storage_root, &config).await?; + assert_eq!(repeated.offloaded.acted, 0); + assert_eq!(repeated.canonicalized_rows, 0); + Ok(()) +} + +#[asupersync::test] +async fn offload_keeps_small_bodies_inline_and_uses_byte_threshold() -> Result<(), String> { + let store = test_store().await?; + let small = "é".repeat(INLINE_BODY_BYTES / 2 - 1); + let small_id = insert_message(&store.conn, 1, 90, &small).await?; + make_projection_durable(&store.conn, small_id).await?; + let config = LcmRetentionConfig { + drop_after_days: None, + ..LcmRetentionConfig::default() + }; + let report = run_apply(&store.conn, &store.storage_root, &config).await?; + assert!(report.errors.is_empty(), "{:?}", report.errors); + assert_eq!(report.offloaded.eligible, 0); + assert_eq!(count(&store.conn, "lcm_external_payloads").await?, 0); + assert!(!payload::payload_dir(&store.storage_root).exists()); + let backlog = read_session_retention_backlog( + &store.conn, + tracedecay_contracts::storage::StoreKeyV1::new("sessions.db") + .map_err(|error| error.to_string())?, + &config, + NOW, + ) + .await + .map_err(|error| error.to_string())?; + assert_eq!(backlog[0].past_window_bytes.get(), 0); + + let boundary = "é".repeat(INLINE_BODY_BYTES / 2); + let boundary_id = insert_message(&store.conn, 2, 90, &boundary).await?; + make_projection_durable(&store.conn, boundary_id).await?; + let report = run_apply(&store.conn, &store.storage_root, &config).await?; + assert!(report.errors.is_empty(), "{:?}", report.errors); + assert_eq!(report.offloaded.eligible, 1); + assert_eq!( + report.offloaded.acted, 0, + "the full UTF-8 snippet would duplicate this body" + ); + assert_eq!(report.offloaded.bytes_reclaimed, 0); + assert_eq!(count(&store.conn, "lcm_external_payloads").await?, 0); + let unchanged = raw::load_raw_message_by_store_id(&store.conn, small_id) + .await + .map_err(|error| error.to_string())?; + assert_eq!(unchanged.content, small); + Ok(()) +} + +#[asupersync::test] +async fn small_offload_restores_verified_text_and_releases_file_only_after_gc_grace() +-> Result<(), String> { + let store = test_store().await?; + let content = "small tool output: café completed"; + let target = seed_retention_offload(&store, 1, content).await?; + let path = payload::payload_dir(&store.storage_root).join(&target.payload_ref); + let config = LcmRetentionConfig { + drop_after_days: None, + ..LcmRetentionConfig::default() + }; + let preview = run_session_retention_authorized( + &store.conn, + &store.storage_root, + "all", + None, + &config, + RetentionMode::DryRun, + NOW, + &|_| Ok(()), + ) + .await + .map_err(|error| error.to_string())?; + assert_eq!(preview.inlined.eligible, 1); + assert_eq!(preview.inlined.acted, 0); + assert_eq!(preview.bytes_reclaimed(), 0); + assert_eq!(count(&store.conn, "lcm_external_payloads").await?, 1); + assert!(path.exists()); + + let report = run_apply(&store.conn, &store.storage_root, &config).await?; + assert!(report.errors.is_empty(), "{:?}", report.errors); + assert_eq!(report.inlined.acted, 1); + assert_eq!(report.offloaded.acted, 0); + assert_eq!(report.bytes_reclaimed(), 0); + assert_eq!(count(&store.conn, "session_canonical_bodies").await?, 0); + let restored = raw::load_raw_message_by_store_id(&store.conn, target.store_id) + .await + .map_err(|error| error.to_string())?; + assert_eq!(restored.content, content); + assert_eq!(restored.content_hash, target.content_hash); + assert_eq!(restored.metadata_json, target.metadata_json); + assert!(restored.payload_ref.is_none()); + assert!(path.exists(), "inlining never deletes the payload file"); + assert_eq!( + fetch_i64( + &store.conn, + "SELECT COUNT(*) FROM lcm_raw_messages_fts + WHERE lcm_raw_messages_fts MATCH 'completed'", + (), + ) + .await?, + 1, + "restoration also makes the original tool text searchable again" + ); + + let marked_at = fetch_i64( + &store.conn, + "SELECT first_seen_at FROM lcm_gc_marks WHERE payload_ref = ?1", + params![target.payload_ref.as_str()], + ) + .await?; + let gc_config = crate::LcmGcConfig::default(); + for (now, retained) in [ + (marked_at + gc_config.grace_seconds as i64 - 1, true), + (marked_at + gc_config.grace_seconds as i64 + 1, false), + ] { + let gc = crate::gc::run_payload_gc_with_apply( + &store.conn, + &store.storage_root, + PROVIDER, + Some(SESSION), + &gc_config, + true, + now, + ) + .await + .map_err(|error| error.to_string())?; + assert!(gc.errors.is_empty(), "{:?}", gc.errors); + assert_eq!(path.exists(), retained); + assert_eq!( + count(&store.conn, "lcm_external_payloads").await?, + i64::from(retained) + ); + } + let restored = raw::load_raw_message_by_store_id(&store.conn, target.store_id) + .await + .map_err(|error| error.to_string())?; + assert_eq!(restored.content, content, "full text survives payload GC"); + Ok(()) +} + +#[asupersync::test] +async fn offload_restoration_rejects_corruption_staleness_and_lost_authority() -> Result<(), String> +{ + for failure in ["payload", "receipt", "stale", "metadata", "authority"] { + let store = test_store().await?; + let content = if failure == "payload" { + "original complete text".to_owned() + } else { + "original complete text ".repeat(1024) + }; + let mut target = seed_retention_offload(&store, 1, &content).await?; + let path = payload::payload_dir(&store.storage_root).join(&target.payload_ref); + match failure { + "payload" => { + std::fs::write(&path, "tampered bytes").map_err(|error| error.to_string())? + } + "receipt" => { + store + .conn + .execute( + "UPDATE lcm_raw_messages SET metadata_json = '{}' WHERE store_id = ?1", + params![target.store_id], + ) + .await + .map_err(|error| error.to_string())?; + target.metadata_json = Some("{}".to_owned()); + } + "stale" => { + store + .conn + .execute( + "UPDATE lcm_raw_messages SET timestamp = ?2 WHERE store_id = ?1", + params![target.store_id, NOW], + ) + .await + .map_err(|error| error.to_string())?; + } + "metadata" => { + store + .conn + .execute( + "UPDATE lcm_raw_messages + SET metadata_json = json_set(metadata_json, '$.changed', 1) + WHERE store_id = ?1", + params![target.store_id], + ) + .await + .map_err(|error| error.to_string())?; + } + _ => {} + } + let convergence_before = mark_convergence_current(&store.conn).await?; + let error = inline_offloaded_one( + RetentionStore::Connection(&store.conn), + &store.storage_root, + &target, + Some(&|intent| { + if failure == "authority" && intent == "commit session retention inline pass" { + Err(LcmError::Cancelled) + } else { + Ok(()) + } + }), + ) + .await + .expect_err("unproven or changed payload must remain external"); + match failure { + "payload" | "receipt" => assert_eq!(error, LcmError::PayloadIntegrityMismatch), + "stale" | "metadata" => { + assert!(error.to_string().contains("compare-and-swap rejected")) + } + "authority" => assert_eq!(error, LcmError::Cancelled), + _ => unreachable!(), + } + assert_eq!( + fetch_i64( + &store.conn, + "SELECT COUNT(*) FROM lcm_raw_messages + WHERE store_id = ?1 AND storage_kind = 'external' + AND content IS NULL AND payload_ref = ?2", + params![target.store_id, target.payload_ref.as_str()], + ) + .await?, + 1, + "{failure} must preserve the original row" + ); + assert_eq!(count(&store.conn, "lcm_external_payloads").await?, 1); + assert_eq!(count(&store.conn, "session_canonical_bodies").await?, 0); + assert!(path.exists(), "{failure} must preserve the payload file"); + assert_eq!(convergence_snapshot(&store.conn).await?, convergence_before); + summary_convergence::validate_raw_update_guard(&store.conn) + .await + .map_err(|error| error.to_string())?; + store + .conn + .execute( + "UPDATE lcm_raw_messages SET timestamp = timestamp + 1 WHERE store_id = ?1", + params![target.store_id], + ) + .await + .map_err(|error| error.to_string())?; + assert_eq!( + count(&store.conn, "lcm_summary_convergence_dirty_raw").await?, + 1, + "ordinary evidence changes still invalidate after {failure}" + ); + } + Ok(()) +} + +#[asupersync::test] +async fn large_offloads_share_verified_canonical_text_and_release_files_after_grace() +-> Result<(), String> { + let store = test_store().await?; + let content = "restored searchable café tool output\n".repeat(1024); + let first = seed_retention_offload(&store, 1, &content).await?; + let second = seed_retention_offload(&store, 2, &content).await?; + let convergence_before = mark_convergence_current(&store.conn).await?; + let config = LcmRetentionConfig { + drop_after_days: None, + ..LcmRetentionConfig::default() + }; + let report = run_apply(&store.conn, &store.storage_root, &config).await?; + assert!(report.errors.is_empty(), "{:?}", report.errors); + assert_eq!(report.inlined.acted, 2); + assert_eq!(report.canonicalized_rows, 2); + assert_eq!(report.offloaded.acted, 0); + assert_eq!( + report.bytes_reclaimed(), + 0, + "restoration does not report moved bytes as reclaimed" + ); + assert_eq!(count(&store.conn, "session_canonical_bodies").await?, 1); + assert_eq!(convergence_snapshot(&store.conn).await?, convergence_before); + assert_eq!(fetch_i64(&store.conn, + "SELECT COUNT(*) FROM lcm_raw_messages_fts WHERE lcm_raw_messages_fts MATCH 'searchable'", () + ).await?, 2); + for target in [&first, &second] { + let restored = raw::load_raw_message_by_store_id(&store.conn, target.store_id) + .await + .map_err(|error| error.to_string())?; + assert_eq!(restored.content, content); + assert_eq!(restored.content_hash, target.content_hash); + assert_eq!(restored.metadata_json, target.metadata_json); + assert!(restored.payload_ref.is_none()); + assert!( + payload::payload_dir(&store.storage_root) + .join(&target.payload_ref) + .exists() + ); + } + let marked_at = fetch_i64( + &store.conn, + "SELECT MAX(first_seen_at) FROM lcm_gc_marks", (), ) .await?; - assert_eq!(live_kind, 1, "live un-projected row stays inline"); + let gc_config = crate::LcmGcConfig::default(); + let gc = crate::gc::run_payload_gc_with_apply( + &store.conn, + &store.storage_root, + PROVIDER, + Some(SESSION), + &gc_config, + true, + marked_at + gc_config.grace_seconds as i64 + 1, + ) + .await + .map_err(|error| error.to_string())?; + assert!(gc.errors.is_empty(), "{:?}", gc.errors); + assert_eq!(count(&store.conn, "lcm_external_payloads").await?, 0); + for target in [&first, &second] { + assert!( + !payload::payload_dir(&store.storage_root) + .join(&target.payload_ref) + .exists() + ); + assert_eq!( + raw::load_raw_message_by_store_id(&store.conn, target.store_id) + .await + .map_err(|error| error.to_string())? + .content, + content + ); + } + summary_convergence::validate_raw_update_guard(&store.conn) + .await + .map_err(|error| error.to_string())?; + store.conn.execute( + "UPDATE lcm_raw_messages SET metadata_json = json_set(metadata_json, '$.provider_note', 'changed') + WHERE store_id = ?1", params![first.store_id], + ).await.map_err(|error| error.to_string())?; + assert_eq!( + count(&store.conn, "lcm_summary_convergence_dirty_raw").await?, + 1, + "ordinary metadata changes still invalidate after successful restoration" + ); + Ok(()) +} + +#[asupersync::test] +async fn canonical_compaction_counts_utf8_snippets_and_marginal_shared_body_cost() +-> Result<(), String> { + let store = test_store().await?; + let mut state = 0x923a_405b_u64; + let ascii: String = (0..6000) + .map(|index| { + state = state.wrapping_mul(6364136223846793005).wrapping_add(1); + if index % 9 == 8 { + ' ' + } else { + char::from(b'a' + ((state >> 32) % 26) as u8) + } + }) + .collect(); + let unicode = "é".repeat(3000); + let mut targets = Vec::new(); + for (index, content) in [&ascii, &unicode].into_iter().enumerate() { + let packed = pack_canonical(content).map_err(|error| error.to_string())?; + let snippet = raw::derived_text_for_snippet(content); + assert!( + packed.blob.len() < content.len(), + "BLOB-only comparison would permit this body" + ); + assert!( + packed.blob.len() + snippet.len() >= content.len(), + "the full representation grows" + ); + targets.push(seed_retention_offload(&store, index as i64 + 1, content).await?); + let direct = insert_message(&store.conn, index as i64 + 3, 90, content).await?; + make_projection_durable(&store.conn, direct).await?; + } + assert!( + raw::derived_text_for_snippet(&unicode).len() + > raw::derived_text_for_snippet(&unicode).chars().count() + ); + let config = LcmRetentionConfig { + drop_after_days: None, + ..LcmRetentionConfig::default() + }; + let report = run_apply(&store.conn, &store.storage_root, &config).await?; + assert!(report.errors.is_empty(), "{:?}", report.errors); + assert_eq!(report.inlined.eligible, 2); + assert_eq!(report.offloaded.eligible, 2); + assert_eq!(report.inlined.acted + report.offloaded.acted, 0); + assert_eq!(count(&store.conn, "session_canonical_bodies").await?, 0); + assert_eq!(count(&store.conn, "lcm_external_payloads").await?, 2); + + // Another ordinary canonical owner makes this BLOB a zero-marginal-cost + // reuse. Only its new snippet and record allowance need to fit. + let packed = pack_canonical(&ascii).map_err(|error| error.to_string())?; + let txn = store + .conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(|error| error.to_string())?; + raw::persist_canonical_body(&txn, &packed) + .await + .map_err(|error| error.to_string())?; + txn.execute( + "UPDATE lcm_raw_messages SET content = NULL, placeholder_text = ?1 WHERE message_id = 'msg-3'", + params![raw::derived_text_for_snippet(&ascii)], + ).await.map_err(|error| error.to_string())?; + txn.commit().await.map_err(|error| error.to_string())?; + assert_eq!( + inline_offloaded_one( + RetentionStore::Connection(&store.conn), + &store.storage_root, + &targets[0], + None + ) + .await + .map_err(|error| error.to_string())?, + Some(true) + ); + let restored = raw::load_raw_message_by_store_id(&store.conn, targets[0].store_id) + .await + .map_err(|error| error.to_string())?; + assert_eq!(restored.content, ascii); + assert_eq!(restored.metadata_json, targets[0].metadata_json); + assert_eq!(count(&store.conn, "session_canonical_bodies").await?, 1); + assert_eq!( + fetch_str( + &store.conn, + "SELECT storage_kind FROM lcm_raw_messages WHERE store_id = ?1", + params![targets[1].store_id] + ) + .await?, + "external" + ); Ok(()) } #[asupersync::test] -async fn offload_cas_preserves_revived_row_and_rolls_back_payload() -> Result<(), String> { +async fn restoration_revisits_old_cursor_and_stops_before_exceeding_byte_budget() +-> Result<(), String> { let store = test_store().await?; - let original = "stale content".repeat(128); + // The sanitizer accepts large structured documents through its bounded + // structured path; unstructured text over 1 MiB is refused at ingestion. + // Keep each valid body around 3 MiB so two fit the 8 MiB storage pass and + // the third must wait. The fourth exceeds the 4 MiB repack limit. + let content = serde_json::json!({ + "body": "bounded historical body ".repeat(3 * 1024 * 1024 / 24) + }) + .to_string(); + let mut targets = Vec::new(); + for ordinal in 1..=3 { + targets.push(seed_retention_offload(&store, ordinal, &content).await?); + } + let too_large = serde_json::json!({ + "body": "x".repeat(MAX_REPACK_BODY_BYTES) + }) + .to_string(); + let oversized = seed_retention_offload(&store, 4, &too_large).await?; + RetentionCursor { + aged_through: (NOW, i64::MAX), + durable_through: i64::MAX, + } + .persist(&store.conn, "inline_offloaded") + .await + .map_err(|error| error.to_string())?; + let config = LcmRetentionConfig { + drop_after_days: None, + ..LcmRetentionConfig::default() + }; + let preview = run_session_retention_authorized( + &store.conn, + &store.storage_root, + "all", + None, + &config, + RetentionMode::DryRun, + NOW, + &|_| Ok(()), + ) + .await + .map_err(|error| error.to_string())?; + assert_eq!(preview.inlined.eligible, 2); + assert_eq!(preview.inlined.acted, 0); + assert_eq!(count(&store.conn, "session_canonical_bodies").await?, 0); + assert!( + schema::get_gc_meta(&store.conn, &RetentionCursor::keys(RESTORE_CURSOR).0) + .await + .map_err(|error| error.to_string())? + .is_none() + ); + for (expected, has_more) in [(2, true), (1, false), (0, false)] { + let report = run_apply(&store.conn, &store.storage_root, &config).await?; + assert!(report.errors.is_empty(), "{:?}", report.errors); + assert_eq!(report.inlined.acted, expected); + assert_eq!(report.canonicalized_rows, expected); + assert_eq!(report.storage_has_more, has_more); + } + for target in &targets { + assert_eq!( + raw::load_raw_message_by_store_id(&store.conn, target.store_id) + .await + .map_err(|error| error.to_string())? + .content, + content + ); + } + assert_eq!( + fetch_str( + &store.conn, + "SELECT payload_ref FROM lcm_raw_messages WHERE store_id = ?1", + params![oversized.store_id] + ) + .await?, + oversized.payload_ref + ); + assert!( + payload::payload_dir(&store.storage_root) + .join(&oversized.payload_ref) + .exists() + ); + // A short continuation also drains newly durable inline rows across its + // row cap without repeating already-compacted work or skipping a source. + let direct_content = "short continuation text\n".repeat(1024); + for ordinal in 5..=7 { + let store_id = insert_message(&store.conn, ordinal, 90, &direct_content).await?; + make_projection_durable(&store.conn, store_id).await?; + } + let config = LcmRetentionConfig { + max_batch_size: 1, + ..config + }; + let mut total = 0; + let mut drained = false; + for _ in 0..6 { + let report = run_apply(&store.conn, &store.storage_root, &config).await?; + assert!(report.errors.is_empty(), "{:?}", report.errors); + assert!(report.offloaded.acted <= 1); + total += report.offloaded.acted; + if !report.storage_has_more { + drained = true; + break; + } + } + assert!(drained, "bounded short continuations must reach the end"); + assert_eq!(total, 3); + Ok(()) +} + +#[asupersync::test] +async fn scoped_single_row_retention_finishes_when_no_lineage_remains() -> Result<(), String> { + let store = test_store().await?; + let config = LcmRetentionConfig { + drop_after_days: None, + max_batch_size: 1, + ..LcmRetentionConfig::default() + }; + let report = run_apply(&store.conn, &store.storage_root, &config).await?; + assert!(report.errors.is_empty(), "{:?}", report.errors); + assert_eq!(report.inlined.eligible, 0); + assert_eq!(report.offloaded.eligible, 0); + assert!( + !report.storage_has_more, + "an empty scoped pass must not hot-loop" + ); + Ok(()) +} + +#[asupersync::test] +async fn canonical_compaction_commit_denial_preserves_original_inline_body() -> Result<(), String> { + let store = test_store().await?; + let content = "complete retained body\n".repeat(1024); + let store_id = insert_message(&store.conn, 1, 90, &content).await?; + make_projection_durable(&store.conn, store_id).await?; + let original = raw::load_raw_message_by_store_id(&store.conn, store_id) + .await + .map_err(|error| error.to_string())?; + let config = LcmRetentionConfig { + drop_after_days: None, + ..LcmRetentionConfig::default() + }; + let report = run_session_retention_authorized( + &store.conn, + &store.storage_root, + "all", + None, + &config, + RetentionMode::Apply, + NOW, + &|intent| { + if intent == "commit session retention canonical pass" { + Err(LcmError::Cancelled) + } else { + Ok(()) + } + }, + ) + .await + .map_err(|error| error.to_string())?; + assert_eq!(report.errors.len(), 1); + assert_eq!(report.offloaded.acted, 0); + assert!( + !report.storage_has_more, + "authority loss must not request a hot-loop retry" + ); + let retained = raw::load_raw_message_by_store_id(&store.conn, store_id) + .await + .map_err(|error| error.to_string())?; + assert_eq!(retained.content, original.content); + assert_eq!(retained.metadata_json, original.metadata_json); + assert_eq!(count(&store.conn, "session_canonical_bodies").await?, 0); + assert_eq!(count(&store.conn, "lcm_external_payloads").await?, 0); + assert_eq!( + RetentionCursor::read(&store.conn, "offload") + .await + .map_err(|error| error.to_string())? + .aged_through, + (i64::MIN, i64::MIN) + ); + Ok(()) +} + +#[asupersync::test] +async fn corrupt_shared_body_cannot_replace_plaintext_or_external_source() -> Result<(), String> { + let store = test_store().await?; + let content = "verified source text\n".repeat(1024); + let external = seed_retention_offload(&store, 1, &content).await?; + let direct = insert_message(&store.conn, 2, 90, &content).await?; + make_projection_durable(&store.conn, direct).await?; + let mut packed = pack_canonical(&content).map_err(|error| error.to_string())?; + packed.blob = vec![0xff]; + raw::persist_canonical_body(&store.conn, &packed) + .await + .map_err(|error| error.to_string())?; + // Both kinds have independent authoritative content and must not fetch an + // unrelated canonical BLOB. The normal canonical path still rejects it. + for store_id in [external.store_id, direct] { + let mut rows = store + .conn + .query( + &format!( + "SELECT {} FROM lcm_raw_messages WHERE store_id = ?1", + raw::RAW_MESSAGE_SELECT_COLUMNS + ), + params![store_id], + ) + .await + .map_err(|error| error.to_string())?; + let row = rows + .next() + .await + .map_err(|error| error.to_string())? + .ok_or("missing fixture row")?; + assert!( + row.get::>>(14) + .map_err(|error| error.to_string())? + .is_none() + ); + raw::verified_raw_message_from_row(&row).map_err(|error| error.to_string())?; + } + let config = LcmRetentionConfig { + drop_after_days: None, + ..LcmRetentionConfig::default() + }; + let report = run_apply(&store.conn, &store.storage_root, &config).await?; + assert_eq!(report.errors.len(), 2); + assert_eq!(report.inlined.acted + report.offloaded.acted, 0); + assert_eq!( + fetch_str( + &store.conn, + "SELECT content FROM lcm_raw_messages WHERE store_id = ?1", + params![direct] + ) + .await?, + content + ); + assert_eq!( + fetch_str( + &store.conn, + "SELECT payload_ref FROM lcm_raw_messages WHERE store_id = ?1", + params![external.store_id] + ) + .await?, + external.payload_ref + ); + assert!( + payload::payload_dir(&store.storage_root) + .join(&external.payload_ref) + .exists() + ); + assert_eq!( + RetentionCursor::read(&store.conn, RESTORE_CURSOR) + .await + .map_err(|error| error.to_string())? + .aged_through, + (i64::MIN, i64::MIN) + ); + store + .conn + .execute( + "UPDATE lcm_raw_messages SET content = NULL, placeholder_text = ?1 WHERE store_id = ?2", + params![raw::derived_text_for_snippet(&content), direct], + ) + .await + .map_err(|error| error.to_string())?; + assert_eq!( + raw::load_raw_message_by_store_id(&store.conn, direct) + .await + .expect_err("actual canonical content must be verified"), + LcmError::PayloadIntegrityMismatch + ); + Ok(()) +} + +#[asupersync::test] +async fn small_offload_restoration_is_bounded_scoped_and_leaves_protected_payloads() +-> Result<(), String> { + let store = test_store().await?; + let first = seed_retention_offload(&store, 1, "first old body").await?; + let second = seed_retention_offload(&store, 2, "second old body").await?; + let protected = seed_retention_offload(&store, 3, "protected body").await?; + store + .conn + .execute( + "UPDATE lcm_external_payloads SET kind = 'quarantined_assistant_output' + WHERE payload_ref = ?1", + params![protected.payload_ref.as_str()], + ) + .await + .map_err(|error| error.to_string())?; + let config = LcmRetentionConfig { + drop_after_days: None, + max_batch_size: 1, + ..LcmRetentionConfig::default() + }; + for disabled in [ + LcmRetentionConfig { + enabled: false, + ..config + }, + LcmRetentionConfig { + offload_after_days: None, + ..config + }, + ] { + let report = run_apply(&store.conn, &store.storage_root, &disabled).await?; + assert!(report.errors.is_empty(), "{:?}", report.errors); + assert_eq!(report.inlined.eligible, 0); + assert_eq!(report.inlined.acted, 0); + } + for (provider, session_id, expected) in [ + ("other-provider", None, 0), + (PROVIDER, Some("other-session"), 0), + (PROVIDER, Some(SESSION), 1), + ("all", None, 1), + ("all", None, 0), + ] { + let report = run_session_retention_authorized( + &store.conn, + &store.storage_root, + provider, + session_id, + &config, + RetentionMode::Apply, + NOW, + &|_| Ok(()), + ) + .await + .map_err(|error| error.to_string())?; + assert!(report.errors.is_empty(), "{:?}", report.errors); + assert_eq!(report.inlined.acted, expected); + assert!(report.inlined.eligible <= 1); + assert_eq!(report.bytes_reclaimed(), 0); + } + for target in [&first, &second] { + assert!( + raw::load_raw_message_by_store_id(&store.conn, target.store_id) + .await + .map_err(|error| error.to_string())? + .payload_ref + .is_none() + ); + } + assert_eq!( + fetch_str( + &store.conn, + "SELECT payload_ref FROM lcm_raw_messages WHERE store_id = ?1", + params![protected.store_id], + ) + .await?, + protected.payload_ref + ); + assert!( + payload::payload_dir(&store.storage_root) + .join(&protected.payload_ref) + .exists() + ); + Ok(()) +} + +#[asupersync::test] +async fn offload_cas_preserves_revived_row_and_rolls_back_canonical_body() -> Result<(), String> { + let store = test_store().await?; + let original = "stale content".repeat(1024); let store_id = insert_message(&store.conn, 1, 90, &original).await?; make_projection_durable(&store.conn, store_id).await?; + let metadata_json = raw::load_raw_message_by_store_id(&store.conn, store_id) + .await + .map_err(|error| error.to_string())? + .metadata_json; let target = OffloadRow { store_id, provider: PROVIDER.to_string(), @@ -546,6 +1352,7 @@ async fn offload_cas_preserves_revived_row_and_rolls_back_payload() -> Result<() message_id: "msg-1".to_string(), timestamp: NOW - 90 * DAY, content: original, + metadata_json, }; let revived = "revived content"; let revived_hash = sha256_hex(revived.as_bytes()); @@ -565,7 +1372,6 @@ async fn offload_cas_preserves_revived_row_and_rolls_back_payload() -> Result<() } let error = offload_one( RetentionStore::Connection(&store.conn), - &store.storage_root, &target, Some(&authorize), ) @@ -592,13 +1398,8 @@ async fn offload_cas_preserves_revived_row_and_rolls_back_payload() -> Result<() "inline" ); assert_eq!(count(&store.conn, "lcm_external_payloads").await?, 0); - assert_eq!( - std::fs::read_dir(crate::payload::payload_dir(&store.storage_root)) - .map_err(|error| error.to_string())? - .count(), - 0, - "failed CAS removes the newly-created payload file" - ); + assert_eq!(count(&store.conn, "session_canonical_bodies").await?, 0); + assert!(!payload::payload_dir(&store.storage_root).exists()); Ok(()) } @@ -709,3 +1510,90 @@ async fn fetch_str( .ok_or_else(|| "no row".to_string())?; row.get::(0).map_err(|e| e.to_string()) } + +async fn convergence_snapshot(conn: &(impl QueryExecutor + ?Sized)) -> Result { + fetch_str( + conn, + "SELECT json_array(state, protection_frontier_store_id, attempted_raw_store_id, + raw_revision_generation, stale_from_store_id, + (SELECT COUNT(*) FROM lcm_summary_convergence_dirty_raw), + (SELECT COUNT(*) FROM lcm_summary_convergence_invalidation_work), + (SELECT COUNT(*) FROM session_summary_sources)) + FROM lcm_summary_convergence_queue", + (), + ) + .await +} + +async fn mark_convergence_current(conn: &(impl Executor + ?Sized)) -> Result { + conn.execute_batch( + "DELETE FROM lcm_summary_convergence_invalidation_work; + DELETE FROM lcm_summary_convergence_dirty_raw; + UPDATE lcm_summary_convergence_queue SET state = 'current', + attempted_raw_store_id = newest_raw_store_id, + protection_frontier_store_id = newest_raw_store_id, + stale_from_store_id = NULL;", + ) + .await + .map_err(|error| error.to_string())?; + convergence_snapshot(conn).await +} + +/// Seeds the prior retention representation through its actual payload writer. +async fn seed_retention_offload( + store: &TestStore, + ordinal: i64, + content: &str, +) -> Result { + let store_id = insert_message(&store.conn, ordinal, 90, content).await?; + make_projection_durable(&store.conn, store_id).await?; + let original = raw::load_raw_message_by_store_id(&store.conn, store_id) + .await + .map_err(|error| error.to_string())?; + assert_eq!( + original.content, content, + "fixture content must survive normal sanitization" + ); + offload_external_one( + RetentionStore::Connection(&store.conn), + &store.storage_root, + &OffloadRow { + store_id, + provider: PROVIDER.to_owned(), + session_id: SESSION.to_owned(), + message_id: format!("msg-{ordinal}"), + timestamp: NOW - 90 * DAY, + content: content.to_owned(), + metadata_json: original.metadata_json, + }, + None, + ) + .await + .map_err(|error| error.to_string())?; + let mut rows = store + .conn + .query( + "SELECT payload_ref, content_hash, placeholder_text, metadata_json + FROM lcm_raw_messages WHERE store_id = ?1", + params![store_id], + ) + .await + .map_err(|error| error.to_string())?; + let row = rows + .next() + .await + .map_err(|error| error.to_string())? + .ok_or_else(|| "missing offloaded row".to_string())?; + Ok(InlineOffloadedRow { + store_id, + timestamp: NOW - 90 * DAY, + provider: PROVIDER.to_owned(), + session_id: SESSION.to_owned(), + message_id: format!("msg-{ordinal}"), + payload_ref: row.get(0).map_err(|error| error.to_string())?, + content_hash: row.get(1).map_err(|error| error.to_string())?, + placeholder_text: row.get(2).map_err(|error| error.to_string())?, + metadata_json: row.get(3).map_err(|error| error.to_string())?, + byte_count: content.len(), + }) +} diff --git a/crates/tracedecay-lcm/src/schema.rs b/crates/tracedecay-lcm/src/schema.rs index 9036550627..4698c8fc77 100644 --- a/crates/tracedecay-lcm/src/schema.rs +++ b/crates/tracedecay-lcm/src/schema.rs @@ -10,6 +10,10 @@ use super::{LcmError, LcmRawMessage, raw}; #[cfg(test)] use super::util; +#[cfg(test)] +#[path = "schema/fts_tests.rs"] +mod fts_tests; + /// Message and raw rows carry no copy of their observation envelope; readers /// join it from the `observations` row. Raw rows store their body once. /// `snippet_text` and `index_text` are virtual columns computing @@ -27,9 +31,9 @@ use super::util; /// `session_summary_sources` authority (session temporal schema) through /// [`SUMMARY_VISIBLE_SQL`]. Session rows LCM creates carry their store's /// scope as project fields (version 14). A message whose host content is a -/// block array is stored as its text (version 15). Version 16 creates every -/// table, index and trigger in its final shape; no in-place upgrade runs on -/// an existing store. Older stores require a profile reset. +/// block array is stored as its text (version 15). Version 16 defines the +/// current durable schema. Derived index maintenance is refreshed on admission; +/// older durable schemas require a profile reset. pub const LCM_SCHEMA_VERSION: i64 = 16; /// Visibility rule for every LCM summary read, over a `session_summary_nodes` @@ -121,9 +125,20 @@ const RAW_FTS_DDL: &str = "CREATE VIRTUAL TABLE IF NOT EXISTS lcm_raw_messages_f 'delete', OLD.store_id, OLD.index_text, OLD.role, OLD.kind, OLD.model, OLD.tool_names ); - END; - CREATE TRIGGER IF NOT EXISTS lcm_raw_messages_fts_update - AFTER UPDATE ON lcm_raw_messages BEGIN + END;"; + +/// Derived index maintenance only runs when an indexed value actually changes. +/// `index_text` is generated, so comparing its values also covers body and +/// placeholder updates without treating metadata-only writes as new documents. +pub const RAW_FTS_UPDATE_TRIGGER_SQL: &str = "CREATE TRIGGER lcm_raw_messages_fts_update + AFTER UPDATE ON lcm_raw_messages + WHEN OLD.store_id IS NOT NEW.store_id + OR OLD.index_text IS NOT NEW.index_text + OR OLD.role IS NOT NEW.role + OR OLD.kind IS NOT NEW.kind + OR OLD.model IS NOT NEW.model + OR OLD.tool_names IS NOT NEW.tool_names + BEGIN INSERT INTO lcm_raw_messages_fts( lcm_raw_messages_fts, rowid, index_text, role, kind, model, tool_names ) @@ -135,6 +150,30 @@ const RAW_FTS_DDL: &str = "CREATE VIRTUAL TABLE IF NOT EXISTS lcm_raw_messages_f VALUES (NEW.store_id, NEW.index_text, NEW.role, NEW.kind, NEW.model, NEW.tool_names); END;"; +async fn ensure_raw_fts_update_trigger(conn: &(impl Executor + ?Sized)) -> Result<(), LcmError> { + let mut rows = conn + .query( + "SELECT sql FROM sqlite_schema + WHERE type = 'trigger' AND name = 'lcm_raw_messages_fts_update'", + (), + ) + .await?; + let existing = match rows.next().await? { + Some(row) => Some(row.get::(0)?), + None => None, + }; + drop(rows); + if existing.as_deref().map(|sql| sql.trim_end_matches(';')) + == Some(RAW_FTS_UPDATE_TRIGGER_SQL.trim_end_matches(';')) + { + return Ok(()); + } + conn.execute_batch("DROP TRIGGER IF EXISTS lcm_raw_messages_fts_update;") + .await?; + conn.execute_batch(RAW_FTS_UPDATE_TRIGGER_SQL).await?; + Ok(()) +} + /// A payload is referenced only by its owner raw row, the /// `(provider, message_id)` its metadata names. Payload GC therefore never /// scans message text for references: a change to the owner row that can drop @@ -257,7 +296,7 @@ pub async fn ensure_lcm_schema_in_transaction( conn: &(impl Executor + ?Sized), ) -> Result<(), LcmError> { match require_admissible_lcm_schema(conn).await? { - LcmSchemaAdmission::Current => return Ok(()), + LcmSchemaAdmission::Current => return ensure_raw_fts_update_trigger(conn).await, LcmSchemaAdmission::Fresh => {} } @@ -379,6 +418,7 @@ pub async fn ensure_lcm_schema_in_transaction( ensure_raw_identity_schema(conn).await?; ensure_payload_gc_candidates(conn).await?; conn.execute_batch(RAW_FTS_DDL).await?; + ensure_raw_fts_update_trigger(conn).await?; super::summary_convergence::ensure_schema(conn).await?; for sql in LCM_STATUS_PERFORMANCE_INDEX_SQL { conn.execute_batch(sql).await?; diff --git a/crates/tracedecay-lcm/src/schema/fts_tests.rs b/crates/tracedecay-lcm/src/schema/fts_tests.rs new file mode 100644 index 0000000000..212bf53fd6 --- /dev/null +++ b/crates/tracedecay-lcm/src/schema/fts_tests.rs @@ -0,0 +1,193 @@ +use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, TestConnection, params}; + +use super::{ensure_lcm_schema, util}; + +async fn fixture() -> Result<(tempfile::TempDir, TestConnection), String> { + let temp = tempfile::tempdir().map_err(|error| error.to_string())?; + let connection = TestConnection::open(&temp.path().join("sessions.db")); + connection + .execute_batch( + "CREATE TABLE sessions ( + provider TEXT NOT NULL, session_id TEXT NOT NULL, + PRIMARY KEY(provider, session_id) + ); + INSERT INTO sessions VALUES ('codex', 'session');", + ) + .await + .map_err(|error| error.to_string())?; + ensure_lcm_schema(&connection) + .await + .map_err(|error| error.to_string())?; + connection + .execute_batch( + "INSERT INTO lcm_raw_messages ( + provider, message_id, session_id, role, ordinal, content, + content_hash, storage_kind + ) VALUES ('codex', 'message', 'session', 'assistant', 1, + 'originalword', 'hash', 'inline');", + ) + .await + .map_err(|error| error.to_string())?; + Ok((temp, connection)) +} + +async fn matches(connection: &TestConnection, query: &str) -> Result { + util::fetch_i64( + &**connection, + "SELECT COUNT(*) FROM lcm_raw_messages_fts WHERE lcm_raw_messages_fts MATCH ?1", + params![query], + "FTS match count", + ) + .await + .map_err(|error| error.to_string()) +} + +#[asupersync::test] +async fn metadata_and_noop_updates_do_not_write_search_segments() -> Result<(), String> { + let (_temp, connection) = fixture().await?; + connection + .execute_batch( + "CREATE TRIGGER reject_search_segment_insert + BEFORE INSERT ON lcm_raw_messages_fts_data + BEGIN SELECT RAISE(ABORT, 'unexpected search segment write'); END; + CREATE TRIGGER reject_search_segment_update + BEFORE UPDATE ON lcm_raw_messages_fts_data + BEGIN SELECT RAISE(ABORT, 'unexpected search segment write'); END; + CREATE TRIGGER reject_search_segment_delete + BEFORE DELETE ON lcm_raw_messages_fts_data + BEGIN SELECT RAISE(ABORT, 'unexpected search segment write'); END;", + ) + .await + .map_err(|error| error.to_string())?; + for statement in [ + "UPDATE lcm_raw_messages SET metadata_json = '{\"receipt\":1}'", + "UPDATE lcm_raw_messages SET ordinal = ordinal + 1, source_path = 'changed'", + "UPDATE lcm_raw_messages SET content = content, role = role, kind = kind, + model = model, tool_names = tool_names", + "UPDATE lcm_raw_messages SET placeholder_text = 'unused while content is inline'", + ] { + connection + .execute_batch(statement) + .await + .map_err(|error| format!("unrelated update wrote search segments: {error}"))?; + } + assert_eq!(matches(&connection, "originalword").await?, 1); + assert!( + connection + .execute_batch("UPDATE lcm_raw_messages SET content = 'replacementword'") + .await + .is_err(), + "the write rejection must detect a real indexed change" + ); + assert_eq!(matches(&connection, "originalword").await?, 1); + + // Model a current store whose derived trigger still rewrites every update. + connection + .execute_batch( + "DROP TRIGGER lcm_raw_messages_fts_update; + CREATE TRIGGER lcm_raw_messages_fts_update + AFTER UPDATE ON lcm_raw_messages BEGIN + INSERT INTO lcm_raw_messages_fts( + lcm_raw_messages_fts, rowid, index_text, role, kind, model, tool_names + ) VALUES ('delete', OLD.store_id, OLD.index_text, OLD.role, + OLD.kind, OLD.model, OLD.tool_names); + INSERT INTO lcm_raw_messages_fts(rowid, index_text, role, kind, model, tool_names) + VALUES (NEW.store_id, NEW.index_text, NEW.role, NEW.kind, NEW.model, NEW.tool_names); + END;", + ) + .await + .map_err(|error| error.to_string())?; + assert!( + connection + .execute_batch("UPDATE lcm_raw_messages SET metadata_json = NULL") + .await + .is_err() + ); + ensure_lcm_schema(&connection) + .await + .map_err(|error| error.to_string())?; + connection + .execute_batch("UPDATE lcm_raw_messages SET metadata_json = NULL") + .await + .map_err(|error| error.to_string())?; + assert_eq!(matches(&connection, "originalword").await?, 1); + let schema_version = + util::fetch_i64(&*connection, "PRAGMA schema_version", (), "schema version") + .await + .map_err(|error| error.to_string())?; + ensure_lcm_schema(&connection) + .await + .map_err(|error| error.to_string())?; + assert_eq!( + util::fetch_i64(&*connection, "PRAGMA schema_version", (), "schema version") + .await + .map_err(|error| error.to_string())?, + schema_version, + "admitting the current trigger must not write schema again" + ); + Ok(()) +} + +#[asupersync::test] +async fn changed_generated_text_fields_nulls_and_rowids_still_reindex() -> Result<(), String> { + let (_temp, connection) = fixture().await?; + for (statement, absent, present) in [ + ( + "UPDATE lcm_raw_messages SET content = 'replacementword'", + "originalword", + "replacementword", + ), + ( + "UPDATE lcm_raw_messages SET content = NULL, placeholder_text = 'placeholderword'", + "replacementword", + "placeholderword", + ), + ( + "UPDATE lcm_raw_messages SET placeholder_text = 'changedplaceholder'", + "placeholderword", + "changedplaceholder", + ), + ( + "UPDATE lcm_raw_messages SET role = 'tool', kind = 'command', + model = 'testmodel', tool_names = 'testtool'", + "role:assistant", + "role:tool AND kind:command AND model:testmodel AND tool_names:testtool", + ), + ( + "UPDATE lcm_raw_messages SET kind = NULL, model = NULL, tool_names = NULL", + "kind:command OR model:testmodel OR tool_names:testtool", + "role:tool", + ), + ] { + connection + .execute_batch(statement) + .await + .map_err(|error| error.to_string())?; + assert_eq!(matches(&connection, absent).await?, 0, "{statement}"); + assert_eq!(matches(&connection, present).await?, 1, "{statement}"); + } + connection + .execute_batch("UPDATE lcm_raw_messages SET store_id = 42") + .await + .map_err(|error| error.to_string())?; + let mut rows = connection + .query( + "SELECT rowid FROM lcm_raw_messages_fts + WHERE lcm_raw_messages_fts MATCH 'changedplaceholder'", + (), + ) + .await + .map_err(|error| error.to_string())?; + assert_eq!( + rows.next().await.unwrap().unwrap().get::(0).unwrap(), + 42 + ); + assert!(rows.next().await.unwrap().is_none()); + drop(rows); + connection + .execute_batch("DELETE FROM lcm_raw_messages") + .await + .map_err(|error| error.to_string())?; + assert_eq!(matches(&connection, "changedplaceholder").await?, 0); + Ok(()) +} diff --git a/crates/tracedecay-lcm/src/summary_convergence.rs b/crates/tracedecay-lcm/src/summary_convergence.rs index bdffe3b99c..acf0cd1e23 100644 --- a/crates/tracedecay-lcm/src/summary_convergence.rs +++ b/crates/tracedecay-lcm/src/summary_convergence.rs @@ -110,6 +110,11 @@ CREATE TRIGGER IF NOT EXISTS lcm_summary_convergence_raw_insert raw_revision_generation = lcm_summary_convergence_queue.raw_revision_generation + 1; END; +"#; + +/// Canonical guard restored by transaction-scoped storage maintenance after it +/// proves that effective content and its sanitizer receipt remain unchanged. +pub(crate) const RAW_UPDATE_TRIGGER_SQL: &str = r#" CREATE TRIGGER IF NOT EXISTS lcm_summary_convergence_raw_unprotected_update AFTER UPDATE OF provider, session_id, role, ordinal, timestamp, content_hash, storage_kind, payload_ref, metadata_json @@ -194,6 +199,53 @@ CREATE TRIGGER IF NOT EXISTS lcm_summary_convergence_raw_unprotected_update END; "#; +pub(crate) async fn validate_raw_update_guard( + conn: &(impl QueryExecutor + ?Sized), +) -> Result<(), LcmError> { + let mut rows = conn + .query( + "SELECT sql FROM sqlite_master WHERE type = 'trigger' AND name = ?1", + params!["lcm_summary_convergence_raw_unprotected_update"], + ) + .await?; + let actual = rows + .next() + .await? + .ok_or_else(|| LcmError::Db("summary convergence guard is missing".to_owned()))? + .get::(0)?; + let expected = RAW_UPDATE_TRIGGER_SQL.trim().replacen( + "CREATE TRIGGER IF NOT EXISTS ", + "CREATE TRIGGER ", + 1, + ); + if actual.trim().trim_end_matches(';') != expected.trim_end_matches(';') { + return Err(LcmError::Db( + "summary convergence guard does not match its contract".to_owned(), + )); + } + Ok(()) +} + +/// Only lossless storage maintenance may call this inside its authorized writer +/// transaction. It must verify the unchanged body/receipt and restore this +/// exact guard before committing. Rollback restores it on any failure. Body +/// ownership, privacy, FTS and payload-reference triggers remain installed. +pub(crate) async fn suspend_raw_update_guard( + conn: &(impl Executor + ?Sized), +) -> Result<(), LcmError> { + validate_raw_update_guard(conn).await?; + conn.execute_batch("DROP TRIGGER lcm_summary_convergence_raw_unprotected_update") + .await?; + Ok(()) +} + +pub(crate) async fn restore_raw_update_guard( + conn: &(impl Executor + ?Sized), +) -> Result<(), LcmError> { + conn.execute_batch(RAW_UPDATE_TRIGGER_SQL).await?; + validate_raw_update_guard(conn).await +} + pub const NEXT_CANDIDATE_SQL: &str = "SELECT provider, session_id, newest_raw_store_id, protection_frontier_store_id, attempted_raw_store_id, failure_count, raw_revision_generation, stale_from_store_id @@ -278,6 +330,7 @@ impl LcmSummaryConvergenceQueueState { pub async fn ensure_schema(conn: &(impl Executor + ?Sized)) -> Result<(), LcmError> { conn.execute_batch(SUMMARY_CONVERGENCE_TABLE_SQL).await?; conn.execute_batch(SUMMARY_CONVERGENCE_WORK_SQL).await?; + conn.execute_batch(RAW_UPDATE_TRIGGER_SQL).await?; Ok(()) } diff --git a/crates/tracedecay-maintenance/src/generation.rs b/crates/tracedecay-maintenance/src/generation.rs index b76a1c6331..70cefa4823 100644 --- a/crates/tracedecay-maintenance/src/generation.rs +++ b/crates/tracedecay-maintenance/src/generation.rs @@ -55,11 +55,11 @@ pub async fn run_project_generation_maintenance( let mut outcome = match code_generation { CodeGenerationRetentionOutcomeV1::Complete => MaintenanceTickOutcome::Complete, CodeGenerationRetentionOutcomeV1::MoreWork => { - MaintenanceTickOutcome::Continue(MaintenanceContinuation::CodeGenerationRetention) + MaintenanceTickOutcome::Continue(MaintenanceContinuation::StoreRetention) } CodeGenerationRetentionOutcomeV1::Failed => MaintenanceTickOutcome::Retry, }; - if continuation == Some(MaintenanceContinuation::CodeGenerationRetention) { + if continuation == Some(MaintenanceContinuation::StoreRetention) { return finalize_generation_outcome(outcome, cancellation); } if !cancellation.is_cancelled() { @@ -124,7 +124,7 @@ pub async fn run_registered_project_generation_maintenance( { CodeGenerationRetentionOutcomeV1::Complete => MaintenanceTickOutcome::Complete, CodeGenerationRetentionOutcomeV1::MoreWork => { - MaintenanceTickOutcome::Continue(MaintenanceContinuation::CodeGenerationRetention) + MaintenanceTickOutcome::Continue(MaintenanceContinuation::StoreRetention) } CodeGenerationRetentionOutcomeV1::Failed => MaintenanceTickOutcome::Retry, }; diff --git a/crates/tracedecay-maintenance/src/loop_run.rs b/crates/tracedecay-maintenance/src/loop_run.rs index 947f11d3d4..2fec3c051f 100644 --- a/crates/tracedecay-maintenance/src/loop_run.rs +++ b/crates/tracedecay-maintenance/src/loop_run.rs @@ -256,7 +256,7 @@ mod tests { drive_loop( MaintenanceCadence::with_retry_delay(DAILY, RETRY), vec![MaintenanceTickOutcome::Continue( - MaintenanceContinuation::CodeGenerationRetention, + MaintenanceContinuation::StoreRetention, )], |_wake, mut observed| async move { let first = next_tick(&mut observed).await; @@ -264,7 +264,7 @@ mod tests { assert_eq!(first.continuation, None); assert_eq!( resumed.continuation, - Some(MaintenanceContinuation::CodeGenerationRetention), + Some(MaintenanceContinuation::StoreRetention), "bounded progress must resume its code-generation phase instead of a full tick" ); assert!( diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 417bc91f9b..7c6d4ead0c 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -407,24 +407,42 @@ pub fn census_sqlite_temp_spill( bound_overrides: &BTreeMap, ) -> Option { let mut bytes = 0u64; - for fd in std::fs::read_dir(fd_directory).ok()?.flatten() { - let Ok(target) = std::fs::read_link(fd.path()) else { - continue; + let mut unavailable_entry_count = 0usize; + for entry in std::fs::read_dir(fd_directory).ok()? { + let fd = match entry { + Ok(fd) => fd, + Err(_) => { + unavailable_entry_count = unavailable_entry_count.saturating_add(1); + continue; + } + }; + let target = match std::fs::read_link(fd.path()) { + Ok(target) => target, + // Descriptors can close between enumeration and observation. + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(_) => { + unavailable_entry_count = unavailable_entry_count.saturating_add(1); + continue; + } }; let unlinked_temp = target .file_name() .and_then(|name| name.to_str()) .is_some_and(|name| name.starts_with("etilqs_") && name.ends_with(" (deleted)")); - // An fd closed since the listing no longer holds bytes. - if let (true, Ok(metadata)) = (unlinked_temp, std::fs::metadata(fd.path())) { - bytes = bytes.saturating_add(metadata.len()); + if unlinked_temp { + match std::fs::metadata(fd.path()) { + Ok(metadata) => bytes = bytes.saturating_add(metadata.len()), + // A closed descriptor no longer consumes space. + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(_) => unavailable_entry_count = unavailable_entry_count.saturating_add(1), + } } } Some(ProfileFootprintEntryV1 { key: SQLITE_TEMP_SPILL_KEY.to_owned(), family: ProfileFamilyV1::SqliteTempSpill, bytes, - unavailable_entry_count: 0, + unavailable_entry_count, bound_bytes: bound_overrides .get(SQLITE_TEMP_SPILL_KEY) .copied() @@ -723,6 +741,65 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn temp_spill_census_reports_unreadable_fd_entries_without_fabricating_health() { + let temporary = tempfile::tempdir().expect("temporary directory"); + let fd_directory = temporary.path().join("fd"); + std::fs::create_dir(&fd_directory).expect("fd directory"); + let live_temp = temporary.path().join("etilqs_active (deleted)"); + sized(&live_temp, 24 * MIB); + std::os::unix::fs::symlink(&live_temp, fd_directory.join("3")).expect("fd link"); + // A real /proc/self/fd directory contains descriptor symlinks. A + // regular entry models an FD whose target cannot be inspected. + std::fs::write(fd_directory.join("4"), b"uninspectable").expect("unreadable fd"); + // A descriptor closed after enumeration is not still consuming space. + std::os::unix::fs::symlink( + temporary.path().join("etilqs_closed (deleted)"), + fd_directory.join("5"), + ) + .expect("closed fd link"); + + let entry = census_sqlite_temp_spill(&fd_directory, &BTreeMap::new()) + .expect("readable fd directory"); + assert_eq!(entry.bytes, 24 * MIB); + assert_eq!(entry.unavailable_entry_count, 1); + let footprint = ProfileFootprintV1 { + entries: vec![entry], + total_bytes: 0, + unavailable_entry_count: 0, + truncated: false, + total_bound_bytes: PROFILE_TOTAL_BOUND_BYTES, + }; + let observed = findings(&footprint); + let finding = &observed[SQLITE_TEMP_SPILL_KEY]; + assert_eq!(finding.finding().state(), DoctorEvidenceStateV1::Partial); + assert!( + finding + .finding() + .coverage() + .statement() + .contains("1 unreadable entries") + ); + + // A measured lower bound above the configured cap still proves overage. + let overrides = BTreeMap::from([(SQLITE_TEMP_SPILL_KEY.to_owned(), MIB)]); + let over = census_sqlite_temp_spill(&fd_directory, &overrides).expect("over-bound census"); + assert_eq!(over.unavailable_entry_count, 1); + assert_eq!( + findings(&ProfileFootprintV1 { + entries: vec![over], + total_bytes: 0, + unavailable_entry_count: 0, + truncated: false, + total_bound_bytes: PROFILE_TOTAL_BOUND_BYTES, + })[SQLITE_TEMP_SPILL_KEY] + .finding() + .state(), + DoctorEvidenceStateV1::Degraded + ); + } + fn entry<'a>(footprint: &'a ProfileFootprintV1, key: &str) -> &'a ProfileFootprintEntryV1 { footprint .entries diff --git a/crates/tracedecay-maintenance/src/retention/registered_store.rs b/crates/tracedecay-maintenance/src/retention/registered_store.rs index 434fc119d5..52236a295c 100644 --- a/crates/tracedecay-maintenance/src/retention/registered_store.rs +++ b/crates/tracedecay-maintenance/src/retention/registered_store.rs @@ -4,7 +4,11 @@ use tracedecay_domain::UtcMicros; use tracedecay_global_db::observation::retention::{ ObservationRetentionConfig, ObservationRetentionReport, RetentionMode, }; +use tracedecay_global_db::observation::{CanonicalBodyGcReport, MAX_CANONICAL_BODY_GC_ROWS}; use tracedecay_global_db::{ObservabilityRetentionReceiptV1, RegisteredGlobalDb}; +use tracedecay_lcm::retention::{ + MAX_REPLAY_METADATA_COMPACTION_ROWS, ReplayMetadataCompactionReport, +}; use tracedecay_lcm::{ LcmGcConfig, LcmGcReport, LcmRetentionConfig, LcmRetentionReport, RetentionMode as LcmRetentionMode, @@ -14,13 +18,18 @@ use tracedecay_lcm::{ pub enum RegisteredStoreRetentionErrorV1 { Observability { diagnostic: String }, PayloadGc { diagnostic: String }, + CanonicalBodyGc { diagnostic: String }, + ReplayMetadataCompaction { diagnostic: String }, } impl RegisteredStoreRetentionErrorV1 { #[must_use] pub fn diagnostic(&self) -> &str { match self { - Self::Observability { diagnostic } | Self::PayloadGc { diagnostic } => diagnostic, + Self::Observability { diagnostic } + | Self::PayloadGc { diagnostic } + | Self::CanonicalBodyGc { diagnostic } + | Self::ReplayMetadataCompaction { diagnostic } => diagnostic, } } } @@ -34,6 +43,11 @@ pub struct RegisteredStoreRetentionReportV1 { /// Applied external-payload GC. It runs after session retention so the /// payloads of rows that pass just dropped are reconciled by the same tick. pub payload_gc: Result, + /// Shared canonical bodies are collected only after both owner indexes + /// and the initial candidate seed are complete. + pub canonical_body_gc: Result, + /// Lossless repair of old provider replay strings duplicated by raw bodies. + pub replay_metadata: Result, } impl RegisteredStoreRetentionReportV1 { @@ -55,6 +69,8 @@ impl RegisteredStoreRetentionReportV1 { && observations_succeeded && self.observability.is_ok() && payload_gc_succeeded + && self.canonical_body_gc.is_ok() + && self.replay_metadata.is_ok() } } @@ -99,12 +115,63 @@ pub async fn run_registered_store_retention( .await .map_err(|diagnostic| RegisteredStoreRetentionErrorV1::Observability { diagnostic }); let payload_gc = run_payload_gc(database, now).await; + let canonical_body_gc = run_canonical_body_gc(database).await; + let replay_metadata = run_replay_metadata_compaction(database).await; RegisteredStoreRetentionReportV1 { session_lcm, observations, observability, payload_gc, + canonical_body_gc, + replay_metadata, + } +} + +/// Resume bounded, lossless storage work without repeating the raw-drop policy. +pub async fn run_session_storage_compaction( + database: &RegisteredGlobalDb, + config: &LcmRetentionConfig, + now: i64, +) -> Option> { + if !config.enabled || config.offload_after_days.is_none() { + return None; } + let config = LcmRetentionConfig { + drop_after_days: None, + ..*config + }; + Some( + database + .run_session_lcm_retention("all", None, &config, LcmRetentionMode::Apply, now) + .await, + ) +} + +/// Continuation ticks advance the bounded body lifecycle, leaving destructive +/// age policies and profile-wide maintenance on their normal cadence. +pub async fn run_canonical_body_gc( + database: &RegisteredGlobalDb, +) -> Result { + database + .run_canonical_body_gc(MAX_CANONICAL_BODY_GC_ROWS) + .await + .map_err(|error| RegisteredStoreRetentionErrorV1::CanonicalBodyGc { + diagnostic: error.to_string(), + }) +} + +/// Runs a bounded replay-metadata repair page without repeating age policies. +pub async fn run_replay_metadata_compaction( + database: &RegisteredGlobalDb, +) -> Result { + database + .compact_replay_metadata(MAX_REPLAY_METADATA_COMPACTION_ROWS) + .await + .map_err( + |error| RegisteredStoreRetentionErrorV1::ReplayMetadataCompaction { + diagnostic: error.to_string(), + }, + ) } async fn run_payload_gc( @@ -171,6 +238,41 @@ mod tests { ..ObservationRetentionConfig::default() }; + let storage = run_session_storage_compaction( + &harness.registered, + &LcmRetentionConfig { + drop_after_days: Some(0), + ..LcmRetentionConfig::default() + }, + 31 * 86_400, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(storage.dropped.window_days, None); + assert_eq!(storage.dropped.acted, 0); + assert!( + run_session_storage_compaction(&harness.registered, &session_lcm, 31 * 86_400) + .await + .is_none(), + "disabling session retention also stops its storage continuation" + ); + assert_eq!( + harness + .registered + .query_analytics_events(&AnalyticsEventQuery { + provider: Some("tracedecay-observability".to_owned()), + project_id: Some("scope:retention".to_owned()), + limit: 10, + ..AnalyticsEventQuery::default() + }) + .await + .unwrap() + .len(), + 1, + "storage continuation must not run profile-wide age policies" + ); + let report = run_registered_store_retention( &harness.registered, &session_lcm, diff --git a/crates/tracedecay-maintenance/src/store_maintenance/graph_replay.rs b/crates/tracedecay-maintenance/src/store_maintenance/graph_replay.rs index eb098b8877..e4f22a9bc0 100644 --- a/crates/tracedecay-maintenance/src/store_maintenance/graph_replay.rs +++ b/crates/tracedecay-maintenance/src/store_maintenance/graph_replay.rs @@ -7,6 +7,7 @@ use tracedecay_code_index_retention::code_index_generations::{ }; use tracedecay_runtime_core::logging::log_daemon_event; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ReconcileOutcome { /// The page from the cursor held no queued release event. Complete, @@ -26,6 +27,20 @@ pub enum ReconcileOutcome { Failed, } +/// Keep a failed staging release visible even if an independent replay-release +/// page succeeds. Otherwise the maintenance owner treats the tick as complete +/// and can park staging cleanup behind the full maintenance interval. +fn with_staging_release_failure( + page_outcome: ReconcileOutcome, + staging_failed: bool, +) -> ReconcileOutcome { + if staging_failed { + ReconcileOutcome::Failed + } else { + page_outcome + } +} + /// Every code-generation degradation is a genuine anomaly: it keeps the tick /// summary loud and logs on every attempt. pub fn log_code_generation_retention_degraded( @@ -166,9 +181,10 @@ pub async fn reconcile_graph_replay_releases( staging_cursor, ) .await; - match staging_release { + let staging_failed = match staging_release { Ok(continuation) => { observations.record_graph_staging_release_cursor(project_root, continuation); + false } Err(error) => { log_code_generation_retention_degraded_with_error( @@ -176,8 +192,9 @@ pub async fn reconcile_graph_replay_releases( "graph_staging_release_failed", &error, ); + true } - } + }; // One bounded page per pass, resumed from the durable-queue cursor of the // previous attempt. Retained releases fall behind the cursor instead of // blocking every later page, and a backlog drains across short-cadence @@ -197,7 +214,7 @@ pub async fn reconcile_graph_replay_releases( }; if page.releases.is_empty() { observations.record_graph_replay_release_served(project_root, page.continuation); - return ReconcileOutcome::Complete; + return with_staging_release_failure(ReconcileOutcome::Complete, staging_failed); } let mut retained = false; let mut released = false; @@ -242,11 +259,39 @@ pub async fn reconcile_graph_replay_releases( } let more_work = page.continuation.is_some(); observations.record_graph_replay_release_served(project_root, page.continuation); - if more_work || released { + let page_outcome = if more_work || released { ReconcileOutcome::MoreWork } else if retained { ReconcileOutcome::Retained } else { ReconcileOutcome::Complete + }; + with_staging_release_failure(page_outcome, staging_failed) +} + +#[cfg(test)] +mod tests { + use super::{ReconcileOutcome, with_staging_release_failure}; + + #[test] + fn staging_release_failure_cannot_be_hidden_by_release_page_result() { + for page_outcome in [ + ReconcileOutcome::Complete, + ReconcileOutcome::MoreWork, + ReconcileOutcome::Retained, + ReconcileOutcome::Deferred, + ReconcileOutcome::Failed, + ] { + assert_eq!( + with_staging_release_failure(page_outcome, true), + ReconcileOutcome::Failed, + "failed staging release must keep the maintenance retry cadence", + ); + assert_eq!( + with_staging_release_failure(page_outcome, false), + page_outcome, + "successful staging release must preserve the independent page result", + ); + } } } diff --git a/crates/tracedecay-maintenance/src/store_maintenance/scope_reconciliation.rs b/crates/tracedecay-maintenance/src/store_maintenance/scope_reconciliation.rs index ab08133ac0..6ef5e64725 100644 --- a/crates/tracedecay-maintenance/src/store_maintenance/scope_reconciliation.rs +++ b/crates/tracedecay-maintenance/src/store_maintenance/scope_reconciliation.rs @@ -137,7 +137,7 @@ pub async fn run_code_index_scope_reconciliation( ), ], ); - MaintenanceTickOutcome::Continue(MaintenanceContinuation::CodeGenerationRetention) + MaintenanceTickOutcome::Continue(MaintenanceContinuation::StoreRetention) } Ok(Err((failure, error))) => scope_reconciliation_failed(failure, error), Err(_) => scope_reconciliation_failed("scope_task_panicked", None), diff --git a/crates/tracedecay-maintenance/src/telemetry.rs b/crates/tracedecay-maintenance/src/telemetry.rs index 581c46765f..62c45ce92f 100644 --- a/crates/tracedecay-maintenance/src/telemetry.rs +++ b/crates/tracedecay-maintenance/src/telemetry.rs @@ -965,7 +965,7 @@ mod retention_tick_log_tests { assert!( registry.admit_retention_tick_log(MaintenanceTickOutcome::Continue( - MaintenanceContinuation::CodeGenerationRetention, + MaintenanceContinuation::StoreRetention, )) ); assert!(registry.admit_retention_tick_log(MaintenanceTickOutcome::Retry)); diff --git a/crates/tracedecay-maintenance/src/tick.rs b/crates/tracedecay-maintenance/src/tick.rs index 5524c2d134..eabd536284 100644 --- a/crates/tracedecay-maintenance/src/tick.rs +++ b/crates/tracedecay-maintenance/src/tick.rs @@ -1,12 +1,14 @@ //! Maintenance tick policy: continuation, cadence, and store-window selection. +use std::collections::BTreeSet; use std::time::{Duration, Instant}; -/// Resume a bounded maintenance phase over the normal graph window. +/// Resume bounded store maintenance over the normal store window. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MaintenanceContinuation { - /// Resume bounded code-generation retention over the normal graph window. - CodeGenerationRetention, + /// Resume code-generation retention and canonical-body collection, without + /// repeating age policies or profile-wide maintenance on the short cadence. + StoreRetention, } /// Outcome of one maintenance tick or per-store unit. @@ -40,9 +42,7 @@ impl MaintenanceTickOutcome { pub fn label(self) -> &'static str { match self { Self::Complete => "complete", - Self::Continue(MaintenanceContinuation::CodeGenerationRetention) => { - "code_generation_progress" - } + Self::Continue(MaintenanceContinuation::StoreRetention) => "store_retention_progress", Self::Retry => "retry", } } @@ -60,6 +60,44 @@ impl MaintenanceTickOutcome { } } +/// Retains unfinished store work across round-robin windows. An idle window +/// must not cancel a continuation another mounted store still needs. +#[derive(Debug, Default)] +pub struct StoreMaintenanceContinuations { + pending: BTreeSet, +} + +impl StoreMaintenanceContinuations { + /// The coordinator's work keys are sorted and represent current mounts. + pub fn retain_stores(&mut self, sorted_keys: &[String]) { + self.pending + .retain(|key| sorted_keys.binary_search(key).is_ok()); + } + + pub fn record(&mut self, key: &str, outcome: MaintenanceTickOutcome) { + match outcome { + MaintenanceTickOutcome::Continue(_) => { + self.pending.insert(key.to_owned()); + } + MaintenanceTickOutcome::Complete => { + self.pending.remove(key); + } + MaintenanceTickOutcome::Retry => {} + } + } + + #[must_use] + pub fn combine(&self, current: MaintenanceTickOutcome) -> MaintenanceTickOutcome { + if self.pending.is_empty() { + current + } else { + current.combine(MaintenanceTickOutcome::Continue( + MaintenanceContinuation::StoreRetention, + )) + } + } +} + /// Interval and retry-delay policy for the maintenance loop. #[derive(Debug)] pub struct MaintenanceCadence { @@ -170,7 +208,7 @@ pub fn cursor_after_attempted_units( mod tests { use super::{ MaintenanceCadence, MaintenanceContinuation, MaintenanceTickOutcome, - cursor_after_attempted_units, select_store_window, + StoreMaintenanceContinuations, cursor_after_attempted_units, select_store_window, }; use std::time::Duration; @@ -198,8 +236,7 @@ mod tests { #[test] fn retry_outcome_takes_precedence_over_bounded_progress() { - let progress = - MaintenanceTickOutcome::Continue(MaintenanceContinuation::CodeGenerationRetention); + let progress = MaintenanceTickOutcome::Continue(MaintenanceContinuation::StoreRetention); assert_eq!( progress.combine(MaintenanceTickOutcome::Retry), @@ -214,7 +251,7 @@ mod tests { #[test] fn code_generation_continuation_dominates_a_complete_tick() { let code_generation = - MaintenanceTickOutcome::Continue(MaintenanceContinuation::CodeGenerationRetention); + MaintenanceTickOutcome::Continue(MaintenanceContinuation::StoreRetention); assert_eq!( code_generation.combine(MaintenanceTickOutcome::Complete), @@ -234,6 +271,32 @@ mod tests { (0..count).map(|index| format!("s:{index:03}")).collect() } + #[test] + fn idle_windows_do_not_forget_another_stores_pending_work() { + let progress = MaintenanceTickOutcome::Continue(MaintenanceContinuation::StoreRetention); + let mut pending = StoreMaintenanceContinuations::default(); + pending.retain_stores(&["a".to_owned(), "b".to_owned()]); + pending.record("a", progress); + pending.record("b", MaintenanceTickOutcome::Complete); + assert_eq!(pending.combine(MaintenanceTickOutcome::Complete), progress); + pending.record("a", MaintenanceTickOutcome::Retry); + assert_eq!( + pending.combine(MaintenanceTickOutcome::Retry), + MaintenanceTickOutcome::Retry + ); + pending.record("a", MaintenanceTickOutcome::Complete); + assert_eq!( + pending.combine(MaintenanceTickOutcome::Complete), + MaintenanceTickOutcome::Complete + ); + pending.record("a", progress); + pending.retain_stores(&["b".to_owned()]); + assert_eq!( + pending.combine(MaintenanceTickOutcome::Complete), + MaintenanceTickOutcome::Complete + ); + } + #[test] fn store_window_round_robin_reaches_every_store_and_never_starves() { // With more stores than the budget, feeding each tick's cursor into the diff --git a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs index 25d074e122..51c9a46022 100644 --- a/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs +++ b/crates/tracedecay-mcp/src/handlers/hook_runtime/admission/tests.rs @@ -80,7 +80,9 @@ fn restart_ledger(data_root: &std::path::Path) { #[test] fn producer_work_commits_with_its_admission_and_redrives_until_completed() { let data_root = tempfile::tempdir().unwrap(); - let now = UtcMicros(1_000); + // Settlement reopens an evicted ledger at the wall clock, and parallel + // tests can evict this root, so admit inside the real retention window. + let now = hook_now(); let envelope = admission_test_envelope(31, 7); let staged = stage_hook_v2_admission( data_root.path(), diff --git a/crates/tracedecay-query/src/retrieval/lexical.rs b/crates/tracedecay-query/src/retrieval/lexical.rs index 45c7c2613f..292aad5b2d 100644 --- a/crates/tracedecay-query/src/retrieval/lexical.rs +++ b/crates/tracedecay-query/src/retrieval/lexical.rs @@ -10,6 +10,7 @@ use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; +use roaring::RoaringBitmap; use serde::{Deserialize, Serialize}; use tracedecay_domain::{ CodeGenerationId, CompactCandidate, ComponentRevision, CursorPayloadDigest, @@ -78,45 +79,52 @@ pub const MAX_LEXICAL_FIELD_FILTERS_V1: usize = 9; /// proximity term. pub const MAX_LEXICAL_QUERY_TERM_BYTES_V1: usize = 512; -/// Summed document-frequency budget for lexical term-source admission. Every -/// candidate is decoded from its row and scored, so the union of the request's -/// term sources, not the winner cap, decides the lane's transient allocation -/// and wall time: unbounded, a natural-language task whose terms include -/// common words hydrated ~74k rows of a 472k-chunk corpus per read (~0.95 GB -/// decoded, 5.7 s) and missed the context deadline. Term sources are admitted -/// in ascending document-frequency order until their summed frequencies would -/// exceed this bound; the most selective source is always admitted so a -/// single common-term query still answers. Phrase sources are admitted separately. +/// Unique-document budget for lexical term-source admission. Every candidate +/// is decoded and scored, so source selection determines transient allocation +/// and wall time. Previously a natural-language query hydrated ~74k rows of +/// a 472k-chunk corpus (~0.95 GB decoded, 5.7 s), missing the deadline. +/// Sources are considered in ascending stored field-frequency order. Only +/// new document IDs charge the budget; the first nonempty source is always +/// admitted so a single common-term query still answers. Phrase sources +/// are admitted separately. /// This is a recall/latency policy, not a hard document or allocation ceiling: -/// documents matching only pruned terms cannot rank. The initial 16,384 value +/// documents matching only pruned sources cannot rank. The 16,384 value /// retains the measured policy (~100 MiB decode churn at ~6.6 KiB per row); /// changing the reader cache must not change candidate eligibility. pub const MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1: usize = 16_384; -/// Admit `(document_frequency, source)` pairs rarest-first while the summed -/// frequency stays within [`MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1`]; the rarest -/// nonempty source is always admitted. Ties keep request order so admission -/// is deterministic. Sources past the bound still weigh admitted candidates -/// through scoring; a document matching only those sources is never hydrated. +/// Visit posting sources rarest-first, counting unique documents rather than +/// charging overlapping whole/subtoken postings twice. The reader supplies a +/// bounded delta; only the rarest nonempty source may exceed the cap. pub(crate) fn admit_candidate_sources( mut sources: Vec<(usize, S)>, + mut new_documents: impl FnMut( + &S, + &RoaringBitmap, + usize, + ) -> Result, RetrievalPortError>, mut on_pruned: impl FnMut(usize, &S), -) -> Vec { +) -> Result { sources.retain(|(frequency, _)| *frequency > 0); sources.sort_by_key(|(frequency, _)| *frequency); - let total = sources.len(); - let mut admitted_documents = 0usize; - let mut admitted = Vec::with_capacity(total); - for (ordinal, (frequency, source)) in sources.into_iter().enumerate() { - let next = admitted_documents.saturating_add(frequency); - if ordinal > 0 && next > MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 { - on_pruned(frequency, &source); - continue; + let mut admitted = RoaringBitmap::new(); + let mut admitted_sources = 0usize; + for (frequency, source) in sources { + let remaining = if admitted_sources == 0 { + usize::MAX + } else { + MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 + .saturating_sub(usize::try_from(admitted.len()).unwrap_or(usize::MAX)) + }; + match new_documents(&source, &admitted, remaining)? { + Some(delta) if delta.len() <= remaining as u64 => { + admitted |= delta; + admitted_sources += 1; + } + _ => on_pruned(frequency, &source), } - admitted_documents = next; - admitted.push(source); } - admitted + Ok(admitted) } fn candidate_admission_outcome( @@ -158,8 +166,10 @@ pub(super) fn normalize_lexical(value: &str) -> String { /// Whole terms are the same maximal technical tokens the code index extracts /// (`tracedecay_domain::technical_tokens`), so a term the indexer keeps whole /// (`foo-bar`, `a::b`, `p/q.rs`) is queried whole, and subtokens decompose -/// through the same shared grammar. Single-token queries intentionally emit -/// no subtokens so one technical term does not fan out to common subtokens. +/// through the same shared grammar. For multiword prose, unsplit content +/// words use the symbol route's policy before searching the Subtoken field, +/// so camelCase symbols can be admitted without expanding stopwords. +/// Single-token queries emit no subtokens, preserving their narrow admission. /// /// Multi-token sanitized input is also retained as a phrase. This gives exact /// diagnostic/error text and natural-language queries a bounded lexical phrase @@ -182,10 +192,11 @@ pub fn lexical_query_parts(query: &str) -> Result LexicalRowScoreV1 { let mut field_scores = BTreeMap::new(); + let mut whole_term_scores = BTreeMap::<&str, u64>::new(); let mut matched_whole_terms = BTreeSet::new(); let mut matched_subtokens = BTreeSet::new(); let mut matched_phrases = BTreeSet::new(); @@ -1052,49 +1053,72 @@ fn score_lexical_row( if !score_name_fields && is_identifier_name_field(field) { continue; } - if field != LexicalFieldV1::Subtoken { - for (query_term, normalized) in &prepared.whole_terms { - let exact_tf = term_frequency(field, normalized); - if exact_tf > 0 { - add_score( - &mut field_scores, - field, - bm25(field, exact_tf, document_frequency(field, normalized)), - ); - matched_whole_terms.insert((*query_term).to_owned()); - collect_term_kinds(exact_terms, normalized, &mut matched_kinds); - } - if let Some(expansions) = fuzzy.by_query.get(*query_term) { - for expansion in expansions { - let fuzzy_tf = term_frequency(field, expansion); - if fuzzy_tf == 0 { - continue; - } - let score = bm25(field, fuzzy_tf, document_frequency(field, expansion)) - .saturating_mul(FUZZY_SCORE_MILLIS) - / 1_000; - add_score(&mut field_scores, field, score); - matched_whole_terms.insert((*query_term).to_owned()); - spelling_variants.insert(LexicalSpellingVariantV1 { - query: (*query_term).to_owned(), - alternative: expansion.clone(), - }); - typo_recovery_applied = true; - collect_term_kinds(exact_terms, expansion, &mut matched_kinds); + // Only an admitted whole-word field can cover subtoken evidence. + // Otherwise a Subtoken-only request loses rows before lane filtering. + if field == LexicalFieldV1::Subtoken || !super::field_admitted(field_filters, field) { + continue; + } + for (query_term, normalized) in &prepared.whole_terms { + let exact_tf = term_frequency(field, normalized); + if exact_tf > 0 { + let score = bm25(field, exact_tf, document_frequency(field, normalized)); + add_score(&mut field_scores, field, score); + let weighted_score = + if field == LexicalFieldV1::Documentation && prepared.whole_terms.len() != 1 { + score.saturating_mul(DOCUMENTATION_NL_SCORE_MILLIS) / 1_000 + } else { + score + }; + whole_term_scores + .entry(normalized.as_str()) + .and_modify(|current| *current = (*current).max(weighted_score)) + .or_insert(weighted_score); + matched_whole_terms.insert((*query_term).to_owned()); + collect_term_kinds(exact_terms, normalized, &mut matched_kinds); + } + if let Some(expansions) = fuzzy.by_query.get(*query_term) { + for expansion in expansions { + let fuzzy_tf = term_frequency(field, expansion); + if fuzzy_tf == 0 { + continue; } + let score = bm25(field, fuzzy_tf, document_frequency(field, expansion)) + .saturating_mul(FUZZY_SCORE_MILLIS) + / 1_000; + add_score(&mut field_scores, field, score); + matched_whole_terms.insert((*query_term).to_owned()); + spelling_variants.insert(LexicalSpellingVariantV1 { + query: (*query_term).to_owned(), + alternative: expansion.clone(), + }); + typo_recovery_applied = true; + collect_term_kinds(exact_terms, expansion, &mut matched_kinds); } } - } else { - for (subtoken, normalized) in &prepared.subtokens { - let tf = term_frequency(field, normalized); - if tf > 0 { - add_score( - &mut field_scores, - field, - bm25(field, tf, document_frequency(field, normalized)), - ); - matched_subtokens.insert((*subtoken).to_owned()); - } + } + } + if field_lengths.contains_key(&LexicalFieldV1::Subtoken) { + for (subtoken, normalized) in &prepared.subtokens { + let tf = term_frequency(LexicalFieldV1::Subtoken, normalized); + if tf == 0 { + continue; + } + // Whole-word and subtoken postings can represent the same + // evidence. Add only the stronger representation's excess. + let score = bm25( + LexicalFieldV1::Subtoken, + tf, + document_frequency(LexicalFieldV1::Subtoken, normalized), + ) + .saturating_sub( + whole_term_scores + .get(normalized.as_str()) + .copied() + .unwrap_or_default(), + ); + if score > 0 { + add_score(&mut field_scores, LexicalFieldV1::Subtoken, score); + matched_subtokens.insert((*subtoken).to_owned()); } } } diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs index 0ee1e6f516..f4cab58639 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/reader/query.rs @@ -236,6 +236,41 @@ impl RequestTermPostingsV1 { } Ok(documents) } + + /// Decode only the documents a source adds to the existing candidate set. + /// Stop as soon as it cannot fit; check cancellation during long postings + /// rather than waiting for the next row-hydration checkpoint. + fn new_documents_bounded( + &self, + term: &str, + admit: impl Fn(LexicalFieldV1) -> bool, + existing: &RoaringBitmap, + remaining: usize, + control: &dyn RetrievalExecutionControl, + ) -> Result, RetrievalPortError> { + let mut added = RoaringBitmap::new(); + let mut visited = 0usize; + for list in self.by_term.get(term).into_iter().flatten() { + if !admit(list.field) { + continue; + } + for posting in PostingListDecoderV1::new(&list.postings, true) { + if visited.is_multiple_of(RETRIEVAL_CANDIDATE_BATCH_SIZE) { + retrieval_checkpoint(control)?; + } + visited += 1; + let (document, _) = posting.map_err(map_query_artifact_error)?; + if !existing.contains(document) { + added.insert(document); + if added.len() > remaining as u64 { + return Ok(None); + } + } + } + } + retrieval_checkpoint(control)?; + Ok(Some(added)) + } } struct PostingCursorV1<'a> { @@ -1169,7 +1204,8 @@ impl<'a> ArtifactQueryV1<'a> { /// The candidate document set for one lexical request: every phrase /// match plus the term sources `admit_candidate_sources` keeps under - /// `MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1`. + /// `MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1`. A term requested through both + /// whole and subtoken channels is charged once for their document union. fn lexical_documents( &self, request: &LexicalLaneRequest<'_>, @@ -1197,34 +1233,31 @@ impl<'a> ArtifactQueryV1<'a> { .iter() .map(|subtoken| normalize_lexical(subtoken)) .collect::>(); - let mut sources = Vec::with_capacity(whole_terms.len() + subtokens.len()); - for term in whole_terms { - if stats.postings.by_term.contains_key(&term) { - sources.push((stats.whole_term_documents(&term), (term, false))); - } - } - for subtoken in subtokens { - if stats.postings.by_term.contains_key(&subtoken) { - sources.push(( - stats.document_frequency(LexicalFieldV1::Subtoken, &subtoken), - (subtoken, true), - )); - } - } - let mut documents = phrase_queries + let sources = stats.candidate_sources(whole_terms, subtokens); + let term_documents = admit_candidate_sources( + sources, + |source, existing, remaining| { + if source.cannot_fit(existing, remaining) { + return Ok(None); + } + stats.postings.new_documents_bounded( + &source.term, + |field| source.fields.admits(field), + existing, + remaining, + request.control, + ) + }, + |frequency, source| { + if !pruned.iter().any(|(term, _)| term == &source.term) { + pruned.push((source.term.clone(), frequency as u64)); + } + }, + )?; + let phrase_documents = phrase_queries .values() .fold(RoaringBitmap::new(), |union, documents| union | documents); - for (term, subtoken) in admit_candidate_sources(sources, |frequency, (term, _)| { - pruned.push((term.clone(), frequency as u64)); - }) { - documents |= stats - .postings - .documents(&term, |field| { - (field == LexicalFieldV1::Subtoken) == subtoken - }) - .map_err(map_query_artifact_error)?; - } - Ok(documents) + Ok(phrase_documents | term_documents) } fn exact_documents( @@ -1607,6 +1640,40 @@ struct LexicalStatsCacheV1 { postings: RequestTermPostingsV1, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CandidateSourceFieldsV1 { + Whole, + Subtoken, +} + +impl CandidateSourceFieldsV1 { + fn admits(self, field: LexicalFieldV1) -> bool { + match self { + Self::Whole => field != LexicalFieldV1::Subtoken, + Self::Subtoken => field == LexicalFieldV1::Subtoken, + } + } +} + +struct LexicalCandidateSourceV1 { + term: String, + fields: CandidateSourceFieldsV1, + minimum_documents: usize, +} + +impl LexicalCandidateSourceV1 { + /// A source cannot fit even if every already-admitted document overlaps. + /// When the first source exceeds the cap, its existing documents may + /// entirely cover a second large channel, so its size alone cannot prune. + fn cannot_fit(&self, existing: &RoaringBitmap, remaining: usize) -> bool { + remaining != usize::MAX + && self.minimum_documents + > usize::try_from(existing.len()) + .unwrap_or(usize::MAX) + .saturating_add(remaining) + } +} + fn lexical_terms( prepared: &PreparedLexicalQueryV1<'_>, fuzzy: &FuzzyExpansionsV1, @@ -1663,6 +1730,59 @@ impl LexicalStatsCacheV1 { .map(|(_, frequencies)| frequencies.get(term).copied().unwrap_or_default()) .fold(0usize, usize::saturating_add) } + + /// Keep whole-word and subtoken channels independently eligible. Their + /// stored field frequencies order sources without eager bitmap decoding; + /// admission later accounts for exact new document IDs across channels. + fn candidate_sources( + &self, + whole_terms: Vec, + subtokens: Vec, + ) -> Vec<(usize, LexicalCandidateSourceV1)> { + let mut seen = BTreeSet::new(); + let mut sources = Vec::with_capacity(whole_terms.len() + subtokens.len()); + for term in whole_terms { + if !seen.insert(term.clone()) || !self.postings.by_term.contains_key(&term) { + continue; + } + let frequency = self.whole_term_documents(&term); + if frequency == 0 { + continue; + } + let minimum_documents = self + .document_frequencies + .iter() + .filter(|(field, _)| **field != LexicalFieldV1::Subtoken) + .map(|(_, terms)| terms.get(&term).copied().unwrap_or_default()) + .max() + .unwrap_or_default(); + sources.push(( + frequency, + LexicalCandidateSourceV1 { + term, + fields: CandidateSourceFieldsV1::Whole, + minimum_documents, + }, + )); + } + for term in subtokens.into_iter().collect::>() { + if !self.postings.by_term.contains_key(&term) { + continue; + } + let frequency = self.document_frequency(LexicalFieldV1::Subtoken, &term); + if frequency > 0 { + sources.push(( + frequency, + LexicalCandidateSourceV1 { + term, + fields: CandidateSourceFieldsV1::Subtoken, + minimum_documents: frequency, + }, + )); + } + } + sources + } } struct LexicalDraftV1 { @@ -1968,3 +2088,259 @@ impl EditDistanceScratchV1 { (self.previous[width - 1] <= limit).then_some(self.previous[width - 1]) } } + +#[cfg(test)] +mod candidate_source_tests { + use std::collections::BTreeMap; + use std::ops::Range; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use roaring::RoaringBitmap; + + use super::super::super::format::PostingListEncoderV1; + use super::{ + CandidateSourceFieldsV1, LexicalStatsCacheV1, RequestTermPostingsV1, TermFieldPostingsV1, + }; + use crate::retrieval::lexical::{LexicalFieldV1, admit_candidate_sources}; + + fn stats(entries: &[(&str, LexicalFieldV1, Range)]) -> LexicalStatsCacheV1 { + let mut stats = LexicalStatsCacheV1 { + field_totals: BTreeMap::new(), + document_frequencies: BTreeMap::new(), + postings: RequestTermPostingsV1::default(), + }; + for (term, field, documents) in entries { + let mut encoder = PostingListEncoderV1::new(true); + for document in documents.clone() { + encoder.push(document, 1).expect("ordered posting"); + } + stats + .document_frequencies + .entry(*field) + .or_default() + .insert((*term).to_owned(), documents.clone().count()); + stats + .postings + .by_term + .entry((*term).to_owned()) + .or_default() + .push(TermFieldPostingsV1 { + field: *field, + postings: encoder.finish().expect("nonempty postings"), + }); + } + stats + } + + struct ActiveControl; + + impl crate::retrieval::ports::RetrievalExecutionControl for ActiveControl { + fn is_cancelled(&self) -> bool { + false + } + + fn elapsed_micros(&self) -> u64 { + 0 + } + } + + fn candidates( + stats: &LexicalStatsCacheV1, + whole: &[&str], + subtokens: &[&str], + ) -> (RoaringBitmap, Vec<(String, usize)>) { + let sources = stats.candidate_sources( + whole.iter().map(|term| (*term).to_owned()).collect(), + subtokens.iter().map(|term| (*term).to_owned()).collect(), + ); + let mut pruned = Vec::new(); + let admitted = admit_candidate_sources( + sources, + |source, existing, remaining| { + if source.cannot_fit(existing, remaining) { + return Ok(None); + } + stats.postings.new_documents_bounded( + &source.term, + |field| source.fields.admits(field), + existing, + remaining, + &ActiveControl, + ) + }, + |frequency, source| { + if !pruned.iter().any(|(term, _)| term == &source.term) { + pruned.push((source.term.clone(), frequency)); + } + }, + ) + .expect("candidate admission"); + (admitted, pruned) + } + + #[test] + fn overlapping_prose_channels_do_not_crowd_out_a_distinct_word() { + let stats = stats(&[ + ("needle", LexicalFieldV1::BodyText, 0..100), + ("needle", LexicalFieldV1::Subtoken, 0..100), + ("target", LexicalFieldV1::BodyText, 100..16_350), + ("target", LexicalFieldV1::Subtoken, 100..16_350), + ]); + let (documents, pruned) = candidates(&stats, &["needle", "target"], &["needle", "target"]); + assert!(pruned.is_empty(), "{pruned:?}"); + assert_eq!(documents.len(), 16_350); + assert!( + documents.contains(16_349), + "the target-only document must remain eligible" + ); + // Admission grouping must leave both channels available to scoring. + let mut cursors = stats.postings.cursors().unwrap(); + let target_channels = cursors + .iter_mut() + .filter(|cursor| cursor.term == "target") + .map(|cursor| (cursor.field, cursor.frequency_at(16_349).unwrap())) + .collect::>(); + assert_eq!( + target_channels, + vec![ + (LexicalFieldV1::BodyText, Some(1)), + (LexicalFieldV1::Subtoken, Some(1)), + ] + ); + } + + #[test] + fn disjoint_channels_are_charged_for_their_full_union() { + let stats = stats(&[ + ("needle", LexicalFieldV1::BodyText, 0..100), + ("needle", LexicalFieldV1::Subtoken, 100..200), + ("target", LexicalFieldV1::BodyText, 200..16_400), + ("target", LexicalFieldV1::Subtoken, 200..16_400), + ]); + let (documents, pruned) = candidates(&stats, &["needle", "target"], &["needle", "target"]); + assert_eq!(documents.len(), 200); + assert_eq!(pruned, vec![("target".to_owned(), 16_200)]); + assert!(!documents.contains(16_399)); + } + + #[test] + fn fully_overlapping_oversized_channels_do_not_claim_partial_evidence() { + let stats = stats(&[ + ("cache", LexicalFieldV1::BodyText, 0..20_000), + ("cache", LexicalFieldV1::Subtoken, 0..20_000), + ]); + let (documents, pruned) = candidates(&stats, &["cache"], &["cache"]); + assert_eq!(documents.len(), 20_000); + assert!(documents.contains(19_999)); + assert!( + pruned.is_empty(), + "the second channel adds no documents despite exceeding the cap: {pruned:?}" + ); + } + + #[test] + fn oversized_disjoint_channels_still_report_omitted_candidates() { + let stats = stats(&[ + ("cache", LexicalFieldV1::BodyText, 0..20_000), + ("cache", LexicalFieldV1::Subtoken, 20_000..40_000), + ]); + let (documents, pruned) = candidates(&stats, &["cache"], &["cache"]); + assert_eq!(documents.len(), 20_000); + assert!(documents.contains(19_999)); + assert!(!documents.contains(20_000)); + assert_eq!(pruned, vec![("cache".to_owned(), 20_000)]); + } + + #[test] + fn rare_whole_word_is_not_lost_behind_a_common_subtoken() { + let stats = stats(&[ + ("cache", LexicalFieldV1::BodyText, 0..1), + ("cache", LexicalFieldV1::Subtoken, 1..20_001), + ("token", LexicalFieldV1::BodyText, 20_001..20_101), + ("token", LexicalFieldV1::Subtoken, 20_001..20_101), + ]); + let (documents, pruned) = candidates(&stats, &["cache", "token"], &["cache", "token"]); + assert_eq!(documents.len(), 101); + assert!(documents.contains(0), "rare whole-word cache must survive"); + assert!(documents.contains(20_100)); + assert_eq!(pruned, vec![("cache".to_owned(), 20_000)]); + } + + #[test] + fn rejected_common_source_is_not_decoded_before_admission() { + let mut stats = stats(&[ + ("needle", LexicalFieldV1::BodyText, 0..1), + ("get", LexicalFieldV1::Subtoken, 1..30_001), + ]); + // The stored frequency alone proves this source cannot fit. If it + // were eagerly decoded, this intentionally invalid payload would fail. + stats.postings.by_term.get_mut("get").unwrap()[0].postings = vec![255]; + let (documents, pruned) = candidates(&stats, &["needle"], &["get"]); + assert_eq!(documents.len(), 1); + assert!(documents.contains(0)); + assert_eq!(pruned, vec![("get".to_owned(), 30_000)]); + } + + #[test] + fn posting_admission_checks_cancellation_during_decode() { + struct CancelAfterBatch(AtomicUsize); + + impl crate::retrieval::ports::RetrievalExecutionControl for CancelAfterBatch { + fn is_cancelled(&self) -> bool { + self.0.fetch_add(1, Ordering::SeqCst) > 0 + } + + fn elapsed_micros(&self) -> u64 { + 0 + } + } + + let stats = stats(&[("needle", LexicalFieldV1::BodyText, 0..1_024)]); + let control = CancelAfterBatch(AtomicUsize::new(0)); + let error = stats + .postings + .new_documents_bounded( + "needle", + |_| true, + &RoaringBitmap::new(), + usize::MAX, + &control, + ) + .expect_err("cancelled during the posting stream"); + assert!(matches!( + error, + crate::retrieval::ports::RetrievalPortError::Cancelled + )); + assert_eq!( + control.0.load(Ordering::SeqCst), + 2, + "the request was interrupted at the second 128-posting checkpoint" + ); + } + + #[test] + fn repeated_sources_are_deduplicated_without_widening_requested_fields() { + let stats = stats(&[ + ("needle", LexicalFieldV1::BodyText, 0..100), + ("needle", LexicalFieldV1::Subtoken, 100..200), + ("part", LexicalFieldV1::Subtoken, 200..210), + ]); + let sources = stats.candidate_sources( + vec!["needle".to_owned(), "needle".to_owned()], + vec!["part".to_owned(), "part".to_owned()], + ); + assert_eq!(sources.len(), 2); + assert_eq!(sources[0].0, 100); + assert_eq!(sources[0].1.fields, CandidateSourceFieldsV1::Whole); + assert_eq!(sources[1].0, 10); + assert_eq!(sources[1].1.fields, CandidateSourceFieldsV1::Subtoken); + let (documents, pruned) = candidates(&stats, &["needle", "needle"], &["part", "part"]); + assert!(pruned.is_empty()); + assert_eq!(documents.len(), 110); + assert!( + !documents.contains(100), + "unrequested subtoken channel stays excluded" + ); + assert!(documents.contains(209)); + } +} diff --git a/crates/tracedecay-query/src/retrieval/lexical/tests.rs b/crates/tracedecay-query/src/retrieval/lexical/tests.rs index 69ef708f8d..b74310511e 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/tests.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/tests.rs @@ -68,11 +68,41 @@ where T::try_from(format!("sha256:{}", byte.to_string().repeat(64))).expect("valid fixture digest") } +fn synthetic_admission( + sources: Vec<(usize, &'static str)>, + ranges: &[(&str, std::ops::Range)], + pruned: &mut Vec<(String, u64)>, +) -> roaring::RoaringBitmap { + admit_candidate_sources( + sources, + |term, existing, remaining| { + let range = ranges + .iter() + .find(|(name, _)| name == term) + .expect("source") + .1 + .clone(); + let mut delta = roaring::RoaringBitmap::new(); + for document in range { + if !existing.contains(document) { + delta.insert(document); + } + if delta.len() > remaining as u64 { + return Ok(None); + } + } + Ok(Some(delta)) + }, + |frequency, term| pruned.push(((*term).to_owned(), frequency as u64)), + ) + .expect("admission") +} + #[test] fn candidate_sources_admit_rarest_first_within_the_document_budget() { let budget = MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1; let mut pruned = Vec::new(); - let admitted = admit_candidate_sources( + let documents = synthetic_admission( vec![ (budget * 3, "the"), (10, "pendingtoolkind"), @@ -80,20 +110,24 @@ fn candidate_sources_admit_rarest_first_within_the_document_budget() { (budget * 2, "tool"), (200, "kind"), ], - |frequency, term| pruned.push((term.to_string(), frequency as u64)), - ); - assert_eq!( - admitted, - ["pendingtoolkind", "kind"], - "`pending` would push the admitted total past the budget, and every later \ - (more common) source is pruned with it" + &[ + ("pendingtoolkind", 0..10), + ("kind", 10..210), + ("pending", 210..(210 + (budget - 100) as u32)), + ("tool", 30_000..(30_000 + (budget * 2) as u32)), + ("the", 70_000..(70_000 + (budget * 3) as u32)), + ], + &mut pruned, ); + assert_eq!(documents.len(), 210); + assert!(documents.contains(209)); + assert!(!documents.contains(210)); assert_eq!( pruned, [ ("pending".to_owned(), (budget - 100) as u64), ("tool".to_owned(), (budget * 2) as u64), - ("the".to_owned(), (budget * 3) as u64) + ("the".to_owned(), (budget * 3) as u64), ] ); let outcome = super::candidate_admission_outcome( @@ -113,24 +147,33 @@ fn candidate_sources_admit_rarest_first_within_the_document_budget() { #[test] fn candidate_sources_ignore_empty_sources_before_admitting_a_common_term() { let budget = MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1; - assert_eq!( - admit_candidate_sources( - vec![(0, "missing"), (budget + 1, "common"), (0, "absent")], - |_, _| {} - ), - ["common"], - "empty sources must not consume the first nonempty source exception" + let documents = synthetic_admission( + vec![(0, "missing"), (budget + 1, "common"), (0, "absent")], + &[("common", 0..(budget + 1) as u32)], + &mut Vec::new(), + ); + assert_eq!(documents.len(), (budget + 1) as u64); + assert!( + synthetic_admission(vec![(0, "missing"), (0, "absent")], &[], &mut Vec::new(),).is_empty() ); - assert!(admit_candidate_sources(vec![(0, "missing"), (0, "absent")], |_, _| {}).is_empty()); } #[test] fn candidate_sources_keep_request_order_across_equal_frequencies() { let half = MAX_LEXICAL_CANDIDATE_DOCUMENTS_V1 / 2; - assert_eq!( - admit_candidate_sources(vec![(half, "b"), (half, "a"), (half, "c")], |_, _| {}), - ["b", "a"] + let mut pruned = Vec::new(); + let documents = synthetic_admission( + vec![(half, "b"), (half, "a"), (half, "c")], + &[ + ("b", 0..half as u32), + ("a", half as u32..(half * 2) as u32), + ("c", (half * 2) as u32..(half * 3) as u32), + ], + &mut pruned, ); + assert_eq!(documents.len(), (half * 2) as u64); + assert!(!documents.contains((half * 2) as u32)); + assert_eq!(pruned, vec![("c".to_owned(), half as u64)]); } #[test] @@ -160,7 +203,7 @@ fn shared_query_parser_retains_phrase_and_identifier_subtokens() { /// query-side subtokens equal the extraction subtokens. #[test] fn query_tokenization_matches_index_tokenization_for_separator_terms() { - let text = "find foo-bar a::b p/q.rs --flag E0308 TS1234 ERR_X"; + let text = "foo-bar a::b p/q.rs --flag E0308 TS1234 ERR_X"; let parts = lexical_query_parts(text).expect("lexical query parts"); let index_tokens: Vec = technical_tokens(text) @@ -181,12 +224,7 @@ fn query_tokenization_matches_index_tokenization_for_separator_terms() { let mut expected_subtokens: Vec = index_tokens .iter() - .flat_map(|token| { - let lowercase = token.to_ascii_lowercase(); - split_subtokens(token) - .into_iter() - .filter(move |part| part != &lowercase) - }) + .flat_map(|token| split_subtokens(token)) .collect(); expected_subtokens.sort(); expected_subtokens.dedup(); @@ -199,6 +237,55 @@ fn query_tokenization_matches_index_tokenization_for_separator_terms() { } } +#[test] +fn prose_words_can_match_identifier_subtokens_without_broadening_single_term_queries() { + let query = "widget auto tag items forward total"; + let parts = lexical_query_parts(query).expect("prose search"); + assert_eq!( + parts.subtokens, + ["auto", "forward", "items", "tag", "total", "widget"] + .map(str::to_owned) + .to_vec() + ); + assert_eq!(parts.whole_terms.len(), 6); + + let hyphenated = + lexical_query_parts("widget-tool auto-tag query rows per item").expect("hyphenated prose"); + for word in [ + "widget", "tool", "auto", "tag", "query", "rows", "per", "item", + ] { + assert!( + hyphenated.subtokens.contains(&word.to_owned()), + "{word} must reach a camelCase subtoken posting" + ); + } + assert!( + lexical_query_parts("fetchWidgetAutoTagItemsForwardTotal") + .expect("single identifier") + .subtokens + .is_empty(), + "single-term queries must retain their narrow admission" + ); +} + +#[test] +fn prose_subtoken_expansion_uses_the_existing_content_word_policy() { + let query = "how do i find the widget auto tag"; + let parts = lexical_query_parts(query).expect("prose search"); + assert_eq!( + parts.subtokens, + ["auto", "tag", "widget"].map(str::to_owned) + ); + for word in ["how", "do", "i", "find", "the"] { + assert!(parts.whole_terms.iter().any(|term| term == word)); + } + assert_eq!(parts.phrases, [query]); + + let technical = lexical_query_parts("TypeOf Widget").expect("technical query"); + assert!(technical.subtokens.iter().any(|term| term == "type")); + assert!(technical.subtokens.iter().any(|term| term == "of")); +} + fn budget(max_candidates_per_lane: u32) -> RetrievalBudget { RetrievalBudget { max_candidates_per_lane, diff --git a/crates/tracedecay-query/src/retrieval/source_tier.rs b/crates/tracedecay-query/src/retrieval/source_tier.rs index ada450aab0..ce8cc7943c 100644 --- a/crates/tracedecay-query/src/retrieval/source_tier.rs +++ b/crates/tracedecay-query/src/retrieval/source_tier.rs @@ -6,11 +6,14 @@ use tracedecay_domain::{CodeSearchChunkGrainV1, RetrievalSourceRoleV1}; use super::lexical::LexicalFieldV1; /// Classify a hit from the existing test-path authority plus whether the -/// matched evidence is a name or signature definition. +/// matched evidence is a name or signature definition. JSON object keys are +/// searchable configuration data, not executable code definitions: treating +/// each key as a production definition fills capped symbol pages ahead of +/// real declarations. Keep the key eligible and its raw score unchanged. pub(crate) fn classify_source_role(path: &str, definition_match: bool) -> RetrievalSourceRoleV1 { if is_test_file(path) { RetrievalSourceRoleV1::TestReference - } else if definition_match { + } else if definition_match && !path.ends_with(".json") { RetrievalSourceRoleV1::ProductionDefinition } else { RetrievalSourceRoleV1::ProductionOther @@ -85,6 +88,20 @@ mod tests { classify_source_role("src/internal/client/ensure-daemon.test.ts", true), RetrievalSourceRoleV1::TestReference ); + assert_eq!( + classify_source_role("config/render.json", true), + RetrievalSourceRoleV1::ProductionOther, + "a JSON key remains searchable but cannot preempt a code definition" + ); + assert_eq!( + classify_source_role("config/render.json", false), + RetrievalSourceRoleV1::ProductionOther + ); + assert_eq!( + classify_source_role("tests/render.json", true), + RetrievalSourceRoleV1::TestReference, + "test source retains its own lower-priority role" + ); assert_eq!( RetrievalSourceRoleV1::ProductionDefinition.admission_rank(), 0 diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index 4cd22cd3ea..f186edc00e 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -6070,6 +6070,111 @@ fn fielded_bm25_keeps_whole_identifiers_and_subtokens_distinct() { })); } +#[test] +fn prose_subtokens_add_identifier_matches_without_rescoring_whole_words() { + let artifact = rust_artifact(&[ + "pub fn reserveStock() {}\n", + "pub fn reserve() { let stock = 1; }\n", + ]); + let lane = artifact.lane(); + let whole = complete( + lane.retrieve_lexical(&artifact.request( + "reserve stock", + &["reserve", "stock"], + &[], + &[], + 0, + 8, + )) + .expect("whole-word retrieval"), + ); + let expanded = complete( + lane.retrieve_lexical(&artifact.request( + "reserve stock", + &["reserve", "stock"], + &["reserve", "stock"], + &[], + 0, + 8, + )) + .expect("expanded retrieval"), + ); + assert!(candidate_files(&whole.candidates).contains(&1)); + assert_eq!( + candidate_files(&expanded.candidates), + BTreeSet::from([0, 1]) + ); + for candidate in &whole.candidates { + if candidate + .file_occurrence_id + .as_ref() + .map(|file| file.as_str()) + != Some("file.fixture.1") + { + continue; + } + let occurrence = &candidate.source_occurrence_id; + let evidence = &whole.evidence_by_occurrence[occurrence]; + assert_eq!( + expanded.evidence_by_occurrence[occurrence].field_scores_micros, + evidence.field_scores_micros, + "an ordinary word already scored in its source field is not new subtoken evidence" + ); + } +} + +#[test] +fn filtered_subtoken_search_keeps_prose_evidence_when_whole_fields_are_excluded() { + let artifact = rust_artifact(&[ + "pub fn process() { let digest = payload; }\n", + "pub fn digestPayloadMismatch() {}\n", + ]); + for filter in [ + LexicalFieldFilterV1 { + field: LexicalFieldV1::Subtoken, + include: true, + }, + LexicalFieldFilterV1 { + field: LexicalFieldV1::BodyText, + include: false, + }, + ] { + let mut request = artifact.request( + "digest payload", + &["digest", "payload"], + &["digest", "payload"], + &[], + 0, + 8, + ); + request.field_filters = Cow::Owned(vec![filter]); + let results = complete( + artifact + .lane() + .retrieve_lexical(&request) + .expect("filtered subtoken search"), + ); + assert!( + results.candidates.iter().any(|candidate| { + candidate.file_occurrence_id.as_ref().is_some_and(|file| { + file.as_str() == "file.fixture.0" + && results + .evidence_by_occurrence + .get(&candidate.source_occurrence_id) + .is_some_and(|evidence| { + evidence.field_scores_micros.iter().any(|(field, score)| { + *field == LexicalFieldV1::Subtoken && *score > 0 + }) && ["digest", "payload"].iter().all(|word| { + evidence.matched_subtokens.iter().any(|term| term == word) + }) + }) + }) + }), + "excluded whole fields must not zero out admitted subtokens: {results:?}" + ); + } +} + #[test] fn lexical_phrase_and_bounded_fuzzy_recovery_are_deterministic() { let artifact = rust_artifact(&[ @@ -6283,6 +6388,128 @@ fn duplicate_whole_terms_do_not_consume_the_global_fuzzy_budget() { ); } +#[test] +fn prose_query_recovers_camel_case_symbol_from_real_sealed_subtoken_postings() { + let artifact = rust_artifact(&[ + "pub fn fetchWidgetAutoTagItemsForwardTotal() {}\n", + "pub fn widget() {}\n", + "pub fn tag() {}\n", + ]); + let query = "widget auto tag items forward total"; + let parts = tracedecay_query::retrieval::lexical::lexical_query_parts(query) + .expect("parse multiword query"); + let mut request = artifact.request(query, &[], &[], &[], 0, 32); + request.whole_terms = Cow::Owned(parts.whole_terms); + request.subtokens = Cow::Owned(parts.subtokens); + request.phrases = Cow::Owned(parts.phrases); + let results = complete( + artifact + .lane() + .retrieve_lexical(&request) + .expect("sealed lexical search"), + ); + assert!( + candidate_files(&results.candidates).contains(&0), + "the camelCase symbol must not be absent from the bounded page: {results:?}" + ); + assert!( + results.candidates.iter().any(|candidate| { + candidate.file_occurrence_id.as_ref().is_some_and(|file| { + file.as_str() == "file.fixture.0" + && ["widget", "auto", "tag", "items", "forward", "total"] + .iter() + .all(|term| { + results.evidence_by_occurrence[&candidate.source_occurrence_id] + .matched_subtokens + .iter() + .any(|matched| matched.as_str() == *term) + }) + }) + }), + "the long symbol must match every query word in the subtoken field" + ); +} + +#[test] +fn prose_words_do_not_add_two_full_scores_for_the_same_evidence() { + // The first row states the words; the second only hides them inside its + // name. Prose subtoken admission exists to reach the second row, and must + // not add two full scores for words the first row already matched. + let artifact = rust_artifact(&[ + "/// Reports when the digest does not match its payload.\npub fn validate() {}\n", + "pub fn digestPayloadMismatch() {}\n", + ]); + let query = "digest payload"; + let parts = tracedecay_query::retrieval::lexical::lexical_query_parts(query) + .expect("parse multiword query"); + let mut request = artifact.request(query, &[], &[], &[], 0, 32); + request.whole_terms = Cow::Owned(parts.whole_terms); + request.subtokens = Cow::Owned(parts.subtokens); + request.phrases = Cow::Owned(parts.phrases); + let results = complete( + artifact + .lane() + .retrieve_lexical(&request) + .expect("sealed lexical search"), + ); + let whole = complete( + artifact + .lane() + .retrieve_lexical(&artifact.request(query, &["digest", "payload"], &[], &[], 0, 32)) + .expect("whole-word control"), + ); + let subtoken = complete( + artifact + .lane() + .retrieve_lexical(&artifact.request(query, &[], &["digest", "payload"], &[], 0, 32)) + .expect("subtoken control"), + ); + let in_file = |ordinal: usize| { + results + .candidates + .iter() + .filter(move |candidate| { + candidate + .file_occurrence_id + .as_ref() + .is_some_and(|file| file.as_str() == format!("file.fixture.{ordinal}")) + }) + .map(|candidate| &results.evidence_by_occurrence[&candidate.source_occurrence_id]) + .collect::>() + }; + let stated = in_file(0); + assert!(!stated.is_empty()); + let total = |evidence: &LexicalLaneEvidence| { + evidence + .field_scores_micros + .iter() + .map(|(_, score)| score) + .sum::() + }; + for evidence in stated { + let occurrence = &evidence.binding.source_occurrence; + let whole_score = total(&whole.evidence_by_occurrence[occurrence]); + let subtoken_score = total(&subtoken.evidence_by_occurrence[occurrence]); + assert!(whole_score > 0 && subtoken_score > 0); + assert!(total(evidence) >= whole_score.max(subtoken_score)); + assert!( + total(evidence) < whole_score + subtoken_score, + "overlapping representations must not add both full scores: {evidence:?}" + ); + } + assert!( + in_file(1).iter().any(|evidence| { + ["digest", "payload"].iter().all(|term| { + evidence + .matched_subtokens + .iter() + .any(|matched| matched == term) + }) + }), + "the camelCase symbol must still match both words as subtokens: {results:?}" + ); +} + #[test] fn lexical_artifact_reports_freshness_coverage_and_page_cutoff() { let sources = ["pub fn target() {}\n"; 3]; diff --git a/crates/tracedecay-query/tests/search_quality_suite/source_roles.rs b/crates/tracedecay-query/tests/search_quality_suite/source_roles.rs index 3372726a74..6505217f00 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/source_roles.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/source_roles.rs @@ -212,3 +212,106 @@ fn name_filtered_routes_keep_undocumented_symbol_hits() { ); } } + +#[test] +fn json_object_keys_remain_searchable_but_do_not_outrank_code_definitions() { + // This fixture travels through the real JSON/Rust extractors, sealed + // lexical artifact, bounded winner heap and emitted source-role evidence. + // A JSON pair is extracted as a Const, but it is configuration data, not + // an executable definition of the same-named Rust function. + let fixture = real_lexical_source_fixture_from_sources(vec![ + ( + "file.json".to_owned(), + "config/render.json".to_owned(), + br#"{"render": true, "other": false}"#.to_vec(), + ), + ( + "file.code".to_owned(), + "src/render.rs".to_owned(), + b"pub fn render() -> bool { true }\n".to_vec(), + ), + ( + "file.test".to_owned(), + "tests/render.rs".to_owned(), + b"#[test] fn render() { assert!(true); }\n".to_vec(), + ), + ]); + let artifact = sealed_artifact(&fixture, fixture.metadata.clone()); + let lane = LexicalLane::new(artifact.reader.clone()); + let request = artifact.request("render", &["render"], &[], &[], 0, 32); + let results = complete( + lane.retrieve_lexical(&request) + .expect("sealed lexical search"), + ); + + let mut json_seen = false; + let mut code_definition_seen = false; + for candidate in &results.candidates { + let file = candidate + .file_occurrence_id + .as_ref() + .expect("file occurrence") + .as_str(); + let role = results.evidence_by_occurrence[&candidate.source_occurrence_id].source_role; + match file { + "file.json" => { + json_seen = true; + assert_ne!( + role, + RetrievalSourceRoleV1::ProductionDefinition, + "JSON object keys must not take the code-definition tier" + ); + } + "file.code" => { + code_definition_seen |= role == RetrievalSourceRoleV1::ProductionDefinition; + } + "file.test" => assert_eq!(role, RetrievalSourceRoleV1::TestReference), + other => panic!("unexpected fixture file {other}"), + } + } + assert!(json_seen, "the JSON key must remain retrievable"); + assert!( + code_definition_seen, + "the Rust function must remain a definition" + ); + + let first = results.candidates.first().expect("named code candidate"); + assert_eq!( + first.file_occurrence_id.as_ref().unwrap().as_str(), + "file.code", + "a production declaration outranks a same-named JSON object key" + ); + let capped = artifact.request("render", &["render"], &[], &[], 0, 1); + let capped_results = complete( + lane.retrieve_lexical(&capped) + .expect("bounded lexical search"), + ); + assert_eq!(capped_results.candidates.len(), 1); + assert_eq!( + capped_results.candidates[0] + .file_occurrence_id + .as_ref() + .unwrap() + .as_str(), + "file.code", + "capped admission must apply the same priority as emitted evidence" + ); + + // A user explicitly searching configuration still finds its actual keys. + let mut scoped = artifact.request("render", &["render"], &[], &[], 0, 32); + scoped.path_prefix = Some("config"); + let scoped_results = complete( + lane.retrieve_lexical(&scoped) + .expect("configuration-scoped search"), + ); + assert!( + !scoped_results.candidates.is_empty(), + "lower priority is not a filter or an opt-out" + ); + assert!(scoped_results.candidates.iter().all(|candidate| { + candidate + .file_occurrence_id + .as_ref() + .is_some_and(|file| file.as_str() == "file.json") + })); +} diff --git a/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs b/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs index 47f9a1194b..a1dc0be714 100644 --- a/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs +++ b/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs @@ -1,6 +1,7 @@ #![allow(clippy::expect_used, clippy::unwrap_used)] -use std::path::Path; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; use std::process::Command; use tempfile::TempDir; @@ -65,6 +66,32 @@ impl RepositoryFixture { self.git(&["rev-parse", &format!("{reference}^{{tree}}")]) } + /// Snapshot actual Git objects, not filesystem block allocation. Git's + /// loose-object size statistic can vary without changing object bytes. + fn object_database_snapshot(&self) -> BTreeMap> { + fn collect(root: &Path, directory: &Path, objects: &mut BTreeMap>) { + for entry in std::fs::read_dir(directory).expect("read object directory") { + let path = entry.expect("object directory entry").path(); + if path.is_dir() { + collect(root, &path, objects); + } else { + let relative = path + .strip_prefix(root) + .expect("object path within repository"); + objects.insert( + relative.to_path_buf(), + std::fs::read(&path).expect("read object bytes"), + ); + } + } + } + + let root = self.path().join(".git/objects"); + let mut objects = BTreeMap::new(); + collect(&root, &root, &mut objects); + objects + } + fn setup_fast_forward(&self) -> (GitOidV1, GitOidV1) { self.write("base.txt", "base\n"); let destination = self.commit("base"); @@ -125,7 +152,7 @@ fn preflight_is_read_only_and_fast_forward_apply_and_rollback_use_exact_ref_cas( let (source, destination) = fixture.setup_fast_forward(); let refs_before = fixture.git(&["show-ref"]); let status_before = fixture.git(&["status", "--porcelain=v1"]); - let objects_before = fixture.git(&["count-objects", "-v"]); + let objects_before = fixture.object_database_snapshot(); let preview = preflight( &fixture, @@ -136,7 +163,11 @@ fn preflight_is_read_only_and_fast_forward_apply_and_rollback_use_exact_ref_cas( assert_eq!(preview.disposition, GitNativePreflightDisposition::Eligible); assert_eq!(fixture.git(&["show-ref"]), refs_before); assert_eq!(fixture.git(&["status", "--porcelain=v1"]), status_before); - assert_eq!(fixture.git(&["count-objects", "-v"]), objects_before); + assert_eq!( + fixture.object_database_snapshot(), + objects_before, + "preflight must not add, remove, or rewrite any Git object", + ); let authority = GitRepositoryAuthority::discover(fixture.path()).unwrap(); let candidate = preview.candidate_tree.as_ref().unwrap(); diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs index 0f4af77d0b..c7192aa660 100644 --- a/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/connection/mod.rs @@ -201,8 +201,22 @@ impl OpenedDatabaseFile { } pub(crate) fn verify_current_path(&self, path: &Path) -> Result<(), OpenedDatabaseFileError> { - let current = File::open(path).map_err(|_| OpenedDatabaseFileError::Open)?; - if opened_file_identity(¤t)? != self.identity { + // POSIX record locks belong to the process: closing any descriptor + // for this inode releases SQLite's locks, even on another connection. + // Inspect the pathname without opening a temporary descriptor. Keep + // using the pinned descriptor's dev/inode identity, and retain both + // checks around SQLITE_FCNTL_HAS_MOVED in verify_connection. + #[cfg(unix)] + let current_identity = { + let current = std::fs::metadata(path).map_err(|_| OpenedDatabaseFileError::Open)?; + metadata_identity(¤t) + }; + #[cfg(not(unix))] + let current_identity = { + let current = File::open(path).map_err(|_| OpenedDatabaseFileError::Open)?; + opened_file_identity(¤t)? + }; + if current_identity != self.identity { return Err(OpenedDatabaseFileError::Replaced); } Ok(()) @@ -276,6 +290,25 @@ impl OpenedDatabaseFile { } } +/// Check a live SQLite worker's physical file without opening and closing a +/// second descriptor for the same inode on Unix. Closing that descriptor can +/// release process-wide SQLite locks owned by other connections, even though +/// SQLite's VFS did not perform the close. +pub(crate) fn live_path_identity(path: &Path) -> Result { + #[cfg(unix)] + { + let metadata = std::fs::metadata(path).map_err(|_| OpenedDatabaseFileError::Open)?; + if !metadata.is_file() { + return Err(OpenedDatabaseFileError::NotFile); + } + Ok(metadata_identity(&metadata)) + } + #[cfg(not(unix))] + { + OpenedDatabaseFile::pin(path).map(|opened| opened.identity()) + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum OpenedDatabaseFileError { Create, @@ -569,10 +602,13 @@ impl std::error::Error for VerifiedReaderError { /// Foreign-database reader bound to the physical file actually /// opened by SQLite rather than a later pathname observation. pub struct VerifiedReader { - _opened_file: OpenedDatabaseFile, + // Keep the connection before the raw pin in declaration order: closing + // any raw descriptor on Unix can release another SQLite connection's + // process-wide record locks. SQLite must close its handle first. connection: Connection, canonical_path: PathBuf, file_identity: u64, + _opened_file: OpenedDatabaseFile, } pub fn default_vfs_max_pathname() -> Option { @@ -592,8 +628,10 @@ impl VerifiedReader { self.file_identity } - pub fn into_parts(self) -> (Connection, PathBuf, u64) { - (self.connection, self.canonical_path, self.file_identity) + /// The verified canonical pathname. It remains bound to this reader's + /// retained physical file until the SQLite connection is closed. + pub fn canonical_path(&self) -> &Path { + &self.canonical_path } } @@ -1269,5 +1307,8 @@ fn policy(stage: &'static str, source: rusqlite::Error) -> ConnectionPolicyError ConnectionPolicyError { stage, source } } +#[cfg(all(test, unix))] +mod path_identity_tests; + #[cfg(test)] mod tests; diff --git a/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs new file mode 100644 index 0000000000..b024e5a454 --- /dev/null +++ b/crates/tracedecay-rusqlite-runtime/src/connection/path_identity_tests.rs @@ -0,0 +1,242 @@ +use std::{env, fs, path::Path, process::Command, time::Duration}; + +use rusqlite::{Connection, Error, ErrorCode, OpenFlags}; +use tempfile::TempDir; + +use super::{OpenedDatabaseFile, OpenedDatabaseFileError, live_path_identity}; + +const PROBE_PATH: &str = "TRACEDECAY_PATH_IDENTITY_PROBE_PATH"; +const PROBE_OPERATION: &str = "TRACEDECAY_PATH_IDENTITY_PROBE_OPERATION"; +const PROBE_EXPECTATION: &str = "TRACEDECAY_PATH_IDENTITY_PROBE_EXPECTATION"; +const PROBE_COMPLETE: &str = "tracedecay-path-identity-probe-complete"; + +// A second connection in this process is not a lock probe: SQLite's Unix VFS +// shares inode/lock bookkeeping within a process and can hide a lost OS lock. +// Execute only the child test in a fresh process, with a marker to reject a +// successful test harness invocation that accidentally selected zero tests. +fn probe_external_connection(path: &Path, operation: &str, blocked: bool) { + let (_, module) = module_path!().split_once("::").expect("test module path"); + let output = Command::new(env::current_exe().expect("test executable")) + .args([ + "--exact", + &format!("{module}::external_connection_probe"), + "--nocapture", + ]) + .env(PROBE_PATH, path) + .env(PROBE_OPERATION, operation) + .env( + PROBE_EXPECTATION, + if blocked { "blocked" } else { "allowed" }, + ) + .output() + .expect("run external SQLite connection"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success() && stdout.contains(PROBE_COMPLETE), + "external {operation} probe failed: {}\n{stdout}\n{stderr}", + output.status, + ); +} + +#[test] +fn external_connection_probe() { + let Some(path) = env::var_os(PROBE_PATH) else { + return; + }; + let operation = env::var(PROBE_OPERATION).expect("probe operation"); + let expectation = env::var(PROBE_EXPECTATION).expect("probe expectation"); + let connection = Connection::open_with_flags( + Path::new(&path), + OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .expect("open existing database in another process"); + connection + .busy_timeout(Duration::ZERO) + .expect("no lock wait"); + let result = match operation.as_str() { + "write" => connection.execute_batch("BEGIN IMMEDIATE; ROLLBACK"), + "journal" => connection + .query_row("PRAGMA journal_mode = DELETE", [], |row| { + row.get::<_, String>(0) + }) + .map(|mode| assert_eq!(mode, "delete")), + _ => panic!("unknown probe operation: {operation}"), + }; + match expectation.as_str() { + "blocked" => assert!( + matches!( + &result, + Err(Error::SqliteFailure(error, _)) if error.code == ErrorCode::DatabaseBusy + ), + "expected SQLITE_BUSY from external {operation}, got {result:?}", + ), + "allowed" => result.expect("external operation after lock release"), + _ => panic!("unknown probe expectation: {expectation}"), + } + println!("{PROBE_COMPLETE}"); +} + +fn assert_committed_data(path: &Path) { + let connection = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY) + .expect("reopen committed database"); + let value: i64 = connection + .query_row("SELECT value FROM sample", [], |row| row.get(0)) + .expect("committed value"); + assert_eq!(value, 2); + let integrity: String = connection + .query_row("PRAGMA integrity_check", [], |row| row.get(0)) + .expect("integrity check"); + assert_eq!(integrity, "ok"); +} + +#[test] +fn worker_identity_checks_keep_cross_process_writer_exclusion() { + let temp = TempDir::new().expect("tempdir"); + let path = temp.path().join("worker.db"); + let connection = Connection::open(&path).expect("open database"); + connection + .execute_batch( + "PRAGMA journal_mode = DELETE; + CREATE TABLE sample(value INTEGER); + INSERT INTO sample VALUES (1); + BEGIN IMMEDIATE; + UPDATE sample SET value = 2;", + ) + .expect("hold live SQLite writer lock"); + // The old worker check opened another raw descriptor and dropped it + // immediately. On POSIX that released this transaction's advisory lock. + probe_external_connection(&path, "write", true); + let identity = live_path_identity(&path).expect("inspect live worker identity"); + assert_ne!(identity, 0); + probe_external_connection(&path, "write", true); + assert_eq!(live_path_identity(&path).unwrap(), identity); + probe_external_connection(&path, "write", true); + + connection.execute_batch("COMMIT").expect("commit"); + probe_external_connection(&path, "write", false); + drop(connection); + assert_committed_data(&path); + + let absent = temp.path().join("absent.db"); + assert_eq!( + live_path_identity(&absent), + Err(OpenedDatabaseFileError::Open) + ); + let directory = temp.path().join("not-a-database"); + fs::create_dir(&directory).expect("directory"); + assert_eq!( + live_path_identity(&directory), + Err(OpenedDatabaseFileError::NotFile) + ); +} + +#[test] +fn path_checks_preserve_rollback_writer_lock() { + let temp = TempDir::new().expect("tempdir"); + let path = temp.path().join("rollback.db"); + // Pin before SQLite takes locks, and keep the pin until after it closes. + // This isolates pathname verification from the separate pin-drop lifetime. + let pinned = OpenedDatabaseFile::create_new(&path).expect("pin database"); + let connection = Connection::open(&path).expect("open database"); + connection + .execute_batch( + "PRAGMA journal_mode = DELETE; + CREATE TABLE sample(value INTEGER); + INSERT INTO sample VALUES (1); + BEGIN IMMEDIATE; + UPDATE sample SET value = 2;", + ) + .expect("hold write transaction"); + probe_external_connection(&path, "write", true); + + pinned.verify_current_path(&path).expect("same inode"); + probe_external_connection(&path, "write", true); + pinned + .verify_connection(&connection, &path) + .expect("same SQLite connection"); + probe_external_connection(&path, "write", true); + + connection.execute_batch("COMMIT").expect("commit"); + probe_external_connection(&path, "write", false); + drop(connection); + drop(pinned); + assert_committed_data(&path); +} + +#[test] +fn path_checks_preserve_live_wal_journal_mode_guard() { + let temp = TempDir::new().expect("tempdir"); + let path = temp.path().join("wal.db"); + let pinned = OpenedDatabaseFile::create_new(&path).expect("pin database"); + let connection = Connection::open(&path).expect("open database"); + connection + .execute_batch( + "PRAGMA journal_mode = WAL; + CREATE TABLE sample(value INTEGER); + INSERT INTO sample VALUES (1); + BEGIN IMMEDIATE; + UPDATE sample SET value = 2;", + ) + .expect("hold WAL write transaction"); + probe_external_connection(&path, "journal", true); + + pinned.verify_current_path(&path).expect("same inode"); + probe_external_connection(&path, "journal", true); + pinned + .verify_connection(&connection, &path) + .expect("same SQLite connection"); + probe_external_connection(&path, "journal", true); + + connection.execute_batch("COMMIT").expect("commit"); + // A live WAL connection still prevents another process switching modes, + // even between transactions. Only closing the connection releases it. + probe_external_connection(&path, "journal", true); + drop(connection); + probe_external_connection(&path, "journal", false); + drop(pinned); + assert_committed_data(&path); +} + +#[test] +fn path_checks_keep_pinned_identity_and_reject_missing_or_replaced_files() { + let temp = TempDir::new().expect("tempdir"); + let path = temp.path().join("identity.db"); + let alias = temp.path().join("same-inode.db"); + let symlink = temp.path().join("symlink.db"); + let pinned = OpenedDatabaseFile::create_new(&path).expect("pin database"); + fs::hard_link(&path, &alias).expect("hard link"); + std::os::unix::fs::symlink(&path, &symlink).expect("symlink"); + pinned.verify_current_path(&path).expect("original inode"); + pinned.verify_current_path(&alias).expect("hard-link inode"); + pinned + .verify_current_path(&symlink) + .expect("symlink target"); + + fs::remove_file(&path).expect("unlink original path"); + assert_eq!( + pinned.verify_current_path(&path), + Err(OpenedDatabaseFileError::Open), + ); + assert_eq!( + pinned.verify_current_path(&symlink), + Err(OpenedDatabaseFileError::Open), + ); + fs::write(&path, b"").expect("create different inode"); + assert_eq!( + pinned.verify_current_path(&path), + Err(OpenedDatabaseFileError::Replaced), + ); + assert_eq!( + pinned.verify_current_path(&symlink), + Err(OpenedDatabaseFileError::Replaced), + ); + pinned.verify_current_path(&alias).expect("retained inode"); + + fs::remove_file(&path).expect("remove replacement"); + fs::create_dir(&path).expect("directory at database path"); + assert_eq!( + pinned.verify_current_path(&path), + Err(OpenedDatabaseFileError::Replaced), + ); +} diff --git a/crates/tracedecay-rusqlite-runtime/src/reader/worker.rs b/crates/tracedecay-rusqlite-runtime/src/reader/worker.rs index 34bbd20ab0..9edee1472e 100644 --- a/crates/tracedecay-rusqlite-runtime/src/reader/worker.rs +++ b/crates/tracedecay-rusqlite-runtime/src/reader/worker.rs @@ -18,7 +18,7 @@ use tracedecay_store::{ StorageRuntimeErrorV1, UnavailableReasonV1, }; -use crate::connection::{self, ConnectionMode, OpenedDatabaseFile}; +use crate::connection::{self, ConnectionMode}; use crate::exact_sql::{ ExactSqlError, ExactSqlRows, ExactSqlStatement, ExactSqlStatementBudget, execute_query, read_across_wal_recovery, @@ -516,8 +516,8 @@ pub(crate) fn spawn( let _ = started.send(Err(error)); return; } - let opened_file_identity = match OpenedDatabaseFile::pin(&worker_open_path) { - Ok(opened) => opened.identity(), + let opened_file_identity = match connection::live_path_identity(&worker_open_path) { + Ok(identity) => identity, Err(error) => { let _ = started.send(Err(ReaderStartError::OpenedDatabaseIdentity(error))); return; diff --git a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs index 1413f65b96..e907ff1118 100644 --- a/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs +++ b/crates/tracedecay-rusqlite-runtime/src/repository/attachment.rs @@ -97,6 +97,7 @@ impl RepositoryPhysicalAttachmentFactory { #[cfg(test)] after_lost_notice: None, close_failure: None, + retained_file_pin: Some(opened_database), })), drain_serial: Mutex::new(()), }) @@ -295,7 +296,11 @@ impl RepositoryPhysicalAttachmentFactory { )); } let opened_file_identity = opened_database.identity(); - let initialization_file = created.then_some(opened_database); + let (initialization_file, retained_file_pin) = if created { + (Some(opened_database), None) + } else { + (None, Some(opened_database)) + }; let state = Arc::new(Mutex::new(RepositoryRuntimePhysicalState { binding, database_path: path, @@ -312,6 +317,7 @@ impl RepositoryPhysicalAttachmentFactory { #[cfg(test)] after_lost_notice: None, close_failure: None, + retained_file_pin, })); let _ = lost_state.set(Arc::downgrade(&state)); Ok(RepositoryRuntimePhysicalAttachment { @@ -398,6 +404,10 @@ struct RepositoryRuntimePhysicalState { #[cfg(test)] after_lost_notice: Option>, close_failure: Option, + /// Kept until all SQLite workers exit: closing any raw fd for the inode + /// while SQLite holds POSIX locks can release those locks process-wide. + /// Keep this last so normal state destruction drops worker owners first. + retained_file_pin: Option, } /// Runs on the writer's pool task once its probe finds the file unlinked or @@ -451,6 +461,11 @@ impl RepositoryRuntimePhysicalAttachment { } state.lost_readers = None; state.lost_writer = None; + if state.file_lost { + // Both worker groups have exited; no live SQLite lock can be + // released by closing the retained identity descriptor. + state.retained_file_pin.take(); + } true } @@ -471,7 +486,9 @@ impl RepositoryRuntimePhysicalAttachment { opened .verify_current_path(&state.database_path) .map_err(|error| error.to_string())?; - state.initialization_file.take(); + // Successful initialization does not end the SQLite worker lifetime. + // Transfer the original pin rather than closing it on commit. + state.retained_file_pin = state.initialization_file.take(); Ok(()) } @@ -502,6 +519,12 @@ impl RepositoryRuntimePhysicalAttachment { } pub fn snapshot(&self) -> RepositoryRuntimePhysicalSnapshot { + // Health observations also complete lost-file cleanup after the last + // SQLite worker exits. Registry lookup is not required to release the + // identity pin for a runtime retained by another owner. + if self.file_lost() { + let _ = self.lost_file_closed(); + } let state = self.lock_state(); let writer = state.writer.as_ref(); let writer_telemetry = writer.map(|writer| writer.telemetry_snapshot()); @@ -774,6 +797,21 @@ impl RepositoryRuntimePhysicalAttachment { return Err("repository physical attachment retained handles after drain".to_owned()); } state.closed = true; + // A lost-file notice moves workers into lost_* before their threads + // finish. Drain can succeed in that interval; only release the raw + // pin once those worker threads have actually exited. + let lost_workers_exited = state + .lost_readers + .as_ref() + .is_none_or(ReaderPool::workers_exited) + && state + .lost_writer + .as_ref() + .is_none_or(|writer| writer.worker_exited()); + if !state.file_lost || lost_workers_exited { + // An uncommitted initialization retains its separate cleanup pin. + state.retained_file_pin.take(); + } Ok(()) } @@ -1080,6 +1118,109 @@ mod tests { .unwrap() } + /// The attachment's original identity descriptor must not be dropped + /// during startup or initialization commit while SQLite workers are live. + /// On Unix, closing it can release the process's unrelated SQLite locks. + #[cfg(target_os = "linux")] + #[test] + fn attachment_retains_identity_pins_until_workers_close() { + let directory = TempDir::new().unwrap(); + let binding = binding(); + + let writable_path = directory.path().join("store-0.sqlite3"); + let writable = attach_seeded_store(directory.path(), &binding, 0); + assert!( + writable + .lock_state() + .retained_file_pin + .as_ref() + .is_some_and(|pin| !pin.is_lost(&writable_path)), + "existing writable stores must keep their original inode pin" + ); + let rows = writable + .exact_sql_handle() + .unwrap() + .query( + statement("SELECT value FROM identity_probe", vec![]), + Duration::from_secs(1), + ) + .unwrap(); + assert_eq!(rows.rows.len(), 1); + writable.drain().unwrap(); + writable.close_and_join().unwrap(); + assert!(writable.lock_state().retained_file_pin.is_none()); + + let read_only_path = directory.path().join("read-only.sqlite3"); + create_identity_database(&read_only_path, "read-only"); + let read_only = RepositoryPhysicalAttachmentFactory + .attach_read_only( + binding.clone(), + locator(&binding), + read_only_path.canonicalize().unwrap(), + AdmissionConfigV1::default(), + ) + .unwrap(); + assert!( + read_only + .lock_state() + .retained_file_pin + .as_ref() + .is_some_and(|pin| !pin.is_lost(&read_only_path)), + "read-only attachments must retain the pin while readers are live" + ); + let rows = read_only + .exact_sql_handle() + .unwrap() + .query( + statement("SELECT value FROM identity_probe", vec![]), + Duration::from_secs(1), + ) + .unwrap(); + assert_eq!(rows.rows.len(), 1); + read_only.drain().unwrap(); + read_only.close_and_join().unwrap(); + assert!(read_only.lock_state().retained_file_pin.is_none()); + + let created_path = directory.path().join("created.sqlite3"); + let created = RepositoryPhysicalAttachmentFactory + .initialize( + &native_runtime(), + binding.clone(), + locator(&binding), + created_path.clone(), + AdmissionConfigV1::default(), + ) + .unwrap(); + assert!( + created + .lock_state() + .initialization_file + .as_ref() + .is_some_and(|pin| !pin.is_lost(&created_path)), + "an uncommitted new store must retain its cleanup pin" + ); + created.commit_initialization().unwrap(); + { + let state = created.lock_state(); + assert!(state.initialization_file.is_none()); + assert!( + state + .retained_file_pin + .as_ref() + .is_some_and(|pin| !pin.is_lost(&created_path)), + "committing an initialized store must transfer, not drop, its pin" + ); + } + created + .exact_sql_handle() + .unwrap() + .execute_batch("CREATE TABLE pinned_identity_probe (value TEXT)".to_owned()) + .unwrap(); + created.drain().unwrap(); + created.close_and_join().unwrap(); + assert!(created.lock_state().retained_file_pin.is_none()); + } + #[cfg(target_os = "linux")] fn alone_in_child_process(test: &str) -> bool { const CHILD: &str = "TRACEDECAY_ATTACHMENT_TEST_CHILD"; @@ -1279,18 +1420,33 @@ mod tests { Ok(false), "the writer task is still running, so its connection is still open" ); + // The lost-file writer no longer occupies the ordinary writer slot. + // Closing that slot must not drop the raw pin while the lost writer + // is still alive and holding a SQLite connection. + attachment.drain().unwrap(); + attachment.close_and_join().unwrap(); + assert!( + attachment.lock_state().retained_file_pin.is_some(), + "closing a lost-file attachment must retain its pin until the writer exits" + ); hold.send(()).unwrap(); let deadline = std::time::Instant::now() + Duration::from_secs(10); - while !attachment.lost_file_closed() { + // Do not poll lost_file_closed: health observation itself must release + // the pin after worker exit, even without another registry lookup. + while attachment.lock_state().retained_file_pin.is_some() { + let _ = attachment.snapshot(); assert!( std::time::Instant::now() < deadline, - "the file stayed unclosed after the writer task returned" + "health observation did not release the pin after worker exit" ); thread::sleep(Duration::from_millis(5)); } - attachment.drain().unwrap(); - attachment.close_and_join().unwrap(); + assert!(attachment.lost_file_closed()); + assert!( + deleted_descriptors(directory.path()).is_empty(), + "the last lost worker exiting must release every deleted-file descriptor" + ); } #[cfg(unix)] diff --git a/crates/tracedecay-rusqlite-runtime/src/writer/worker/mod.rs b/crates/tracedecay-rusqlite-runtime/src/writer/worker/mod.rs index 92ba85b5c1..b18573ea65 100644 --- a/crates/tracedecay-rusqlite-runtime/src/writer/worker/mod.rs +++ b/crates/tracedecay-rusqlite-runtime/src/writer/worker/mod.rs @@ -434,8 +434,8 @@ impl Worker { } let opened_file_identity = match self.expected_file_identity { Some(expected) => { - let actual = match OpenedDatabaseFile::pin(&self.path) { - Ok(opened) => opened.identity(), + let actual = match connection::live_path_identity(&self.path) { + Ok(identity) => identity, Err(error) => { return self.fail_start(WriterStartError::OpenedDatabaseIdentity(error)); } @@ -1178,9 +1178,19 @@ fn run_incremental_vacuum( return; } }; - if let Err(error) = - transaction.pragma_update(None, "incremental_vacuum", command.max_pages.max(1)) - { + // Each result row releases one page. pragma_update only steps once, so + // drain this bounded pragma before deciding whether its transaction commits. + let reclaimed = transaction + .prepare(&format!( + "PRAGMA incremental_vacuum({})", + command.max_pages.max(1) + )) + .and_then(|mut statement| { + let mut rows = statement.query([])?; + while rows.next()?.is_some() {} + Ok(()) + }); + if let Err(error) = reclaimed { command.settle(Err(WriterActorError::IncrementalVacuumFailed( error.to_string(), ))); @@ -2025,12 +2035,71 @@ mod auxiliary_scheduling_tests { ); } - #[test] - fn incremental_vacuum_rolls_back_when_authority_is_revoked() { - let mut connection = rusqlite::Connection::open_in_memory().unwrap(); + fn vacuum_connection_with_free_pages() -> rusqlite::Connection { + let connection = rusqlite::Connection::open_in_memory().unwrap(); connection .pragma_update(None, "auto_vacuum", "INCREMENTAL") .unwrap(); + connection + .execute_batch( + "CREATE TABLE retained(value TEXT NOT NULL); + INSERT INTO retained VALUES ('keep the live evidence'); + CREATE TABLE released(value BLOB NOT NULL); + WITH RECURSIVE copies(n) AS ( + VALUES(1) UNION ALL SELECT n + 1 FROM copies WHERE n < 64 + ) INSERT INTO released SELECT zeroblob(16384) FROM copies; + DELETE FROM released;", + ) + .unwrap(); + assert!(vacuum_page_counts(&connection).1 > 32); + connection + } + + fn vacuum_page_counts(connection: &rusqlite::Connection) -> (i64, i64) { + connection + .query_row( + "SELECT page_count, freelist_count FROM pragma_page_count, pragma_freelist_count", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap() + } + + #[test] + fn incremental_vacuum_reclaims_the_requested_page_budget() { + let mut connection = vacuum_connection_with_free_pages(); + for max_pages in [0, 8, 3] { + let before = vacuum_page_counts(&connection); + let authority = Arc::new(RecordingAuthority { + stages: Arc::new(Mutex::new(Vec::new())), + deny_before_commit: false, + }); + let (reply, mut response) = oneshot::channel(); + + run_incremental_vacuum( + &mut connection, + IncrementalVacuumCommand::new(max_pages, authority, reply), + ); + + assert!(response.try_recv().unwrap().is_ok()); + let after = vacuum_page_counts(&connection); + assert_eq!(before.1 - after.1, i64::from(max_pages.max(1))); + assert!(after.0 < before.0, "reclaimed pages leave the database"); + assert!(after.1 > 0, "one call must not drain the entire freelist"); + assert_eq!( + connection + .query_row("SELECT value FROM retained", [], |row| row + .get::<_, String>(0)) + .unwrap(), + "keep the live evidence" + ); + } + } + + #[test] + fn incremental_vacuum_rolls_back_when_authority_is_revoked() { + let mut connection = vacuum_connection_with_free_pages(); + let before = vacuum_page_counts(&connection); let authority = Arc::new(RecordingAuthority { stages: Arc::new(Mutex::new(Vec::new())), deny_before_commit: true, @@ -2049,5 +2118,6 @@ mod auxiliary_scheduling_tests { }) )); assert!(connection.is_autocommit()); + assert_eq!(vacuum_page_counts(&connection), before); } } diff --git a/crates/tracedecay-search-eval/src/report_tests.rs b/crates/tracedecay-search-eval/src/report_tests.rs index b40af42ef7..5d87916362 100644 --- a/crates/tracedecay-search-eval/src/report_tests.rs +++ b/crates/tracedecay-search-eval/src/report_tests.rs @@ -75,16 +75,27 @@ fn assert_baseline_report_retains_raw_fallback_current_and_exact_ten_x_samples( [ "train-016", "train-019", - "train-020", "train-026", "validation-015", - "validation-017", "validation-023", "validation-024", "validation-025", "validation-027", ] ); + for query_id in ["train-020", "validation-018", "validation-019"] { + let query = report + .profiles + .iter() + .flat_map(|profile| &profile.queries) + .find(|query| query.query_id == query_id) + .expect("ranking regression query is present"); + assert_eq!( + query.status, + crate::DirectEvaluationStatusV1::Pass, + "{query_id} must retain useful top-ten retrieval" + ); + } assert!(report.profiles.iter().all(|profile| { profile.quality.protected_recall_at_10.denominator > 0 && profile.quality.protected_recall_at_10.numerator diff --git a/crates/tracedecay-session-memory/src/session/tests/harness.rs b/crates/tracedecay-session-memory/src/session/tests/harness.rs index 5752ecd429..3d1d32c7d2 100644 --- a/crates/tracedecay-session-memory/src/session/tests/harness.rs +++ b/crates/tracedecay-session-memory/src/session/tests/harness.rs @@ -230,6 +230,11 @@ impl RegisteredTemporalHarness { } pub(super) async fn seed_quarantined_legacy_fixture(&self) { + // The unavailable legacy source is not a canonical inline body. An + // inline row with NULL content must reference a verified canonical + // body, and the canonical ownership trigger correctly rejects one + // without that owner. Keep the untrusted metadata and missing bytes + // outside the admitted inline representation. self.registered .writer_connection() .expect("registered writer") @@ -246,13 +251,25 @@ impl RegisteredTemporalHarness { ) VALUES ( 'claude', 'message.temporal.legacy', 'session.temporal.legacy', 'user', 1, 1, NULL, - 'sha256:quarantined', 'inline', NULL, + 'sha256:quarantined', 'external', 'legacy-quarantined-unavailable', 'quarantined legacy record', '{\"payload_access\":\"quarantined\",\"migration\":\"legacy-unsanitized\",\"legacy_preview\":\"sk-proj-private-canary\"}' );", ) .await .expect("seed quarantined legacy source"); + assert_eq!( + self.count( + "SELECT COUNT(*) FROM lcm_raw_messages + WHERE provider = 'claude' AND message_id = 'message.temporal.legacy' + AND storage_kind = 'external' AND content IS NULL + AND payload_ref = 'legacy-quarantined-unavailable' + AND instr(metadata_json, 'sk-proj-private-canary') > 0", + ) + .await, + 1, + "the legacy canary must remain stored but unavailable to full-text readers" + ); } /// Reopens the profile sessions store through the registry, so a test can diff --git a/crates/tracedecay-session-runtime/src/session_sync/git_topology.rs b/crates/tracedecay-session-runtime/src/session_sync/git_topology.rs index 39285e392a..27ec0dd156 100644 --- a/crates/tracedecay-session-runtime/src/session_sync/git_topology.rs +++ b/crates/tracedecay-session-runtime/src/session_sync/git_topology.rs @@ -62,6 +62,11 @@ impl SessionSyncProjectContext { } }; let Some(runtime) = project_sessions.project_graph_runtime() else { + tracing::warn!( + event = "git_topology_publication_unavailable", + stage = "project graph runtime", + "git topology publication failed" + ); return GitTopologySyncOutcome::Finished(Err(GitTopologySyncFailure::Unavailable)); }; let runtime: Arc = Arc::new(runtime.clone()); @@ -113,6 +118,22 @@ impl GraphCancellation for GitTopologySyncCancellation { } } +/// Keeps the operator-facing failure code stable while recording which step +/// failed and why, which the code alone cannot say. +fn unavailable( + stage: &'static str, +) -> impl FnOnce(E) -> GitTopologySyncFailure { + move |error| { + tracing::warn!( + event = "git_topology_publication_unavailable", + stage, + ?error, + "git topology publication failed" + ); + GitTopologySyncFailure::Unavailable + } +} + pub fn publish_native_topology( runtime: Arc, project_root: PathBuf, @@ -122,23 +143,23 @@ pub fn publish_native_topology( cancelled: Arc, ) -> Result<(), GitTopologySyncFailure> { let identity = git_topology_projection_identity( - git_topology_namespace(&repository).map_err(|_| GitTopologySyncFailure::Unavailable)?, + git_topology_namespace(&repository).map_err(unavailable("projection namespace"))?, ) - .map_err(|_| GitTopologySyncFailure::Unavailable)?; + .map_err(unavailable("projection identity"))?; let read_cancelled = Arc::clone(&cancelled); let current = runtime .verified_snapshot( &identity, FactReadControl::new(Arc::new(move || read_cancelled.load(Ordering::Acquire))), ) - .map_err(|_| GitTopologySyncFailure::Unavailable)?; + .map_err(unavailable("verified snapshot read"))?; let (branch_stacks, worktree_occupancies) = match current { Some(snapshot) => { let store = GitTopologyProjectionStore::from_verified_snapshot_verified( snapshot, Arc::new(GitTopologySyncCancellation(Arc::clone(&cancelled))), ) - .map_err(|_| GitTopologySyncFailure::Unavailable)?; + .map_err(unavailable("retained projection load"))?; validate_retained_declared_topology( &repository, &project_root, @@ -156,11 +177,11 @@ pub fn publish_native_topology( let adapter = NativeGitIntelligence::new(project_root, repository.clone(), worktree); let projection = adapter .topology_projection(GIT_HISTORY_MAX_COUNT_LIMIT) - .map_err(|_| GitTopologySyncFailure::Unavailable)? + .map_err(unavailable("git topology read"))? .with_declared_topology(branch_stacks, worktree_occupancies) - .map_err(|_| GitTopologySyncFailure::Unavailable)?; + .map_err(unavailable("declared topology merge"))?; let revision = GraphProjectorRevision::try_from(GIT_TOPOLOGY_PROJECTOR_REVISION_V1.to_owned()) - .map_err(|_| GitTopologySyncFailure::Unavailable)?; + .map_err(unavailable("projector revision"))?; let check = || { if cancelled.load(Ordering::Relaxed) { Err(GraphDbError::Cancelled) @@ -169,12 +190,12 @@ pub fn publish_native_topology( } }; let manifest = build_git_topology_manifest_checked(identity, &projection, &revision, &check) - .map_err(|_| GitTopologySyncFailure::Unavailable)?; + .map_err(unavailable("manifest build"))?; let idempotency = git_topology_idempotency_key(&projection, &revision) - .map_err(|_| GitTopologySyncFailure::Unavailable)?; + .map_err(unavailable("idempotency key"))?; runtime .publish_verified_manifest(&manifest, idempotency, cancelled) - .map_err(|_| GitTopologySyncFailure::Unavailable)?; + .map_err(unavailable("manifest publication"))?; Ok(()) } diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/history.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/history.rs index ba2411b151..d33a70ebd5 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/history.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/history.rs @@ -393,9 +393,9 @@ fn classify_transcript_ingest_outcome( SessionHistoricalIngestOutcome::Complete } -/// Names the pass blocker(s) per provider. A single-provider pass keeps the -/// raw reason code so existing status strings stay stable. Multi-provider -/// passes use `provider:reason` so rotation order cannot hide the real stall. +/// Names every distinct pass blocker in stable provider/reason order. A +/// single-provider pass keeps raw reason codes; multi-provider passes qualify +/// each reason so neither provider rotation nor source order hides a fault. fn pass_reason_code( failures: &[tracedecay_sessions::runtime::TranscriptCatchUpFailure], retryable: bool, @@ -405,28 +405,19 @@ fn pass_reason_code( .map(|failure| failure.provider) .collect::>() .len(); - let mut blockers = BTreeMap::new(); - for failure in failures { - if failure.retryable != retryable { - continue; - } - blockers - .entry(failure.provider) - .or_insert(failure.reason_code); - } - if blockers.is_empty() { - return String::new(); - } - if provider_count <= 1 { - let mut reasons = blockers.into_values(); - return match reasons.next() { - Some(reason) => reason.to_owned(), - None => String::new(), - }; - } - blockers + failures + .iter() + .filter(|failure| failure.retryable == retryable) + .map(|failure| (failure.provider, failure.reason_code)) + .collect::>() .into_iter() - .map(|(provider, reason)| format!("{provider}:{reason}")) + .map(|(provider, reason)| { + if provider_count <= 1 { + reason.to_owned() + } else { + format!("{provider}:{reason}") + } + }) .collect::>() .join(",") } @@ -443,7 +434,10 @@ mod tests { use tracedecay_sessions::runtime::{ IngestPassCoverage, TranscriptCatchUpFailure, TranscriptIngestOutcome, }; - use tracedecay_sessions::serving::SessionProjectionServingStatusPort; + use tracedecay_sessions::serving::{ + SessionProjectionServingState, SessionProjectionServingStatusPort, + SessionProjectionStaleReason, + }; #[test] fn completed_history_records_committed_progress_but_idle_passes_do_not() { @@ -675,4 +669,113 @@ mod tests { ); assert!(!outcome.needs_another_pass()); } + + fn failure( + provider: &'static str, + reason_code: &'static str, + retryable: bool, + ) -> TranscriptCatchUpFailure { + TranscriptCatchUpFailure { + provider, + source: "observation", + reason_code, + retryable, + source_locator: None, + } + } + + #[test] + fn same_provider_corruption_survives_source_order_and_reaches_status() { + let expected = "invalid_observation_contract,projection_store_corrupt"; + for reverse in [false, true] { + let mut failures = vec![ + failure("claude", "invalid_observation_contract", false), + failure("claude", "projection_store_corrupt", false), + failure("claude", "projection_store_corrupt", false), + ]; + if reverse { + failures.reverse(); + } + let outcome = classify_transcript_ingest_outcome( + ingest_outcome_with_failures(failures), + &ObservationCancellation::default(), + ); + assert_eq!( + outcome, + SessionHistoricalIngestOutcome::Blocked { + reason_code: expected.to_owned(), + made_progress: false, + } + ); + assert!(!outcome.needs_another_pass()); + + let state = Arc::new(SessionTemporalRefreshWakeState::default()); + state.mark_running(); + state.record_history_outcome(outcome); + assert_eq!( + state.handle().serving_status().state, + SessionProjectionServingState::Stale { + reason: SessionProjectionStaleReason::HistoricalBlocked { + reason_code: expected.to_owned(), + }, + } + ); + state.record_history_outcome(SessionHistoricalIngestOutcome::Complete); + assert_eq!( + state.handle().serving_status().state, + SessionProjectionServingState::Current + ); + } + } + + #[test] + fn same_provider_retry_reasons_are_sorted_and_deduplicated() { + for reverse in [false, true] { + let mut failures = vec![ + failure("codex", "projection_storage_retry_scheduled", true), + failure("codex", "cursor_conflict", true), + failure("codex", "cursor_conflict", true), + ]; + if reverse { + failures.reverse(); + } + let outcome = classify_transcript_ingest_outcome( + ingest_outcome_with_failures(failures), + &ObservationCancellation::default(), + ); + assert_eq!( + outcome, + SessionHistoricalIngestOutcome::Retryable { + reason_code: "cursor_conflict,projection_storage_retry_scheduled".to_owned(), + made_progress: false, + } + ); + assert!(outcome.needs_another_pass()); + } + } + + #[test] + fn all_terminal_reasons_keep_provider_identity_and_progress() { + let mut ingested = ingest_outcome_with_failures(vec![ + failure("cursor", "projection_store_corrupt", false), + failure("claude", "invalid_observation_contract", false), + failure("claude", "projection_store_corrupt", false), + failure("claude", "cursor_conflict", true), + ]); + ingested.stats.messages_upserted = 2; + let pass = historical_pass(ingested, &ObservationCancellation::default()); + assert!(pass.made_progress); + assert_eq!( + pass.outcome, + SessionHistoricalIngestOutcome::Blocked { + reason_code: concat!( + "claude:invalid_observation_contract,", + "claude:projection_store_corrupt,cursor:projection_store_corrupt" + ) + .to_owned(), + made_progress: true, + } + ); + assert!(!pass.outcome.needs_another_pass()); + } } diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs index 6e41dcd610..51a6b1a8d9 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/worker.rs @@ -104,6 +104,28 @@ fn history_continuation(outcome: Option) -> Hist } } +/// Only an actually executed, unblocked history pass clears terminal backoff. +/// A suppressed wake has no evidence of recovery; a blocked pass may have +/// advanced healthy providers while the corrupt one remains blocked. +/// Conversely, a completed, pending, or retryable pass may clear the terminal +/// blocker without committing a source row. +fn clears_blocked_history_backoff( + outcome: Option<&SessionHistoricalIngestOutcome>, + release: Option<&HistoryRelease>, +) -> bool { + // Admission refusal returns Retryable without ever running the ingestor. + // Only a held history capacity subscription proves that a pass executed. + matches!(release, Some(HistoryRelease::Capacity(_))) + && matches!( + outcome, + Some( + SessionHistoricalIngestOutcome::Complete + | SessionHistoricalIngestOutcome::Pending { .. } + | SessionHistoricalIngestOutcome::Retryable { .. } + ) + ) +} + /// What a history pass that committed nothing waits on before it runs again. enum HistoryRelease { /// The daemon-wide historical admission had no free permit. @@ -333,17 +355,10 @@ pub(super) async fn run_session_temporal_refresh_scheduler( ) || history_outcome .as_ref() .is_some_and(SessionHistoricalIngestOutcome::made_progress); - // A pass that made progress does NOT prove the blocked cause - // healed: one healthy provider advancing while another stays - // blocked reports Blocked { made_progress: true }, and the - // blocked provider would be re-read at full cadence again. The - // backoff resets only on an outcome with no blocker at all. - if history_requires_projection - && !matches!( - history_outcome, - Some(SessionHistoricalIngestOutcome::Blocked { .. }) - ) - { + // A nonblocked attempt clears old terminal backoff even when it + // committed no new rows. A suppressed wake or progress by healthy + // providers while another remains Blocked cannot clear it. + if clears_blocked_history_backoff(history_outcome.as_ref(), history_release.as_ref()) { blocked_history_attempts = 0; blocked_history_until = None; } @@ -397,10 +412,14 @@ pub(super) async fn run_session_temporal_refresh_scheduler( // The bounded predecessor rewrite retains its history fairness // turn; it performs no model call. if matches!(convergence_admission, LcmConvergenceAdmission::Admitted) - && projection_still_unpublished(&report) - && (projection_published_work(&report) - || (!projection_joins.is_empty() && report.discovery_advanced)) + && (report.retry_class.is_some() + || (projection_still_unpublished(&report) + && (projection_published_work(&report) + || (!projection_joins.is_empty() && report.discovery_advanced)))) { + // A retryable projection has not settled its generation. Retry + // publication before competing for summary admission, even + // when this attempt committed no progress. convergence_admission = LcmConvergenceAdmission::Deferred; } let ( @@ -2604,6 +2623,60 @@ mod tests { assert!(!state.has_pending_work()); } + #[test] + fn terminal_history_backoff_clears_only_after_an_admitted_unblocked_pass() { + let admitted = HistoryRelease::Capacity(SessionHistoricalCapacityRelease::new(Vec::new())); + let refused = HistoryRelease::Admission; + assert!(!clears_blocked_history_backoff(None, Some(&admitted))); + for made_progress in [false, true] { + let blocked = SessionHistoricalIngestOutcome::Blocked { + reason_code: "projection_store_corrupt".to_owned(), + made_progress, + }; + assert!( + !clears_blocked_history_backoff(Some(&blocked), Some(&admitted)), + "a healthy sibling's progress cannot reset a corrupt provider's backoff" + ); + let pending = SessionHistoricalIngestOutcome::Pending { made_progress }; + assert!(clears_blocked_history_backoff( + Some(&pending), + Some(&admitted) + )); + assert!(!clears_blocked_history_backoff( + Some(&pending), + Some(&refused) + )); + let retryable = SessionHistoricalIngestOutcome::Retryable { + reason_code: "provider_temporarily_unavailable".to_owned(), + made_progress, + }; + assert!(clears_blocked_history_backoff( + Some(&retryable), + Some(&admitted) + )); + assert!(!clears_blocked_history_backoff( + Some(&retryable), + Some(&refused) + )); + } + let admission_saturated = SessionHistoricalIngestOutcome::Retryable { + reason_code: HISTORY_ADMISSION_SATURATED_REASON.to_owned(), + made_progress: false, + }; + assert!( + !clears_blocked_history_backoff(Some(&admission_saturated), Some(&refused)), + "blocked, admission-refused, then blocked again must retain escalating backoff" + ); + assert!(clears_blocked_history_backoff( + Some(&SessionHistoricalIngestOutcome::Complete), + Some(&admitted) + )); + assert!(!clears_blocked_history_backoff( + Some(&SessionHistoricalIngestOutcome::Cancelled), + Some(&admitted) + )); + } + #[test] fn a_window_that_committed_coverage_continues_and_one_that_did_not_awaits_release() { assert_eq!( diff --git a/crates/tracedecay-session-runtime/tests/session_retention_read_cost.rs b/crates/tracedecay-session-runtime/tests/session_retention_read_cost.rs index 3c0fba5903..8463502dfd 100644 --- a/crates/tracedecay-session-runtime/tests/session_retention_read_cost.rs +++ b/crates/tracedecay-session-runtime/tests/session_retention_read_cost.rs @@ -8,8 +8,8 @@ pub mod support; use support::{ BASE_SESSIONS, DrainFixture, GROWN_SESSIONS, SEED_TIMESTAMP, advance_to_wal_restart, - probe_one_message, probe_retention_tick_read_bytes, retention_tick_read_bytes, seed_sessions, - session_cursor, + probe_one_message, probe_retention_tick_read_bytes, publish_retention_probe_summary, + seed_sessions, session_cursor, settle_retention_backlog, }; use tracedecay_sessions::admission::HostAdmissionScope; @@ -34,7 +34,8 @@ async fn retention_tick_reads_do_not_scale_with_the_session_store() { // Settle padding before the warm-up so each measured tick still follows // exactly STREAMED new messages at a comparable checkpoint phase. advance_to_wal_restart(&facade, &fixture, &mut padding, probe_timestamp).await; - retention_tick_read_bytes(database).await; + settle_retention_backlog(database).await; + let base_start = streamed.next_ordinal; for offset in 0..STREAMED { probe_one_message( &facade, @@ -45,11 +46,20 @@ async fn retention_tick_reads_do_not_scale_with_the_session_store() { ) .await; } - let base_read = probe_retention_tick_read_bytes(database).await; + publish_retention_probe_summary( + database, + &streamed, + base_start, + STREAMED, + "summary.retention-cost.base", + ) + .await; + let base_read = probe_retention_tick_read_bytes(database, STREAMED).await; seed_sessions(&facade, project, &scope, BASE_SESSIONS..GROWN_SESSIONS).await; advance_to_wal_restart(&facade, &fixture, &mut padding, probe_timestamp).await; - retention_tick_read_bytes(database).await; + settle_retention_backlog(database).await; + let grown_start = streamed.next_ordinal; for offset in STREAMED..2 * STREAMED { probe_one_message( &facade, @@ -60,7 +70,15 @@ async fn retention_tick_reads_do_not_scale_with_the_session_store() { ) .await; } - let grown_read = probe_retention_tick_read_bytes(database).await; + publish_retention_probe_summary( + database, + &streamed, + grown_start, + STREAMED, + "summary.retention-cost.grown", + ) + .await; + let grown_read = probe_retention_tick_read_bytes(database, STREAMED).await; eprintln!("retention tick read bytes: base={base_read} grown={grown_read}"); assert!( diff --git a/crates/tracedecay-session-runtime/tests/support/session_store_read_cost.rs b/crates/tracedecay-session-runtime/tests/support/session_store_read_cost.rs index 0c24dedd4c..34f69d73b5 100644 --- a/crates/tracedecay-session-runtime/tests/support/session_store_read_cost.rs +++ b/crates/tracedecay-session-runtime/tests/support/session_store_read_cost.rs @@ -35,6 +35,8 @@ use tracedecay_global_db::{RegisteredGlobalDb, RegisteredGlobalDbLeaseV1}; use tracedecay_host_admission::session_ingest_authority::GlobalDbSessionIngestAuthority; use tracedecay_host_admission::{HostAdmissionAuthorities, HostAdmissionFacade}; use tracedecay_lcm::LcmRetentionConfig; +use tracedecay_lcm::types::LcmImmutableSummaryPublication; +use tracedecay_lcm::{LcmSourceRef, LcmSummaryNodeDraft}; use tracedecay_maintenance::retention::registered_store::run_registered_store_retention; use tracedecay_privacy::{ObservationRecordParseErrorV1, parse_normalized_observation_record_v1}; use tracedecay_runtime_core::background_cpu::ProcessBackgroundCpuV1; @@ -53,6 +55,7 @@ use tracedecay_sessions::repository_provenance::RepositoryProvenanceAdmissionCon use tracedecay_sessions::runtime::{ TranscriptIngestOutcome, ingest_project_sources_for_provider, with_transcript_source_profile, }; +use tracedecay_temporal_query::execution::ExecutionControl; pub const PROVIDER: &str = "codex"; pub const MESSAGES_PER_SESSION: u64 = 5; @@ -750,11 +753,60 @@ pub async fn retention_tick_read_bytes(database: &RegisteredGlobalDb) -> u64 { process_read_bytes() - before } -/// Measures retention after aligning the WAL and settling the seeded backlog. -/// Warm-up ticks may checkpoint; neither measured tick may include that copy. -pub async fn probe_retention_tick_read_bytes(database: &RegisteredGlobalDb) -> u64 { +/// Discharge an older seeded backlog before comparing equal-size streamed +/// batches. One retention pass is bounded (500 rows by default), so a single +/// warm-up would leave a grown store with many more eligible deletes than the +/// base store, measuring backlog volume instead of per-tick read amplification. +pub async fn settle_retention_backlog(database: &RegisteredGlobalDb) { + let seeded_rows = GROWN_SESSIONS * MESSAGES_PER_SESSION + DRAIN_WINDOW as u64; + let batch = LcmRetentionConfig::default().max_batch_size as u64; + let max_ticks = seeded_rows.div_ceil(batch) + 2; + for _ in 0..max_ticks { + let report = run_registered_store_retention( + database, + &LcmRetentionConfig::default(), + &ObservationRetentionConfig::default(), + RETENTION_NOW, + ) + .await; + assert!(report.succeeded(), "retention warm-up must succeed"); + let lcm = report.session_lcm.as_ref().unwrap().as_ref().unwrap(); + if lcm.dropped.eligible == 0 { + return; + } + } + panic!("prior retained rows must drain within the derived bounded tick count"); +} + +/// Measures one tick with only the expected newly streamed messages eligible +/// for the destructive age policy, and excludes unrelated WAL checkpoints. +pub async fn probe_retention_tick_read_bytes( + database: &RegisteredGlobalDb, + expected_dropped: u64, +) -> u64 { let checkpointed = writer_telemetry(database).wal.checkpointed_frames; - let read = retention_tick_read_bytes(database).await; + let before = process_read_bytes(); + let report = run_registered_store_retention( + database, + &LcmRetentionConfig::default(), + &ObservationRetentionConfig::default(), + RETENTION_NOW, + ) + .await; + assert!(report.succeeded(), "retention must succeed"); + let actual_dropped = report + .session_lcm + .as_ref() + .unwrap() + .as_ref() + .unwrap() + .dropped + .acted; + assert_eq!( + actual_dropped, expected_dropped, + "the compared ticks must perform identical bounded raw-row deletion work" + ); + let read = process_read_bytes() - before; assert_eq!( writer_telemetry(database).wal.checkpointed_frames, checkpointed, @@ -762,3 +814,50 @@ pub async fn probe_retention_tick_read_bytes(database: &RegisteredGlobalDb) -> u ); read } + +/// Make the streamed batch genuinely eligible for destructive retention: +/// only raw sources backed by a published, durable summary may be deleted. +/// Build the lineage through the registered production publication journey. +pub async fn publish_retention_probe_summary( + database: &RegisteredGlobalDb, + cursor: &SessionCursor, + from_ordinal: u64, + count: u64, + summary_id: &str, +) { + let mut source_refs = Vec::with_capacity(usize::try_from(count).unwrap()); + for ordinal in from_ordinal..from_ordinal + count { + let message_id = format!("{}.message.{ordinal:05}", cursor.session_id.as_str()); + let store_id = database + .lcm_raw_message_store_id(PROVIDER, &message_id) + .await + .unwrap() + .expect("streamed raw message must exist before summary publication"); + source_refs.push(LcmSourceRef::RawMessage { store_id }); + } + database + .lcm_publish_immutable_summary_guarded( + LcmImmutableSummaryPublication { + summary_id: summary_id.to_owned(), + predecessor_summary_id: None, + draft: LcmSummaryNodeDraft { + provider: PROVIDER.to_owned(), + conversation_id: cursor.session_id.as_str().to_owned(), + session_id: cursor.session_id.as_str().to_owned(), + depth: 0, + summary_text: "Summary of the newly streamed retention probe".to_owned(), + source_refs, + source_token_count: i64::try_from(count).unwrap(), + summary_token_count: 8, + source_time_start: Some(SEED_TIMESTAMP), + source_time_end: Some(RETENTION_NOW), + expand_hint: None, + metadata_json: None, + }, + }, + &ExecutionControl::default(), + || Ok(()), + ) + .await + .expect("publish durable retention lineage through registered authority"); +} diff --git a/crates/tracedecay-session-temporal-store/src/hydration.rs b/crates/tracedecay-session-temporal-store/src/hydration.rs index 84471ba546..eff354a606 100644 --- a/crates/tracedecay-session-temporal-store/src/hydration.rs +++ b/crates/tracedecay-session-temporal-store/src/hydration.rs @@ -1589,9 +1589,16 @@ mod tests { let writer = database .writer_connection() .expect("registered profile writer"); - Executor::execute_batch(&writer, "DROP TRIGGER observations_immutable_update;") - .await - .expect("drop observation immutability for corruption fixture"); + // Corruption is outside normal writer admission: bypass both + // immutability and canonical-body ownership validation in this + // isolated fixture, without weakening either production guard. + Executor::execute_batch( + &writer, + "DROP TRIGGER observations_immutable_update; + DROP TRIGGER canonical_body_gc_observation_update;", + ) + .await + .expect("drop observation immutability for corruption fixture"); Executor::execute( &writer, "UPDATE observations diff --git a/crates/tracedecay-session-temporal-store/src/query.rs b/crates/tracedecay-session-temporal-store/src/query.rs index e88575bae6..bb8ed52b63 100644 --- a/crates/tracedecay-session-temporal-store/src/query.rs +++ b/crates/tracedecay-session-temporal-store/src/query.rs @@ -9,9 +9,10 @@ use tracedecay_domain::{ UtcMicros, }; use tracedecay_runtime_core::db::engine::{Row, params, params_from_iter}; +use tracedecay_store::canonical_body::LOAD_CANONICAL_BODIES_SQL; use tracedecay_store::{ - CanonicalBodyError, LOAD_CANONICAL_BODY_SQL, SessionFrozenWatermarksV1, SessionStoreError, - SessionStoreResult, collect_body_refs, parse_stored_observation, unpack_body, + CanonicalBodyError, SessionFrozenWatermarksV1, SessionStoreError, SessionStoreResult, + collect_body_refs, parse_stored_observation, unpack_body, }; use crate::sql::{ @@ -262,38 +263,82 @@ pub(super) async fn read_observation( Ok((sequence, observation)) } +/// Hydrate one observation's distinct canonical bodies in bounded SQL pages. +/// The shared ordered query preserves missing/corrupt verdict order; pages +/// split only when their combined bodies exceed SQL materialization capacity. pub(super) async fn decode_stored_observation( conn: &impl crate::handle::SessionTemporalQuery, encoded: &str, operation: &'static str, ) -> SessionStoreResult { + const BODY_READ_BATCH: usize = 16; let hashes = collect_body_refs(encoded).map_err(|error| storage(operation, error))?; - let mut bodies = HashMap::new(); - for hash in hashes { - let mut rows = conn - .query(LOAD_CANONICAL_BODY_SQL, params![hash.as_str()]) - .await - .map_err(|error| storage(operation, error))?; - let row = rows - .next() - .await - .map_err(|error| storage(operation, error))? - .ok_or_else(|| { - storage( - operation, - CanonicalBodyError::Missing { - content_hash: hash.clone(), - }, - ) - })?; - let encoding: String = row.get(0).map_err(|error| storage(operation, error))?; - let blob: Vec = row.get(1).map_err(|error| storage(operation, error))?; - let uncompressed: i64 = row.get(2).map_err(|error| storage(operation, error))?; - bodies.insert( - hash.clone(), - unpack_body(&hash, &encoding, &blob, uncompressed) - .map_err(|error| storage(operation, error))?, - ); + let mut bodies = HashMap::with_capacity(hashes.len()); + for page in hashes.chunks(BODY_READ_BATCH) { + let mut pending = vec![(0, page.len())]; + while let Some((start, end)) = pending.pop() { + let chunk = &page[start..end]; + let references = + serde_json::to_string(chunk).map_err(|error| storage(operation, error))?; + let mut rows = match conn + .query(LOAD_CANONICAL_BODIES_SQL, params![references]) + .await + { + Ok(rows) => rows, + Err(error) => { + let error = storage(operation, error); + // The original single-hash path can read oversized + // combinations. Split only on the exact-SQL materialization + // ceiling, preserving typed errors for every other failure. + if chunk.len() > 1 + && observation_prefetch_exceeded_materialization_limit(&error) + { + let mid = start + chunk.len() / 2; + pending.push((mid, end)); + pending.push((start, mid)); + continue; + } + return Err(error); + } + }; + let mut stored = HashMap::with_capacity(chunk.len()); + while let Some(row) = rows + .next() + .await + .map_err(|error| storage(operation, error))? + { + let hash: String = row.get(0).map_err(|error| storage(operation, error))?; + stored.insert(hash, row); + } + // Decode and verify in requested-hash order, independently of + // SQLite's IN row ordering and any later row's invalid encoding. + for hash in chunk { + let row = stored.remove(hash).ok_or_else(|| { + storage( + operation, + CanonicalBodyError::Missing { + content_hash: hash.clone(), + }, + ) + })?; + let encoding = row + .get::>(1) + .map_err(|error| storage(operation, error))? + .ok_or_else(|| { + storage( + operation, + CanonicalBodyError::Missing { + content_hash: hash.clone(), + }, + ) + })?; + let blob: Vec = row.get(2).map_err(|error| storage(operation, error))?; + let uncompressed: i64 = row.get(3).map_err(|error| storage(operation, error))?; + let unpacked = unpack_body(hash, &encoding, &blob, uncompressed) + .map_err(|error| storage(operation, error))?; + bodies.insert(hash.clone(), unpacked); + } + } } parse_stored_observation(encoded, |hash| { bodies @@ -420,6 +465,191 @@ pub(super) fn missing_observation(observation_id: &CanonicalObservationIdV1) -> #[cfg(test)] mod tests { use super::{PERSIST_OPERATION, observation_prefetch_exceeded_materialization_limit, storage}; + use std::future::Future; + use std::sync::atomic::{AtomicUsize, Ordering}; + use tracedecay_runtime_core::db::engine; + + struct CountedRead<'a> { + inner: crate::sql::TemporalSqlRead<'a>, + queries: AtomicUsize, + } + + impl crate::handle::SessionTemporalQuery for CountedRead<'_> { + fn query

( + &self, + sql: &str, + params: P, + ) -> impl Future> + Send + where + P: engine::IntoParams + Send, + { + self.queries.fetch_add(1, Ordering::Relaxed); + self.inner.query(sql, params) + } + } + + #[asupersync::test] + async fn observation_hydration_batches_distinct_bodies_and_preserves_integrity() { + use tracedecay_domain::{ + ComponentVersion, DurableObservationV1, ObservationId, ObservationIdentityMaterialV1, + ObservationOrderingDomainV1, ObservationScopeV1, ObservationSourceGenerationV1, + ObservationSourceIdentityV1, ObservationSourceRangeV1, PayloadReferenceV1, + RetentionClass, SanitizationReceiptId, SanitizationReceiptRefV1, SanitizationReceiptV1, + SanitizerDispositionV1, SensitivityV1, SessionId, + }; + use tracedecay_runtime_core::db::engine::{TestConnection, params}; + use tracedecay_store::{ + CANONICAL_BODIES_TABLE_SQL, CanonicalBodyError, UPSERT_CANONICAL_BODY_SQL, + collect_body_refs, slim_stored_json, + }; + + let mut payload = serde_json::Map::new(); + for index in 0..18 { + payload.insert( + format!("part-{index:02}"), + serde_json::json!(format!("body-{index:02}-{}", "z".repeat(4096))), + ); + } + let repeated = payload["part-00"].clone(); + payload.insert("repeat".to_owned(), repeated); + let payload = serde_json::Value::Object(payload); + let identity = ObservationIdentityMaterialV1::for_native_record( + ObservationSourceIdentityV1::new(SessionId::new("session.batch").unwrap()).unwrap(), + ObservationScopeV1::Profile, + ObservationSourceGenerationV1::new(7).unwrap(), + ObservationSourceRangeV1::new(12, 34).unwrap(), + ObservationOrderingDomainV1::FileBytes, + ObservationId::new("record.batch").unwrap(), + ) + .unwrap(); + let receipt = SanitizationReceiptV1::new( + SanitizationReceiptRefV1::new( + SanitizationReceiptId::new("receipt.batch").unwrap(), + ComponentVersion::new("sanitizer.batch-test.v1").unwrap(), + ) + .unwrap(), + SanitizerDispositionV1::Accepted, + SensitivityV1::NonSensitive, + Some(PayloadReferenceV1::for_payload(&payload).unwrap()), + ) + .unwrap(); + let original = DurableObservationV1::new( + identity, + receipt, + RetentionClass::new("transcript.batch-test").unwrap(), + payload, + ) + .unwrap(); + let (stored, packed) = + slim_stored_json(&serde_json::to_string(&original).unwrap()).unwrap(); + let hashes = collect_body_refs(&stored).unwrap(); + assert_eq!(packed.len(), 19, "repeat reference shares a content hash"); + assert_eq!(hashes.len(), 18); + + let directory = tempfile::TempDir::new().unwrap(); + let connection = TestConnection::open(&directory.path().join("canonical-read.db")); + connection + .execute_batch(CANONICAL_BODIES_TABLE_SQL) + .await + .unwrap(); + for body in &packed { + connection + .execute( + UPSERT_CANONICAL_BODY_SQL, + params![ + body.content_hash.as_str(), + body.encoding, + body.blob.as_slice(), + body.uncompressed_bytes + ], + ) + .await + .unwrap(); + } + let counted = CountedRead { + inner: crate::sql::TemporalSqlRead::engine_connection(&connection), + queries: AtomicUsize::new(0), + }; + let decoded = super::decode_stored_observation(&counted, &stored, super::PERSIST_OPERATION) + .await + .unwrap(); + assert_eq!( + serde_json::to_value(decoded).unwrap(), + serde_json::to_value(&original).unwrap() + ); + assert_eq!( + counted.queries.load(Ordering::Relaxed), + 2, + "18 distinct references need two bounded SQL reads, not 18" + ); + + let inline = serde_json::to_string(&original).unwrap(); + let decoded = super::decode_stored_observation(&counted, &inline, super::PERSIST_OPERATION) + .await + .unwrap(); + assert_eq!( + serde_json::to_value(decoded).unwrap(), + serde_json::to_value(&original).unwrap() + ); + assert_eq!( + counted.queries.load(Ordering::Relaxed), + 2, + "inline observations need no canonical-body read" + ); + + let first = &hashes[0]; + connection + .execute( + "DELETE FROM session_canonical_bodies WHERE content_hash = ?1", + params![first.as_str()], + ) + .await + .unwrap(); + let error = super::decode_stored_observation(&counted, &stored, super::PERSIST_OPERATION) + .await + .unwrap_err(); + let tracedecay_store::SessionStoreError::Storage { source, .. } = error else { + panic!("expected missing canonical-body storage error"); + }; + assert!(matches!( + source.downcast_ref::(), + Some(CanonicalBodyError::Missing { content_hash }) if content_hash == first + )); + + let original_body = packed + .iter() + .find(|body| &body.content_hash == first) + .unwrap(); + connection + .execute( + UPSERT_CANONICAL_BODY_SQL, + params![ + original_body.content_hash.as_str(), + original_body.encoding, + original_body.blob.as_slice(), + original_body.uncompressed_bytes + ], + ) + .await + .unwrap(); + connection + .execute( + "UPDATE session_canonical_bodies SET encoding = 'identity', body = ?1 WHERE content_hash = ?2", + params![b"corrupt".as_slice(), first.as_str()], + ) + .await + .unwrap(); + let error = super::decode_stored_observation(&counted, &stored, super::PERSIST_OPERATION) + .await + .unwrap_err(); + let tracedecay_store::SessionStoreError::Storage { source, .. } = error else { + panic!("expected corrupted canonical-body storage error"); + }; + assert!(matches!( + source.downcast_ref::(), + Some(CanonicalBodyError::Corrupt { content_hash, .. }) if content_hash == first + )); + } #[test] fn materialization_limit_is_the_prefetch_split_signal() { diff --git a/crates/tracedecay-sessions/src/runtime/host_sqlite.rs b/crates/tracedecay-sessions/src/runtime/host_sqlite.rs index 2775366cc0..b48ccd3cf0 100644 --- a/crates/tracedecay-sessions/src/runtime/host_sqlite.rs +++ b/crates/tracedecay-sessions/src/runtime/host_sqlite.rs @@ -1,14 +1,37 @@ use std::sync::{Arc, Mutex, PoisonError}; +/// A plain SQLite read or a verified foreign read. The verified case retains +/// its inode pin for the whole connection lifetime instead of unpacking and +/// dropping it immediately after admission. +enum SqliteReadHandle { + Plain(rusqlite::Connection), + Verified(tracedecay_rusqlite_runtime::VerifiedReader), +} + +impl SqliteReadHandle { + fn connection(&self) -> &rusqlite::Connection { + match self { + Self::Plain(connection) => connection, + Self::Verified(reader) => reader.connection(), + } + } +} + #[derive(Clone)] pub(crate) struct SqliteReadConn { - inner: Arc>, + inner: Arc>, } impl SqliteReadConn { pub(crate) fn new(connection: rusqlite::Connection) -> Self { Self { - inner: Arc::new(Mutex::new(connection)), + inner: Arc::new(Mutex::new(SqliteReadHandle::Plain(connection))), + } + } + + pub(crate) fn new_verified(reader: tracedecay_rusqlite_runtime::VerifiedReader) -> Self { + Self { + inner: Arc::new(Mutex::new(SqliteReadHandle::Verified(reader))), } } @@ -21,9 +44,69 @@ impl SqliteReadConn { let inner = Arc::clone(&self.inner); tracedecay_runtime_core::native_runtime::spawn_blocking(move || { let guard = inner.lock().unwrap_or_else(PoisonError::into_inner); - body(&guard) + body(guard.connection()) }) .await .ok() } } + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use std::os::unix::fs::MetadataExt; + use std::path::Path; + + use super::SqliteReadConn; + + fn descriptors_for(path: &Path) -> usize { + let identity = std::fs::metadata(path).expect("database metadata"); + std::fs::read_dir("/proc/self/fd") + .expect("process file descriptors") + .filter_map(Result::ok) + .filter(|entry| { + std::fs::metadata(entry.path()).is_ok_and(|metadata| { + metadata.dev() == identity.dev() && metadata.ino() == identity.ino() + }) + }) + .count() + } + + #[test] + fn verified_foreign_reader_retains_its_pin_until_the_last_handle_closes() { + let dir = tempfile::tempdir().expect("test directory"); + let path = dir.path().join("foreign.sqlite3"); + { + let conn = rusqlite::Connection::open(&path).expect("create database"); + conn.execute_batch( + "CREATE TABLE sample(value INTEGER); INSERT INTO sample VALUES (42)", + ) + .expect("seed database"); + } + assert_eq!(descriptors_for(&path), 0); + + let verified = tracedecay_rusqlite_runtime::open_verified_immutable_reader(&path) + .expect("open verified foreign database"); + let pinned_descriptors = descriptors_for(&path); + assert!( + pinned_descriptors >= 2, + "SQLite and its verified identity pin each retain an inode handle" + ); + + let reader = SqliteReadConn::new_verified(verified); + assert_eq!(descriptors_for(&path), pinned_descriptors); + let other = reader.clone(); + drop(reader); + assert_eq!(descriptors_for(&path), pinned_descriptors); + let value: i64 = other + .inner + .lock() + .expect("reader lock") + .connection() + .query_row("SELECT value FROM sample", [], |row| row.get(0)) + .expect("read verified database"); + assert_eq!(value, 42); + + drop(other); + assert_eq!(descriptors_for(&path), 0); + } +} diff --git a/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer/sqlite.rs b/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer/sqlite.rs index c96205b831..e0821235fa 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer/sqlite.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/cursor_composer/sqlite.rs @@ -114,11 +114,11 @@ pub(super) async fn open_readonly_immutable(db_path: &Path) -> Result pending, - Ok(None) => continue, + let pending = match SourceClaim::observe_file_discovery_blocking( + ledger, &candidate, + ) { + Ok(FileSourceObservation::Read(pending)) => pending, + Ok(FileSourceObservation::Skipped) => continue, + Ok(FileSourceObservation::Quarantined) => { + discovery.quarantined_headers = + discovery.quarantined_headers.saturating_add(1); + continue; + } Err(TranscriptIngestError::ScanIo { source, .. }) => { discovery.record_failure( PiDiscoveryFailureKind::SessionHeaderUnavailable, @@ -306,6 +316,7 @@ impl PiSource { path: candidate, witness, error_kind: error.kind(), + pending, }); continue; } @@ -888,6 +899,7 @@ async fn reconcile_invalid_headers( discovery.quarantined_headers = discovery.quarantined_headers.saturating_add(1); let memo = PathMemo::read(facade, scope, &invalid.path).await?; if memo.disposition(Some(invalid.witness)) == Some(Disposition::InvalidHeader) { + invalid.pending.quarantined()?; continue; } if !budget.try_charge_input(invalid.witness.len()) { @@ -902,9 +914,11 @@ async fn reconcile_invalid_headers( break; } if matches!(current, Ok(HeaderRead::Invalid { witness, .. }) if witness == invalid.witness) + && memo + .record(facade, scope, invalid.witness, Disposition::InvalidHeader) + .await? { - memo.record(facade, scope, invalid.witness, Disposition::InvalidHeader) - .await?; + invalid.pending.quarantined()?; } discovery.failure_count = discovery.failure_count.saturating_add(1); if discovery.failures.len() < MAX_DISCOVERY_FAILURE_EVIDENCE { diff --git a/crates/tracedecay-sessions/src/runtime/hosts/pi_tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/pi_tests.rs index 6b119e7a3a..80b705befb 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/pi_tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/pi_tests.rs @@ -10,12 +10,17 @@ use tracedecay_domain::{ use super::{ MAX_DISCOVERY_CANDIDATES, MAX_DISCOVERY_FAILURE_EVIDENCE, MAX_DISCOVERY_INPUT_BYTES, MAX_DISCOVERY_UNITS, PI_AGENT_RELATIVE, PiDiscoveryReport, PiSource, capture_pi_observations, - capture_pi_session, pi_agent_dir_for, session_dir_name, warn_discovery_failures, + capture_pi_session, capture_pi_sweep, pi_agent_dir_for, session_dir_name, + warn_discovery_failures, }; use crate::admission::{HostAdmission, test_support::MemoryHostAdmission}; use crate::observation::ObservationCancellation; use crate::runtime::host_scan::{HOST_SCAN_WINDOW, HostScanBudget}; -use crate::runtime::source::{FileDiscoveryReport, TranscriptDiscoveryBounds}; +use crate::runtime::hosts::codex::CodexDiscoveryHub; +use crate::runtime::ingest::quarantine::SourceSweep; +use crate::runtime::source::{ + FileDiscoveryReport, TranscriptDiscoveryBounds, spin_until_jsonl_change_settled, +}; const FIXTURE_NAME: &str = "2026-09-25T16-00-00-000Z_5f0c2a8e-3b1d-4c7e-9a2f-6d8e1b4c7a90.jsonl"; const FIXTURE: &str = include_str!( @@ -280,6 +285,85 @@ async fn repeated_discovery_summary_keeps_malformed_sources_and_coverage_refused ); } +#[tokio::test] +async fn a_quarantined_pi_header_skips_unchanged_discovery_without_masking_coverage() { + let layout = Layout::new(); + let path = layout.session_dir().join(FIXTURE_NAME); + std::fs::write(&path, b"{\"not\":\"a Pi session header\"}\n").unwrap(); + assert!( + spin_until_jsonl_change_settled(&path), + "the initial file must have a settled witness" + ); + + let hub = CodexDiscoveryHub::default(); + hub.register("pi-headers", Some(&layout.agent_dir)); + let admission = MemoryHostAdmission::default(); + let scope = ObservationScopeV1::Profile; + let run = || async { + let mut sweep = SourceSweep::new("pi", Some((&hub, "pi-headers"))); + capture_pi_sweep( + &admission, + &layout.source(), + &layout.project, + scope.clone(), + None, + &ObservationCancellation::default(), + &mut sweep, + ) + .await + .unwrap() + }; + + let first = run().await; + assert_eq!(first.discovery_failures, 1); + assert!(first.deferred); + assert!(admission.observations().is_empty()); + + // A second discovery with the same source ledger must not open/parse the + // quarantined header again, but its evidence still denies complete coverage. + let sweep = SourceSweep::new("pi", Some((&hub, "pi-headers"))); + let (discovery, _) = layout + .source() + .discover( + &layout.project, + TranscriptDiscoveryBounds::from_discovered_units(MAX_DISCOVERY_CANDIDATES), + None, + HostScanBudget::new( + MAX_DISCOVERY_INPUT_BYTES, + MAX_DISCOVERY_UNITS, + std::time::Instant::now() + HOST_SCAN_WINDOW, + ObservationCancellation::default(), + ), + sweep.ledger().as_ref(), + ) + .unwrap(); + assert!(discovery.invalid_headers.is_empty()); + assert_eq!(discovery.quarantined_headers, 1); + assert_eq!(discovery.failure_count, 0); + + let second = run().await; + assert_eq!(second.discovery_failures, 0); + assert!(second.deferred); + let coverage = admission + .get_parse_offset(&scope, "host-coverage://pi/v1") + .await + .unwrap() + .unwrap(); + assert_eq!(coverage.file_id, 2, "quarantine must remain partial"); + + // Changed source evidence releases the in-memory quarantine; the durable + // path memo then re-evaluates the repaired header and admits the session. + layout.write_fixture(); + assert!( + spin_until_jsonl_change_settled(&path), + "the repaired file must have a settled witness" + ); + let recovered = run().await; + assert!(!recovered.deferred, "{recovered:?}"); + assert_eq!(recovered.discovery_failures, 0); + assert_eq!(admission.observations().len(), 9); +} + #[tokio::test] async fn empty_host_is_observed_once_without_revising_unchanged_frontier() { let layout = Layout::new(); diff --git a/crates/tracedecay-sessions/src/runtime/ingest/project_provider.rs b/crates/tracedecay-sessions/src/runtime/ingest/project_provider.rs index 6e7cf03d5a..b968165347 100644 --- a/crates/tracedecay-sessions/src/runtime/ingest/project_provider.rs +++ b/crates/tracedecay-sessions/src/runtime/ingest/project_provider.rs @@ -302,14 +302,17 @@ impl<'a> ProjectProviderRun<'a> { continue; }; if persisted_day.is_some_and(|day| path.parent() != Some(day)) { - let membership = run_blocking_transcript_section("codex", { - let path = path.clone(); - let project_root = self.project_root.to_path_buf(); - move || { - codex::codex_rollout_project_membership(&path, &project_root) - == Some(ProjectMembership::NoMatch) - } - }) + let membership = tracing::Instrument::instrument( + run_blocking_transcript_section("codex", { + let path = path.clone(); + let project_root = self.project_root.to_path_buf(); + move || { + codex::codex_rollout_project_membership(&path, &project_root) + == Some(ProjectMembership::NoMatch) + } + }), + tracing::trace_span!("sessions.ingest.project.codex.membership_check"), + ) .await; match membership { Ok(false) => {} @@ -371,7 +374,10 @@ impl<'a> ProjectProviderRun<'a> { { Ok(progress) => { if let Err(error) = - pending.admitted(progress.source_deferred, progress.covered_through) + tracing::trace_span!("sessions.ingest.project.codex.convergence_settle") + .in_scope(|| { + pending.admitted(progress.source_deferred, progress.covered_through) + }) { outcome.add_failure(warn_transcript_catch_up_failure( "codex", diff --git a/crates/tracedecay-sessions/src/runtime/ingest/quarantine.rs b/crates/tracedecay-sessions/src/runtime/ingest/quarantine.rs index ce06c06466..d898bbd948 100644 --- a/crates/tracedecay-sessions/src/runtime/ingest/quarantine.rs +++ b/crates/tracedecay-sessions/src/runtime/ingest/quarantine.rs @@ -56,11 +56,18 @@ impl SourceKey { impl SourceLedger { pub(crate) fn holds(&self, key: &SourceKey, witness: Witness) -> bool { + self.observed_quarantine(key, witness).is_some() + } + + /// An exact in-memory file witness can be settled or quarantined. Callers + /// that skip opening a quarantined header must retain partial coverage. + pub(crate) fn observed_quarantine(&self, key: &SourceKey, witness: Witness) -> Option { self.inner .lock() .unwrap_or_else(PoisonError::into_inner) .get(key) - .is_some_and(|recorded| recorded.witness == witness) + .filter(|recorded| recorded.witness == witness) + .map(|recorded| recorded.quarantined) } fn recorded_converged(&self, key: &SourceKey) -> bool { @@ -175,6 +182,14 @@ pub(crate) struct SourceClaim { settled: Option<(SourceLedger, Witness)>, } +/// Discovery may avoid opening a known-bad file, but must never report that +/// quarantined input as complete coverage. +pub(crate) enum FileSourceObservation { + Read(SourceClaim), + Skipped, + Quarantined, +} + impl SourceClaim { fn unless_recorded( ledger: Option<&SourceLedger>, @@ -220,17 +235,43 @@ impl SourceClaim { ledger: Option<&SourceLedger>, path: &Path, ) -> TranscriptIngestResult> { - let key = SourceKey::file(path); - if ledger.is_none() { - return Ok(Some(Self { key, settled: None })); + match Self::observe_file_discovery_blocking(ledger, path)? { + FileSourceObservation::Read(claim) => Ok(Some(claim)), + FileSourceObservation::Skipped | FileSourceObservation::Quarantined => Ok(None), } + } + + /// Preserve the reason a file was skipped for discovery sources that must + /// report quarantined evidence as partial, rather than complete, coverage. + pub(crate) fn observe_file_discovery_blocking( + ledger: Option<&SourceLedger>, + path: &Path, + ) -> TranscriptIngestResult { + let key = SourceKey::file(path); + let Some(ledger) = ledger else { + return Ok(FileSourceObservation::Read(Self { key, settled: None })); + }; // Codex archives a session by moving its rollout away; failing on the // vanished path would replay the unacknowledged discovery pass forever. let Some(metadata) = stat_if_present(path, "stat transcript for convergence")? else { - return Ok(None); + return Ok(FileSourceObservation::Skipped); }; let witness = Witness::settled_file(path, &metadata)?; - Ok(Self::unless_recorded(ledger, key, witness)) + if let Some(witness) = witness { + if let Some(quarantined) = ledger.observed_quarantine(&key, witness) { + return Ok(if quarantined { + FileSourceObservation::Quarantined + } else { + FileSourceObservation::Skipped + }); + } + } else if ledger.recorded_converged(&key) { + return Ok(FileSourceObservation::Skipped); + } + Ok(FileSourceObservation::Read(Self { + key, + settled: witness.map(|witness| (ledger.clone(), witness)), + })) } pub(crate) fn admitted( @@ -252,6 +293,15 @@ impl SourceClaim { } } + /// Cache a durable invalid-source disposition only after the canonical + /// source memo has confirmed it; this ledger is never its authority. + pub(crate) fn quarantined(self) -> TranscriptIngestResult<()> { + match self.settled { + Some((ledger, witness)) => ledger.record(&self.key, witness, true), + None => Ok(()), + } + } + pub(crate) fn admission_settles(&self, source_deferred: bool, covered_through: u64) -> bool { self.settled .as_ref() diff --git a/crates/tracedecay-sessions/src/runtime/observation/jsonl_observation_admission.rs b/crates/tracedecay-sessions/src/runtime/observation/jsonl_observation_admission.rs index cdc39a07e2..5b76239710 100644 --- a/crates/tracedecay-sessions/src/runtime/observation/jsonl_observation_admission.rs +++ b/crates/tracedecay-sessions/src/runtime/observation/jsonl_observation_admission.rs @@ -2238,7 +2238,12 @@ impl ActiveAdmission<'_> { requests.push(self.capture_request(batch_expected, frame, retention_class)?); batch_expected = next_expected; } - match self.admission.capture_observations(requests).await { + match tracing::Instrument::instrument( + self.admission.capture_observations(requests), + tracing::trace_span!("sessions.observation.admit_jsonl.capture_batch"), + ) + .await + { Ok(outcomes) => { if outcomes.len() != checkpoints.len() { return Err(CaptureWindowError::Ingest( @@ -2344,17 +2349,18 @@ pub(in crate::runtime) async fn admit_jsonl_observations( if cancellation.is_cancelled() { return Err(TranscriptIngestError::Cancelled { provider }); } - let mut expected_cursor = - admission - .get_source_cursor(&source, &scope) - .await - .map_err(|outcome| { - if is_admission_cancellation(&outcome, &cancellation) { - TranscriptIngestError::Cancelled { provider } - } else { - host_admission_error(provider, outcome) - } - })?; + let mut expected_cursor = tracing::Instrument::instrument( + admission.get_source_cursor(&source, &scope), + tracing::trace_span!("sessions.observation.admit_jsonl.cursor_lookup"), + ) + .await + .map_err(|outcome| { + if is_admission_cancellation(&outcome, &cancellation) { + TranscriptIngestError::Cancelled { provider } + } else { + host_admission_error(provider, outcome) + } + })?; if cancellation.is_cancelled() { return Err(TranscriptIngestError::Cancelled { provider }); } diff --git a/crates/tracedecay-store-runtime/src/session_registry/code_graph_manifest.rs b/crates/tracedecay-store-runtime/src/session_registry/code_graph_manifest.rs index d255981371..1739c89b54 100644 --- a/crates/tracedecay-store-runtime/src/session_registry/code_graph_manifest.rs +++ b/crates/tracedecay-store-runtime/src/session_registry/code_graph_manifest.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, VecDeque}; use std::fs::File; use std::io::{BufReader, Read, Seek, SeekFrom}; use std::path::PathBuf; @@ -779,9 +779,14 @@ struct PinnedPartitionedSegmentV1 { file: File, } +/// Keep a bounded set of pinned files during interleaved graph page reads. +/// A single last-used handle still reopens segments when pages alternate. +const MAX_GRAPH_SEGMENT_OPEN_HANDLES: usize = 8; + struct GraphSegmentReadPassV1<'a> { roots: &'a [PathBuf], - pinned: Option, + /// Least-recently used first; the most recent handle is at the back. + pinned: VecDeque, open_attempts: usize, reads: usize, } @@ -790,7 +795,7 @@ impl<'a> GraphSegmentReadPassV1<'a> { fn new(roots: &'a [PathBuf]) -> Self { Self { roots, - pinned: None, + pinned: VecDeque::new(), open_attempts: 0, reads: 0, } @@ -802,17 +807,27 @@ impl<'a> GraphSegmentReadPassV1<'a> { buffer: &mut Vec, ) -> Result<(), CodeIndexProductionErrorV1> { let (digest, _, _, _) = partitioned_segment_request(request)?; - if !self + if let Some(position) = self .pinned - .as_ref() - .is_some_and(|pinned| pinned.digest == digest) + .iter() + .position(|pinned| pinned.digest == digest) { - self.pinned = None; + let pinned = self.pinned.remove(position).ok_or_else(|| { + CodeIndexProductionErrorV1::Contract( + "sealed graph segment cache lost its retained handle".to_owned(), + ) + })?; + self.pinned.push_back(pinned); + } else { let root = select_partitioned_segment_root(self.roots, request)?; self.open_attempts += 1; - self.pinned = Some(open_partitioned_segment(root, request)?); + let pinned = open_partitioned_segment(root, request)?; + if self.pinned.len() == MAX_GRAPH_SEGMENT_OPEN_HANDLES { + self.pinned.pop_front(); + } + self.pinned.push_back(pinned); } - let pinned = self.pinned.as_mut().ok_or_else(|| { + let pinned = self.pinned.back_mut().ok_or_else(|| { CodeIndexProductionErrorV1::Contract( "sealed graph segment handle was not pinned".to_owned(), ) @@ -2328,6 +2343,117 @@ mod tests { ); } + #[test] + fn graph_segment_pass_reuses_interleaved_sealed_handles() { + let temporary = TempDir::new().unwrap(); + let first: &[u8] = b"first authenticated graph segment"; + let second: &[u8] = b"second authenticated graph segment"; + let digests = [first, second].map(|bytes| { + let digest = ManifestDigest::from_sha256_bytes(&Sha256::digest(bytes)).unwrap(); + let path = temporary.path().join(format!( + "segment-{}.json", + sha256_hex_suffix(digest.as_str()).unwrap() + )); + std::fs::write(path, bytes).unwrap(); + digest + }); + let seal_path = temporary.path().join("seal"); + std::fs::write(&seal_path, b"live").unwrap(); + let routes = SealRoutes { + seal_paths: vec![seal_path], + segment_roots: vec![temporary.path().to_path_buf()], + }; + SEGMENT_OPENS.with(|opens| opens.set(0)); + with_verified_segments(&digests[0], &routes, &|| Ok(()), |read| { + let mut buffer = Vec::new(); + for index in [0, 1, 0, 1, 0, 1] { + let expected = [first, second][index]; + read( + SealedGenerationSegmentReadV1::Whole { + digest: &digests[index], + size_bytes: expected.len() as u64, + }, + &mut buffer, + )?; + assert_eq!(buffer.as_slice(), expected); + assert_eq!( + ManifestDigest::from_sha256_bytes(&Sha256::digest(&buffer)).unwrap(), + digests[index] + ); + } + Ok(()) + }) + .unwrap(); + assert_eq!( + SEGMENT_OPENS.with(|opens| opens.get()), + 2, + "interleaved pages must not re-open previously pinned segments" + ); + } + + #[test] + fn graph_segment_pass_bounds_handles_and_reopens_evicted_segments() { + let temporary = TempDir::new().unwrap(); + let roots = vec![temporary.path().to_path_buf()]; + let segments = (0..super::MAX_GRAPH_SEGMENT_OPEN_HANDLES + 1) + .map(|index| { + let bytes = format!("sealed graph segment {index}").into_bytes(); + let digest = ManifestDigest::from_sha256_bytes(&Sha256::digest(&bytes)).unwrap(); + let path = temporary.path().join(format!( + "segment-{}.json", + sha256_hex_suffix(digest.as_str()).unwrap() + )); + std::fs::write(path, &bytes).unwrap(); + (digest, bytes) + }) + .collect::>(); + let mut pass = super::GraphSegmentReadPassV1::new(&roots); + SEGMENT_OPENS.with(|opens| opens.set(0)); + for (digest, expected) in &segments { + let mut bytes = Vec::new(); + pass.read( + SealedGenerationSegmentReadV1::Whole { + digest, + size_bytes: expected.len() as u64, + }, + &mut bytes, + ) + .unwrap(); + assert_eq!(bytes.as_slice(), expected.as_slice()); + assert!(pass.pinned.len() <= super::MAX_GRAPH_SEGMENT_OPEN_HANDLES); + } + assert_eq!(pass.open_attempts, segments.len()); + assert_eq!(pass.pinned.len(), super::MAX_GRAPH_SEGMENT_OPEN_HANDLES); + let (newest_digest, newest) = segments.last().unwrap(); + assert!( + pass.read( + SealedGenerationSegmentReadV1::Range { + digest: newest_digest, + size_bytes: newest.len() as u64, + offset: 1, + length: newest.len() as u64, + }, + &mut Vec::new(), + ) + .is_err(), + "a cached handle must still reject an over-bound read" + ); + let (oldest_digest, oldest) = &segments[0]; + let mut bytes = Vec::new(); + pass.read( + SealedGenerationSegmentReadV1::Whole { + digest: oldest_digest, + size_bytes: oldest.len() as u64, + }, + &mut bytes, + ) + .unwrap(); + assert_eq!(bytes.as_slice(), oldest.as_slice()); + assert_eq!(pass.open_attempts, segments.len() + 1); + assert_eq!(pass.pinned.len(), super::MAX_GRAPH_SEGMENT_OPEN_HANDLES); + assert_eq!(SEGMENT_OPENS.with(|opens| opens.get()), segments.len() + 1); + } + #[test] fn graph_build_segment_ranges_open_once_per_pass() { let fixture = partitioned_seal_fixture("range-opens"); diff --git a/crates/tracedecay-store/src/canonical_body.rs b/crates/tracedecay-store/src/canonical_body.rs index c71a631060..abf7c219aa 100644 --- a/crates/tracedecay-store/src/canonical_body.rs +++ b/crates/tracedecay-store/src/canonical_body.rs @@ -38,6 +38,16 @@ pub const UPSERT_CANONICAL_BODY_SQL: &str = "INSERT OR IGNORE INTO session_canon pub const LOAD_CANONICAL_BODY_SQL: &str = "SELECT encoding, body, uncompressed_bytes FROM session_canonical_bodies WHERE content_hash = ?1"; +/// Reads the distinct references of one observation in one SQL dispatch. The +/// JSON array parameter avoids a variable per hash; the left join keeps a +/// missing body visible, in reference order, instead of silently dropping it. +/// Callers still verify every returned body with [`unpack_body`]. +pub const LOAD_CANONICAL_BODIES_SQL: &str = "SELECT requested.value, body.encoding, + body.body, body.uncompressed_bytes + FROM json_each(?1) AS requested + LEFT JOIN session_canonical_bodies AS body ON body.content_hash = requested.value + ORDER BY requested.key"; + #[derive(Clone, Debug, PartialEq, Eq)] pub struct StoredCanonicalBody { pub content_hash: String, diff --git a/crates/tracedecay/src/daemon/maintenance.rs b/crates/tracedecay/src/daemon/maintenance.rs index 35fdc012d2..dec31a63b5 100644 --- a/crates/tracedecay/src/daemon/maintenance.rs +++ b/crates/tracedecay/src/daemon/maintenance.rs @@ -20,8 +20,8 @@ use tracedecay_maintenance::telemetry::{ StoreFileIdentityStateV1, StoreTelemetrySamplingOutcome, StoreTelemetrySamplingRegistry, }; use tracedecay_maintenance::tick::{ - MaintenanceContinuation, MaintenanceTickOutcome, cursor_after_attempted_units, - select_store_window, + MaintenanceContinuation, MaintenanceTickOutcome, StoreMaintenanceContinuations, + cursor_after_attempted_units, select_store_window, }; use tracedecay_runtime_core::storage::{RESPONSE_HANDLES_DIRECTORY, profile_sharded_data_root}; @@ -207,7 +207,7 @@ async fn join_cancelled_maintenance_task( async fn run_registered_store_retention( database: &tracedecay_global_db::RegisteredGlobalDb, config: &tracedecay_configuration::RetentionConfig, -) -> bool { +) -> MaintenanceTickOutcome { let now = match now_secs_i64() { Ok(now) => now, Err(failure) => { @@ -218,7 +218,7 @@ async fn run_registered_store_retention( ("failure", failure.to_owned()), ], ); - return false; + return MaintenanceTickOutcome::Retry; } }; let report = @@ -229,29 +229,7 @@ async fn run_registered_store_retention( now, ) .await; - match &report.session_lcm { - Some(Ok(session_lcm)) => { - let reclaimed = session_lcm.bytes_reclaimed(); - if reclaimed > 0 || !session_lcm.errors.is_empty() { - log_daemon_event( - "retention_session_lcm", - &[ - ("store", "mounted_sessions".to_owned()), - ("bytes_reclaimed", reclaimed.to_string()), - ("errors", session_lcm.errors.len().to_string()), - ], - ); - } - } - Some(Err(_)) => log_daemon_event( - "retention_degraded", - &[ - ("pass", "session_lcm".to_owned()), - ("failure", "retention_pass_failed".to_owned()), - ], - ), - None => {} - } + let session_storage_outcome = record_session_lcm_outcome(&report.session_lcm); match &report.observations { Some(Ok(observations)) => { let reclaimed = observations.bytes_reclaimed(); @@ -289,15 +267,198 @@ async fn run_registered_store_retention( Ok(_) => {} } log_payload_gc_outcome(&report.payload_gc); - let mut succeeded = report.succeeded(); + let canonical_body_outcome = record_canonical_body_gc_outcome(&report.canonical_body_gc); + let replay_metadata_outcome = record_replay_metadata_outcome(&report.replay_metadata); + let outcome = if report.succeeded() { + session_storage_outcome + .combine(canonical_body_outcome) + .combine(replay_metadata_outcome) + } else { + MaintenanceTickOutcome::Retry + }; if let Some(compaction) = &config.compaction { let outcome = tracedecay_maintenance::retention::live_compaction::compact_registered_store( database, compaction, ) .await; - succeeded &= record_live_compaction_outcome("mounted_sessions", outcome); + if !record_live_compaction_outcome("mounted_sessions", outcome) { + return MaintenanceTickOutcome::Retry; + } + } + outcome +} + +fn record_session_lcm_outcome( + outcome: &Option>, +) -> MaintenanceTickOutcome { + match outcome { + Some(Ok(session_lcm)) => { + let reclaimed = session_lcm.bytes_reclaimed(); + if reclaimed > 0 + || session_lcm.inlined.acted > 0 + || session_lcm.canonicalized_rows > 0 + || !session_lcm.errors.is_empty() + { + log_daemon_event( + "retention_session_lcm", + &[ + ("store", "mounted_sessions".to_owned()), + ("bytes_reclaimed", reclaimed.to_string()), + ("inlined_rows", session_lcm.inlined.acted.to_string()), + ( + "canonicalized_rows", + session_lcm.canonicalized_rows.to_string(), + ), + ("errors", session_lcm.errors.len().to_string()), + ], + ); + } + } + Some(Err(_)) => log_daemon_event( + "retention_degraded", + &[ + ("pass", "session_lcm".to_owned()), + ("failure", "retention_pass_failed".to_owned()), + ], + ), + None => {} + } + match outcome { + Some(Ok(report)) if report.errors.is_empty() => { + if report.storage_has_more { + MaintenanceTickOutcome::Continue(MaintenanceContinuation::StoreRetention) + } else { + MaintenanceTickOutcome::Complete + } + } + None => MaintenanceTickOutcome::Complete, + _ => MaintenanceTickOutcome::Retry, + } +} + +async fn run_registered_storage_continuation( + database: &tracedecay_global_db::RegisteredGlobalDb, + config: &tracedecay_configuration::RetentionConfig, +) -> MaintenanceTickOutcome { + let now = match now_secs_i64() { + Ok(now) => now, + Err(failure) => { + log_daemon_event( + "retention_degraded", + &[ + ("pass", "session_storage".to_owned()), + ("failure", failure.to_owned()), + ], + ); + return MaintenanceTickOutcome::Retry; + } + }; + let session_lcm = + tracedecay_maintenance::retention::registered_store::run_session_storage_compaction( + database, + &config.session_lcm, + now, + ) + .await; + let canonical_body_gc = + tracedecay_maintenance::retention::registered_store::run_canonical_body_gc(database).await; + let replay_metadata = + tracedecay_maintenance::retention::registered_store::run_replay_metadata_compaction( + database, + ) + .await; + record_session_lcm_outcome(&session_lcm) + .combine(record_canonical_body_gc_outcome(&canonical_body_gc)) + .combine(record_replay_metadata_outcome(&replay_metadata)) +} + +fn record_canonical_body_gc_outcome( + outcome: &Result< + tracedecay_global_db::observation::CanonicalBodyGcReport, + tracedecay_maintenance::retention::registered_store::RegisteredStoreRetentionErrorV1, + >, +) -> MaintenanceTickOutcome { + match outcome { + Ok(report) => { + if report.roots_scanned > 0 || report.bodies_scanned > 0 { + log_daemon_event( + "retention_canonical_bodies", + &[ + ("roots_scanned", report.roots_scanned.to_string()), + ("bodies_scanned", report.bodies_scanned.to_string()), + ("bodies_deleted", report.bodies_deleted.to_string()), + ( + "stored_bytes_deleted", + report.stored_bytes_deleted.to_string(), + ), + ("bootstrap_complete", report.bootstrap_complete.to_string()), + ("has_more", report.has_more.to_string()), + ], + ); + } + if report.has_more { + MaintenanceTickOutcome::Continue(MaintenanceContinuation::StoreRetention) + } else { + MaintenanceTickOutcome::Complete + } + } + Err(error) => { + log_daemon_event( + "retention_degraded", + &[ + ("pass", "canonical_body_gc".to_owned()), + ("failure", error.diagnostic().to_owned()), + ], + ); + MaintenanceTickOutcome::Retry + } + } +} + +fn record_replay_metadata_outcome( + outcome: &Result< + tracedecay_lcm::retention::ReplayMetadataCompactionReport, + tracedecay_maintenance::retention::registered_store::RegisteredStoreRetentionErrorV1, + >, +) -> MaintenanceTickOutcome { + match outcome { + Ok(report) => { + if report.rows_scanned > 0 { + log_daemon_event( + "retention_replay_metadata", + &[ + ("rows_scanned", report.rows_scanned.to_string()), + ("rows_compacted", report.rows_compacted.to_string()), + ("rows_skipped", report.rows_skipped.to_string()), + ( + "rows_skipped_oversized", + report.rows_skipped_oversized.to_string(), + ), + ( + "metadata_bytes_removed", + report.metadata_bytes_removed.to_string(), + ), + ("has_more", report.has_more.to_string()), + ], + ); + } + if report.has_more { + MaintenanceTickOutcome::Continue(MaintenanceContinuation::StoreRetention) + } else { + MaintenanceTickOutcome::Complete + } + } + Err(error) => { + log_daemon_event( + "retention_degraded", + &[ + ("pass", "replay_metadata_compaction".to_owned()), + ("failure", error.diagnostic().to_owned()), + ], + ); + MaintenanceTickOutcome::Retry + } } - succeeded } fn log_payload_gc_outcome( @@ -456,6 +617,7 @@ pub(super) struct MaintenanceCoordinator { /// last store processed. The next tick resumes immediately after it so no /// store is starved when the mounted set exceeds `MAINTENANCE_STORE_PAGE_LIMIT`. store_cursor: Arc>>, + store_continuations: Arc>, /// Instant of the last branch-store GC pass that succeeded for every /// mounted project. `None` keeps the daily cadence retry-eligible. last_branch_gc: Arc>>, @@ -476,6 +638,7 @@ impl Default for MaintenanceCoordinator { task: Arc::new(Mutex::new(None)), metrics: Arc::new(Mutex::new(MaintenanceMetricsV1::default())), store_cursor: Arc::new(Mutex::new(None)), + store_continuations: Arc::new(Mutex::new(StoreMaintenanceContinuations::default())), last_branch_gc: Arc::new(Mutex::new(None)), resident_memory_sampler: Arc::new(Mutex::new(None)), resident_memory_log: Arc::new(std::sync::Mutex::new( @@ -801,16 +964,12 @@ impl MaintenanceCoordinator { administration .store_telemetry_sampling() .begin_retention_tick_log_window(); - let session_databases = if continuation.is_none() { - administration - .mounted_registered_session_databases() - .await - .iter() - .map(|database| database.db_path().to_path_buf()) - .collect() - } else { - Vec::new() - }; + let session_databases = administration + .mounted_registered_session_databases() + .await + .iter() + .map(|database| database.db_path().to_path_buf()) + .collect::>(); let project_graphs = administration .mounted_project_graphs() .await @@ -900,6 +1059,7 @@ impl MaintenanceCoordinator { } work.sort_by(|left, right| left.0.cmp(&right.0)); let keys = work.iter().map(|(key, _)| key.clone()).collect::>(); + self.store_continuations.lock().await.retain_stores(&keys); let after = self.store_cursor.lock().await.clone(); let (window, _) = select_store_window(&keys, after.as_deref(), MAINTENANCE_STORE_PAGE_LIMIT); @@ -940,8 +1100,8 @@ impl MaintenanceCoordinator { // Bounded, round-robin slice of mounted stores. Writer admission is // per unit so one busy store defers only itself, and the cursor // advances past attempted units even on cancellation. A continuation - // omits session stores, but remains phase-scoped over the same bounded - // graph window rather than pinning one project. + // advances bounded lossless storage work and code-generation retention + // over the same bounded store window rather than pinning one project. let mut attempted = 0usize; let mut deferred = 0u64; let mut outcome = MaintenanceTickOutcome::Complete; @@ -966,10 +1126,10 @@ impl MaintenanceCoordinator { else { return MaintenanceTickOutcome::Complete; }; - if run_registered_store_retention(&database, retention).await { - MaintenanceTickOutcome::Complete + if continuation.is_some() { + run_registered_storage_continuation(&database, retention).await } else { - MaintenanceTickOutcome::Retry + run_registered_store_retention(&database, retention).await } } MaintenanceStoreWork::Graph(_) => { @@ -1007,7 +1167,13 @@ impl MaintenanceCoordinator { .await; attempted = attempted.saturating_add(1); match admitted { - Some(unit_outcome) => outcome = outcome.combine(unit_outcome), + Some(unit_outcome) => { + self.store_continuations + .lock() + .await + .record(&keys[index], unit_outcome); + outcome = outcome.combine(unit_outcome); + } None => { deferred = deferred.saturating_add(1); outcome = MaintenanceTickOutcome::Retry; @@ -1020,6 +1186,7 @@ impl MaintenanceCoordinator { } *self.store_cursor.lock().await = cursor_after_attempted_units(&keys, &window, attempted, after.as_deref()); + outcome = self.store_continuations.lock().await.combine(outcome); if !registry_listed { outcome = MaintenanceTickOutcome::Retry; } @@ -1646,7 +1813,7 @@ mod tests { &schedulers, &retention, branch_gc, - Some(MaintenanceContinuation::CodeGenerationRetention), + Some(MaintenanceContinuation::StoreRetention), ) .await; assert_eq!(continuation, MaintenanceTickOutcome::Complete); diff --git a/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs b/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs index 723ab51d0d..7affbade6f 100644 --- a/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs @@ -262,7 +262,7 @@ async fn mounted_code_generation_retention_continues_capped_segment_reclamation( }; assert_eq!( outcome, - MaintenanceTickOutcome::Continue(MaintenanceContinuation::CodeGenerationRetention), + MaintenanceTickOutcome::Continue(MaintenanceContinuation::StoreRetention), "a capped segment-only pass must keep the production maintenance cadence short" ); assert_eq!( diff --git a/crates/tracedecay/src/daemon/tests/rmcp_route/host_index_handoff.rs b/crates/tracedecay/src/daemon/tests/rmcp_route/host_index_handoff.rs index 4bbc7e900b..15c146b9a9 100644 --- a/crates/tracedecay/src/daemon/tests/rmcp_route/host_index_handoff.rs +++ b/crates/tracedecay/src/daemon/tests/rmcp_route/host_index_handoff.rs @@ -222,8 +222,10 @@ async fn readiness_budget_bounds_canonical_broker_runtime_contention() { let broker = Arc::clone(&broker); move || { let mut entered = Some(entered); + // Establish the held runtime deterministically: a try-lock probe + // can legitimately refuse unrelated concurrent spool activity. broker - .pending_replay_matches(|_| { + .pending_replay_matches_blocking(|_| { entered.take().unwrap().send(()).unwrap(); let _ = released.recv(); Ok(false) diff --git a/crates/tracedecay/src/mcp/server.rs b/crates/tracedecay/src/mcp/server.rs index bfb4279a5f..e662233c61 100644 --- a/crates/tracedecay/src/mcp/server.rs +++ b/crates/tracedecay/src/mcp/server.rs @@ -810,7 +810,7 @@ impl McpServer { tracing::warn!( project_root = %cg.project_root().display(), %error, - "MCP token accounting is unavailable because its durable baseline could not be read" + "MCP token accounting display baseline is unavailable; durable writes will still be attempted" ); None } diff --git a/crates/tracedecay/src/mcp/server/freshness_tests.rs b/crates/tracedecay/src/mcp/server/freshness_tests.rs index f437b049f0..49aefd7055 100644 --- a/crates/tracedecay/src/mcp/server/freshness_tests.rs +++ b/crates/tracedecay/src/mcp/server/freshness_tests.rs @@ -1,6 +1,6 @@ use super::{DatabaseOwnerReconciler, McpServer, McpServerConstructionContext}; use crate::test_support::git::git; -use std::sync::atomic::Ordering; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; use tempfile::TempDir; @@ -77,6 +77,32 @@ async fn init_indexed_repo() -> (TraceDecay, TempDir, FreshnessFixtureAuthority) ) } +/// A failed startup baseline read must not permanently turn off durable +/// accounting. The missing cache is not an authorization or write failure. +#[tokio::test] +async fn savings_without_a_display_baseline_still_commit_to_the_project_store() { + let (cg, _dir, _authority) = init_indexed_repo().await; + assert_eq!(cg.get_tokens_saved().await.unwrap(), 0); + + super::ledger::commit_token_savings(&cg, None, super::ledger::LedgerSink::NotMounted, 13).await; + assert_eq!(cg.get_tokens_saved().await.unwrap(), 13); + assert_eq!(cg.get_local_counter().await.unwrap(), 13); + + // A later projected completion must observe the committed lifetime total, + // not restart its accounting from the absent startup display baseline. + let display = AtomicU64::new(0); + super::ledger::commit_token_savings( + &cg, + Some(&display), + super::ledger::LedgerSink::NotMounted, + 7, + ) + .await; + assert_eq!(display.load(Ordering::Relaxed), 20); + assert_eq!(cg.get_tokens_saved().await.unwrap(), 20); + assert_eq!(cg.get_local_counter().await.unwrap(), 20); +} + /// Serve-old/await-new: the request that notices the drift answers on the /// snapshot it already had, and the reopen (plus its owner reconcile) lands /// behind it. diff --git a/crates/tracedecay/src/mcp/server/ledger.rs b/crates/tracedecay/src/mcp/server/ledger.rs index fecd44f2ce..764a5abedd 100644 --- a/crates/tracedecay/src/mcp/server/ledger.rs +++ b/crates/tracedecay/src/mcp/server/ledger.rs @@ -86,6 +86,43 @@ pub(crate) struct McpToolErrorAnalyticsRequest<'a> { pub(crate) connection_instance_id: Option<&'a str>, } +/// Commits a response delta even when startup could not read the display +/// baseline. The optional cache and global projection consume committed data; +/// neither can suppress the authoritative project-store write. +pub(super) async fn commit_token_savings( + cg: &TraceDecay, + tokens_saved: Option<&AtomicU64>, + sink: LedgerSink, + delta: u64, +) { + match cg.record_token_savings(delta).await { + Ok(committed_total) => { + // Two completions may resume in reverse order after their commits. + if let Some(tokens_saved) = tokens_saved { + tokens_saved.fetch_max(committed_total, Ordering::Relaxed); + } + if let LedgerSink::Mounted(gdb) = sink + && let Err(error) = gdb + .try_upsert_project_tokens(cg.project_root(), committed_total) + .await + { + tracing::warn!( + project_root = %cg.project_root().display(), + %error, + "background token-accounting projection failed" + ); + } + } + Err(error) => { + tracing::warn!( + project_root = %cg.project_root().display(), + %error, + "background token-accounting commit failed" + ); + } + } +} + impl McpServer { /// Resolves the ledger sink: the dedicated accounting database when the /// daemon mounted one, otherwise the registry handle it shares with, and @@ -132,11 +169,14 @@ impl McpServer { net_saved_tokens: u64, raw_file_tokens: u64, ) { - let persist = self - .tokens_saved - .as_ref() - .filter(|_| net_saved_tokens != 0) - .map(|tokens_saved| (Arc::clone(tokens_saved), self.ledger_sink())); + // An unreadable baseline suppresses only the approximate display cache. + // It must not disable future durable deltas if that read failure clears. + let persist = (net_saved_tokens != 0).then(|| { + ( + self.tokens_saved.as_ref().map(Arc::clone), + self.ledger_sink(), + ) + }); let monitor_project_root = monitor_project_root.to_path_buf(); let monitor_dir = self .owner_profile() @@ -144,31 +184,7 @@ impl McpServer { let tool_name = tool_name.to_owned(); self.spawn_observed_ledger_write(async move { if let Some((tokens_saved, sink)) = persist { - match cg.record_token_savings(net_saved_tokens).await { - Ok(committed_total) => { - // Two completions may resume in reverse order after - // their commits. The cache is a monotonic projection. - tokens_saved.fetch_max(committed_total, Ordering::Relaxed); - if let LedgerSink::Mounted(gdb) = sink - && let Err(error) = gdb - .try_upsert_project_tokens(cg.project_root(), committed_total) - .await - { - tracing::warn!( - project_root = %cg.project_root().display(), - %error, - "background token-accounting projection failed" - ); - } - } - Err(error) => { - tracing::warn!( - project_root = %cg.project_root().display(), - %error, - "background token-accounting commit failed" - ); - } - } + commit_token_savings(&cg, tokens_saved.as_deref(), sink, net_saved_tokens).await; } // The monitor entry opens, locks, and mmaps a file; keep that // off the async workers. diff --git a/crates/tracedecay/tests/hermes_suite/lcm_bridge.rs b/crates/tracedecay/tests/hermes_suite/lcm_bridge.rs index 4c8cfd3113..de49a82b08 100644 --- a/crates/tracedecay/tests/hermes_suite/lcm_bridge.rs +++ b/crates/tracedecay/tests/hermes_suite/lcm_bridge.rs @@ -292,10 +292,9 @@ class NoToolCtx: ctx = NoToolCtx() plugin.register(ctx) -assert [name for name, _ in ctx.hooks] == ["pre_llm_call", "post_tool_call"] +assert [name for name, _ in ctx.hooks] == ["pre_llm_call", "on_session_end", "post_tool_call"] assert len(ctx.memory_providers) == 1 -assert len(ctx.context_engines) == 1 -assert isinstance(ctx.context_engines[0], plugin.TraceDecayContextEngine) +assert ctx.context_engines == [], "TraceDecay must leave Hermes' compressor selected" "#, "generated plugin registration should continue when host lacks register_tool", ); @@ -332,9 +331,9 @@ ctx = RaisingToolCtx() plugin.register(ctx) assert ctx.tool_calls -assert [name for name, _ in ctx.hooks] == ["pre_llm_call", "post_tool_call"] +assert [name for name, _ in ctx.hooks] == ["pre_llm_call", "on_session_end", "post_tool_call"] assert len(ctx.memory_providers) == 1 -assert len(ctx.context_engines) == 1 +assert ctx.context_engines == [] "#, "generated plugin registration should continue when register_tool raises", ); @@ -399,15 +398,6 @@ assert receipt_event == { assert "secret.txt" not in repr(captured) assert "cat " not in repr(captured) -# The engine's session-end boundary drops per-session state without emitting -# a host receipt; the daemon owns any stop-side transcript work. -engine = plugin.TraceDecayContextEngine() -engine.initialize(session_id="session-hermes-native", project_root="/workspace/project") -assert engine.active_session_id == "session-hermes-native" -engine.on_session_end(session_id="session-hermes-native") -assert engine.active_session_id is None -assert len(captured) == 1 - class Ctx: def __init__(self): self.hooks = [] @@ -418,7 +408,10 @@ class Ctx: ctx = Ctx() plugin.register(ctx) -assert [name for name, _ in ctx.hooks] == ["pre_llm_call", "post_tool_call"] +assert [name for name, _ in ctx.hooks] == ["pre_llm_call", "on_session_end", "post_tool_call"] +# Ending a session flushes queued receipts without taking over compaction. +dict(ctx.hooks)["on_session_end"](session_id="session-hermes-native") +assert len(captured) == 1 "#, "generated Hermes callbacks must forward terminal receipt identity without content", ); @@ -499,67 +492,48 @@ finally: } #[test] -fn generated_registration_skips_tools_without_message_forwarding_capability() { +fn generated_registration_keeps_lcm_readers_without_message_forwarding() { run_generated_plugin_script( - "check_registration_capability_gate.py", + "check_generated_registration_keeps_lcm_readers_without_message_forwarding.py", r#" -class UnsafeRegisteredToolCtx: +class Ctx: context_engine_tool_handlers_receive_messages = False - def __init__(self): - self.tools = [] + self.registered = [] self.context_engines = [] - - def register_tool(self, **kwargs): - self.tools.append(kwargs["name"]) - - def register_hook(self, name, handler): - pass - - def register_memory_provider(self, provider): + def register_hook(self, *args, **kwargs): pass - + def register_tool(self, **kwargs): + self.registered.append(kwargs) def register_context_engine(self, engine): self.context_engines.append(engine) -ctx = UnsafeRegisteredToolCtx() +ctx = Ctx() plugin.register(ctx) - -# Code-graph / memory / transcript tools register even without message -# forwarding; only the live-ingest LCM verbs whose schemas carry the -# in-memory messages list (and the context-engine tool mirrors) are gated. -expected_without_messages = { - schema["name"] - for schema in plugin.schemas.TOOL_SCHEMAS - if schema["name"] not in plugin.MESSAGE_DEPENDENT_TOOLS -} -assert expected_without_messages.issubset(ctx.tools) -assert "tracedecay_lcm_compress" not in ctx.tools -assert "tracedecay_lcm_preflight" not in ctx.tools -assert "lcm_grep" not in ctx.tools -assert len(ctx.context_engines) == 1 -engine = ctx.context_engines[0] -assert engine.name == "tracedecay" -assert "lcm_grep" in {schema["name"] for schema in engine.get_tool_schemas()} +expected = {schema["name"] for schema in plugin.schemas.TOOL_SCHEMAS} +expected.update(schema["name"] for schema in plugin.LCM_NATIVE_SCHEMAS) +registered = {entry["name"] for entry in ctx.registered} +assert expected == registered +assert "tracedecay_lcm_compress" not in registered +assert "tracedecay_lcm_preflight" not in registered +assert ctx.context_engines == [] +assert all(entry["toolset"] == "tracedecay" for entry in ctx.registered) "#, - "generated plugin should gate only message-dependent tools when host does not forward messages", + "LCM readers must register independently of live-message forwarding and compaction", ); } #[test] -fn generated_context_engine_exposes_native_lcm_surface_and_dispatch() { +fn generated_lcm_tools_expose_native_surface_and_dispatch() { run_generated_plugin_script( "check_context_engine_native_surface.py", r#" import json plugin._resolved_project_scope = lambda path, *_args: path -engine = plugin.TraceDecayContextEngine() -engine.initialize(session_id="session-1", project_root="/tmp/project") - -assert engine.name == "tracedecay" - -schemas = engine.get_tool_schemas() +from functools import partial +call_tool = partial(plugin._handle_lcm_tool_call, session_id="session-1", project_root="/tmp/project") +schemas = plugin.LCM_NATIVE_SCHEMAS schemas_by_name = {schema["name"]: schema for schema in schemas} schema_names = {schema["name"] for schema in schemas} expected_native = { @@ -615,24 +589,6 @@ doctor_params = schemas_by_name["lcm_doctor"]["parameters"] assert status_params["properties"] == {} assert doctor_params["properties"] == {} -status = engine.get_status() -assert status["engine"] == "tracedecay" -assert status["session_id"] == "session-1" -assert status["active_session_id"] == "session-1" -assert "storage_scope" not in status -assert "hermes_home" not in status -assert "lcm_project_root" not in status -assert status["project_root"] == "/tmp/project" -assert status["tracedecay_binary_path"] == plugin.tools.TRACEDECAY_BIN -assert isinstance(status["tracedecay_binary_available"], bool) -assert status["context_engine_tool_names"] == sorted(schema_names) -assert status["last_compress_result"] == {"status": "never_ran"} -assert status["live_ingest"]["registered_tool_names"] == [] -assert status["live_ingest"]["context_tool_names"] == [] -assert status["live_ingest"]["host_forwards_messages"] is None -assert status["live_ingest"]["message_dependent_tools_registered"] is False -assert status["live_ingest"]["gate_reason"] == "not_registered" - calls = [] def fake_call_tracedecay_tool(name, args, **kwargs): @@ -640,7 +596,7 @@ def fake_call_tracedecay_tool(name, args, **kwargs): return json.dumps({"ok": True, "tool": name}) plugin.tools.call_tracedecay_tool = fake_call_tracedecay_tool -native_result = engine.handle_tool_call( +native_result = call_tool( "lcm_grep", { "query": "orchard", @@ -653,19 +609,19 @@ native_result = engine.handle_tool_call( }, messages=[{"role": "user", "content": "current turn"}], ) -load_result = engine.handle_tool_call( +load_result = call_tool( "lcm_load_session", {"session_id": "session-1", "max_content_chars": 123, "roles": ["user", "tool"], "time_from": 1, "time_to": 2}, messages=[{"role": "assistant", "content": "load turn"}], ) -describe_node_result = engine.handle_tool_call("lcm_describe", {"node_id": 7}) -describe_payload_result = engine.handle_tool_call("lcm_describe", {"externalized_ref": "payload_123.payload"}) -expand_result = engine.handle_tool_call( +describe_node_result = call_tool("lcm_describe", {"node_id": 7}) +describe_payload_result = call_tool("lcm_describe", {"externalized_ref": "payload_123.payload"}) +expand_result = call_tool( "lcm_expand", {"store_id": 42, "session_id": "session-foreign", "max_tokens": 77, "source_offset": 3, "source_limit": 2}, ) -direct_result = engine.handle_tool_call("tracedecay_lcm_grep", {"query": "direct", "session_scope": "all"}) -implicit_current_result = engine.handle_tool_call("lcm_grep", {"query": "implicit"}) +direct_result = call_tool("tracedecay_lcm_grep", {"query": "direct", "session_scope": "all"}) +implicit_current_result = call_tool("lcm_grep", {"query": "implicit"}) assert json.loads(native_result) == {"ok": True, "tool": "tracedecay_lcm_grep"} assert json.loads(load_result) == {"ok": True, "tool": "tracedecay_lcm_load_session"} @@ -731,84 +687,40 @@ assert "project_root" not in calls[6][1] assert calls[6][2] == {"project_root": "/tmp/project"} assert calls[6][1]["session_id"] == "session-1" "#, - "generated context engine should expose Hermes-style native LCM surface", + "generated tools should expose Hermes-style native LCM retrieval", ); } #[test] -fn generated_context_engine_never_uses_hermes_home_as_storage_identity() { +fn generated_lcm_tools_never_use_hermes_home_as_storage_identity() { run_generated_plugin_script( - "check_context_engine_env_home.py", + "check_generated_lcm_tools_never_use_hermes_home_as_storage_identity.py", r#" -import importlib.machinery -import importlib.util import json import os -import pathlib -import sys - -plugin_dir = pathlib.Path(sys.argv[1]) -parent_name = "_hermes_user_env_home" -parent_spec = importlib.machinery.ModuleSpec(parent_name, None, is_package=True) -parent_spec.submodule_search_locations = [] -parent_module = importlib.util.module_from_spec(parent_spec) -sys.modules[parent_name] = parent_module - -module_name = f"{parent_name}.tracedecay" -spec = importlib.util.spec_from_file_location( - module_name, - plugin_dir / "__init__.py", - submodule_search_locations=[str(plugin_dir)], -) -plugin = importlib.util.module_from_spec(spec) -sys.modules[module_name] = plugin -spec.loader.exec_module(plugin) os.environ["HERMES_HOME"] = "/tmp/hermes-from-env" - calls = [] - -def fake_call_tracedecay_tool(name, args, **kwargs): +def fake_call(name, args, **kwargs): calls.append((name, args, kwargs)) - return json.dumps({"content": [{"type": "text", "text": json.dumps({"status": "ok"})}]}) - -plugin.tools.call_tracedecay_tool = fake_call_tracedecay_tool -plugin._resolved_project_scope = lambda path, *_args: ( - str(path) if path and str(path) == "/tmp/project" else None -) - -engine = plugin.TraceDecayContextEngine() -engine.initialize(session_id="session-1") -assert os.path.normcase(os.path.realpath(engine.hermes_home)) == os.path.normcase( - os.path.realpath(str(plugin_dir.parent.parent)) -) -status = engine.get_status() -assert "storage_scope" not in status -assert "hermes_home" not in status -assert "lcm_project_root" not in status -assert status["project_root"] is None - -engine.handle_tool_call( - "lcm_grep", - {"query": "orchard"}, - messages=[{"role": "user", "content": "profile current turn"}], -) - -assert len(calls) == 1 -assert calls[0][0] == "tracedecay_lcm_grep" -assert "project_root" not in calls[0][1] -assert calls[0][1]["storage_scope"] == "user" -assert calls[0][2] == {} -assert "messages" not in calls[0][1] - + return "{}" +plugin.tools.call_tracedecay_tool = fake_call +plugin._project_scope_resolution = lambda *_args: ("rejected", None) +handler = plugin._make_wrapped_lcm_handler("lcm_grep", hermes_home="/tmp/hermes-from-config") +handler({"query": "orchard"}, session_id="session-1", cwd="/tmp/hermes-from-config", + messages=[{"role": "user", "content": "profile current turn"}]) +assert calls == [("tracedecay_lcm_grep", { + "query": "orchard", "scope": "current", "provider": "hermes", + "sort": "relevance", "include_summaries": False, "temporal_mode": {"kind": "current"}, + "session_id": "session-1", "storage_scope": "user", +}, {})], calls os.environ["HERMES_HOME"] = "/tmp/another-hermes-home" -other = plugin.TraceDecayContextEngine() -other.initialize(session_id="session-2") -other.status() +handler({"query": "orchard"}, session_id="session-2") +assert calls[-1][1]["session_id"] == "session-2" assert calls[-1][1]["storage_scope"] == "user" assert calls[-1][2] == {} "#, - "generated context engine must not use HERMES_HOME as a TraceDecay storage identity", + "LCM readers must not use Hermes home as a TraceDecay storage identity", ); } @@ -861,173 +773,67 @@ assert args == {"format": "json"} } #[test] -fn generated_context_engine_resolves_configured_hermes_home_on_registration() { +fn generated_registration_resolves_configured_hermes_home() { run_generated_plugin_script( - "check_context_engine_config_home.py", + "check_generated_registration_resolves_configured_hermes_home.py", r#" -import importlib.machinery -import importlib.util -import pathlib -import sys - -plugin_dir = pathlib.Path(sys.argv[1]) -parent_name = "_hermes_user_config_home" -parent_spec = importlib.machinery.ModuleSpec(parent_name, None, is_package=True) -parent_spec.submodule_search_locations = [] -parent_module = importlib.util.module_from_spec(parent_spec) -sys.modules[parent_name] = parent_module - -module_name = f"{parent_name}.tracedecay" -spec = importlib.util.spec_from_file_location( - module_name, - plugin_dir / "__init__.py", - submodule_search_locations=[str(plugin_dir)], -) -plugin = importlib.util.module_from_spec(spec) -sys.modules[module_name] = plugin -spec.loader.exec_module(plugin) - class Ctx: - def __init__(self): - self.config = {"hermes_home": "/tmp/hermes-from-config"} - self.context_engines = [] + config = {"hermes_home": "/tmp/hermes-from-config"} def register_hook(self, name, handler): pass - def register_context_engine(self, engine): - self.context_engines.append(engine) + def register_memory_provider(self, provider): + self.provider = provider ctx = Ctx() plugin.register(ctx) - -assert len(ctx.context_engines) == 1 -engine = ctx.context_engines[0] -assert engine.name == "tracedecay" -assert engine.hermes_home == "/tmp/hermes-from-config" +ctx.provider.initialize(session_id="session-1") +assert ctx.provider.hermes_home == "/tmp/hermes-from-config" "#, - "generated registration should resolve configured hermes_home", + "generated registration should resolve configured hermes_home without a context engine", ); } #[test] -fn generated_context_engine_registers() { - run_python_check( - "check_context_engine.py", +fn generated_registration_preserves_native_compaction_and_lcm_tools() { + run_generated_plugin_script( + "check_generated_registration_preserves_native_compaction_and_lcm_tools.py", r#" -import importlib.machinery -import importlib.util -import os -import pathlib -import sys -import types - -plugin_dir = pathlib.Path(sys.argv[1]) - -parent_name = "_hermes_user_context" -parent_spec = importlib.machinery.ModuleSpec(parent_name, None, is_package=True) -parent_spec.submodule_search_locations = [] -parent_module = importlib.util.module_from_spec(parent_spec) -sys.modules[parent_name] = parent_module - -class ContextEngine: - pass - -agent_module = types.ModuleType("agent") -agent_module.__path__ = [] -context_engine_module = types.ModuleType("agent.context_engine") -context_engine_module.ContextEngine = ContextEngine -agent_module.context_engine = context_engine_module -sys.modules["agent"] = agent_module -sys.modules["agent.context_engine"] = context_engine_module - -module_name = f"{parent_name}.tracedecay" -spec = importlib.util.spec_from_file_location( - module_name, - plugin_dir / "__init__.py", - submodule_search_locations=[str(plugin_dir)], -) -plugin = importlib.util.module_from_spec(spec) -sys.modules[module_name] = plugin -spec.loader.exec_module(plugin) - -class FullCtx: +import json +class Ctx: + # A retrieval plugin must load on a host that exposes no context-engine API. + config = {"context": {"engine": "compressor"}, "memory": {"provider": "tracedecay"}} def __init__(self): - self.tools = [] - self.hooks = [] - self.context_engines = [] - - def register_tool(self, **kwargs): - self.tools.append(kwargs) - + self.tools = {} def register_hook(self, name, handler): - self.hooks.append((name, handler)) - - def register_context_engine(self, engine): - self.context_engines.append(engine) + pass + def register_tool(self, **kwargs): + self.tools[kwargs["name"]] = kwargs["handler"] + def register_memory_provider(self, provider): + self.provider = provider -ctx = FullCtx() +ctx = Ctx() plugin.register(ctx) -assert len(ctx.context_engines) == 1 -engine = ctx.context_engines[0] -assert isinstance(engine, plugin.TraceDecayContextEngine) -assert isinstance(engine, ContextEngine) - -plugin._resolved_project_scope = lambda path, *_args: path -engine.initialize( - session_id="session-123", - hermes_home="/tmp/hermes-profile", - project_root="/tmp/project", -) -assert engine.active_session_id == "session-123" -assert engine.hermes_home == "/tmp/hermes-profile" -assert engine.project_root == "/tmp/project" - -# Project routing is transport-only and never derived from Hermes home. -assert not hasattr(plugin, "_storage_args") -assert plugin._project_call_kwargs("/tmp/project") == { - "project_root": "/tmp/project", -} - +assert ctx.config["context"]["engine"] == "compressor" +assert ctx.provider.name == "tracedecay" +assert set(plugin.LCM_TOOL_ALIASES).issubset(ctx.tools) calls = [] - -def fake_call_tracedecay_tool(name, args, **kwargs): - calls.append((name, args, kwargs)) - return "{}" - -plugin.tools.call_tracedecay_tool = fake_call_tracedecay_tool -plugin._resolved_project_scope = lambda path, *_args: ( - str(path) if path and str(path) == "/tmp/project" else None -) - -profile_engine = plugin.TraceDecayContextEngine() -profile_engine.on_session_start(session_id="session-1", hermes_home="/tmp/hermes") -assert profile_engine.should_compress_preflight(messages=[], current_tokens=123) is False - -project_engine = plugin.TraceDecayContextEngine() -project_engine.on_session_start( - session_id="session-2", - hermes_home="/tmp/hermes", - project_root="/tmp/project", +plugin._project_scope_resolution = lambda path, *_: ("registered", path) +plugin.tools.call_tracedecay_tool = lambda name, args, **kwargs: ( + calls.append((name, dict(args), kwargs)) or "{}" ) -assert project_engine.should_compress_preflight(messages=[], current_tokens=456) is False - -project_engine = plugin.TraceDecayContextEngine() -project_engine.initialize(session_id="initial", project_root="/tmp/project") -project_engine.on_session_start(session_id="next", project_root="/tmp/project") -assert project_engine.should_compress_preflight(messages=[], current_tokens=789) is False - -profile_engine = plugin.TraceDecayContextEngine() -profile_engine.initialize(session_id="initial", hermes_home="/tmp/hermes") -profile_engine.on_session_start(session_id="next") -assert profile_engine.should_compress_preflight(messages=[], current_tokens=321) is False -assert calls == [] - +ctx.tools["lcm_grep"]({"query": "orchard"}, session_id="session-123", project_root="/tmp/project") +assert calls == [("tracedecay_lcm_grep", { + "query": "orchard", "scope": "current", "provider": "hermes", "session_id": "session-123", + "sort": "relevance", "include_summaries": False, "temporal_mode": {"kind": "current"}, +}, {"project_root": "/tmp/project"})], calls +assert not hasattr(plugin, "_storage_args") "#, - "generated plugin should register a Hermes context engine", + "generated registration must preserve native compaction while exposing LCM retrieval", ); } #[test] -fn context_engine_expand_query_synthesizes_and_degrades() { +fn lcm_expand_query_synthesizes_and_degrades() { run_python_check( "check_expand_query_synthesis.py", r#" @@ -1108,11 +914,12 @@ class Aux: return "hiddenFinal answer from context" agent = type("Agent", (), {"auxiliary_client": Aux()})() -engine = plugin.TraceDecayContextEngine() -engine.initialize(session_id="session-1", project_root="/tmp/project", agent=agent) +def expand_query(**args): + args["session_id"] = "session-1" + return json.loads(plugin._handle_lcm_expand_query(args, project_root="/tmp/project", agent=agent)) responses.append(needs_synthesis()) -answer = engine.expand_query(prompt="What changed?", query="orchard") +answer = expand_query(prompt="What changed?", query="orchard") assert answer["status"] == "ok" assert answer["needs_synthesis"] is False assert answer["answer"] == "Final answer from context" @@ -1127,7 +934,7 @@ assert "EXPANDED CONTEXT" in agent.auxiliary_client.calls[0]["messages"][1]["con agent.auxiliary_client.mode = "timeout" responses.append(needs_synthesis()) -timeout_payload = engine.expand_query(prompt="What changed?", query="orchard") +timeout_payload = expand_query(prompt="What changed?", query="orchard") assert timeout_payload["degraded"] is True assert "timed out" in timeout_payload["error"] assert timeout_payload["timeout_seconds"] > 0 @@ -1135,7 +942,7 @@ assert timeout_payload["needs_synthesis"] is False agent.auxiliary_client.mode = "empty" responses.append(needs_synthesis()) -empty_payload = engine.expand_query(prompt="What changed?", query="orchard") +empty_payload = expand_query(prompt="What changed?", query="orchard") assert empty_payload["degraded"] is True assert "empty answer" in empty_payload["error"] assert empty_payload["needs_synthesis"] is False @@ -1145,13 +952,13 @@ assert empty_payload["needs_synthesis"] is False # exception that loses the retrieval behind a generic registry error. agent.auxiliary_client.mode = "unexpected" responses.append(needs_synthesis()) -failed_payload = engine.expand_query(prompt="What changed?", query="orchard") +failed_payload = expand_query(prompt="What changed?", query="orchard") assert failed_payload["degraded"] is True assert "schema bug" in failed_payload["error"] assert failed_payload["needs_synthesis"] is False assert failed_payload["matches"], "retrieval must survive synthesis failures" "#, - "generated context engine should synthesize and degrade expand-query answers", + "generated retrieval should synthesize and degrade expand-query answers", ); } @@ -1295,7 +1102,7 @@ assert provider.project_root is None } #[test] -fn context_engine_records_all_forwarded_native_lcm_tools() { +fn registration_records_all_native_lcm_tools() { run_generated_plugin_script( "check_context_tool_names_bookkeeping.py", r#" @@ -1324,10 +1131,10 @@ ctx = Ctx() plugin.register(ctx) expected = sorted(schema["name"] for schema in plugin.LCM_NATIVE_SCHEMAS) -assert sorted(plugin._CONTEXT_TOOL_NAMES) == expected +assert set(expected).issubset(plugin._REGISTERED_TOOL_NAMES) assert set(expected).issubset(set(ctx.registered)) "#, - "generated plugin should record every forwarded native LCM context tool", + "generated plugin should record every native LCM retrieval tool", ); } @@ -1397,7 +1204,7 @@ assert "cursor" not in raw } #[test] -fn context_engine_lcm_expand_query_tolerates_forwarded_agent_kwarg() { +fn lcm_expand_query_tolerates_forwarded_agent_kwarg() { run_generated_plugin_script( "check_expand_query_forwarded_agent.py", r#" @@ -1419,24 +1226,23 @@ class Aux: raise AssertionError("synthesis should not run") forwarded_agent = type("ForwardedAgent", (), {"auxiliary_client": Aux()})() -engine = plugin.TraceDecayContextEngine() -engine.initialize(session_id="session-1", project_root="/tmp/project", agent=forwarded_agent) - -raw = engine.handle_tool_call( +raw = plugin._handle_lcm_tool_call( "lcm_expand_query", {"prompt": "What changed?"}, + session_id="session-1", + project_root="/tmp/project", agent=forwarded_agent, ) payload = json.loads(raw) assert payload["status"] == "ok" assert payload["answer"] == "retrieval-only answer" "#, - "generated context engine should not pass duplicate agent kwargs through lcm_expand_query", + "generated retrieval must not pass duplicate agent kwargs through lcm_expand_query", ); } #[test] -fn context_engine_expand_query_uses_expansion_model_context_and_timeout_knobs() { +fn lcm_expand_query_uses_expansion_model_context_and_timeout_knobs() { run_generated_plugin_script( "check_expand_query_expansion_knobs.py", r#" @@ -1501,16 +1307,15 @@ os.environ["LCM_EXPANSION_CONTEXT_TOKENS"] = "4321" os.environ["LCM_EXPANSION_TIMEOUT_MS"] = "9000" agent = FakeAgent() -engine = plugin.TraceDecayContextEngine( +result = json.loads(plugin._handle_lcm_expand_query( + {"prompt": "What changed?", "query": "orchard", "session_id": "session-1"}, + project_root="/tmp/project", agent=agent, config={ "expansion_model": "cfg-expansion-model", "expansion_context_tokens": 32000, "expansion_timeout_ms": 120000, - } -) -engine.initialize(session_id="session-1", project_root="/tmp/project", agent=agent) - -result = engine.expand_query(prompt="What changed?", query="orchard") + }, +)) assert result["status"] == "ok" assert result["needs_synthesis"] is False assert result["answer"] == "synthetic answer" @@ -1702,89 +1507,6 @@ assert tools.call_tracedecay_tool("tracedecay_lcm_status", {}) == "{}" ); } -/// Newer Hermes declares `ContextEngine.update_from_response(usage)` as an -/// abstract method; the generated engine must implement it or the plugin -/// fails to load with "Can't instantiate abstract class". -#[test] -fn generated_context_engine_satisfies_abstract_update_from_response() { - run_python_check( - "check_update_from_response.py", - r#" -import abc -import importlib.machinery -import importlib.util -import pathlib -import sys -import types - -plugin_dir = pathlib.Path(sys.argv[1]) - -# Mimic the newer Hermes ABC *before* the plugin module is executed. -class ContextEngine(abc.ABC): - @abc.abstractmethod - def update_from_response(self, usage): - raise NotImplementedError - -agent_module = types.ModuleType("agent") -agent_module.__path__ = [] -context_engine_module = types.ModuleType("agent.context_engine") -context_engine_module.ContextEngine = ContextEngine -agent_module.context_engine = context_engine_module -sys.modules["agent"] = agent_module -sys.modules["agent.context_engine"] = context_engine_module - -parent_name = "_hermes_user_abc" -parent_spec = importlib.machinery.ModuleSpec(parent_name, None, is_package=True) -parent_spec.submodule_search_locations = [] -parent_module = importlib.util.module_from_spec(parent_spec) -sys.modules[parent_name] = parent_module - -module_name = f"{parent_name}.tracedecay" -spec = importlib.util.spec_from_file_location( - module_name, - plugin_dir / "__init__.py", - submodule_search_locations=[str(plugin_dir)], -) -plugin = importlib.util.module_from_spec(spec) -sys.modules[module_name] = plugin -spec.loader.exec_module(plugin) - -class Ctx: - def __init__(self): - self.context_engines = [] - def register_hook(self, name, handler): - pass - def register_context_engine(self, engine): - self.context_engines.append(engine) - -ctx = Ctx() -# This instantiates TraceDecayContextEngine; an unimplemented abstract method -# raises TypeError here. -plugin.register(ctx) -assert len(ctx.context_engines) == 1 -engine = ctx.context_engines[0] - -engine.update_from_response({"prompt_tokens": 11, "completion_tokens": 7}) -assert engine.last_prompt_tokens == 11 -assert engine.last_completion_tokens == 7 -assert engine.last_total_tokens == 18 - -engine.update_from_response( - {"input_tokens": "3", "output_tokens": "4", "total_tokens": 9} -) -assert engine.last_prompt_tokens == 3 -assert engine.last_completion_tokens == 4 -assert engine.last_total_tokens == 9 - -engine.update_from_response(None) -assert engine.last_prompt_tokens == 0 -assert engine.last_completion_tokens == 0 -assert engine.last_total_tokens == 0 -"#, - "generated engine must satisfy the abstract update_from_response contract", - ); -} - /// Newer Hermes derives the skill namespace from the plugin name and rejects /// ':' inside skill names, so registration must use the bare "tracedecay". #[test] @@ -1926,46 +1648,45 @@ assert "tracedecay:tracedecay" in text, text /// Host runtime config may identify a project, but a real session cwd or /// explicit call wins. The plugin-owned config block is filtered separately. #[test] -fn generated_context_engine_prefers_runtime_project_context() { +fn generated_lcm_tools_scope_each_call_to_its_runtime_session() { run_generated_plugin_script( - "check_context_engine_project_root.py", + "check_generated_lcm_tools_scope_each_call_to_its_runtime_session.py", r#" +import json +from concurrent.futures import ThreadPoolExecutor class Ctx: + config = {"project_root": "/tmp/stale-pinned-project"} def __init__(self): - self.config = {"project_root": "/tmp/pinned-project"} - self.context_engines = [] + self.tools = {} def register_hook(self, name, handler): pass - def register_context_engine(self, engine): - self.context_engines.append(engine) + def register_tool(self, **kwargs): + self.tools[kwargs["name"]] = kwargs["handler"] ctx = Ctx() -plugin._resolved_project_scope = lambda path, *_args: path plugin.register(ctx) -engine = ctx.context_engines[0] - -# Host runtime config applies at registration time. -assert engine.project_root == "/tmp/pinned-project", engine.project_root - -# A real session cwd overrides stale host config. -engine.on_session_start(session_id="s1", cwd="/somewhere/else") -assert engine.project_root == "/somewhere/else", engine.project_root - -# An explicit project_root remains highest priority. -engine.on_session_start(session_id="s2", project_root="/explicit/root") -assert engine.project_root == "/explicit/root", engine.project_root - -# Returning to a prior session restores its route instead of inheriting the -# most recently active session or the host default. -engine.on_session_start(session_id="s1") -assert engine.project_root == "/somewhere/else", engine.project_root - -# Without explicit runtime context, cwd is the fallback. -unpinned = plugin.TraceDecayContextEngine(config={}) -assert unpinned.project_root is None -unpinned.on_session_start(session_id="s3", cwd="/cwd/fallback") -assert unpinned.project_root == "/cwd/fallback", unpinned.project_root +plugin._project_scope_resolution = lambda path, *_: ("registered", path) +plugin.tools.call_tracedecay_tool = lambda name, args, **kwargs: json.dumps({ + "tool": name, "args": args, "route": kwargs, +}) +handler = ctx.tools["lcm_grep"] +def recall(index): + sid = "session-" + str(index) + root = "/tmp/project-" + str(index) + result = json.loads(handler({"query": "q"}, session_id=sid, cwd=root)) + assert result["args"]["session_id"] == sid + assert result["route"] == {"project_root": root} +with ThreadPoolExecutor(max_workers=4) as pool: + list(pool.map(recall, range(16))) +# Explicit selectors and runtime identity are resolved for every call; no +# preceding conversation's session or project can become a fallback. +result = json.loads(handler({"query": "q", "session_id": "explicit"}, + session_id="runtime", cwd="/tmp/cwd", project_root="/tmp/explicit")) +assert result["args"]["session_id"] == "explicit" +assert result["route"] == {"project_root": "/tmp/explicit"} +missing = json.loads(handler({"query": "q"}, cwd="/tmp/other")) +assert "session_id" in missing["error"] "#, - "generated engine must prefer explicit project and real cwd over host config", + "LCM retrieval must preserve runtime project and session scope across concurrent calls", ); } diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/skill_list_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/skill_list_test.rs index 2ac3b321c3..ce3a183d29 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/skill_list_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/skill_list_test.rs @@ -241,7 +241,7 @@ fn active_listing() -> Value { "improvement": false, "improvement_skill_id": "skill-active", "improvement_recommendation": "none", - "improvement_reason": "no repeated correction or failed-use signal is present", + "improvement_reason": "skill use evidence is unavailable; patch counts alone do not establish a repair need", "improvement_priority": "none", }) } diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/skill_view_behavior_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/skill_view_behavior_test.rs index 10751a1efa..ceebf91a48 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/skill_view_behavior_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/skill_view_behavior_test.rs @@ -291,7 +291,7 @@ async fn skill_view_returns_the_requested_package_and_withholds_support_bytes() assert_eq!(summary["improvement_recommendation"]["priority"], "none"); assert_eq!( summary["improvement_recommendation"]["reason"], - "no repeated correction or failed-use signal is present" + "skill use evidence is unavailable; patch counts alone do not establish a repair need" ); let included = call_skill_view( diff --git a/crates/tracedecay/tests/session_suite/lcm_compression/overflow.rs b/crates/tracedecay/tests/session_suite/lcm_compression/overflow.rs index 116d762ce8..1eb7abab3f 100644 --- a/crates/tracedecay/tests/session_suite/lcm_compression/overflow.rs +++ b/crates/tracedecay/tests/session_suite/lcm_compression/overflow.rs @@ -633,6 +633,51 @@ async fn overflow_recovery_keeps_preserved_objective_scaffold_when_evicting_tail })); } +#[tokio::test] +async fn pinned_objective_is_not_reinjected_outside_the_replay_budget() { + let tmp = TempDir::new().unwrap(); + let db = open_lcm_db(&tmp).await; + let objective = + "[Current user objective preserved from compacted history]\nShip OAuth login now"; + let messages = [ + ("system", "policy anchor"), + ("system", objective), + ("assistant", "still working"), + ("assistant", "recent update"), + ]; + insert_raw_messages_with_roles(&db, "cursor", "session-pinned-objective", &messages).await; + let cap = messages + .iter() + .map(|(_, content)| budget_tokens(content)) + .sum::(); + let mut request = limited_compress_request( + "cursor", + "session-pinned-objective", + provided_summary("unused"), + None, + None, + Some(cap), + ); + request.current_tokens = None; + request.overrides.fresh_tail_count = Some(2); + + let response = db.lcm_compress(request).await.unwrap(); + + assert_eq!(response.summary_nodes_created, 0); + assert_eq!( + response + .replay_messages + .iter() + .filter(|message| message["content"] == objective) + .count(), + 1, + "the objective is already present among the pinned anchors" + ); + assert_eq!(response.replay_messages.len(), messages.len()); + assert_eq!(response.replay_token_estimate, cap); + assert!(!response.replay_over_budget); +} + // Mirrors hermes-lcm `_assemble_overflow_recovery_context`: with no backlog // to compact, forced overflow evicts droppable assistant/tool tail turns that // do not fit under the cap while keeping anchors and budgetable user intent. diff --git a/crates/tracedecay/tests/session_suite/lcm_compression/replay.rs b/crates/tracedecay/tests/session_suite/lcm_compression/replay.rs index 72d17d0eff..429fb6c79b 100644 --- a/crates/tracedecay/tests/session_suite/lcm_compression/replay.rs +++ b/crates/tracedecay/tests/session_suite/lcm_compression/replay.rs @@ -214,6 +214,75 @@ async fn threshold_pressure_summarizes_short_huge_active_context() { ); } +#[tokio::test] +async fn active_tool_result_body_is_stored_once_with_lossless_replay() { + let tmp = TempDir::new().unwrap(); + let db = open_lcm_db(&tmp).await; + insert_session(&db, "cursor", "session-tool-body").await; + + let body = format!( + "tool-output-start\n{}tool-output-tail", + "test result: module completed successfully\n".repeat(2_048) + ); + assert!(!tracedecay_lcm::security::should_externalize( + "tool", + Some("tool_result"), + &body, + )); + let mut messages = active_multi_tool_transaction(); + messages[1]["content"] = json!(body); + let initial = + ingest_active_messages(&db, "cursor", "session-tool-body", messages.clone()).await; + let raw = db + .lcm_load_raw_message("cursor", "tool-a") + .await + .expect("tool result should be stored"); + assert_eq!(raw.storage_kind, LcmStorageKind::Inline); + assert_eq!(raw.content, body); + let stored_metadata = raw + .metadata_json + .as_deref() + .expect("active replay metadata"); + let metadata: Value = serde_json::from_str(stored_metadata).unwrap(); + assert!( + metadata["active_replay"].get("content").is_none(), + "the canonical body must not also be stored in SQL metadata" + ); + assert!(stored_metadata.len() < tracedecay_store::INLINE_BODY_BYTES); + assert_eq!(metadata["active_replay"]["tool_call_id"], "call-a"); + assert_eq!(initial.replay_messages[1]["content"], body); + assert_eq!(initial.replay_messages[1]["tool_call_id"], "call-a"); + + let preflight = db + .lcm_preflight(preflight_request( + "cursor", + "session-tool-body", + Vec::new(), + Some(100), + )) + .await + .unwrap(); + assert_eq!(preflight.replay_messages, initial.replay_messages); + + for input in [messages, initial.replay_messages.clone()] { + let replay = ingest_active_messages(&db, "cursor", "session-tool-body", input).await; + assert_eq!(replay.replay_messages, initial.replay_messages); + assert_eq!(replay.replay_token_estimate, initial.replay_token_estimate); + assert_eq!( + db.lcm_load_raw_message("cursor", "tool-a").await.unwrap(), + raw, + "original input and replay must preserve body, identity, and metadata" + ); + assert_eq!( + db.lcm_status("cursor", Some("session-tool-body")) + .await + .unwrap() + .raw_message_count, + 3 + ); + } +} + #[tokio::test] async fn active_structured_content_survives_compress_ingest_and_preflight_replay() { let tmp = TempDir::new().unwrap(); diff --git a/crates/tracedecay/tests/session_suite/lcm_compression/tool_transactions.rs b/crates/tracedecay/tests/session_suite/lcm_compression/tool_transactions.rs index 4d7781399a..173b3dda75 100644 --- a/crates/tracedecay/tests/session_suite/lcm_compression/tool_transactions.rs +++ b/crates/tracedecay/tests/session_suite/lcm_compression/tool_transactions.rs @@ -22,7 +22,12 @@ async fn fresh_tail_boundary_keeps_multi_tool_transaction_atomic_and_shrinking() Some(400), ); request.messages = with_authoritative_timestamps(messages); - request.current_tokens = Some(107); + let original_tokens = request + .messages + .iter() + .map(|message| budget_tokens(&message.to_string())) + .sum::(); + request.current_tokens = Some(original_tokens); request.overrides.threshold_tokens = Some(80); request.overrides.fresh_tail_count = Some(3); @@ -47,7 +52,81 @@ async fn fresh_tail_boundary_keeps_multi_tool_transaction_atomic_and_shrinking() .collect::>(), vec!["call-a", "call-b"] ); - assert!(response.replay_token_estimate < 107); + assert!(response.replay_token_estimate < original_tokens); +} + +#[tokio::test] +async fn tool_arguments_use_token_budget_when_selecting_atomic_replay() { + let tmp = TempDir::new().unwrap(); + let db = open_lcm_db(&tmp).await; + insert_session(&db, "cursor", "session-tool-token-budget").await; + + let arguments = json!({"source": "f(x:u32)->u32{x+1};".repeat(256)}).to_string(); + let cap = budget_tokens(&arguments) / 2; + let mut messages = active_multi_tool_transaction(); + messages[0]["tool_calls"][0]["function"]["arguments"] = json!(arguments); + let mut request = limited_compress_request( + "cursor", + "session-tool-token-budget", + provided_summary("unused"), + None, + None, + Some(cap), + ); + request.messages = with_authoritative_timestamps(messages); + request.current_tokens = None; + request.overrides.fresh_tail_count = Some(3); + + let response = db.lcm_compress(request).await.unwrap(); + + assert_eq!(response.reason, "no_backlog_to_compress"); + assert_eq!(response.summary_nodes_created, 0); + assert!( + response.replay_messages.is_empty(), + "tool arguments exceed the cap, so the whole transaction must be evicted" + ); + assert!(!response.replay_over_budget); +} + +#[tokio::test] +async fn overflow_recovery_reports_oversized_tool_arguments() { + let tmp = TempDir::new().unwrap(); + let db = open_lcm_db(&tmp).await; + insert_session(&db, "cursor", "session-tool-overflow").await; + + let arguments = json!({"source": "f(x:u32)->u32{x+1};".repeat(256)}).to_string(); + let argument_tokens = budget_tokens(&arguments); + let mut messages = active_multi_tool_transaction(); + messages[0]["tool_calls"][0]["function"]["arguments"] = json!(arguments); + ingest_active_messages(&db, "cursor", "session-tool-overflow", messages).await; + + // Recover from the canonical raw history, with no backlog to summarize. + // The newest tool transaction must remain intact even when it cannot fit. + let mut request = limited_compress_request( + "cursor", + "session-tool-overflow", + provided_summary("unused"), + None, + None, + Some(64), + ); + request.current_tokens = Some(argument_tokens); + request.overrides.fresh_tail_count = Some(3); + + let response = db.lcm_compress(request).await.unwrap(); + + assert_eq!(response.status, "best_effort"); + assert_eq!(response.reason, "irreducible_overflow_no_backlog"); + assert_eq!(response.summary_nodes_created, 0); + assert!(response.replay_over_budget); + assert!(response.replay_token_estimate >= argument_tokens); + assert_eq!(response.replay_messages.len(), 3); + assert_eq!( + response.replay_messages[0]["tool_calls"][0]["function"]["arguments"], + arguments + ); + assert_eq!(response.replay_messages[1]["tool_call_id"], "call-a"); + assert_eq!(response.replay_messages[2]["tool_call_id"], "call-b"); } #[tokio::test] diff --git a/scripts/bazel/patches/grafeo-core-read-allocation.patch b/scripts/bazel/patches/grafeo-core-read-allocation.patch deleted file mode 100644 index 1effd1e0ff..0000000000 --- a/scripts/bazel/patches/grafeo-core-read-allocation.patch +++ /dev/null @@ -1,473 +0,0 @@ -diff --git a/src/graph/compact/column.rs b/src/graph/compact/column.rs -index 861a9595..098e900f 100644 ---- a/src/graph/compact/column.rs -+++ b/src/graph/compact/column.rs -@@ -291,6 +291,28 @@ pub enum ColumnCodec { - } - - impl ColumnCodec { -+ /// Reads borrowed codec storage. It never decodes a Value. -+ pub fn allocation_bytes(&self, index: usize) -> usize { -+ use std::mem::size_of; -+ use grafeo_common::memory::heap::arc_slice_bytes; -+ match self { -+ Self::Dict(dict) => dict.get(index).map_or(0, |entry| { -+ // A malformed marker falls back to a literal string. Include -+ // the partial hex vector as well as that fallback allocation. -+ entry.len().saturating_mul(2).saturating_add(2 * size_of::()) -+ }), -+ Self::Int8Vector { dimensions, .. } => { -+ usize::from(*dimensions).saturating_mul(size_of::()) -+ .saturating_add(arc_slice_bytes::(usize::from(*dimensions))) -+ } -+ Self::Float32Vector { dimensions, .. } => { -+ usize::from(*dimensions).saturating_mul(size_of::()) -+ .saturating_add(arc_slice_bytes::(usize::from(*dimensions))) -+ } -+ _ => 0, -+ } -+ } -+ - // ── Phase 3a: Bytes-backed constructors ───────────────────────── - - /// Constructs a [`RawI64`](Self::RawI64) column from a `Vec` -diff --git a/src/graph/compact/graph_store_impl.rs b/src/graph/compact/graph_store_impl.rs -index 341460db..75dfac1b 100644 ---- a/src/graph/compact/graph_store_impl.rs -+++ b/src/graph/compact/graph_store_impl.rs -@@ -19,6 +19,40 @@ use crate::graph::traits::{GraphStore, GraphStoreSearch}; - use crate::statistics::Statistics; - - impl GraphStore for CompactStore { -+ fn node_allocation_bytes(&self, id: NodeId) -> Option { -+ let Some((table, offset)) = self.resolve_node(id) else { return Some(0); }; -+ let table = self.resolve_node_table(table)?; -+ Some(table.allocation_bytes(usize::try_from(offset).ok()?).saturating_add(table.label().len())) -+ } -+ -+ fn edge_allocation_bytes(&self, id: EdgeId) -> Option { -+ let Some((table, offset)) = self.resolve_edge(id) else { return Some(0); }; -+ Some(self.resolve_rel_table(table)?.allocation_bytes(usize::try_from(offset).ok()?)) -+ } -+ -+ fn node_property_allocation_bytes(&self, id: NodeId, key: &PropertyKey) -> Option { -+ let Some((table, offset)) = self.resolve_node(id) else { return Some(0); }; -+ Some(self.resolve_node_table(table)?.property_allocation_bytes(usize::try_from(offset).ok()?, key)) -+ } -+ -+ fn label_allocation_bytes(&self) -> Option { -+ Some(self.table_id_to_label.iter().fold( -+ self.table_id_to_label.len().saturating_mul(std::mem::size_of::()), -+ |bytes, label| bytes.saturating_add(label.len()), -+ )) -+ } -+ -+ fn property_lookup_allocation_bytes(&self, key: &PropertyKey, value: &Value) -> Option { -+ let index = self.property_value_index(key)?; -+ let mut count = 0_usize; -+ for (table, order) in index.iter() { -+ let col = self.resolve_node_table(*table)?.column(key)?; -+ count = count.saturating_add(super::value_order::equality_count(col, order, value)?); -+ } -+ let payload = match value { Value::String(value) => value.len(), Value::Bytes(value) => value.len() * 2, _ => 0 }; -+ Some(count.saturating_mul(8 * std::mem::size_of::()).saturating_add(payload.saturating_mul(3)).saturating_add(128)) -+ } -+ - fn get_node(&self, id: NodeId) -> Option { - let (table_id, offset) = self.resolve_node(id)?; - let nt = self.resolve_node_table(table_id)?; -@@ -29,10 +63,7 @@ impl GraphStore for CompactStore { - - let mut node = Node::new(id); - node.add_label(nt.label()); -- let props = nt.get_all_properties(row); -- for (k, v) in props { -- node.set_property(k, v); -- } -+ nt.for_each_property(row, |key, value| node.set_property(key.clone(), value)); - Some(node) - } - -@@ -48,10 +79,7 @@ impl GraphStore for CompactStore { - let edge_type = rt.edge_type().clone(); - - let mut edge = Edge::new(id, src, dst, edge_type); -- let props = rt.get_all_edge_properties(pos as usize); -- for (k, v) in props { -- edge.set_property(k, v); -- } -+ rt.for_each_property(pos as usize, |key, value| edge.set_property(key.clone(), value)); - Some(edge) - } - -diff --git a/src/graph/compact/layered.rs b/src/graph/compact/layered.rs -index 6eb25cb7..ed189b50 100644 ---- a/src/graph/compact/layered.rs -+++ b/src/graph/compact/layered.rs -@@ -437,6 +437,31 @@ impl LayeredStore { - // ── GraphStore implementation ────────────────────────────────────── - - impl GraphStore for LayeredStore { -+ fn node_allocation_bytes(&self, id: NodeId) -> Option { -+ Some(self.base.load().node_allocation_bytes(id)? -+ .saturating_add(self.overlay.load().node_allocation_bytes(id)?)) -+ } -+ -+ fn edge_allocation_bytes(&self, id: EdgeId) -> Option { -+ Some(self.base.load().edge_allocation_bytes(id)? -+ .saturating_add(self.overlay.load().edge_allocation_bytes(id)?)) -+ } -+ -+ fn node_property_allocation_bytes(&self, id: NodeId, key: &PropertyKey) -> Option { -+ Some(self.base.load().node_property_allocation_bytes(id, key)? -+ .saturating_add(self.overlay.load().node_property_allocation_bytes(id, key)?)) -+ } -+ -+ fn label_allocation_bytes(&self) -> Option { -+ Some(self.base.load().label_allocation_bytes()? -+ .saturating_add(self.overlay.load().label_allocation_bytes()?).saturating_mul(4)) -+ } -+ -+ fn property_lookup_allocation_bytes(&self, key: &PropertyKey, value: &Value) -> Option { -+ Some(self.base.load().property_lookup_allocation_bytes(key, value)? -+ .saturating_add(self.overlay.load().property_lookup_allocation_bytes(key, value)?).saturating_mul(2)) -+ } -+ - fn get_node(&self, id: NodeId) -> Option { - if self.is_node_deleted_from_base(id) { - return None; -diff --git a/src/graph/compact/node_table.rs b/src/graph/compact/node_table.rs -index 95ee80e5..103db9d0 100644 ---- a/src/graph/compact/node_table.rs -+++ b/src/graph/compact/node_table.rs -@@ -135,6 +135,34 @@ impl NodeTable { - /// A `true` row is one the builder padded because the node never had the - /// property; its column body still decodes to a value there, which no - /// read may surface. -+ pub fn allocation_bytes(&self, offset: usize) -> usize { -+ use std::mem::size_of; -+ if offset >= self.len { return 0; } -+ self.columns.iter().fold( -+ self.columns.len().saturating_mul(4 * size_of::<(PropertyKey, Value)>()), -+ |bytes, (key, col)| { -+ if self.is_null(offset, key) { bytes } else { -+ bytes.saturating_add(col.allocation_bytes(offset)).saturating_add(key.as_str().len()) -+ } -+ }, -+ ) -+ } -+ -+ pub fn for_each_property(&self, offset: usize, mut visit: impl FnMut(&PropertyKey, Value)) { -+ for (key, col) in &self.columns { -+ if !self.is_null(offset, key) { -+ if let Some(value) = col.get(offset) { -+ visit(key, value); -+ } -+ } -+ } -+ } -+ -+ pub fn property_allocation_bytes(&self, offset: usize, key: &PropertyKey) -> usize { -+ if self.is_null(offset, key) { return 0; } -+ self.columns.get(key).map_or(0, |col| col.allocation_bytes(offset)) -+ } -+ - #[must_use] - pub fn is_null(&self, offset: usize, key: &PropertyKey) -> bool { - self.null_masks -diff --git a/src/graph/compact/rel_table.rs b/src/graph/compact/rel_table.rs -index f5e5df76..55eb73e3 100644 ---- a/src/graph/compact/rel_table.rs -+++ b/src/graph/compact/rel_table.rs -@@ -182,6 +182,28 @@ impl RelTable { - - /// Returns the property value for a specific edge (by CSR position) and - /// key; `None` when the edge holds no value for it. -+ pub fn allocation_bytes(&self, offset: usize) -> usize { -+ use std::mem::size_of; -+ self.properties.iter().fold( -+ self.properties.len().saturating_mul(4 * size_of::<(PropertyKey, Value)>()), -+ |bytes, (key, col)| { -+ if self.is_null(offset, key) { bytes } else { -+ bytes.saturating_add(col.allocation_bytes(offset)).saturating_add(key.as_str().len()) -+ } -+ }, -+ ) -+ } -+ -+ pub fn for_each_property(&self, offset: usize, mut visit: impl FnMut(&PropertyKey, Value)) { -+ for (key, col) in &self.properties { -+ if !self.is_null(offset, key) { -+ if let Some(value) = col.get(offset) { -+ visit(key, value); -+ } -+ } -+ } -+ } -+ - #[must_use] - pub fn get_edge_property(&self, csr_position: usize, key: &PropertyKey) -> Option { - if self.is_null(csr_position, key) { -diff --git a/src/graph/compact/value_order.rs b/src/graph/compact/value_order.rs -index 230b3a4d..83a3c523 100644 ---- a/src/graph/compact/value_order.rs -+++ b/src/graph/compact/value_order.rs -@@ -137,6 +137,46 @@ pub(crate) fn build_row_order( - Some(RowOrder::Inline(rows)) - } - -+/// The number of rows whose value equals `value`, found by binary search -+/// over `order`, mirroring [`find_eq`] without collecting the rows. -+/// -+/// `None` when the codec and value are not an orderable pair this index -+/// serves, so the caller can decline rather than assume zero. -+pub(crate) fn equality_count(codec: &ColumnCodec, order: &RowOrder, value: &Value) -> Option { -+ let entry; -+ let compare: Box Option + '_> = match (codec, value) { -+ (ColumnCodec::Dict(dict), Value::String(_) | Value::Bytes(_)) => { -+ entry = encode_dict_entry(value); -+ Box::new(|row| { -+ let code = dict.code_at(row)?; -+ Some(dict.entry(code as usize)?.cmp(entry.as_str())) -+ }) -+ } -+ (ColumnCodec::BitPacked(packed), &Value::Int64(target)) => { -+ let target = u64::try_from(target).ok()?; -+ Box::new(move |row| Some(packed.get(row)?.cmp(&target))) -+ } -+ (ColumnCodec::RawI64(store), &Value::Int64(target)) => { -+ Box::new(move |row| Some(store.get(row)?.cmp(&target))) -+ } -+ _ => return None, -+ }; -+ let bound = |inclusive: bool| -> Option { -+ let (mut low, mut high) = (0, order.len()); -+ while low < high { -+ let mid = low + (high - low) / 2; -+ let ordering = order.row(mid).and_then(&compare)?; -+ if ordering == Ordering::Less || (inclusive && ordering == Ordering::Equal) { -+ low = mid + 1; -+ } else { -+ high = mid; -+ } -+ } -+ Some(low) -+ }; -+ Some(bound(true)?.saturating_sub(bound(false)?)) -+} -+ - /// The rows of `codec` whose value equals `value`, in row order, found by - /// binary search over `order`. - pub(crate) fn find_eq(codec: &ColumnCodec, order: &RowOrder, value: &Value) -> Vec { -diff --git a/src/graph/lpg/property.rs b/src/graph/lpg/property.rs -index 415d1498..e22be7fd 100644 ---- a/src/graph/lpg/property.rs -+++ b/src/graph/lpg/property.rs -@@ -302,6 +302,27 @@ impl PropertyStorage { - } - - /// Gets all properties for an entity. -+ #[must_use] -+ /// Borrowed current values, without a map, Value clone, or payload decode. -+ pub fn allocation_bytes(&self, id: Id, key: Option<&PropertyKey>) -> usize { -+ use std::mem::size_of; -+ let columns = self.columns.read(); -+ let mut bytes = 0_usize; -+ for (name, col) in columns.iter() { -+ if key.is_some_and(|key| key != name) { continue; } -+ if let Some(value) = col.borrowed(id) { -+ bytes = bytes.saturating_add(4 * size_of::<(PropertyKey, Value)>()) -+ .saturating_add(name.as_str().len()) -+ .saturating_add(match value { -+ Value::String(value) => value.len(), -+ Value::Bytes(value) => value.len(), -+ _ => 0, -+ }); -+ } -+ } -+ bytes -+ } -+ - #[must_use] - pub fn get_all(&self, id: Id) -> FxHashMap { - let columns = self.columns.read(); -@@ -705,6 +726,16 @@ impl PropertyStorage { - } - } - -+ /// Visits the same visible epoch values as get_all_at. -+ pub fn for_each_at(&self, id: Id, epoch: EpochId, mut visit: impl FnMut(&PropertyKey, Value)) { -+ let columns = self.columns.read(); -+ for (key, col) in columns.iter() { -+ if let Some(value) = col.get_at(id, epoch) { -+ visit(key, value); -+ } -+ } -+ } -+ - /// Removes all PENDING entries from all columns (transaction rollback). - pub fn remove_pending(&self) { - let mut columns = self.columns.write(); -@@ -1020,6 +1051,12 @@ impl PropertyColumn { - /// - /// First checks the hot buffer (uncompressed values), then falls back - /// to the compressed data if present. -+ #[must_use] -+ #[cfg(not(feature = "temporal"))] -+ fn borrowed(&self, id: Id) -> Option<&Value> { -+ self.values.get(&id) -+ } -+ - #[must_use] - pub fn get(&self, id: Id) -> Option { - // First check hot buffer -@@ -1685,6 +1722,12 @@ impl PropertyColumn { - } - - /// Gets the latest value for an entity, filtering out tombstones (Null). -+ #[must_use] -+ #[cfg(feature = "temporal")] -+ fn borrowed(&self, id: Id) -> Option<&Value> { -+ self.values.get(&id).and_then(|log| log.latest()).filter(|value| !value.is_null()) -+ } -+ - #[must_use] - pub fn get(&self, id: Id) -> Option { - self.values -diff --git a/src/graph/lpg/store/edge_ops.rs b/src/graph/lpg/store/edge_ops.rs -index 2aa96b6c..e9a49569 100644 ---- a/src/graph/lpg/store/edge_ops.rs -+++ b/src/graph/lpg/store/edge_ops.rs -@@ -20,7 +20,9 @@ impl LpgStore { - id_to_type.get(record.type_id as usize)?.clone() - }; - let mut edge = Edge::new(id, record.src, record.dst, edge_type); -- edge.properties = self.edge_properties.get_all(id).into_iter().collect(); -+ self.edge_properties.for_each(id, |key, value| { -+ edge.set_property(key.clone(), value); -+ }); - Some(edge) - } - -@@ -76,11 +78,7 @@ impl LpgStore { - id_to_type.get(record.type_id as usize)?.clone() - }; - let mut edge = Edge::new(id, record.src, record.dst, edge_type); -- edge.properties = self -- .edge_properties -- .get_all_at(id, epoch) -- .into_iter() -- .collect(); -+ self.edge_properties.for_each_at(id, epoch, |key, value| edge.set_property(key.clone(), value)); - Some(edge) - } - -diff --git a/src/graph/lpg/store/graph_store_impl.rs b/src/graph/lpg/store/graph_store_impl.rs -index f19a3e9e..8fb6e2e8 100644 ---- a/src/graph/lpg/store/graph_store_impl.rs -+++ b/src/graph/lpg/store/graph_store_impl.rs -@@ -20,6 +20,41 @@ use grafeo_common::utils::hash::FxHashMap; - use std::sync::Arc; - - impl GraphStore for LpgStore { -+ fn node_allocation_bytes(&self, id: NodeId) -> Option { -+ if !self.node_visible(id) { return Some(0); } -+ let labels = self.node_labels.read(); -+ #[cfg(not(feature = "temporal"))] -+ let count = labels.get(&id).map_or(0, |labels| labels.len()); -+ #[cfg(feature = "temporal")] -+ let count = labels.get(&id).and_then(|log| log.latest()).map_or(0, |labels| labels.len()); -+ Some(self.node_properties.allocation_bytes(id, None) -+ .saturating_add(count.saturating_mul(4 * std::mem::size_of::()))) -+ } -+ -+ fn property_lookup_allocation_bytes(&self, key: &PropertyKey, value: &Value) -> Option { -+ let indexes = self.property_indexes.read(); -+ let index = indexes.get(key)?; -+ let value = grafeo_common::types::HashableValue::new(value.clone()); -+ Some(index.get(&value).map_or(0, |nodes| nodes.len()).saturating_mul(std::mem::size_of::())) -+ } -+ -+ fn edge_allocation_bytes(&self, id: EdgeId) -> Option { -+ if self.edge_header(id).is_none() { return Some(0); } -+ Some(self.edge_properties.allocation_bytes(id, None)) -+ } -+ -+ fn node_property_allocation_bytes(&self, id: NodeId, key: &PropertyKey) -> Option { -+ Some(self.node_properties.allocation_bytes(id, Some(key))) -+ } -+ -+ fn label_allocation_bytes(&self) -> Option { -+ let registry = self.label_registry.read(); -+ Some(registry.names().iter().fold( -+ registry.names().len().saturating_mul(std::mem::size_of::()), -+ |bytes, label| bytes.saturating_add(label.len()), -+ )) -+ } -+ - fn get_node(&self, id: NodeId) -> Option { - LpgStore::get_node(self, id) - } -diff --git a/src/graph/lpg/store/node_ops.rs b/src/graph/lpg/store/node_ops.rs -index 3cd1b525..bd3a02d9 100644 ---- a/src/graph/lpg/store/node_ops.rs -+++ b/src/graph/lpg/store/node_ops.rs -@@ -102,7 +102,9 @@ impl LpgStore { - } - } - -- node.properties = self.node_properties.get_all(id).into_iter().collect(); -+ self.node_properties.for_each(id, |key, value| { -+ node.set_property(key.clone(), value); -+ }); - node - } - -@@ -181,11 +183,7 @@ impl LpgStore { - } - } - -- node.properties = self -- .node_properties -- .get_all_at(id, epoch) -- .into_iter() -- .collect(); -+ self.node_properties.for_each_at(id, epoch, |key, value| node.set_property(key.clone(), value)); - node - } - -diff --git a/src/graph/traits.rs b/src/graph/traits.rs -index 4b226ec2..cb2154e6 100644 ---- a/src/graph/traits.rs -+++ b/src/graph/traits.rs -@@ -44,6 +44,30 @@ use std::sync::Arc; - pub trait GraphStore: Send + Sync { - // --- Point lookups --- - -+ /// Allocation demand of materializing a native row, without materializing it. -+ /// None means this store cannot supply allocation metadata. -+ fn node_allocation_bytes(&self, _id: NodeId) -> Option { -+ None -+ } -+ -+ fn edge_allocation_bytes(&self, _id: EdgeId) -> Option { -+ None -+ } -+ -+ /// Allocation demand of one property lookup, including dictionary decoding. -+ fn node_property_allocation_bytes(&self, _id: NodeId, _key: &PropertyKey) -> Option { -+ None -+ } -+ -+ /// Allocation demand of enumerating the label universe. -+ fn label_allocation_bytes(&self) -> Option { -+ None -+ } -+ -+ fn property_lookup_allocation_bytes(&self, _key: &PropertyKey, _value: &Value) -> Option { -+ None -+ } -+ - /// Returns a node by ID (latest visible version at current epoch). - fn get_node(&self, id: NodeId) -> Option; - diff --git a/scripts/hermes_plugin_unit_check.py b/scripts/hermes_plugin_unit_check.py index 6b09936b97..b5912b2062 100644 --- a/scripts/hermes_plugin_unit_check.py +++ b/scripts/hermes_plugin_unit_check.py @@ -6,12 +6,12 @@ plugin context and a fake tracedecay binary, so it runs anywhere Python 3 exists. It asserts the host contracts the 2026-06 review found broken: - 1. compress() returns a MESSAGE LIST on success / no-op / error - (hermes ContextEngine ABC), never the raw result dict. + 1. TraceDecay leaves the native context engine selected and registers + LCM retrieval tools independently, scoped to each calling session. 2. Payloads >128 KiB round-trip through `--args @file` and the spill tempfile is cleaned up. 3. Code-graph tools register WITHOUT the message-forwarding capability - flag; the messages-dependent LCM verbs stay gated. + flag; LCM readers use stored transcripts and forwarded session identity. 4. The memory provider implements sync_turn / prefetch / on_memory_write and calls the right subprocess verbs. 5. pre_llm_call returns None on non-first turns and on first-turn greetings @@ -30,7 +30,6 @@ (default: bazel-bin/crates/tracedecay-cli/tracedecay). """ -import copy import json import os import shutil @@ -330,9 +329,6 @@ def raise_resolution(*_args, **_kwargs): ) is None finally: plugin.call_tracedecay_json = real_json - home_engine = plugin.TraceDecayContextEngine(hermes_home=str(host_home)) - home_engine.on_session_start(session_id="home-scope", cwd=str(host_home)) - assert home_engine.project_root is None home_provider = plugin.TracedecayMemoryProvider() home_provider.initialize( session_id="home-scope", hermes_home=str(host_home), cwd=str(host_home) @@ -351,10 +347,8 @@ def _check_registration_split( # duplicates; transcript search has no provider twin and stays. ctx = StubCtx() plugin.register(ctx) - assert ctx.engine.hermes_home == str(host_home), ( - ctx.engine.hermes_home, - str(host_home), - ) + assert ctx.engine is None, "TraceDecay must not replace Hermes' native compressor" + assert ctx.provider.hermes_home == str(host_home) assert ctx.skills == { "managed-test": managed_skill, "tracedecay": plugin_dir / "skills" / "tracedecay" / "SKILL.md", @@ -377,9 +371,7 @@ def _check_registration_split( custom_ctx = StubCtx() custom_ctx.hermes_home = str(custom_home) plugin.register(custom_ctx) - assert custom_ctx.engine.hermes_home == str(custom_home) - custom_ctx.engine.on_session_start(session_id="custom-home", cwd=str(custom_home)) - assert custom_ctx.engine.project_root is None + assert custom_ctx.engine is None custom_ctx.provider.initialize(session_id="custom-home", cwd=str(custom_home)) assert custom_ctx.provider.hermes_home == str(custom_home) assert custom_ctx.provider.project_root is None @@ -433,8 +425,8 @@ def _check_registration_split( ok("registered tools and providers honor a custom Hermes home") assert "tracedecay_lcm_compress" not in ctx.tools, sorted(ctx.tools) assert "tracedecay_lcm_preflight" not in ctx.tools, sorted(ctx.tools) - # Context-engine native mirrors stay gated without the capability flag. - assert "lcm_grep" not in ctx.tools, sorted(ctx.tools) + # Readers work through the ordinary registry without taking over compaction. + assert set(plugin.LCM_TOOL_ALIASES).issubset(ctx.tools), sorted(ctx.tools) assert ctx.provider.get_config_schema() == [], ctx.provider.get_config_schema() schemas = json.dumps([entry["schema"] for entry in ctx.tools.values()]) assert "storage_scope" not in schemas and "hermes_home" not in schemas, schemas @@ -459,7 +451,7 @@ class ForwardingCtx(StubCtx): plugin.register(fwd) assert "tracedecay_lcm_compress" not in fwd.tools, sorted(fwd.tools) assert "lcm_grep" in fwd.tools, sorted(fwd.tools) - ok("read-only LCM mirrors register when the host forwards messages") + ok("LCM readers register regardless of live-message forwarding") return ctx @@ -572,7 +564,7 @@ def _check_canonical_metadata(plugin, plugin_dir: Path, ctx): assert plugin.READ_ONLY_TOOL_NAMES.isdisjoint( {"tracedecay_str_replace", "tracedecay_fact_store_add", "tracedecay_fact_store_curate"} ), sorted(plugin.READ_ONLY_TOOL_NAMES) - assert set(plugin.FACT_STORE_EXACT_ROUTES) >= set(plugin.MEMORY_FACT_ACTIONS.values()) + assert {"add", "search"} <= plugin.FACT_STORE_EXACT_ROUTES.keys() assert all( route == f"tracedecay_fact_store_{action}" and route in plugin.TOOL_SCHEMAS_BY_NAME for action, route in plugin.FACT_STORE_EXACT_ROUTES.items() @@ -701,151 +693,130 @@ def _scripted(name, args, **kwargs): plugin.tools.call_tracedecay_tool = real_tool -def _check_engine_state_and_compress( - plugin, ctx, host_home: Path, runtime_project: Path -) -> list: - # ── 1. compress() message-list contract ────────────────────────────── - engine = ctx.engine - assert engine is not None - engine.initialize(session_id="check-session", hermes_home=str(host_home)) - engine.project_root = str(runtime_project) - engine.context_length = 200_000 - engine.threshold_tokens = 150_000 - engine.agent = object() - cloned_engine = copy.deepcopy(engine) - assert cloned_engine is not engine - assert cloned_engine._state_lock is not engine._state_lock - assert cloned_engine.project_root == str(runtime_project) - assert cloned_engine.context_length == 200_000 - assert cloned_engine.threshold_tokens == 150_000 - assert cloned_engine.agent is None - cloned_engine.context_length = 100_000 - assert engine.context_length == 200_000 - ok("context engine safely deep-copies per-agent budget state") - - real_resolver = plugin._resolved_project_scope +def _check_lcm_retrieval_scope(plugin, ctx, host_home: Path, runtime_project: Path): + assert ctx.engine is None, "TraceDecay must leave native compaction enabled" + calls = [] + real_tool = plugin.tools.call_tracedecay_tool + real_resolution = plugin._project_scope_resolution try: - plugin._resolved_project_scope = lambda path, *_args: ( - str(runtime_project) - if path - and os.path.realpath(str(path)) == os.path.realpath(str(runtime_project)) - else None + plugin._project_scope_resolution = lambda path, *_args: ( + ("rejected", None) if str(path) == str(host_home) else ("registered", str(path)) ) - engine.initialize(session_id="project-session", project_root=str(runtime_project)) - assert engine.project_root == str(runtime_project) - engine.initialize(session_id="untethered-session", cwd=str(host_home)) - assert engine.project_root is None - engine.initialize(session_id="project-session") - assert engine.project_root == str(runtime_project) - engine.initialize(session_id="untethered-session") - assert engine.project_root is None - finally: - plugin._resolved_project_scope = real_resolver - ok("context engine isolates project routing per Hermes session") - - status_calls = [] - real_json = plugin.call_tracedecay_json - try: - plugin.call_tracedecay_json = lambda name, args, **kwargs: ( - status_calls.append((name, args, kwargs)) - or {"status": "not_ingested"} + plugin.tools.call_tracedecay_tool = lambda name, args, **kwargs: ( + calls.append((name, args, kwargs)) or "{}" ) - engine.active_session_id = "status-session" - engine.project_root = str(runtime_project) - assert engine.status() == {"status": "not_ingested"} - finally: - plugin.call_tracedecay_json = real_json - assert status_calls == [ - ( + handler = ctx.tools["lcm_grep"]["handler"] + handler({"query": "orchard"}, session_id="project-session", cwd=str(runtime_project)) + handler({"query": "orchard"}, session_id="chat-session", cwd=str(host_home)) + handler({"query": "orchard"}, session_id="project-session", cwd=str(runtime_project)) + assert [args["session_id"] for _, args, _ in calls] == [ + "project-session", "chat-session", "project-session", + ] + assert calls[0][2] == calls[2][2] == {"project_root": str(runtime_project)} + assert calls[1][2] == {} + assert calls[1][1]["storage_scope"] == "user" + missing = json.loads(handler({"query": "orchard"}, cwd=str(runtime_project))) + assert "session_id" in missing["error"] + assert len(calls) == 3, "missing session identity must never reuse a previous session" + ctx.tools["lcm_status"]["handler"]({}, session_id="status-session", cwd=str(runtime_project)) + assert calls[-1] == ( "tracedecay_lcm_status", {"provider": "hermes", "session_id": "status-session"}, {"project_root": str(runtime_project)}, ) - ], status_calls - ok("context-engine status binds its Hermes provider and session target") - - messages = [ + finally: + plugin.tools.call_tracedecay_tool = real_tool + plugin._project_scope_resolution = real_resolution + ok("retrieval tools retain per-call session/project scope without replacing compaction") + return [ {"role": "user", "content": "hello"}, {"role": "assistant", "content": "hi"}, ] - out = engine.compress(list(messages), current_tokens=10) - assert out == messages, out - assert engine._last_compress_aborted is True - assert engine.last_compress_result == { - "status": "unavailable", - "reason": "host_raw_compression_unavailable", - "semantic_error": True, + + +def _check_lcm_handler_configuration(plugin, host_home: Path, runtime_project: Path): + ctx = StubCtx() + ctx.hermes_home = str(host_home) + ctx.config = { + "expansion_model": "registered-model", + "expansion_context_tokens": 2048, + "expansion_timeout_ms": 3000, } - ok("compress() returns the unchanged transcript with typed unavailability") - - # Host-contract token attrs (minimum-context guard reads these). - engine.update_model("check-model", 128000) - assert engine.context_length == 128000 - assert 0 < engine.threshold_tokens <= 128000 - ok("update_model populates context_length/threshold_tokens") - return messages - - -def _check_engine_thread_isolation(engine): - # Per-session isolation: the singleton engine must not leak one - # conversation's state into another (gateway sessions, delegate_task - # children all share this instance). - import threading as _threading - - seen = {} - - def _other_session(): - engine.on_session_start(session_id="check-session-b") - engine.update_model("other-model", 64000) - engine.update_from_response({"prompt_tokens": 7, "completion_tokens": 3}) - seen["b_context"] = engine.context_length - seen["b_total"] = engine.last_total_tokens - - worker = _threading.Thread(target=_other_session) - worker.start() - worker.join(timeout=10) - assert seen == {"b_context": 64000, "b_total": 10}, seen - # The original thread stays bound to its own session. - assert engine.context_length == 128000, engine.context_length - assert engine.model == "check-model", engine.model - assert engine._last_compress_aborted is True # from the exception check above - ok("per-session engine state stays isolated across threads") - - -def _check_should_compress_gating(plugin, engine): - # Hermes exposes no authenticated raw-compaction payload, so no pressure - # probe can claim that compression is available. - probes = [] - real_probe = plugin.TraceDecayContextEngine._preflight_probe - try: - def _spy_probe(self, *args, **kwargs): - probes.append(args) - return {"status": "ok", "should_compress": True} - - plugin.TraceDecayContextEngine._preflight_probe = _spy_probe - assert engine.should_compress(prompt_tokens=10) is False - assert probes == [], probes - assert engine.should_compress(prompt_tokens=engine.threshold_tokens) is False - assert probes == [], probes - finally: - plugin.TraceDecayContextEngine._preflight_probe = real_probe - ok("should_compress stays false without an authentic host payload") + plugin.register(ctx) + handler = ctx.tools["lcm_expand_query"]["handler"] + requests, config_homes, syntheses = [], [], [] + real_tool = plugin.tools.call_tracedecay_tool + real_block = plugin.tools.plugin_config_block + env_keys = ("LCM_EXPANSION_MODEL", "LCM_EXPANSION_CONTEXT_TOKENS", "LCM_EXPANSION_TIMEOUT_MS") + saved_env = {key: os.environ.pop(key, None) for key in env_keys} + + class Client: + def call_llm(self, **kwargs): + syntheses.append(kwargs) + return "grounded answer" + + class Agent: + auxiliary_client = Client() - # ABC contract: should_compress_preflight returns a BOOL; an error dict - # from the probe must read as False, never truthy. - real_probe = plugin.TraceDecayContextEngine._preflight_probe + def retrieve(name, args, **kwargs): + requests.append((name, dict(args), kwargs)) + payload = {"needs_synthesis": True, "prompt": args["prompt"], "context_blocks": []} + return json.dumps({"content": [{"type": "text", "text": json.dumps(payload)}]}) + + def profile_config(home): + config_homes.append(home) + return { + "expansion_model": "profile-model", + "expansion_context_tokens": 1024, + "expansion_timeout_ms": 2000, + } + + override = { + "expansion_model": "call-model", + "expansion_context_tokens": 4096, + "expansion_timeout_ms": 6000, + } + args = {"prompt": "What did we decide?"} try: - plugin.TraceDecayContextEngine._preflight_probe = ( - lambda self, *a, **k: {"error": "boom"} - ) - assert engine.should_compress_preflight([{"role": "user", "content": "x"}]) is False - plugin.TraceDecayContextEngine._preflight_probe = ( - lambda self, *a, **k: {"status": "ok", "should_compress": True} - ) - assert engine.should_compress_preflight([]) is False + plugin.tools.call_tracedecay_tool = retrieve + plugin.tools.plugin_config_block = profile_config + for index, (kwargs, model, budget, timeout) in enumerate(( + ({}, "registered-model", 2048, 3), + ({"config": override}, "call-model", 4096, 6), + ({"config": None}, "profile-model", 1024, 2), + ({"config": {}}, "profile-model", 1024, 2), + ({}, "registered-model", 2048, 3), + )): + session = f"configuration-session-{index}" + result = json.loads(handler( + args, session_id=session, cwd=str(host_home), + hermes_home=str(runtime_project), agent=Agent(), **kwargs, + )) + assert result["answer"] == "grounded answer", result + assert requests[-1] == ( + "tracedecay_lcm_expand_query", + { + "prompt": args["prompt"], "provider": "hermes", "session_id": session, + "storage_scope": "user", "context_max_tokens": budget, + }, + {}, + ), requests[-1] + assert syntheses[-1]["model"] == model, syntheses[-1] + assert syntheses[-1]["timeout"] == timeout, syntheses[-1] + assert config_homes[-1] == str(host_home), config_homes + assert len(requests) == len(syntheses) == len(config_homes) == 5 + assert args == {"prompt": "What did we decide?"} + assert ctx.config["expansion_model"] == "registered-model" + assert override["expansion_model"] == "call-model" finally: - plugin.TraceDecayContextEngine._preflight_probe = real_probe - ok("should_compress_preflight reports typed capability absence") + plugin.tools.call_tracedecay_tool = real_tool + plugin.tools.plugin_config_block = real_block + for key, value in saved_env.items(): + if value is not None: + os.environ[key] = value + else: + os.environ.pop(key, None) + ok("LCM call configuration overrides registration defaults without changing profile or session") def _check_expand_query_degradation(plugin): @@ -1173,7 +1144,7 @@ def _stop_daemon(proc: subprocess.Popen): proc.wait(timeout=5) -def _check_hermes_lcm_session_census(plugin, ctx, work: Path): +def _check_hermes_lcm_session_census(plugin, work: Path): """One ingested Hermes session shows its message; a session that was not ingested shows none. Status is asked only for the session id.""" home = work / "home" @@ -1196,9 +1167,6 @@ def _check_hermes_lcm_session_census(plugin, ctx, work: Path): stderr=subprocess.PIPE, text=True, ) - engine = ctx.engine - saved_root = engine.project_root - saved_session = engine.active_session_id try: deadline = time.monotonic() + 10 while not (profile / "daemon.sock").exists(): @@ -1207,8 +1175,6 @@ def _check_hermes_lcm_session_census(plugin, ctx, work: Path): if time.monotonic() >= deadline: raise AssertionError("isolated daemon socket did not appear") time.sleep(0.05) - engine.project_root = None - engine.active_session_id = "not-the-queried-session" ingested = _call_until_tool_ready( lambda: plugin.call_tracedecay_json( "tracedecay_hook_runtime", @@ -1230,17 +1196,19 @@ def _check_hermes_lcm_session_census(plugin, ctx, work: Path): ) assert ingested.get("status") in ("accepted", "committed"), ingested present = _call_until_tool_ready( - lambda: engine.status(session_id="gate-hermes-session") + lambda: plugin.call_tracedecay_json("tracedecay_lcm_status", { + "provider": "hermes", "session_id": "gate-hermes-session", "storage_scope": "user", + }) ) absent = _call_until_tool_ready( - lambda: engine.status(session_id="gate-empty-session") + lambda: plugin.call_tracedecay_json("tracedecay_lcm_status", { + "provider": "hermes", "session_id": "gate-empty-session", "storage_scope": "user", + }) ) assert _session_message_count(present, "gate-hermes-session") == 1, present assert _session_message_count(absent, "gate-empty-session") == 0, absent - ok("context-engine status counts the ingested session only") + ok("LCM status counts the ingested session only") finally: - engine.project_root = saved_root - engine.active_session_id = saved_session _stop_daemon(daemon) for key, value in saved_env.items(): if value is None: @@ -1262,16 +1230,15 @@ def run_checks(work: Path): _check_nudge_kill_switch(plugin, ctx) _check_canonical_metadata(plugin, plugin_dir, ctx) _check_response_envelope_table(plugin) - messages = _check_engine_state_and_compress(plugin, ctx, host_home, runtime_project) - _check_engine_thread_isolation(ctx.engine) - _check_should_compress_gating(plugin, ctx.engine) + messages = _check_lcm_retrieval_scope(plugin, ctx, host_home, runtime_project) + _check_lcm_handler_configuration(plugin, host_home, runtime_project) _check_expand_query_degradation(plugin) _check_no_hermes_storage_routing(plugin) _check_provider_verbs(plugin, ctx, host_home, messages) _check_prefetch_cache(plugin, ctx) _check_system_prompt_block(ctx) _check_args_file_spill(plugin, work) - _check_hermes_lcm_session_census(plugin, ctx, work) + _check_hermes_lcm_session_census(plugin, work) if __name__ == "__main__": diff --git a/scripts/hermes_stock_check.py b/scripts/hermes_stock_check.py index f1d6bc7897..7c28ead60d 100644 --- a/scripts/hermes_stock_check.py +++ b/scripts/hermes_stock_check.py @@ -10,16 +10,15 @@ Asserts the surfaces stock Hermes actually exposes: 1. the general PluginManager loads + enables the plugin (hook, command), - 2. the context engine registers and is selected via `context.engine`, + 2. Hermes keeps its native compressor while LCM readers register as tools, 3. the memory provider is discovered via `memory.provider` config (stock routes providers through plugins/memory, not PluginContext), 4. real tool dispatch round-trips through the tracedecay binary - (memory facts, LCM status/preflight/compress, graph status). + (memory facts, LCM status/retrieval, graph status). -Everything runs offline: no model calls (compress stays below threshold). +Everything runs offline: compression pressure checks do not call a model. """ -import copy import json import os import subprocess @@ -88,7 +87,7 @@ def assert_tool_dispatch_success_when_ready(dispatch, *, timeout_seconds=15): def assert_lcm_status_argv(argv, project_root): - """Validate the canonical CLI route used by the stock context engine.""" + """Validate the canonical CLI route used by LCM retrieval.""" assert len(argv) >= 6, argv assert argv[1] == "tool", argv name_index = 2 @@ -162,16 +161,15 @@ def main(): assert managed_view.get("success") is True, managed_view assert "Hermes can load this exported managed skill" in managed_view.get("content", "") ok("plugin-native managed skills resolve through qualified skill_view") - # Code-graph / memory / transcript tools register unconditionally; only - # the live-ingest LCM verbs (whose schemas take the in-memory messages - # list) depend on the context_engine_tool_handlers_receive_messages - # capability, which stock does not advertise. + # Retrieval uses stored transcripts, so all LCM readers register through + # stock Hermes without live-message forwarding or context-engine takeover. registered = set(loaded.tools_registered) assert "tracedecay_search" in registered, sorted(registered) assert "tracedecay_context" in registered, sorted(registered) assert "tracedecay_message_search" in registered, sorted(registered) assert "tracedecay_lcm_compress" not in registered, sorted(registered) assert "tracedecay_lcm_preflight" not in registered, sorted(registered) + assert set(plugin.LCM_TOOL_ALIASES).issubset(registered), sorted(registered) # memory.provider is tracedecay here, so the provider-owned fact trio # must not register as direct duplicates. assert "tracedecay_fact_store" not in registered, sorted(registered) @@ -179,7 +177,7 @@ def main(): assert "tracedecay_fact_feedback" not in registered, sorted(registered) assert "tracedecay_memory_status" not in registered, sorted(registered) ok( - "code-graph tools register on stock; LCM + provider-owned tools stay gated", + "code-graph and LCM readers register on stock; provider-owned tools deduplicate", f"{len(registered)} tools", ) from model_tools import get_tool_definitions @@ -239,50 +237,37 @@ def main(): ) ok("TraceDecay tools participate in stock progressive tool discovery") - # 2. Context engine: registered through the plugin and selected the way - # stock agent/agent_init.py selects it (config-driven, plugin fallback). + # 2. Exercise the real host selector and compressor. Registering a no-op + # plugin engine here would silently disable native automatic compaction. from hermes_cli.config import load_config + from agent.agent_init import _select_context_engine + from agent.context_compressor import ContextCompressor config = load_config() - engine_name = (config.get("context") or {}).get("engine") - assert engine_name == "tracedecay", f"context.engine = {engine_name!r}" - ok("config.yaml selects context.engine: tracedecay") - - from plugins.context_engine import load_context_engine - - assert load_context_engine(engine_name) is None - engine = get_plugin_context_engine() - assert engine is not None and engine.name == engine_name - from agent.context_engine import ContextEngine - - assert isinstance(engine, ContextEngine) - ok("context engine activates via stock plugin fallback") - - engine.initialize(session_id="stock-check-session", project_root=project_root) - assert engine.project_root is not None - assert os.path.realpath(engine.project_root) == os.path.realpath(project_root) - engine.update_model("stock-check-model", 128000) - engine.update_from_response({"prompt_tokens": 120, "completion_tokens": 30}) - assert engine.last_total_tokens == 150 - ok("stock ContextEngine ABC surface works", "update_from_response") - - # Stock Hermes deep-copies the registered plugin singleton for every - # AIAgent. The copy must retain routing/budget state without sharing locks - # or live agent references, otherwise Hermes falls back to its compressor. - engine.agent = object() - cloned_engine = copy.deepcopy(engine) - assert cloned_engine is not engine - assert cloned_engine._state_lock is not engine._state_lock - assert cloned_engine.project_root == engine.project_root - assert cloned_engine.context_length == 128000 - assert cloned_engine.last_total_tokens == 150 - assert cloned_engine.agent is None - ok("context engine deep-copies through the stock Hermes agent contract") - - assert engine.should_compress(1000) is False - ok("should_compress gates locally below the tracked threshold") - assert engine.should_compress_preflight([], current_tokens=1000) is False - ok("should_compress_preflight honors the bool ABC contract") + engine_name = (config.get("context") or {}).get("engine") or "compressor" + assert engine_name == "compressor", f"context.engine = {engine_name!r}" + assert get_plugin_context_engine() is None + assert _select_context_engine(config) is None + compressor = ContextCompressor( + model="stock-check-model", config_context_length=128000, + threshold_percent=0.75, quiet_mode=True, + ) + assert compressor.should_compress(1000) is False + assert compressor.should_compress(compressor.threshold_tokens) is True + ok("stock Hermes retains native compaction and responds to context pressure") + + from tools.registry import registry + + def lcm_status(): + raw = registry.dispatch( + "lcm_status", {}, session_id="stock-check-session", project_root=project_root, + ) + outer = assert_tool_dispatch_success(raw) + payloads = plugin._content_json_payloads(outer["content"]) + assert len(payloads) == 1, outer + payload, error = plugin._application_envelope_payload(payloads[0]) + assert error is None, error + return payload lcm_status_argv = [] real_subprocess_run = plugin.tools.subprocess.run @@ -292,10 +277,10 @@ def capture_lcm_status_argv(argv, *args, **kwargs): return real_subprocess_run(argv, *args, **kwargs) plugin.tools.subprocess.run = capture_lcm_status_argv - status = engine.status() + status = lcm_status() finally: plugin.tools.subprocess.run = real_subprocess_run - assert lcm_status_argv, "context engine did not invoke tracedecay_lcm_status" + assert lcm_status_argv, "registered reader did not invoke tracedecay_lcm_status" status_payload = json.loads(assert_lcm_status_argv(lcm_status_argv[-1], project_root)) assert status_payload == { "provider": "hermes", @@ -305,7 +290,7 @@ def capture_lcm_status_argv(argv, *args, **kwargs): assert_decoded_result(status, "status") if status.get("status") == "not_ingested": assert status.get("store_exists") is False, status - ok("lcm_status dispatch round-trips", "not_ingested before compress") + ok("lcm_status dispatch round-trips", "not_ingested before transcript sync") else: assert status.get("session_id") == "stock-check-session", status ok("lcm_status dispatch round-trips") @@ -314,25 +299,6 @@ def capture_lcm_status_argv(argv, *args, **kwargs): {"role": "user", "content": "hello"}, {"role": "assistant", "content": "hi there"}, ] - compressed = engine.compress(list(messages), current_tokens=50) - # Host ABC contract: compress() returns a MESSAGE LIST the host adopts as - # the live transcript. Hermes exposes no authentic raw-compression - # protocol, so the daemon-owned LCM authority reports host raw compaction - # as a typed unavailable capability and the transcript passes through - # unchanged (the transcript itself still ingests through the daemon). - assert isinstance(compressed, list), type(compressed) - assert compressed == messages, compressed - result = engine.last_compress_result - assert isinstance(result, dict) and result == { - "status": "unavailable", - "reason": "host_raw_compression_unavailable", - "semantic_error": True, - }, result - ok( - "compress returns the unchanged transcript with typed unavailability", - f"reason={result.get('reason')}", - ) - # 3. Memory provider: stock discovers providers via plugins/memory and the # memory.provider config key (the general PluginContext has no # register_memory_provider, so this is the only stock activation path). @@ -510,14 +476,14 @@ def capture_lcm_status_argv(argv, *args, **kwargs): "hello", "hi there", session_id="stock-check-session", messages=messages ) ingest_deadline = time.time() + 30 - ingest_status = engine.status() + ingest_status = lcm_status() while time.time() < ingest_deadline and ( not isinstance(ingest_status, dict) or ingest_status.get("status") in (None, "not_ingested") or _retryable_retained_pressure(ingest_status) ): time.sleep(0.25) - ingest_status = engine.status() + ingest_status = lcm_status() assert isinstance(ingest_status, dict), ingest_status assert ingest_status.get("status") not in (None, "not_ingested"), ingest_status grep = call_when_ready( diff --git a/scripts/test-configure-ci-fleet.py b/scripts/test-configure-ci-fleet.py index ab20eb25c7..aa8904906b 100644 --- a/scripts/test-configure-ci-fleet.py +++ b/scripts/test-configure-ci-fleet.py @@ -33,11 +33,22 @@ def setUp(self) -> None: self.endpoint.bind(("127.0.0.1", 0)) def configure( - self, outcome: str = "success", runner_os: str = "Linux", arch: str = "X64" + self, + outcome: str = "success", + runner_os: str = "Linux", + arch: str = "X64", + remote_download_outputs: str = "minimal", ) -> bool: host, port = self.endpoint.getsockname() return FLEET.configure_fleet( - self.workspace, self.output, outcome, runner_os, arch, host, port + self.workspace, + self.output, + outcome, + runner_os, + arch, + host, + port, + remote_download_outputs, ) def options(self) -> dict[str, str]: @@ -60,6 +71,7 @@ def test_linux_executor_shares_compilation_with_bounded_hosted_cpu(self) -> None self.assertTrue(self.configure()) options = self.options() self.assertEqual(options["--config"], "rbe-linux") + self.assertEqual(options["--remote_download_outputs"], "minimal") self.assertIn("--remote_cache_compression", options) self.assertIn("--remote_upload_local_results", options) self.assertIn("--noremote_cache_async", options) @@ -84,6 +96,7 @@ def test_windows_executor_compiles_remotely_with_bounded_local_fallback( self.assertTrue(self.configure(runner_os="Windows")) options = self.options() self.assertEqual(options["--config"], "rbe-windows") + self.assertEqual(options["--remote_download_outputs"], "minimal") self.assertIn("--remote_cache_compression", options) self.assertIn("--remote_upload_local_results", options) self.assertIn("--noremote_cache_async", options) @@ -111,6 +124,7 @@ def test_macos_executor_shares_compilation_with_bounded_hosted_cpu(self) -> None self.assertTrue(self.configure(runner_os="macOS", arch="ARM64")) options = self.options() self.assertEqual(options["--config"], "rbe-macos") + self.assertEqual(options["--remote_download_outputs"], "minimal") self.assertIn("--remote_cache_compression", options) self.assertIn("--remote_upload_local_results", options) self.assertIn("--noremote_cache_async", options) @@ -122,6 +136,18 @@ def test_macos_executor_shares_compilation_with_bounded_hosted_cpu(self) -> None self.assertNotIn("--experimental_local_lockfree_output", options) self.assertIn("build --strategy=Rustc=dynamic", self.rc.read_text()) + def test_packaging_materializes_top_level_outputs(self) -> None: + self.endpoint.listen() + self.assertTrue(self.configure(remote_download_outputs="toplevel")) + self.assertEqual(self.options()["--remote_download_outputs"], "toplevel") + + def test_invalid_download_mode_preserves_existing_configuration(self) -> None: + self.rc.write_text("# Existing configuration\n") + with self.assertRaises(ValueError): + self.configure(remote_download_outputs="minimal\nbuild --jobs=1000") + self.assertEqual(self.rc.read_text(), "# Existing configuration\n") + self.assertFalse(self.output.exists()) + def test_unsupported_windows_architecture_does_not_contact_executor(self) -> None: self.endpoint.listen() self.endpoint.setblocking(False) diff --git a/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json b/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json index b7e73cd319..a3f50a061a 100644 --- a/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json +++ b/tests/fixtures/search_quality/query-lexical-graph-workload-v1.json @@ -133,8 +133,8 @@ } ], "expected_query_fallback_digests": { - "train": "sha256:7f52a7213e6caab00e6a2bfca4a24ccdfe3f37fcd2312fd86da34807a397b846", - "validation": "sha256:f8f6e5973a2c07c2e3dc46ec270c75e819f6dc333030f8acabdf1580311577eb" + "train": "sha256:469266a120472b3203ad5a4d520b1a4e539e4c44ae6f5cf9babe1fb64ddfc1a3", + "validation": "sha256:94e8499d1c6237a7c7cb2f5008f34edb17e76cb6a9b1144b79851595f569fd09" }, "profile_matrix": [ {