DeusData · astandrik · Sep 13, 2026
@@ -9,6 +9,14 @@ LANG="${2:?}"
 REPO="${3:?}"
 RESULTS_DIR="${4:?}"
 
+# The index must run against a daemon rendezvous and cache this run owns: only
+# CBM_RUNTIME_DIR moves the rendezvous, and without a private cache the
+# benchmark repository was indexed into the operator's live store (#1696).
+# shellcheck source=test-runtime.sh
+source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"
+cbm_test_runtime_init
+trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT
+
 # Resolve symlinks
 REPO=$(cd "$REPO" && pwd -P)
 
@@ -33,6 +41,16 @@ LOC=$(find "$REPO" -type f \
 echo "$FILE_COUNT" > "$OUT/file-count.txt"
 echo "$LOC" > "$OUT/loc.txt"
 
+# Start the private daemon before timing so index-time.txt measures the index
+# alone. setup-time.txt keeps the activation cost attributable and
+# total-time.txt is their sum — the figure comparable with earlier runs, which
+# paid activation inside the index timing whenever no daemon was already warm.
+SETUP_START_MS=$(python3 -c "import time; print(int(time.time()*1000))")
+if ! "$BINARY" daemon start >/dev/null 2>&1; then
+  echo "  $LANG: private daemon did not start" >&2
+  exit 1
+fi
+
 # Index via CLI and capture timing
 START_MS=$(python3 -c "import time; print(int(time.time()*1000))")
 
@@ -43,6 +61,8 @@ ELAPSED=$((END_MS - START_MS))
 
 echo "$INDEX_JSON" > "$OUT/00-index.json"
 echo "$ELAPSED" > "$OUT/index-time.txt"
+echo "$((START_MS - SETUP_START_MS))" > "$OUT/setup-time.txt"
+echo "$((END_MS - SETUP_START_MS))" > "$OUT/total-time.txt"
 
 # Extract node/edge counts (CLI wraps in MCP content envelope)
 NODES=$(echo "$INDEX_JSON" | python3 -c "

@@ -3,18 +3,46 @@
 # codebase-memory-mcp binary to measure the regex / LIKE pre-filter performance.
 #
 # Usage:
-#   scripts/benchmark-search-graph.sh <binary-path> <project-name>
+#   scripts/benchmark-search-graph.sh <binary-path> <repo-path>
 #
 # Example:
-#   scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp my-project
+#   scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp ~/src/my-project
+#
+# The repository is indexed (untimed) into a private runtime and cache first;
+# the queries then run against that index through a daemon this run keeps warm,
+# so a timing never includes daemon activation and never touches the operator's
+# live store (#1696).
 
 set -euo pipefail
 
-BINARY="${1:?Usage: $0 <binary-path> <project-name>}"
-PROJECT="${2:?Usage: $0 <binary-path> <project-name>}"
+BINARY="${1:?Usage: $0 <binary-path> <repo-path>}"
+REPO="${2:?Usage: $0 <binary-path> <repo-path>}"
+REPO=$(cd "$REPO" && pwd -P)
+
+# shellcheck source=test-runtime.sh
+source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"
+cbm_test_runtime_init
+trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT
+
+if ! "$BINARY" daemon start >/dev/null 2>&1; then
+    echo "private daemon did not start" >&2
+    exit 1
+fi
+INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{}')
+PROJECT=$(echo "$INDEX_JSON" | python3 -c "
+import json, sys
+d = json.load(sys.stdin)
+if 'content' in d:
+    d = json.loads(d['content'][0]['text'])
+print(d.get('project', ''))
+" 2>/dev/null || echo "")
+if [ -z "$PROJECT" ]; then
+    echo "index of $REPO did not report a project" >&2
+    exit 1
+fi
 
 echo "Binary:  $BINARY"
-echo "Project: $PROJECT"
+echo "Project: $PROJECT (indexed from $REPO)"
 echo ""
 
 run_case() {

@@ -233,6 +233,9 @@ bash "$ROOT/tests/test_smoke_fixture_contract.sh"
 echo "=== Step 0i: parallel suite scheduler contract ==="
 bash "$ROOT/tests/test_parallel_harness_contract.sh"
 
+echo "=== Step 0i2: benchmark harness runtime isolation contract (#1696) ==="
+bash "$ROOT/tests/test_benchmark_runtime_isolation_contract.sh"
+
 echo "=== Step 0j: venue parity contract (one harness, every venue) ==="
 bash "$ROOT/tests/test_venue_parity_contract.sh"
 

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Runtime-isolation contract for the benchmark harnesses (#1696, follow-up to
+# #1691).
+#
+# scripts/benchmark-index.sh and scripts/benchmark-search-graph.sh ran the
+# product with no runtime or cache of their own: the index landed in the
+# operator's live store, every one-shot joined the operator's account daemon,
+# and the timings depended on whatever that daemon was doing. Drive both with
+# an environment-probe fixture and require that no product process ever
+# receives the caller's runtime or cache, and that the index benchmark records
+# the setup cost it now pays explicitly.
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "$WORKDIR"' EXIT
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+
+normalize_path() {
+    local path=${1%$'\r'}
+    if command -v cygpath >/dev/null 2>&1; then
+        cygpath -u "$path" 2>/dev/null && return 0
+    fi
+    printf '%s\n' "${path//\\//}"
+}
+
+ENV_PROBE="$WORKDIR/environment-probe"
+cat > "$ENV_PROBE" <<'EOF'
+#!/usr/bin/env bash
+printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_BENCH_ENV_PROBE"
+[[ "${1-} ${2-}" == "daemon status" ]] && exit 1
+exit 0
+EOF
+chmod +x "$ENV_PROBE"
+
+CALLER_CACHE="$WORKDIR/caller-cache"
+CALLER_RUNTIME="$WORKDIR/caller-runtime"
+REPO="$WORKDIR/repo"
+mkdir -p "$CALLER_CACHE" "$CALLER_RUNTIME" "$REPO"
+echo 'def bench(): return 1' > "$REPO/bench.py"
+CALLER_CACHE_NORMALIZED=$(normalize_path "$CALLER_CACHE")
+CALLER_RUNTIME_NORMALIZED=$(normalize_path "$CALLER_RUNTIME")
+
+# The fixture answers nothing, so the search benchmark stops once the index
+# reports no project; only the environment handed to the product is under test.
+assert_isolated() {
+    local harness="$1" env_log="$2" private_root=""
+    [[ -s "$env_log" ]] || fail "$harness did not execute the environment-probe fixture"
+    while IFS=$'\t' read -r child_cache_raw child_runtime_raw; do
+        local child_cache child_runtime
+        child_cache=$(normalize_path "$child_cache_raw")
+        child_runtime=$(normalize_path "$child_runtime_raw")
+        if [[ -z "$child_runtime" || "$child_runtime" == "$CALLER_RUNTIME_NORMALIZED" ]]; then
+            fail "$harness exposed the caller CBM_RUNTIME_DIR to a product process"
+        fi
+        if [[ -z "$child_cache" || "$child_cache" == "$CALLER_CACHE_NORMALIZED" ]]; then
+            fail "$harness exposed the caller CBM_CACHE_DIR to a product process"
+        fi
+        if [[ "${child_runtime%/*}" != "${child_cache%/*}" ||
+              "${child_runtime##*/}" != "runtime" || "${child_cache##*/}" != "cache" ]]; then
+            fail "$harness runtime/cache were not isolated beneath one private root"
+        fi
+        if [[ -n "$private_root" && "$private_root" != "${child_runtime%/*}" ]]; then
+            fail "$harness switched private roots mid-run"
+        fi
+        private_root="${child_runtime%/*}"
+    done < "$env_log"
+    [[ ! -e "$private_root" ]] || fail "$harness left its private root behind: $private_root"
+}
+
+INDEX_LOG="$WORKDIR/index-environment.log"
+CBM_CACHE_DIR="$CALLER_CACHE" \
+CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+CBM_BENCH_ENV_PROBE="$INDEX_LOG" \
+    "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" probe "$REPO" "$WORKDIR/results" \
+    > "$WORKDIR/index.out" 2>&1 || true
+assert_isolated "benchmark-index" "$INDEX_LOG"
+for metric in setup-time total-time index-time; do
+    [[ -s "$WORKDIR/results/probe/$metric.txt" ]] ||
+        fail "benchmark-index did not record $metric.txt"
+done
+
+SEARCH_LOG="$WORKDIR/search-environment.log"
+CBM_CACHE_DIR="$CALLER_CACHE" \
+CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+CBM_BENCH_ENV_PROBE="$SEARCH_LOG" \
+    "$ROOT/scripts/benchmark-search-graph.sh" "$ENV_PROBE" "$REPO" \
+    > "$WORKDIR/search.out" 2>&1 || true
+assert_isolated "benchmark-search-graph" "$SEARCH_LOG"
+
+echo "PASS: benchmark harnesses isolate their daemon runtime and cache from the caller"