From b5b69af7f8d0d1083974e993f0e9b92a435fa253 Mon Sep 17 00:00:00 2001 From: Anton Standrik Date: Sun, 13 Sep 2026 13:52:16 +0300 Subject: [PATCH] fix(test): isolate benchmark harness daemon runtimes scripts/benchmark-index.sh and scripts/benchmark-search-graph.sh ran the product with no runtime or cache of their own: the benchmark repository was indexed into the operator's live store and every one-shot joined the operator's account daemon. Source scripts/test-runtime.sh in both, start the private daemon before timing, record setup-time.txt and total-time.txt beside index-time.txt, and index from a repository path instead of querying a project in the live store. Add tests/test_benchmark_runtime_isolation_contract.sh, which fails before this change, and wire it into scripts/test.sh. Part of #1696. Signed-off-by: Anton Standrik --- scripts/benchmark-index.sh | 20 ++++ scripts/benchmark-search-graph.sh | 38 +++++++- scripts/test.sh | 3 + ...st_benchmark_runtime_isolation_contract.sh | 96 +++++++++++++++++++ 4 files changed, 152 insertions(+), 5 deletions(-) create mode 100755 tests/test_benchmark_runtime_isolation_contract.sh diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh index 756bda06e..71e694c17 100755 --- a/scripts/benchmark-index.sh +++ b/scripts/benchmark-index.sh @@ -9,6 +9,14 @@ LANG="${2:?}" REPO="${3:?}" RESULTS_DIR="${4:?}" +# The index must run against a daemon rendezvous and cache this run owns: only +# CBM_RUNTIME_DIR moves the rendezvous, and without a private cache the +# benchmark repository was indexed into the operator's live store (#1696). +# shellcheck source=test-runtime.sh +source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh" +cbm_test_runtime_init +trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT + # Resolve symlinks REPO=$(cd "$REPO" && pwd -P) @@ -33,6 +41,16 @@ LOC=$(find "$REPO" -type f \ echo "$FILE_COUNT" > "$OUT/file-count.txt" echo "$LOC" > "$OUT/loc.txt" +# Start the private daemon before timing so index-time.txt measures the index +# alone. setup-time.txt keeps the activation cost attributable and +# total-time.txt is their sum — the figure comparable with earlier runs, which +# paid activation inside the index timing whenever no daemon was already warm. +SETUP_START_MS=$(python3 -c "import time; print(int(time.time()*1000))") +if ! "$BINARY" daemon start >/dev/null 2>&1; then + echo " $LANG: private daemon did not start" >&2 + exit 1 +fi + # Index via CLI and capture timing START_MS=$(python3 -c "import time; print(int(time.time()*1000))") @@ -43,6 +61,8 @@ ELAPSED=$((END_MS - START_MS)) echo "$INDEX_JSON" > "$OUT/00-index.json" echo "$ELAPSED" > "$OUT/index-time.txt" +echo "$((START_MS - SETUP_START_MS))" > "$OUT/setup-time.txt" +echo "$((END_MS - SETUP_START_MS))" > "$OUT/total-time.txt" # Extract node/edge counts (CLI wraps in MCP content envelope) NODES=$(echo "$INDEX_JSON" | python3 -c " diff --git a/scripts/benchmark-search-graph.sh b/scripts/benchmark-search-graph.sh index cc94147ec..1d67b9581 100755 --- a/scripts/benchmark-search-graph.sh +++ b/scripts/benchmark-search-graph.sh @@ -3,18 +3,46 @@ # codebase-memory-mcp binary to measure the regex / LIKE pre-filter performance. # # Usage: -# scripts/benchmark-search-graph.sh +# scripts/benchmark-search-graph.sh # # Example: -# scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp my-project +# scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp ~/src/my-project +# +# The repository is indexed (untimed) into a private runtime and cache first; +# the queries then run against that index through a daemon this run keeps warm, +# so a timing never includes daemon activation and never touches the operator's +# live store (#1696). set -euo pipefail -BINARY="${1:?Usage: $0 }" -PROJECT="${2:?Usage: $0 }" +BINARY="${1:?Usage: $0 }" +REPO="${2:?Usage: $0 }" +REPO=$(cd "$REPO" && pwd -P) + +# shellcheck source=test-runtime.sh +source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh" +cbm_test_runtime_init +trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT + +if ! "$BINARY" daemon start >/dev/null 2>&1; then + echo "private daemon did not start" >&2 + exit 1 +fi +INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{}') +PROJECT=$(echo "$INDEX_JSON" | python3 -c " +import json, sys +d = json.load(sys.stdin) +if 'content' in d: + d = json.loads(d['content'][0]['text']) +print(d.get('project', '')) +" 2>/dev/null || echo "") +if [ -z "$PROJECT" ]; then + echo "index of $REPO did not report a project" >&2 + exit 1 +fi echo "Binary: $BINARY" -echo "Project: $PROJECT" +echo "Project: $PROJECT (indexed from $REPO)" echo "" run_case() { diff --git a/scripts/test.sh b/scripts/test.sh index ecfc26998..e8e262854 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -233,6 +233,9 @@ bash "$ROOT/tests/test_smoke_fixture_contract.sh" echo "=== Step 0i: parallel suite scheduler contract ===" bash "$ROOT/tests/test_parallel_harness_contract.sh" +echo "=== Step 0i2: benchmark harness runtime isolation contract (#1696) ===" +bash "$ROOT/tests/test_benchmark_runtime_isolation_contract.sh" + echo "=== Step 0j: venue parity contract (one harness, every venue) ===" bash "$ROOT/tests/test_venue_parity_contract.sh" diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh new file mode 100755 index 000000000..b944df768 --- /dev/null +++ b/tests/test_benchmark_runtime_isolation_contract.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Runtime-isolation contract for the benchmark harnesses (#1696, follow-up to +# #1691). +# +# scripts/benchmark-index.sh and scripts/benchmark-search-graph.sh ran the +# product with no runtime or cache of their own: the index landed in the +# operator's live store, every one-shot joined the operator's account daemon, +# and the timings depended on whatever that daemon was doing. Drive both with +# an environment-probe fixture and require that no product process ever +# receives the caller's runtime or cache, and that the index benchmark records +# the setup cost it now pays explicitly. + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +normalize_path() { + local path=${1%$'\r'} + if command -v cygpath >/dev/null 2>&1; then + cygpath -u "$path" 2>/dev/null && return 0 + fi + printf '%s\n' "${path//\\//}" +} + +ENV_PROBE="$WORKDIR/environment-probe" +cat > "$ENV_PROBE" <<'EOF' +#!/usr/bin/env bash +printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_BENCH_ENV_PROBE" +[[ "${1-} ${2-}" == "daemon status" ]] && exit 1 +exit 0 +EOF +chmod +x "$ENV_PROBE" + +CALLER_CACHE="$WORKDIR/caller-cache" +CALLER_RUNTIME="$WORKDIR/caller-runtime" +REPO="$WORKDIR/repo" +mkdir -p "$CALLER_CACHE" "$CALLER_RUNTIME" "$REPO" +echo 'def bench(): return 1' > "$REPO/bench.py" +CALLER_CACHE_NORMALIZED=$(normalize_path "$CALLER_CACHE") +CALLER_RUNTIME_NORMALIZED=$(normalize_path "$CALLER_RUNTIME") + +# The fixture answers nothing, so the search benchmark stops once the index +# reports no project; only the environment handed to the product is under test. +assert_isolated() { + local harness="$1" env_log="$2" private_root="" + [[ -s "$env_log" ]] || fail "$harness did not execute the environment-probe fixture" + while IFS=$'\t' read -r child_cache_raw child_runtime_raw; do + local child_cache child_runtime + child_cache=$(normalize_path "$child_cache_raw") + child_runtime=$(normalize_path "$child_runtime_raw") + if [[ -z "$child_runtime" || "$child_runtime" == "$CALLER_RUNTIME_NORMALIZED" ]]; then + fail "$harness exposed the caller CBM_RUNTIME_DIR to a product process" + fi + if [[ -z "$child_cache" || "$child_cache" == "$CALLER_CACHE_NORMALIZED" ]]; then + fail "$harness exposed the caller CBM_CACHE_DIR to a product process" + fi + if [[ "${child_runtime%/*}" != "${child_cache%/*}" || + "${child_runtime##*/}" != "runtime" || "${child_cache##*/}" != "cache" ]]; then + fail "$harness runtime/cache were not isolated beneath one private root" + fi + if [[ -n "$private_root" && "$private_root" != "${child_runtime%/*}" ]]; then + fail "$harness switched private roots mid-run" + fi + private_root="${child_runtime%/*}" + done < "$env_log" + [[ ! -e "$private_root" ]] || fail "$harness left its private root behind: $private_root" +} + +INDEX_LOG="$WORKDIR/index-environment.log" +CBM_CACHE_DIR="$CALLER_CACHE" \ +CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ +CBM_BENCH_ENV_PROBE="$INDEX_LOG" \ + "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" probe "$REPO" "$WORKDIR/results" \ + > "$WORKDIR/index.out" 2>&1 || true +assert_isolated "benchmark-index" "$INDEX_LOG" +for metric in setup-time total-time index-time; do + [[ -s "$WORKDIR/results/probe/$metric.txt" ]] || + fail "benchmark-index did not record $metric.txt" +done + +SEARCH_LOG="$WORKDIR/search-environment.log" +CBM_CACHE_DIR="$CALLER_CACHE" \ +CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ +CBM_BENCH_ENV_PROBE="$SEARCH_LOG" \ + "$ROOT/scripts/benchmark-search-graph.sh" "$ENV_PROBE" "$REPO" \ + > "$WORKDIR/search.out" 2>&1 || true +assert_isolated "benchmark-search-graph" "$SEARCH_LOG" + +echo "PASS: benchmark harnesses isolate their daemon runtime and cache from the caller"