diff --git a/.machine_readable/scorecards/0-ai-gatekeeper-protocol.scorecard.a2ml b/.machine_readable/scorecards/0-ai-gatekeeper-protocol.scorecard.a2ml index 5ca9013d..43b66da2 100644 --- a/.machine_readable/scorecards/0-ai-gatekeeper-protocol.scorecard.a2ml +++ b/.machine_readable/scorecards/0-ai-gatekeeper-protocol.scorecard.a2ml @@ -66,8 +66,8 @@ effects = "/home/user/standards/0-ai-gatekeeper-protocol/CONTRIBUTING.md instruc [[should]] id = "S3" -text = "The mcp-repo-guardian Deno/TypeScript test suite (manifest_test.js, 36 Deno.test cases covering parsing, security/injection, session and guard logic) SHOULD be executed automatically in CI on every push/PR." -system = "none (no workflow in .github/workflows/ or .gitlab-ci.yml under mcp-repo-guardian/ invokes `deno test`; deno.json defines a `test` task but nothing calls it in CI)" +text = "The mcp-repo-guardian Deno test suite (manifest_test.js, 36 Deno.test cases covering parsing, security/injection, session and guard logic) SHOULD be executed automatically in CI on every push/PR." +system = "Root `just test` invokes `deno test --allow-read test/manifest_test.js` directly; no deno.json task is required. No workflow in .github/workflows/ or .gitlab-ci.yml under mcp-repo-guardian/ invokes that command directly." status = "fail" effects = "Regressions in the TypeScript/ReScript-compiled manifest/session/guard logic (src/Manifest.mjs, Session.mjs, Guards.mjs) that ships as the actual MCP server enforcement path would not be caught until manually run; Claude/MCP-based consumers of mcp-repo-guardian are the primary enforcement channel per README.adoc, so this is the most consumer-facing untested surface." diff --git a/Justfile b/Justfile index 5286d2de..c30faf82 100644 --- a/Justfile +++ b/Justfile @@ -167,24 +167,14 @@ build: # Run all sub-project test suites test: - @echo "=== Standards Monorepo Test Runner ===" - @echo "" - @echo "[1/5] groove-protocol/reference/ipv6t (Zig — 10 tests)" - @cd groove-protocol/reference/ipv6t && zig build test 2>&1 || echo " SKIP: zig not available" - @echo "" - @echo "[2/5] 0-ai-gatekeeper-protocol/mcp-repo-guardian (Deno — 36 tests)" - @cd 0-ai-gatekeeper-protocol/mcp-repo-guardian && deno task test 2>&1 || echo " SKIP: deno not available or tests failed" - @echo "" - @echo "[3/5] axel-protocol (Deno — 14 tests)" - @cd axel-protocol && deno task test 2>&1 || echo " SKIP: deno not available or tests failed" - @echo "" - @echo "[4/5] a2ml/bindings/rust (Rust — 47 tests)" - @cd a2ml/bindings/rust && cargo test 2>&1 || echo " SKIP: cargo not available or tests failed" - @echo "" - @echo "[5/5] k9-svc/bindings/rust (Rust — 45 tests)" - @cd k9-svc/bindings/rust && cargo test 2>&1 || echo " SKIP: cargo not available or tests failed" - @echo "" - @echo "=== Test run complete ===" + @bash scripts/run-required-test-suite.sh "MCP repo guardian (Deno — 36 tests)" "0-ai-gatekeeper-protocol/mcp-repo-guardian" deno test --allow-read test/manifest_test.js + @bash scripts/run-required-test-suite.sh "Repo guardian offline core (Rust — 29 tests)" "0-ai-gatekeeper-protocol/repo-guardian-fs/tests-offline" cargo test + @bash scripts/run-required-test-suite.sh "A2ML Rust binding (47 tests + 3 doctests)" "a2ml/bindings/rust" cargo test + @bash scripts/run-required-test-suite.sh "K9 Rust binding (42 tests + 3 doctests)" "k9-svc/bindings/rust" cargo test + +# Regression test: test aggregation fails for missing prerequisites and test failures. +test-runner-test: + @bash scripts/tests/run-required-test-suite-test.sh # Format sub-project code fmt: diff --git a/TEST-NEEDS.adoc b/TEST-NEEDS.adoc index 7cd7d0db..49198e91 100644 --- a/TEST-NEEDS.adoc +++ b/TEST-NEEDS.adoc @@ -43,7 +43,7 @@ Zig: grv6; Deno: manifest (mcp-repo-guardian, repo-guardian-fs), a2ml, axel-protocol, groove-protocol, contractiles, and many more sub-projects -|*Unit tests* |158+ |Real tests across 6 test suites (see breakdown +|*Runnable automated tests* |154 |Four executable suites (see breakdown below) |*P2P (property) tests* |15+ |proptest in a2ml + k9-svc integration test @@ -61,13 +61,25 @@ files |*Fuzz tests* |0 |Placeholder removed; real fuzz TODO |=== -=== Test Suite Breakdown (as of 2026-04-04) +=== Test Suite Breakdown (current runnable suites) -==== groove-protocol/reference/ipv6t — 10 tests (Zig) +The root `+just test+` aggregate runs the four suites below and fails loudly +when a required test executable, target directory, or suite fails. It does not +claim a skipped suite as a pass. -Run: `+zig build test+` from `+groove-protocol/reference/ipv6t/+` +NOTE: `+build+`, `+fmt+`, and `+lint+` retain their older failure-swallowing +recipes (and `+build+` still refers to extracted Groove). They are adjacent +developer-experience debt, deliberately outside this focused test-runner fix. -All 10 tests pass. Cover all 5 spec validation scenarios + 5 property +==== groove-protocol — extracted from this monorepo + +The former `+groove-protocol/reference/ipv6t/+` Zig implementation was +extracted into the separate `+hyperpolymath/groove-protocol+` repository. It is +not a standards-local runnable suite. + +The following is the historical standards-local description of its 10 tests, +not a current result for this repository: all 10 covered 5 spec validation +scenarios + 5 property tests: - [x] Positive: correct type hash accepted - [x] Negative: wrong type hash rejected before payload parsing - [x] Provenance: 3 chained frames produce verifiable hash chain - [x] Fallback: raw bytes without @@ -84,7 +96,7 @@ hex format: ~57ns/iter (17.5 M/s) ==== mcp-repo-guardian — 36 tests (Deno/JS) -Run: `+deno task test+` from +Run: `+deno test --allow-read test/manifest_test.js+` from `+0-ai-gatekeeper-protocol/mcp-repo-guardian/+` All 36 tests pass. Tests cover: - [x] Manifest parsing (hash, canonical @@ -103,11 +115,14 @@ manifest handled - [x] *E2E dogfood*: parses standards repo’s own ~4µs/iter - Full manifest build: ~5.6µs/iter (hash + 5 regex + date) - Session lifecycle: ~7.7µs/iter -==== axel-protocol — 14 tests (Deno/TS) +==== axel-protocol — test infrastructure pending -Run: `+deno task test+` from `+axel-protocol/+` +The former Deno/TypeScript tests are now AffineScript sources. They are not +part of `+just test+`: the native AffineScript runner is not yet available and +the policy-validator fixture still needs an Ajv-compatible test shim. This is +test-infrastructure debt, not a passing or skipped suite. -All 14 tests pass. Tests cover: - [x] Valid AXEL1 DNS TXT record parsing +The pending tests cover: - [x] Valid AXEL1 DNS TXT record parsing - [x] Extra whitespace handling - [x] Unknown keys ignored - [x] id with equals signs (base64) - [x] Reject: missing version, empty payload, whitespace-only - [x] Reject: wrong version (AXEL2) - [x] Reject: @@ -148,7 +163,8 @@ Session: idempotent get_or_create Run: `+cargo test+` from `+a2ml/bindings/rust/+` -All 47 tests pass (11 inline unit tests + 36 CRG C integration tests). +All 47 non-documentation tests pass (11 inline unit tests + 36 CRG C +integration tests), alongside 3 doctests. CRG C integration tests (`+tests/crg_c_tests.rs+`): - [x] *Smoke*: version directive roundtrip, empty document, TrustLevel display (3 @@ -214,7 +230,10 @@ benchmarks (6 bench functions) covering all CRG C categories * [x] Created `+k9-svc/bindings/rust/benches/k9_bench.rs+` — Criterion benchmarks (6 bench functions) -* [x] All 158+ tests pass across all 6 test suites +* [x] The four runnable suites (MCP guardian, offline guardian core, A2ML Rust, + K9 Rust) pass through the fail-loud `+just test+` aggregate +* [ ] Restore an executable AXEL AffineScript test harness +* [ ] Verify the extracted Groove suite in its dedicated repository === What Was Fixed in Previous Session (2026-04-04, session 1) diff --git a/scripts/run-required-test-suite.sh b/scripts/run-required-test-suite.sh new file mode 100755 index 00000000..a3999191 --- /dev/null +++ b/scripts/run-required-test-suite.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# +# Run one required test suite without conflating a missing prerequisite with a +# passing (or skipped) suite. The aggregate `just test` recipe is an +# attestation about these suites, so each prerequisite and test failure must +# remain observable to its caller. + +set -uo pipefail + +if [ "$#" -lt 4 ]; then + echo "usage: $0 [tool-arg ...]" >&2 + exit 64 +fi + +suite_label="$1" +working_directory="$2" +tool="$3" +shift 3 + +if [ ! -d "$working_directory" ]; then + echo "MISSING TARGET: $suite_label: $working_directory is not a directory" >&2 + exit 66 +fi + +if ! command -v "$tool" >/dev/null 2>&1; then + echo "UNAVAILABLE: $suite_label: required executable '$tool' is not on PATH" >&2 + exit 127 +fi + +echo "=== $suite_label ===" +( + cd "$working_directory" || exit 66 + "$tool" "$@" +) +status=$? + +if [ "$status" -eq 0 ]; then + echo "PASS: $suite_label" + exit 0 +fi + +echo "FAILED (exit $status): $suite_label" >&2 +exit "$status" diff --git a/scripts/tests/run-required-test-suite-test.sh b/scripts/tests/run-required-test-suite-test.sh new file mode 100755 index 00000000..a8cef163 --- /dev/null +++ b/scripts/tests/run-required-test-suite-test.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# +# Behavioral regression test for the fail-loud aggregate test-suite helper. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +RUNNER="$ROOT/scripts/run-required-test-suite.sh" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +pass=0 +fail=0 + +ok() { + echo "PASS: $1" + pass=$((pass + 1)) +} + +bad() { + echo "FAIL: $1" >&2 + fail=$((fail + 1)) +} + +assert_case() { # label expected_status required_text command... + local label="$1" expected_status="$2" required_text="$3" + shift 3 + local output status + set +e + output="$("$@" 2>&1)" + status=$? + set -e + + if [ "$status" -eq "$expected_status" ] && printf '%s' "$output" | grep -Fq "$required_text" && ! printf '%s' "$output" | grep -Fq 'SKIP'; then + ok "$label" + else + bad "$label (wanted exit $expected_status and '$required_text'; got exit $status: $output)" + fi +} + +mkdir -p "$TMP/suite" "$TMP/bin" +cat > "$TMP/bin/passing-tool" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF +cat > "$TMP/bin/failing-tool" <<'EOF' +#!/usr/bin/env bash +exit 23 +EOF +chmod +x "$TMP/bin/passing-tool" "$TMP/bin/failing-tool" + +assert_case "available passing suite succeeds" 0 "PASS: passing suite" \ + env "PATH=$TMP/bin:$PATH" "$RUNNER" "passing suite" "$TMP/suite" passing-tool --test +assert_case "missing tool fails loudly" 127 "UNAVAILABLE: missing tool suite" \ + env "PATH=$TMP/bin:$PATH" "$RUNNER" "missing tool suite" "$TMP/suite" absent-tool --test +assert_case "failing test preserves its exit status" 23 "FAILED (exit 23): failing suite" \ + env "PATH=$TMP/bin:$PATH" "$RUNNER" "failing suite" "$TMP/suite" failing-tool --test +assert_case "missing target fails before tool invocation" 66 "MISSING TARGET: missing target suite" \ + env "PATH=$TMP/bin:$PATH" "$RUNNER" "missing target suite" "$TMP/absent" passing-tool --test + +test_recipe="$(sed -n '/^test:/,/^# Regression test/p' "$ROOT/Justfile")" +if printf '%s\n' "$test_recipe" | grep -Eq '\|\|[[:space:]]*echo.*SKIP|not available or tests failed'; then + bad "Justfile does not reintroduce a false-soft test aggregate" +else + ok "Justfile does not reintroduce a false-soft test aggregate" +fi + +echo "run-required-test-suite regression: $pass passed, $fail failed" +[ "$fail" -eq 0 ]