From 05061d4920c05543a78b86a6b979eeda05aaa813 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Mon, 31 Aug 2026 16:46:20 +0100 Subject: [PATCH 01/11] ci: centralize conformance in cf-integration Signed-off-by: lucarlig --- .github/workflows/ci.yml | 62 ++- .github/workflows/mcp_conformance.yml | 138 ------ .gitignore | 8 +- Makefile | 48 +- _context/wiki/getting-started.md | 16 +- _context/wiki/testing.md | 120 +---- docker/Dockerfile | 3 - docker/conformance.Dockerfile | 32 -- .../modern/client/external-data-plane.yml | 61 +++ .../2026-07-28/modern/external-data-plane.yml | 1 + .../2026-07-28/modern/fixture-direct.yml | 93 ++++ .../conformance/bless-client-baseline-test.sh | 51 --- tests/conformance/bless-client-baseline.sh | 99 ---- .../conformance/client-expected-failures.yml | 10 - tests/conformance/client-under-test-test.sh | 64 --- tests/conformance/client-under-test.sh | 100 ---- .../conformance/disable-flaky-progress.patch | 11 - tests/conformance/docker-compose.yml | 135 ------ tests/conformance/expected-failures.yml | 32 -- tests/conformance/fixture-proxy.conf | 18 - tests/conformance/nginx.conf.template | 65 --- tests/conformance/register-fixture.sh | 123 ----- .../conformance/report-baseline-diff-test.sh | 194 -------- tests/conformance/report-baseline-diff.sh | 433 ------------------ .../resolve-control-plane-image.sh | 33 -- tests/conformance/run-client-conformance.sh | 44 -- tests/conformance/run-conformance.sh | 30 -- tests/conformance/run-local.sh | 143 ------ .../conformance/start-dataplane-and-nginx.sh | 56 --- .../start-fixture-and-control-plane.sh | 28 -- tests/conformance/stop-live-stack.sh | 13 - .../conformance/upstream-fixture-failures.yml | 14 - tests/conformance/write_client_config.py | 71 --- 33 files changed, 278 insertions(+), 2071 deletions(-) delete mode 100644 .github/workflows/mcp_conformance.yml delete mode 100644 docker/conformance.Dockerfile create mode 100644 tests/conformance/baselines/2026-07-28/modern/client/external-data-plane.yml create mode 100644 tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml create mode 100644 tests/conformance/baselines/2026-07-28/modern/fixture-direct.yml delete mode 100755 tests/conformance/bless-client-baseline-test.sh delete mode 100755 tests/conformance/bless-client-baseline.sh delete mode 100644 tests/conformance/client-expected-failures.yml delete mode 100755 tests/conformance/client-under-test-test.sh delete mode 100755 tests/conformance/client-under-test.sh delete mode 100644 tests/conformance/disable-flaky-progress.patch delete mode 100644 tests/conformance/docker-compose.yml delete mode 100644 tests/conformance/expected-failures.yml delete mode 100644 tests/conformance/fixture-proxy.conf delete mode 100644 tests/conformance/nginx.conf.template delete mode 100755 tests/conformance/register-fixture.sh delete mode 100755 tests/conformance/report-baseline-diff-test.sh delete mode 100755 tests/conformance/report-baseline-diff.sh delete mode 100755 tests/conformance/resolve-control-plane-image.sh delete mode 100755 tests/conformance/run-client-conformance.sh delete mode 100755 tests/conformance/run-conformance.sh delete mode 100755 tests/conformance/run-local.sh delete mode 100755 tests/conformance/start-dataplane-and-nginx.sh delete mode 100755 tests/conformance/start-fixture-and-control-plane.sh delete mode 100755 tests/conformance/stop-live-stack.sh delete mode 100644 tests/conformance/upstream-fixture-failures.yml delete mode 100755 tests/conformance/write_client_config.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 506211d4..70b90020 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,20 +81,6 @@ jobs: - name: Build the workspace run: cargo build --locked --workspace --all-features - - name: Prepare the conformance binary artifact - run: | - mkdir -p target/conformance - cp target/debug/contextforge-data-plane target/conformance/contextforge-data-plane - strip target/conformance/contextforge-data-plane - - - name: Upload the conformance binary - uses: actions/upload-artifact@v7.0.1 - with: - name: contextforge-data-plane-conformance - path: target/conformance/contextforge-data-plane - if-no-files-found: error - retention-days: 1 - bench: runs-on: ubuntu-latest steps: @@ -104,5 +90,51 @@ jobs: - run: cargo bench --locked --workspace --no-run conformance: + name: conformance needs: build - uses: ./.github/workflows/mcp_conformance.yml + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Check out data plane + uses: actions/checkout@v6.0.2 + with: + fetch-depth: 0 + + - name: Check out cf-integration + uses: actions/checkout@v6.0.2 + with: + repository: contextforge-org/contextforge-dev-tools + ref: a056b269d28c25702339c81f35afcce33649beb7 + path: target/cf-integration-src + persist-credentials: false + + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2.9.1 + with: + workspaces: target/cf-integration-src -> .integration/cargo-target + + - name: Install Node.js 22 + uses: actions/setup-node@v6.5.0 + with: + node-version: 22 + + - name: Build cf-integration + working-directory: target/cf-integration-src + run: cargo build --release --locked --bin cf-integration + + - name: Run strict modern conformance + env: + CF_INTEGRATION: ${{ github.workspace }}/target/cf-integration-src/.integration/cargo-target/release/cf-integration + run: make conformance + + - name: Upload conformance diagnostics + if: failure() + uses: actions/upload-artifact@v7.0.1 + with: + name: conformance-diagnostics-${{ github.run_id }}-${{ github.run_attempt }} + path: | + .integration/conformance/ + .integration/reports/ + if-no-files-found: warn + include-hidden-files: true + retention-days: 7 diff --git a/.github/workflows/mcp_conformance.yml b/.github/workflows/mcp_conformance.yml deleted file mode 100644 index 6b114bf8..00000000 --- a/.github/workflows/mcp_conformance.yml +++ /dev/null @@ -1,138 +0,0 @@ -name: MCP Conformance - -on: - workflow_call: - -permissions: - contents: read - -env: - MCP_CONFORMANCE_COLOR: always - MCP_CONFORMANCE_VERSION: 0.2.0-alpha.11 - MCP_CONFORMANCE_SOURCE_SHA: c321dd32035556e6769d3724a8ee97d87c3faaac # pragma: allowlist secret - MCP_CONFORMANCE_SPEC_VERSION: 2026-07-28 - MCP_CONFORMANCE_SERVER_ID: 3f33286667d34b65a31c3bafd30e4c21 - CF_DATAPLANE_IMAGE: contextforge-data-plane:conformance - -jobs: - conformance: - name: MCP 2026-07-28 through nginx and dataplane - runs-on: ubuntu-latest - continue-on-error: true - timeout-minutes: 30 - - steps: - - name: Check out data plane - uses: actions/checkout@v6.0.2 - - - name: Test conformance reporter - run: tests/conformance/report-baseline-diff-test.sh - - - name: Test client conformance adapter - run: | - python3 -m py_compile tests/conformance/write_client_config.py - tests/conformance/client-under-test-test.sh - tests/conformance/bless-client-baseline-test.sh - - - name: Download the conformance binary - uses: actions/download-artifact@v8.0.1 - with: - name: contextforge-data-plane-conformance - path: target/conformance - - - name: Make the conformance binary executable - run: chmod +x target/conformance/contextforge-data-plane - - - name: Package the data plane conformance image - run: >- - docker buildx build --load - --target conformance-prebuilt - --build-context prebuilt=target/conformance - --tag "${CF_DATAPLANE_IMAGE}" - --file docker/Dockerfile . - - - name: Check out the alpha.11 conformance fixture - uses: actions/checkout@v6.0.2 - with: - repository: modelcontextprotocol/conformance - ref: ${{ env.MCP_CONFORMANCE_SOURCE_SHA }} - path: .conformance-suite - persist-credentials: false - - - name: Install Node.js 22 - uses: actions/setup-node@v6.5.0 - with: - node-version: 22 - cache: npm - cache-dependency-path: .conformance-suite/package-lock.json - - - name: Install official conformance dependencies - working-directory: .conformance-suite - run: | - test "$(node -p "require('./package.json').version")" = "${MCP_CONFORMANCE_VERSION}" - npm ci --ignore-scripts - - # The pinned conformance CLI has no scenario-exclusion option. Apply an - # exact patch so this flaky scenario is not run until forwarding is ordered. - - name: Temporarily disable the flaky progress scenario - working-directory: .conformance-suite - run: git apply ../tests/conformance/disable-flaky-progress.patch - - - name: Resolve latest control-plane main image - env: - GITHUB_TOKEN: ${{ github.token }} - run: echo "CF_CONTROLPLANE_IMAGE=$(tests/conformance/resolve-control-plane-image.sh)" >> "${GITHUB_ENV}" - - - name: Pull external stack images - env: - MCP_CONFORMANCE_TOKEN: pull-only - run: >- - docker compose -f tests/conformance/docker-compose.yml - pull redis fixture-proxy control-plane nginx - - - name: Start the fixture and control plane - env: - MCP_CONFORMANCE_TOKEN: bootstrap-only - run: tests/conformance/start-fixture-and-control-plane.sh - - - name: Register the official fixture through the control plane - env: - MCP_CONFORMANCE_TOKEN: bootstrap-only - run: tests/conformance/register-fixture.sh - - - name: Start dataplane and nginx - run: tests/conformance/start-dataplane-and-nginx.sh - - - name: Run MCP 2026-07-28 requirements - id: runner - run: tests/conformance/run-conformance.sh - - - name: Run scoped MCP 2026-07-28 client conformance - run: tests/conformance/run-client-conformance.sh - - - name: Report conformance baseline diff - if: always() - run: tests/conformance/report-baseline-diff.sh conformance-results/server - - - name: Print live stack logs - if: always() - env: - MCP_CONFORMANCE_TOKEN: diagnostics-only - run: >- - docker compose -f tests/conformance/docker-compose.yml - logs --no-color || true - - - name: Print official fixture log - if: always() - run: | - if [ -f conformance-logs/reference-server.log ]; then - sed -n '1,240p' conformance-logs/reference-server.log - else - echo "No official fixture log was produced." - fi - - - name: Stop the live stack - if: always() - env: - MCP_CONFORMANCE_TOKEN: cleanup-only - run: tests/conformance/stop-live-stack.sh diff --git a/.gitignore b/.gitignore index 5c8c16ae..1f66ef2f 100644 --- a/.gitignore +++ b/.gitignore @@ -15,10 +15,8 @@ target # Generated by gateway local runs contextforge-data-plane.log.* -# Generated by local MCP conformance runs -.conformance-suite/ -conformance-logs/ -conformance-results/ +# Generated by cf-integration +.integration/ # Generated by mdBook (wiki) _context/wiki/book/ @@ -34,4 +32,4 @@ _context/wiki/book/ # Personal dev tooling (wt worktree manager, etc.) -.config/ \ No newline at end of file +.config/ diff --git a/Makefile b/Makefile index 78ea1645..d2aba0f0 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,11 @@ IMAGE_NAME := contextforge-data-plane:latest -CF_DATAPLANE_IMAGE ?= contextforge-data-plane:conformance SERVICES ?= nginx control-plane redis postgres pgbouncer data-plane fast_time_server register_fast_time ARGS ?= +CF_INTEGRATION ?= cf-integration +CF_INTEGRATION_DIR ?= $(CURDIR)/.integration +CF_DATAPLANE_REPO ?= $(CURDIR) +CF_DATAPLANE_REF ?= $(shell git -C "$(CF_DATAPLANE_REPO)" rev-parse HEAD) +CONFORMANCE_BASELINE_DIR := $(CURDIR)/tests/conformance/baselines # IBM detect-secrets hardened fork — pinned to the same commit used in mcp-context-forge. DETECT_SECRETS_SPEC ?= git+https://github.com/ibm/detect-secrets.git@076672a9a01abdfc7ecee2e7d14f08cdccb73976 @@ -25,12 +29,44 @@ compose-up: ## Launch stack: nginx, control plane, redis, postgres, pgbouncer, d compose-down: ## Tear down the stack docker compose -f docker/docker-compose.yml stop $(SERVICES) $(ARGS) -conformance: ## Build the data plane and run official MCP 2026-07-28 conformance locally - docker build -t "$(CF_DATAPLANE_IMAGE)" -f docker/conformance.Dockerfile . - CF_DATAPLANE_IMAGE="$(CF_DATAPLANE_IMAGE)" tests/conformance/run-local.sh +conformance: ## Run strict modern MCP conformance against the committed data-plane HEAD + @if ! command -v "$(CF_INTEGRATION)" >/dev/null 2>&1; then \ + echo "cf-integration not found: set CF_INTEGRATION to the built CLI path."; \ + exit 1; \ + fi + @if [ -n "$$(git -C "$(CF_DATAPLANE_REPO)" status --porcelain --untracked-files=no)" ]; then \ + echo "Tracked data-plane changes are not committed; commit or stash them before conformance."; \ + exit 1; \ + fi + @CF_INTEGRATION_DIR="$(CF_INTEGRATION_DIR)" \ + CF_DATAPLANE_REPO="$(CF_DATAPLANE_REPO)" \ + CF_DATAPLANE_REF="$(CF_DATAPLANE_REF)" \ + "$(CF_INTEGRATION)" conformance run \ + --client-era modern \ + --server-era modern \ + --lane external-data-plane \ + --baseline-dir "$(CONFORMANCE_BASELINE_DIR)" \ + --output-dir "$(CF_INTEGRATION_DIR)/reports" -conformance-bless: ## Run conformance and update the server and client expected-failure baselines - MCP_CONFORMANCE_BLESS=true $(MAKE) conformance +conformance-bless: ## Run strict modern conformance and atomically update its baselines + @if ! command -v "$(CF_INTEGRATION)" >/dev/null 2>&1; then \ + echo "cf-integration not found: set CF_INTEGRATION to the built CLI path."; \ + exit 1; \ + fi + @if [ -n "$$(git -C "$(CF_DATAPLANE_REPO)" status --porcelain --untracked-files=no)" ]; then \ + echo "Tracked data-plane changes are not committed; commit or stash them before conformance."; \ + exit 1; \ + fi + @CF_INTEGRATION_DIR="$(CF_INTEGRATION_DIR)" \ + CF_DATAPLANE_REPO="$(CF_DATAPLANE_REPO)" \ + CF_DATAPLANE_REF="$(CF_DATAPLANE_REF)" \ + "$(CF_INTEGRATION)" conformance run \ + --client-era modern \ + --server-era modern \ + --lane external-data-plane \ + --baseline-dir "$(CONFORMANCE_BASELINE_DIR)" \ + --output-dir "$(CF_INTEGRATION_DIR)/reports" \ + --bless docs-serve: ## Serve the wiki book locally at http://127.0.0.1:3000 mdbook serve _context/wiki --hostname 127.0.0.1 --port 3000 --open diff --git a/_context/wiki/getting-started.md b/_context/wiki/getting-started.md index 31bf95dd..979fa444 100644 --- a/_context/wiki/getting-started.md +++ b/_context/wiki/getting-started.md @@ -26,18 +26,18 @@ docker compose -f docker/docker-compose.yml logs -f register_fast_time Teardown: `make compose-down` (stops containers; volumes kept). -## cf-integration Harness (full end-to-end) +## cf-integration Conformance ```bash -scripts/cf-integration.sh up # checkout Python control/built-in repo, pull external-dataplane image, start full stack -scripts/cf-integration.sh probe # smoke: 401 check → initialize → tools/list → tools/call -scripts/cf-integration.sh test-all # all lanes: live-mcp, live-rbac, live-protocol -scripts/cf-integration.sh down +CF_INTEGRATION=../contextforge-dev-tools/.integration/cargo-target/release/cf-integration \ + make conformance ``` -Admin UI (control-plane): `http://localhost:8080/admin` — `admin@example.com` / `changeme` - -Key env overrides: `CF_DATAPLANE_IMAGE`, `CF_DATAPLANE_VERSION`, `NGINX_PORT` (default `8080`). +This runs the modern client and modern server eras through the committed +external-dataplane `HEAD`, including fixture-direct server comparison and the +scoped client suite. Use `make conformance-bless` to replace all selected +baselines transactionally after a fully successful run. Generated checkouts, +results, reports, and logs stay under `.integration/`. ## Local Cargo Dev Workflow diff --git a/_context/wiki/testing.md b/_context/wiki/testing.md index 3d811d40..c4139416 100644 --- a/_context/wiki/testing.md +++ b/_context/wiki/testing.md @@ -34,115 +34,41 @@ Protocol-sensitive tests and fixtures must cover MCP `2026-07-28` and `2025-11-2 These run in `cargo nextest run` with no Docker dependencies. -## MCP Conformance CI - -`.github/workflows/mcp_conformance.yml` runs the pinned official conformance -suite `0.2.0-alpha.11` for MCP `2026-07-28` in both directions. The server leg -is official client → nginx → checked-out external dataplane → fixture proxy -→ official server, with the newest available image built from the control plane's `main` -branch registering and publishing the fixture through Redis. The backend-only -proxy rewrites `Host` to -`localhost:3000`, which the official fixture's DNS-rebinding protection -requires, while leaving external-dataplane header protections unchanged. The -control plane uses ephemeral SQLite, so PostgreSQL is unnecessary. The harness lives -in `tests/conformance/`. - -The scoped client leg then treats the external dataplane as an MCP client: the -official runner starts a scenario backend, the adapter publishes an isolated -route to Redis, and a downstream `tools/call` makes the external dataplane -connect to that backend. It covers tool calls, per-request client metadata and -protocol-version retry, standard MCP headers, and custom parameter headers. The -control plane is stopped first so its periodic publisher cannot replace the -scenario route or probe the observation backend. OAuth client scenarios remain -a control-plane responsibility. Server and client results are written below -`server/` and `client/`, with separate `expected-failures.yml` and -`client-expected-failures.yml` baselines. - -The official fixture keeps some diagnostic tools out of `tools/list`. Because -the external dataplane fails closed unless the control plane publishes a tool -schema, checks that require those hidden tools remain explicit server-leg -baseline entries rather than bypassing schema validation in the harness. - -`make conformance` runs both legs locally, while `make conformance-bless` runs -both and refreshes both expected-failure baselines from that run. - -Because this conformance CLI cannot set a bearer header, nginx adds an -ephemeral control-plane token when one is absent; there is no auth proxy or -repository-owned JavaScript. A route probe prevents built-in-dataplane fallback. -Counts and the official fixture log appear directly in the Actions log. The -job does not retain a separate conformance artifact. -`upstream-fixture-failures.yml` records the pinned fixture's seven scored -failures and one warning; its other 47 failures are extension or pending -scenarios and are already unscored. CI prints the exact server -actual-versus-baseline diff, adds annotations for unexpected and stale entries, -and writes the same comparison to the job summary. - -## Full-Stack Integration Harness +## MCP Conformance [`cf-integration`](https://github.com/contextforge-org/contextforge-dev-tools) -wires the ContextForge control plane, built-in dataplane, and this ContextForge -external dataplane together. The stock Python Compose stack contains both the -control plane and built-in dataplane. The harness adds two intentional -differences: nginx routes the selected `/servers/{virtual_host_id}/mcp` path to -the external dataplane as `/contextforge-rs/servers/{virtual_host_id}/mcp`, and -the control plane runs with `DATAPLANE_PUBLISHER=true` so virtual-server config -reaches the external dataplane through Redis. +owns the official fixture, control-plane registration, Compose topology, server +and client runners, result rendering, and transactional baseline handling. This +repository keeps only the CI invocation, Make targets, and expected findings. -### Quick Start +The blocking CI job runs the modern client and modern server eras through the +external dataplane. Selecting that lane also runs the fixture-direct server +leg and the scoped external-dataplane client leg: ```bash -scripts/cf-integration.sh up +make conformance ``` -This checks out the Python control-plane/built-in-dataplane repository under -`.integration/mcp-context-forge`, pulls the published external-dataplane image, -and starts the combined stack plus a local MCP counter backend. The admin UI is -at `http://localhost:8080/admin` (`admin@example.com` / `changeme`). A Fast Time -backend is auto-registered as a fixed virtual server, so the commands below -work with no manual UI step. - -### Route Probe +The Make target tests the committed data-plane `HEAD`. It rejects tracked +uncommitted changes because the CLI clones the selected repository and commit +into `.integration/`. To use a locally built CLI from a devtools branch: ```bash -scripts/cf-integration.sh probe +CF_INTEGRATION=../contextforge-dev-tools/.integration/cargo-target/release/cf-integration \ + make conformance ``` -Verifies the public nginx-to-external-dataplane route end to end: a 401 negative check, `initialize`, session reuse, `tools/list`, and `tools/call`. - -### Full Test Runs - -| Command | What it runs | -| --- | --- | -| `scripts/cf-integration.sh test-all` | Every live lane against the running stack, with per-test result rows and full output in a timestamped log under `.integration/test-logs/`. | -| `CF_TEST_ALL_LOCUST=true scripts/cf-integration.sh test-all` | Same, plus the full Locust load run as a final lane. | -| `scripts/cf-integration.sh test-all-up` | Start or update the stack, then `test-all` without the load lane. | -| `scripts/cf-integration.sh test-all-up-load` | Start or update the stack, then `test-all` with the load lane. | - -Individual lanes: `live-mcp`, `live-rbac`, `live-protocol`, and `live-all`. -`live-mcp` is the green lane: the full MCP protocol end-to-end suite passes -against this harness. Remaining failures in other lanes measure known -external-dataplane feature gaps; the harness `reports/` directory keeps the -current classification. - -### Built-In-Dataplane Baseline - -To separate external-dataplane regressions from Python behavior, the harness -can run the stock `IBM/mcp-context-forge` stack. MCP traffic then uses the -ContextForge built-in dataplane; the external dataplane, nginx split, and -publisher are absent: +Update every selected baseline atomically only after all operational work and +baseline evaluation succeeds: ```bash -scripts/cf-integration.sh down # frees the shared host ports -scripts/cf-integration.sh controlplane-test-all # up + live core + locust +CF_INTEGRATION=../contextforge-dev-tools/.integration/cargo-target/release/cf-integration \ + make conformance-bless ``` -Individual steps: `controlplane-up`, `controlplane-live-core`, `controlplane-live-all`, `controlplane-locust`, and `controlplane-down`. The baseline load run is covered in [Performance](performance.md). - -### Key Settings - -| Variable | Purpose | -| --- | --- | -| `CF_DATAPLANE_IMAGE` / `CF_DATAPLANE_VERSION` | Which published external-dataplane image the stack runs. | -| `CF_CONTROLPLANE_IMAGE` / `CF_CONTROLPLANE_REF` | Which `IBM/mcp-context-forge` Python image and git ref to use for the control plane and built-in dataplane. | -| `NGINX_PORT` | Public front-door port (default `8080`). | -| `CF_TEST_LOG_DIR` | Where `test-all` writes timestamped logs. | +Baselines are partitioned beneath +`tests/conformance/baselines///`. Server findings +use `fixture-direct.yml` and `external-data-plane.yml`; scoped client findings +use `client/external-data-plane.yml`. Operational failures are always failures +and cannot be blessed. Runtime checkouts, logs, results, and reports remain +beneath `.integration/`. diff --git a/docker/Dockerfile b/docker/Dockerfile index 7f8a9b6f..4be00c22 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -26,8 +26,5 @@ LABEL org.opencontainers.image.source=https://github.com/contextforge-org/contex LABEL org.opencontainers.image.description="contextforge-data-plane - open source experimental data plane for ContextForge." ENTRYPOINT ["/contextforge-data-plane"] -FROM runtime AS conformance-prebuilt -COPY --from=prebuilt /contextforge-data-plane /contextforge-data-plane - FROM runtime COPY --from=builder /app/target/release/contextforge-data-plane /contextforge-data-plane diff --git a/docker/conformance.Dockerfile b/docker/conformance.Dockerfile deleted file mode 100644 index 494cd43d..00000000 --- a/docker/conformance.Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# syntax=docker/dockerfile:1 - -FROM rust:1.96.1 AS builder -WORKDIR /app - -RUN < "${baseline_file}" <<'EOF' -client: [] -EOF -for scenario in tools_call request-metadata http-standard-headers; do - cat > "${results_dir}/${scenario}-2026-08-20T00-00-00-000Z/checks.json" <<'EOF' -[{"id":"passes","status":"SUCCESS"}] -EOF -done -cat > "${results_dir}/http-custom-headers-2026-08-20T00-00-00-000Z/checks.json" <<'EOF' -[ - {"id":"mirrors","status":"SUCCESS"}, - {"id":"mirrors","status":"FAILURE"}, - {"id":"encodes","status":"WARNING"}, - {"id":"omits-null","status":"SUCCESS"} -] -EOF - -"${script_dir}/bless-client-baseline.sh" "${results_dir}" "${baseline_file}" - -cat > "${state_dir}/expected.yml" <<'EOF' -# Generated by `make conformance-bless` from scoped client findings. -# OAuth client scenarios are control-plane responsibilities and are not run here. -client: - - http-custom-headers:encodes - - http-custom-headers:mirrors -EOF -diff -u "${state_dir}/expected.yml" "${baseline_file}" - -second_output="$("${script_dir}/bless-client-baseline.sh" "${results_dir}" "${baseline_file}")" -grep --fixed-strings --quiet -- 'was already current' <<< "${second_output}" - -echo 'client conformance baseline tests passed' diff --git a/tests/conformance/bless-client-baseline.sh b/tests/conformance/bless-client-baseline.sh deleted file mode 100755 index d27af6e4..00000000 --- a/tests/conformance/bless-client-baseline.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [ "$#" -gt 2 ]; then - echo "Usage: bless-client-baseline.sh [results-dir [baseline-file]]" >&2 - exit 2 -fi - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(cd -- "${script_dir}/../.." && pwd)" -results_dir="${1:-${repo_root}/conformance-results/client}" -baseline_file="${2:-${script_dir}/client-expected-failures.yml}" -scenarios=(tools_call request-metadata http-standard-headers http-custom-headers) - -for command in awk cmp cp jq mktemp sed sort; do - if ! command -v "${command}" > /dev/null 2>&1; then - echo "Required command not found: ${command}" >&2 - exit 2 - fi -done -if [ ! -d "${results_dir}" ]; then - echo "No client conformance results directory: ${results_dir}" >&2 - exit 2 -fi -if [ ! -f "${baseline_file}" ]; then - echo "Client conformance baseline not found: ${baseline_file}" >&2 - exit 2 -fi - -state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-client-baseline.XXXXXX")" -findings="${state_dir}/findings.txt" -baseline_entries="${state_dir}/baseline-entries.txt" -candidate="${state_dir}/client-expected-failures.yml" - -cleanup() { - rm -f -- "${findings}" "${baseline_entries}" "${candidate}" - rmdir -- "${state_dir}" -} -trap cleanup EXIT INT TERM - -: > "${findings}" -for scenario in "${scenarios[@]}"; do - matches=("${results_dir}/${scenario}-"*/checks.json) - if [ ! -e "${matches[0]}" ]; then - echo "No client conformance results found for scenario: ${scenario}" >&2 - exit 2 - fi - checks_file="${matches[$((${#matches[@]} - 1))]}" - jq --raw-output --arg scenario "${scenario}" ' - def severity($status): - if $status == "FAILURE" then 3 - elif $status == "WARNING" then 2 - elif $status == "SUCCESS" then 1 - else 0 - end; - - reduce (.[] | select(.status != "INFO")) as $check - ({}; - ($check.id) as $id | - if .[$id] == null or severity($check.status) >= severity(.[$id].status) - then .[$id] = $check - else . - end) | - to_entries[] | - select(.value.status == "FAILURE" or .value.status == "WARNING") | - ($scenario + ":" + .key) - ' "${checks_file}" >> "${findings}" -done -LC_ALL=C sort -u -o "${findings}" "${findings}" - -awk ' - /^client:$/ { in_client = 1; next } - in_client && /^[^[:space:]]/ { exit } - in_client && /^[[:space:]]*-[[:space:]]+/ { - line = $0 - sub(/^[[:space:]]*-[[:space:]]+/, "", line) - sub(/[[:space:]]+#.*$/, "", line) - print line - } -' "${baseline_file}" | LC_ALL=C sort -u > "${baseline_entries}" - -{ - # shellcheck disable=SC2016 # Backticks are literal Markdown. - echo '# Generated by `make conformance-bless` from scoped client findings.' - echo '# OAuth client scenarios are control-plane responsibilities and are not run here.' - if [ -s "${findings}" ]; then - echo 'client:' - sed 's/^/ - /' "${findings}" - else - echo 'client: []' - fi -} > "${candidate}" - -if cmp --silent "${findings}" "${baseline_entries}"; then - echo "Client conformance baseline was already current: ${baseline_file}" -else - cp "${candidate}" "${baseline_file}" - echo "Client conformance baseline updated: ${baseline_file}" -fi diff --git a/tests/conformance/client-expected-failures.yml b/tests/conformance/client-expected-failures.yml deleted file mode 100644 index 5f19d937..00000000 --- a/tests/conformance/client-expected-failures.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Dataplane-owned upstream MCP client findings for the scoped client lane. -# OAuth scenarios are control-plane responsibilities and are not run here. -client: - # The shell adapter drives the dataplane's outbound client path but is not a - # full MCP client: it does not discover x-mcp-header annotations or generate - # Mcp-Param-* headers. Header forwarding is covered by gateway integration tests. - - http-custom-headers:sep-2243-client-supports-custom-headers - - http-custom-headers:sep-2243-client-mirrors-designated-params - - http-custom-headers:sep-2243-client-encode-values - - http-custom-headers:sep-2243-client-base64-unsafe diff --git a/tests/conformance/client-under-test-test.sh b/tests/conformance/client-under-test-test.sh deleted file mode 100755 index 3387f24a..00000000 --- a/tests/conformance/client-under-test-test.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-client-adapter-test.XXXXXX")" -fake_bin="${state_dir}/bin" -docker_args="${state_dir}/docker-args" -curl_bodies="${state_dir}/curl-bodies" - -cleanup() { - rm -rf -- "${state_dir}" -} -trap cleanup EXIT INT TERM - -mkdir -p "${fake_bin}" -cat > "${fake_bin}/docker" <<'EOF' -#!/usr/bin/env bash -printf '%s\n' "$*" > "${FAKE_DOCKER_ARGS}" -EOF -cat > "${fake_bin}/curl" <<'EOF' -#!/usr/bin/env bash -while [ "$#" -gt 0 ]; do - if [ "$1" = "--data" ]; then - shift - printf '%s\n' "$1" >> "${FAKE_CURL_BODIES}" - fi - shift -done -printf '%s\n' 'data: {"jsonrpc":"2.0","id":1,"result":{"content":[]}}' -EOF -chmod +x "${fake_bin}/docker" "${fake_bin}/curl" - -export PATH="${fake_bin}:${PATH}" -export FAKE_DOCKER_ARGS="${docker_args}" -export FAKE_CURL_BODIES="${curl_bodies}" -export MCP_CONFORMANCE_PROTOCOL_VERSION=2026-07-28 -export MCP_CONFORMANCE_SUBJECT=test-subject -export MCP_CONFORMANCE_CLIENT_SERVER_ID=test-client-server -export MCP_CONFORMANCE_PORT=18080 -export MCP_CONFORMANCE_SCENARIO=http-custom-headers -export MCP_CONFORMANCE_CONTEXT='{ - "name": "http-custom-headers", - "toolCalls": [ - {"name": "first", "arguments": {"region": "west"}}, - {"name": "second", "arguments": {"verbose": null}} - ] -}' - -"${script_dir}/client-under-test.sh" "http://localhost:43123/mcp" - -grep --fixed-strings --quiet -- 'http://host.docker.internal:43123/mcp' "${docker_args}" -grep --fixed-strings --quiet -- '["first","second"]' "${docker_args}" -test "$(wc -l < "${curl_bodies}" | tr -d '[:space:]')" -eq 2 -jq --exit-status --slurp ' - length == 2 and - .[0].method == "tools/call" and - .[0].params.name == "first" and - .[0].params.arguments.region == "west" and - .[0].params._meta["io.modelcontextprotocol/protocolVersion"] == "2026-07-28" and - .[1].params.name == "second" and - .[1].params.arguments.verbose == null -' "${curl_bodies}" > /dev/null - -echo 'client conformance adapter tests passed' diff --git a/tests/conformance/client-under-test.sh b/tests/conformance/client-under-test.sh deleted file mode 100755 index 909ee2f1..00000000 --- a/tests/conformance/client-under-test.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -: "${MCP_CONFORMANCE_SCENARIO:?MCP_CONFORMANCE_SCENARIO must be set by the conformance runner}" -: "${MCP_CONFORMANCE_PROTOCOL_VERSION:?MCP_CONFORMANCE_PROTOCOL_VERSION must be set by the conformance runner}" -: "${MCP_CONFORMANCE_SUBJECT:?MCP_CONFORMANCE_SUBJECT must be set}" - -if [ "$#" -ne 1 ]; then - echo "Usage: client-under-test.sh " >&2 - exit 2 -fi -if [ "${MCP_CONFORMANCE_PROTOCOL_VERSION}" != "2026-07-28" ]; then - echo "Only MCP 2026-07-28 client conformance is supported" >&2 - exit 2 -fi - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -compose_file="${script_dir}/docker-compose.yml" -virtual_host_id="${MCP_CONFORMANCE_CLIENT_SERVER_ID:-dataplane-client-conformance}" -conformance_port="${MCP_CONFORMANCE_PORT:-8080}" -scenario_server_url="$1" -backend_url="${scenario_server_url/\/\/localhost:/\/\/host.docker.internal:}" -backend_url="${backend_url/\/\/127.0.0.1:/\/\/host.docker.internal:}" - -case "${MCP_CONFORMANCE_SCENARIO}" in - tools_call) - tool_calls='[{"name":"add_numbers","arguments":{"a":2,"b":3}}]' - ;; - request-metadata) - tool_calls='[{"name":"metadata_probe","arguments":{}}]' - ;; - http-standard-headers) - tool_calls='[{"name":"test_headers","arguments":{}}]' - ;; - http-custom-headers) - : "${MCP_CONFORMANCE_CONTEXT:?MCP_CONFORMANCE_CONTEXT is required for http-custom-headers}" - tool_calls="$(jq --exit-status --compact-output '.toolCalls' <<< "${MCP_CONFORMANCE_CONTEXT}")" - ;; - *) - echo "Unsupported dataplane client conformance scenario: ${MCP_CONFORMANCE_SCENARIO}" >&2 - exit 2 - ;; -esac - -tool_names="$(jq --exit-status --compact-output '[.[].name] | unique' <<< "${tool_calls}")" -docker compose -f "${compose_file}" run --rm --no-deps \ - --entrypoint python3 control-plane \ - /opt/contextforge-conformance/write_client_config.py \ - "${MCP_CONFORMANCE_SUBJECT}" \ - "${virtual_host_id}" \ - "${backend_url}" \ - "${tool_names}" \ - > /dev/null - -endpoint="http://127.0.0.1:${conformance_port}/servers/${virtual_host_id}/mcp" -while IFS= read -r tool_call; do - tool_name="$(jq --exit-status --raw-output '.name' <<< "${tool_call}")" - arguments="$(jq --exit-status --compact-output '.arguments' <<< "${tool_call}")" - request="$(jq --null-input --compact-output \ - --arg name "${tool_name}" \ - --argjson arguments "${arguments}" \ - --arg version "${MCP_CONFORMANCE_PROTOCOL_VERSION}" \ - '{ - jsonrpc: "2.0", - id: 1, - method: "tools/call", - params: { - name: $name, - arguments: $arguments, - _meta: { - "io.modelcontextprotocol/protocolVersion": $version, - "io.modelcontextprotocol/clientInfo": { - name: "dataplane-client-conformance-driver", - version: "1.0.0" - }, - "io.modelcontextprotocol/clientCapabilities": {} - } - } - }')" - - response="$(curl --silent --show-error --fail-with-body \ - --request POST \ - --header 'Content-Type: application/json' \ - --header 'Accept: application/json, text/event-stream' \ - --header "MCP-Protocol-Version: ${MCP_CONFORMANCE_PROTOCOL_VERSION}" \ - --header 'MCP-Method: tools/call' \ - --header "MCP-Name: ${tool_name}" \ - --data "${request}" \ - "${endpoint}")" - - response_json="$(sed -n 's/^data: //p' <<< "${response}" | head -n 1)" - if [ -z "${response_json}" ]; then - response_json="${response}" - fi - if ! jq --exit-status '.result != null and .error == null' <<< "${response_json}" > /dev/null; then - echo "Dataplane rejected client conformance tool call ${tool_name}:" >&2 - echo "${response}" >&2 - exit 1 - fi -done < <(jq --compact-output '.[]' <<< "${tool_calls}") diff --git a/tests/conformance/disable-flaky-progress.patch b/tests/conformance/disable-flaky-progress.patch deleted file mode 100644 index 41b4fc11..00000000 --- a/tests/conformance/disable-flaky-progress.patch +++ /dev/null @@ -1,11 +0,0 @@ -diff --git a/requirements/2026-07-28.yaml b/requirements/2026-07-28.yaml ---- a/requirements/2026-07-28.yaml -+++ b/requirements/2026-07-28.yaml -@@ -45,7 +45,6 @@ server: - - tools-call-embedded-resource - - tools-call-mixed-content - - tools-call-error -- - tools-call-with-progress - - server-sse-multiple-streams - - resources-list - - resources-read-text diff --git a/tests/conformance/docker-compose.yml b/tests/conformance/docker-compose.yml deleted file mode 100644 index d16e8fcb..00000000 --- a/tests/conformance/docker-compose.yml +++ /dev/null @@ -1,135 +0,0 @@ -name: contextforge-conformance - -services: - redis: - image: redis:8.8.1-alpine3.23 - command: [redis-server, --save, "", --appendonly, "no"] - networks: [contextforge] - healthcheck: - test: [CMD, redis-cli, ping] - interval: 2s - timeout: 2s - retries: 30 - - fixture-proxy: - image: nginx:1.30.4-alpine3.24 - networks: [contextforge] - extra_hosts: - - host.docker.internal:host-gateway - volumes: - - ./fixture-proxy.conf:/etc/nginx/conf.d/default.conf:ro - healthcheck: - test: - - CMD-SHELL - - >- - curl --fail --silent --show-error --output /dev/null - --request POST - --header 'Content-Type: application/json' - --header 'Accept: application/json, text/event-stream' - --header 'MCP-Protocol-Version: 2026-07-28' - --header 'MCP-Method: server/discover' - --data '{"jsonrpc":"2.0","id":"health","method":"server/discover","params":{"_meta":{"io.modelcontextprotocol/protocolVersion":"2026-07-28","io.modelcontextprotocol/clientCapabilities":{}}}}' - http://127.0.0.1/mcp - interval: 1s - timeout: 5s - retries: 60 - - control-plane: - image: ${CF_CONTROLPLANE_IMAGE:-ghcr.io/ibm/mcp-context-forge:latest} - ports: - - "127.0.0.1:4444:4444" - networks: [contextforge] - environment: - HOST: 0.0.0.0 - PORT: "4444" - DATABASE_URL: sqlite:////tmp/contextforge-conformance.db - CACHE_TYPE: redis - REDIS_URL: redis://redis:6379/0 - JWT_SECRET_KEY: contextforge-conformance-jwt-secret-at-least-32-bytes - JWT_ALGORITHM: RS256 - JWT_PUBLIC_KEY_PATH: /keys/jwt.key.pub - JWT_PRIVATE_KEY_PATH: /keys/jwt.key - AUTH_ENCRYPTION_SECRET: contextforge-conformance-encryption-secret-at-least-32-bytes # pragma: allowlist secret - AUTH_REQUIRED: "true" - BASIC_AUTH_USER: admin - BASIC_AUTH_PASSWORD: conformance-admin-password # pragma: allowlist secret - PLATFORM_ADMIN_EMAIL: admin@example.com - PLATFORM_ADMIN_PASSWORD: conformance-admin-password # pragma: allowlist secret - MCPGATEWAY_SKIP_MIGRATIONS: "false" - MCPGATEWAY_UI_ENABLED: "false" - MCPGATEWAY_ADMIN_API_ENABLED: "true" - SECURE_COOKIES: "false" - SSRF_ALLOW_PRIVATE_NETWORKS: "true" - DATAPLANE_PUBLISHER: "true" - DATAPLANE_PUBLISHER_INTERVAL_SECONDS: "2" - PASSWORD_CHANGE_ENFORCEMENT_ENABLED: "false" - ADMIN_REQUIRE_PASSWORD_CHANGE_ON_BOOTSTRAP: "false" - REQUIRE_PASSWORD_CHANGE_FOR_DEFAULT_PASSWORD: "false" - GATEWAY_TOOL_NAME_SEPARATOR: _ - GUNICORN_WORKERS: "1" - LOG_LEVEL: INFO - volumes: - - ../../assets:/keys:ro - - ./write_client_config.py:/opt/contextforge-conformance/write_client_config.py:ro - depends_on: - redis: - condition: service_healthy - healthcheck: - test: [CMD, curl, --fail, http://127.0.0.1:4444/health] - interval: 5s - timeout: 5s - retries: 60 - start_period: 10s - - data-plane: - image: ${CF_DATAPLANE_IMAGE:-ghcr.io/contextforge-org/contextforge-data-plane:latest} - networks: [contextforge] - extra_hosts: - - host.docker.internal:host-gateway - environment: - CONTEXTFORGE_DATA_PLANE_ADDRESS: 0.0.0.0:4445 - CONTEXTFORGE_DATA_PLANE_REDIS_HOSTNAME: redis - CONTEXTFORGE_DATA_PLANE_REDIS_PORT: "6379" - CONTEXTFORGE_DATA_PLANE_REDIS_CONNECTION_MODE: plain-text - CONTEXTFORGE_DATA_PLANE_UPSTREAM_CONNECTION_MODE: plain-text-or-tls - CONTEXTFORGE_DATA_PLANE_TOKEN_VERIFICATION_PUBLIC_KEY: /keys/jwt.key.pub - CONTEXTFORGE_DATA_PLANE_TOKEN_VERIFICATION_PRIVATE_KEY: /keys/jwt.key # pragma: allowlist secret - CONTEXTFORGE_DATA_PLANE_USER_CONFIG_CACHE_EXPIRY_SECONDS: "0" - CONTEXTFORGE_GATEWAY_RS_MCP_ALLOWED_HOSTS: >- - 127.0.0.1:${MCP_CONFORMANCE_PORT:-8080},localhost:${MCP_CONFORMANCE_PORT:-8080} - CONTEXTFORGE_GATEWAY_RS_MCP_ALLOWED_ORIGINS: >- - http://127.0.0.1:${MCP_CONFORMANCE_PORT:-8080},http://localhost:${MCP_CONFORMANCE_PORT:-8080} - RUST_LOG: info - volumes: - - ../../assets:/keys:ro - depends_on: - redis: - condition: service_healthy - - nginx: - image: nginx:1.30.4-alpine3.24 - ports: - - "127.0.0.1:${MCP_CONFORMANCE_PORT:-8080}:80" - networks: [contextforge] - environment: - MCP_CONFORMANCE_TOKEN: ${MCP_CONFORMANCE_TOKEN:?} - NGINX_ENVSUBST_TEMPLATE_DIR: /templates - NGINX_ENVSUBST_TEMPLATE_SUFFIX: .template - NGINX_ENVSUBST_OUTPUT_DIR: /etc/nginx - volumes: - - ./nginx.conf.template:/templates/nginx.conf.template:ro - depends_on: - control-plane: - condition: service_healthy - # Full readiness, routing, and auth are verified by start-dataplane-and-nginx.sh. - data-plane: - condition: service_started - healthcheck: - test: [CMD, curl, --fail, http://127.0.0.1/health] - interval: 2s - timeout: 2s - retries: 30 - -networks: - contextforge: - driver: bridge diff --git a/tests/conformance/expected-failures.yml b/tests/conformance/expected-failures.yml deleted file mode 100644 index c38bf4cd..00000000 --- a/tests/conformance/expected-failures.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Generated by `make conformance-bless` from scored dataplane findings. -# Pinned fixture findings are excluded; see upstream-fixture-failures.yml. -server: - - caching:sep-2549-cache-scope-valid - - caching:sep-2549-prompts-list-caching-hints - - caching:sep-2549-resources-list-caching-hints - - caching:sep-2549-resources-templates-list-caching-hints - - caching:sep-2549-tools-list-caching-hints - - caching:sep-2549-ttl-non-negative - - completion-complete:completion-complete - - input-required-result-basic-elicitation:sep-2322-elicitation-incomplete - - input-required-result-basic-list-roots:sep-2322-list-roots-incomplete - - input-required-result-basic-sampling:sep-2322-sampling-incomplete - - input-required-result-capability-check:sep-2322-respect-client-capabilities - - input-required-result-ignore-extra-params:sep-2322-ignore-unexpected-params - - input-required-result-missing-input-response:sep-2322-missing-response-rerequests - - input-required-result-multi-round:sep-2322-multi-round-r1 - - input-required-result-multiple-input-requests:sep-2322-multiple-inputs-incomplete - - input-required-result-non-tool-request:sep-2322-non-tool-incomplete - - input-required-result-request-state:sep-2322-request-state-incomplete - - input-required-result-result-type:sep-2322-result-type-included - - input-required-result-tampered-state:sep-2322-reject-tampered-state - - prompts-list:prompts-list - - resources-list:resources-list - - server-stateless:sep-2575-discover-capabilities-match-handlers - - server-stateless:sep-2575-http-server-no-independent-requests-on-stream - - server-stateless:sep-2575-missing-capability-http-400 - - server-stateless:sep-2575-server-declares-prompts-in-discover - - server-stateless:sep-2575-server-no-log-without-loglevel - - server-stateless:sep-2575-server-rejects-undeclared-capability - - tools-call-with-progress:tools-call-with-progress - - tools-list:tools-list diff --git a/tests/conformance/fixture-proxy.conf b/tests/conformance/fixture-proxy.conf deleted file mode 100644 index 7fac91be..00000000 --- a/tests/conformance/fixture-proxy.conf +++ /dev/null @@ -1,18 +0,0 @@ -server { - listen 80 default_server; - server_name _; - - location / { - proxy_pass http://host.docker.internal:3000; - proxy_http_version 1.1; - proxy_set_header Host localhost:3000; - proxy_set_header Connection ""; - proxy_request_buffering on; - proxy_buffering off; - proxy_cache off; - proxy_connect_timeout 30s; - proxy_send_timeout 1h; - proxy_read_timeout 1h; - add_header X-Accel-Buffering "no" always; - } -} diff --git a/tests/conformance/nginx.conf.template b/tests/conformance/nginx.conf.template deleted file mode 100644 index 404e8d17..00000000 --- a/tests/conformance/nginx.conf.template +++ /dev/null @@ -1,65 +0,0 @@ -user nginx; -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - access_log /var/log/nginx/access.log; - resolver 127.0.0.11 valid=5s ipv6=off; - - map $http_authorization $conformance_authorization { - default $http_authorization; - "" "Bearer ${MCP_CONFORMANCE_TOKEN}"; - } - - server { - listen 80 default_server; - server_name _; - - proxy_http_version 1.1; - proxy_set_header Host $http_host; - proxy_set_header Authorization $conformance_authorization; - proxy_set_header Mcp-Session-Id $http_mcp_session_id; - proxy_set_header Mcp-Protocol-Version $http_mcp_protocol_version; - proxy_set_header Origin $http_origin; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $http_host; - proxy_set_header Connection ""; - - location ~ ^/servers/([^/]+)/mcp/?$ { - set $dataplane http://data-plane:4445; - proxy_pass $dataplane/contextforge-rs/servers/$1/mcp$is_args$args; - proxy_request_buffering on; - proxy_buffering off; - proxy_cache off; - proxy_connect_timeout 30s; - proxy_send_timeout 1h; - proxy_read_timeout 1h; - add_header X-Accel-Buffering "no" always; - add_header X-CF-Conformance-Backend dataplane always; - } - - location = /health { - set $control_plane http://control-plane:4444; - proxy_pass $control_plane/health; - proxy_connect_timeout 5s; - proxy_read_timeout 5s; - } - - location / { - set $control_plane http://control-plane:4444; - proxy_pass $control_plane; - proxy_buffering off; - proxy_connect_timeout 30s; - proxy_send_timeout 120s; - proxy_read_timeout 120s; - } - } -} diff --git a/tests/conformance/register-fixture.sh b/tests/conformance/register-fixture.sh deleted file mode 100755 index dca91dcc..00000000 --- a/tests/conformance/register-fixture.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -: "${GITHUB_ENV:?GITHUB_ENV must be set}" -: "${MCP_CONFORMANCE_SERVER_ID:?MCP_CONFORMANCE_SERVER_ID must be set}" - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -compose_file="${script_dir}/docker-compose.yml" - -bootstrap_token="$({ - docker compose -f "${compose_file}" exec -T control-plane \ - python3 -m mcpgateway.utils.create_jwt_token \ - --username admin@example.com --admin --exp 120 -} 2>/dev/null | tail -n 1)" -test -n "${bootstrap_token}" -if [ "${GITHUB_ACTIONS:-}" = "true" ]; then - echo "::add-mask::${bootstrap_token}" -fi - -api_request() { - local method="$1" - local path="$2" - local body="${3-}" - local args=( - --silent --show-error --fail-with-body - --request "${method}" - --header "Authorization: Bearer ${bootstrap_token}" - --header "Content-Type: application/json" - "http://127.0.0.1:4444${path}" - ) - if [ -n "${body}" ]; then - args+=(--data "${body}") - fi - curl "${args[@]}" -} - -gateway="$(api_request POST /gateways '{ - "name": "_", - "url": "http://fixture-proxy/mcp", - "transport": "STREAMABLEHTTP", - "authType": "authheaders", - "authHeaders": [{"key": "Host", "value": "localhost:3000"}], - "description": "Official MCP alpha.11 conformance fixture through the test-only Host proxy" -}')" -gateway_id="$(jq --exit-status --raw-output '.id' <<< "${gateway}")" - -api_request POST \ - "/gateways/${gateway_id}/tools/refresh?include_resources=true&include_prompts=true" \ - '{}' > /dev/null - -tool_ids='[]' -resource_ids='[]' -prompt_ids='[]' -has_tool=0 -has_resource=0 -has_prompt=0 -for _ in $(seq 1 120); do - tools="$(api_request GET /tools)" - resources="$(api_request GET /resources)" - prompts="$(api_request GET /prompts)" - - tool_ids="$(jq --compact-output --arg id "${gateway_id}" \ - '[.[] | select((.gateway_id // .gatewayId) == $id) | .id]' <<< "${tools}")" - resource_ids="$(jq --compact-output --arg id "${gateway_id}" \ - '[.[] | select((.gateway_id // .gatewayId) == $id) | .id]' <<< "${resources}")" - prompt_ids="$(jq --compact-output --arg id "${gateway_id}" \ - '[.[] | select((.gateway_id // .gatewayId) == $id) | .id]' <<< "${prompts}")" - - has_tool="$(jq --arg id "${gateway_id}" \ - '[.[] | select((.gateway_id // .gatewayId) == $id and .name == "test_simple_text")] | length' \ - <<< "${tools}")" - has_resource="$(jq --arg id "${gateway_id}" \ - '[.[] | select((.gateway_id // .gatewayId) == $id and .uri == "test://static-text")] | length' \ - <<< "${resources}")" - has_prompt="$(jq --arg id "${gateway_id}" \ - '[.[] | select((.gateway_id // .gatewayId) == $id and .name == "test_simple_prompt")] | length' \ - <<< "${prompts}")" - if [ "${has_tool}" -gt 0 ] && [ "${has_resource}" -gt 0 ] && [ "${has_prompt}" -gt 0 ]; then - break - fi - sleep 0.5 -done -test "${has_tool}" -gt 0 -test "${has_resource}" -gt 0 -test "${has_prompt}" -gt 0 - -server_payload="$(jq --null-input --compact-output \ - --arg id "${MCP_CONFORMANCE_SERVER_ID}" \ - --argjson tools "${tool_ids}" \ - --argjson resources "${resource_ids}" \ - --argjson prompts "${prompt_ids}" \ - '{server: { - id: $id, - name: "Official MCP Conformance Server", - description: "Virtual server for alpha.11 conformance", - associated_tools: $tools, - associated_resources: $resources, - associated_prompts: $prompts - }}')" -api_request POST /servers "${server_payload}" > /dev/null - -token_response="$(api_request POST /v1/tokens '{ - "name": "MCP conformance CI", - "description": "Ephemeral dataplane token", - "expires_in_days": 1, - "user_email": "admin@example.com" -}')" -conformance_token="$(jq --exit-status --raw-output '.access_token' <<< "${token_response}")" -if [ "${GITHUB_ACTIONS:-}" = "true" ]; then - echo "::add-mask::${conformance_token}" -fi - -jwt_payload="$(cut -d. -f2 <<< "${conformance_token}")" -jwt_payload="${jwt_payload//-/+}" -jwt_payload="${jwt_payload//_/\/}" -case $((${#jwt_payload} % 4)) in - 2) jwt_payload="${jwt_payload}==" ;; - 3) jwt_payload="${jwt_payload}=" ;; -esac -conformance_subject="$(jq --raw-input --exit-status --raw-output '@base64d | fromjson | .sub' <<< "${jwt_payload}")" - -echo "MCP_CONFORMANCE_TOKEN=${conformance_token}" >> "${GITHUB_ENV}" -echo "MCP_CONFORMANCE_SUBJECT=${conformance_subject}" >> "${GITHUB_ENV}" diff --git a/tests/conformance/report-baseline-diff-test.sh b/tests/conformance/report-baseline-diff-test.sh deleted file mode 100755 index 525f357b..00000000 --- a/tests/conformance/report-baseline-diff-test.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -reporter="${script_dir}/report-baseline-diff.sh" -state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-baseline-test.XXXXXX")" -suite_dir="${state_dir}/suite" -results_dir="${state_dir}/results" -baseline_file="${state_dir}/expected-failures.yml" -upstream_file="${state_dir}/upstream-fixture-failures.yml" -summary_file="${state_dir}/summary.md" - -cleanup() { - rm -rf -- "${state_dir}" -} -trap cleanup EXIT INT TERM - -mkdir -p "${suite_dir}/requirements" "${results_dir}" - -cat > "${suite_dir}/requirements/2026-07-28.yaml" <<'EOF' -server: - - expected-check - - expected-whole - - regression - - xpass-check - - xpass-whole - - absent-check - - upstream - - duplicate - - normal-pass -EOF - -cat > "${baseline_file}" <<'EOF' -server: - - expected-check:known - - expected-whole - - xpass-check:fixed - - xpass-whole - - absent-check:not-emitted -EOF - -cat > "${upstream_file}" <<'EOF' -server: - - upstream:fixture-defect -EOF - -write_checks() { - scenario="$1" - checks="$2" - result_dir="${results_dir}/server-${scenario}-2026-08-18T12-00-00-000Z" - mkdir -p "${result_dir}" - printf '%s\n' "${checks}" > "${result_dir}/checks.json" -} - -write_checks expected-check '[{"id":"known","status":"FAILURE"}]' -write_checks expected-whole '[{"id":"any-failure","status":"WARNING"}]' -write_checks regression '[{"id":"new-failure","status":"FAILURE"}]' -write_checks xpass-check '[{"id":"fixed","status":"SUCCESS"}]' -write_checks xpass-whole '[{"id":"all-good","status":"SUCCESS"}]' -write_checks absent-check '[{"id":"other","status":"SUCCESS"}]' -write_checks upstream '[{"id":"fixture-defect","status":"FAILURE","errorMessage":"must not be reported as a dataplane failure"}]' -write_checks duplicate '[{"id":"repeated","status":"FAILURE"},{"id":"repeated","status":"SUCCESS"}]' -write_checks normal-pass '[{"id":"good","status":"SUCCESS"},{"id":"not-applicable","status":"SKIPPED"}]' - -assert_contains() { - haystack="$1" - needle="$2" - if [[ "${haystack}" != *"${needle}"* ]]; then - echo "Expected output to contain: ${needle}" >&2 - echo "${haystack}" >&2 - exit 1 - fi -} - -assert_not_contains() { - haystack="$1" - needle="$2" - if [[ "${haystack}" == *"${needle}"* ]]; then - echo "Expected output not to contain: ${needle}" >&2 - echo "${haystack}" >&2 - exit 1 - fi -} - -set +e -output="$( - GITHUB_ACTIONS=true \ - GITHUB_STEP_SUMMARY="${summary_file}" \ - MCP_CONFORMANCE_COLOR=never \ - MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \ - "${reporter}" "${results_dir}" "${baseline_file}" "${upstream_file}" 2>&1 -)" -status="$?" -set -e - -if [ "${status}" -ne 1 ]; then - echo "Expected mismatch status 1, got ${status}" >&2 - echo "${output}" >&2 - exit 1 -fi - -assert_contains "${output}" 'XFAIL expected-check:known' -assert_contains "${output}" 'XFAIL expected-whole' -assert_contains "${output}" 'UPSTREAM upstream:fixture-defect' -assert_contains "${output}" 'FAIL duplicate:repeated (expected PASS, got FAILURE)' -assert_contains "${output}" 'FAIL regression:new-failure (expected PASS, got FAILURE)' -assert_contains "${output}" 'XPASS xpass-check:fixed (expected FAILURE, got PASS)' -assert_contains "${output}" 'XPASS xpass-whole (expected FAILURE, got PASS)' -assert_not_contains "${output}" 'XPASS absent-check:not-emitted' -assert_not_contains "${output}" '::error title=Expected conformance pass failed::upstream:fixture-defect' - -summary="$(cat "${summary_file}")" -assert_contains "${summary}" '| Pinned fixture findings ignored | 1 |' -assert_not_contains "${summary}" 'upstream:fixture-defect' - -cat > "${state_dir}/unmatched-upstream.yml" <<'EOF' -server: - - never-seen:fixture-defect -EOF -set +e -unmatched_upstream_output="$( - MCP_CONFORMANCE_COLOR=never \ - MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \ - "${reporter}" \ - "${results_dir}" \ - "${baseline_file}" \ - "${state_dir}/unmatched-upstream.yml" 2>&1 -)" -unmatched_upstream_status="$?" -set -e -if [ "${unmatched_upstream_status}" -ne 1 ]; then - echo "Expected unmatched-upstream status 1, got ${unmatched_upstream_status}" >&2 - exit 1 -fi -assert_contains "${unmatched_upstream_output}" 'FAIL upstream:fixture-defect' - -echo 'server: []' > "${state_dir}/empty-baseline.yml" -set +e -empty_baseline_output="$( - MCP_CONFORMANCE_COLOR=never \ - MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \ - "${reporter}" \ - "${results_dir}" \ - "${state_dir}/empty-baseline.yml" \ - "${upstream_file}" 2>&1 -)" -empty_baseline_status="$?" -set -e -if [ "${empty_baseline_status}" -ne 1 ]; then - echo "Expected empty-baseline status 1, got ${empty_baseline_status}" >&2 - exit 1 -fi -assert_contains "${empty_baseline_output}" 'FAIL regression:new-failure' - -bless_output="$( - MCP_CONFORMANCE_COLOR=never \ - MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \ - "${reporter}" --bless "${results_dir}" "${baseline_file}" "${upstream_file}" -)" -assert_contains "${bless_output}" "BLESS updated ${baseline_file}" - -cat > "${state_dir}/expected-after-bless.yml" <<'EOF' -# Generated by `make conformance-bless` from scored dataplane findings. -# Pinned fixture findings are excluded; see upstream-fixture-failures.yml. -server: - - duplicate:repeated - - expected-check:known - - expected-whole:any-failure - - regression:new-failure -EOF -diff -u "${state_dir}/expected-after-bless.yml" "${baseline_file}" - -MCP_CONFORMANCE_COLOR=never \ -MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \ - "${reporter}" "${results_dir}" "${baseline_file}" "${upstream_file}" > /dev/null - -baseline_before_missing="$(cat "${baseline_file}")" -set +e -MCP_CONFORMANCE_COLOR=never \ -MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \ - "${reporter}" --bless "${state_dir}/missing" "${baseline_file}" "${upstream_file}" > /dev/null 2>&1 -missing_status="$?" -set -e - -if [ "${missing_status}" -ne 2 ]; then - echo "Expected missing-results status 2, got ${missing_status}" >&2 - exit 1 -fi -if [ "$(cat "${baseline_file}")" != "${baseline_before_missing}" ]; then - echo 'Bless changed the baseline without results' >&2 - exit 1 -fi - -echo 'conformance reporter tests passed' diff --git a/tests/conformance/report-baseline-diff.sh b/tests/conformance/report-baseline-diff.sh deleted file mode 100755 index 6f7eeb92..00000000 --- a/tests/conformance/report-baseline-diff.sh +++ /dev/null @@ -1,433 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -usage() { - cat <<'EOF' -Usage: report-baseline-diff.sh [--bless] [results-dir [baseline-file [upstream-file]]] - -Compare scored MCP conformance checks with the expected-failure baseline. -With --bless, replace the baseline with the current dataplane-owned findings. -EOF -} - -bless=false -case "${1:-}" in - --bless) - bless=true - shift - ;; - --help|-h) - usage - exit 0 - ;; -esac - -if [ "$#" -gt 3 ]; then - usage >&2 - exit 2 -fi - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(cd -- "${script_dir}/../.." && pwd)" -results_dir="${1:-${repo_root}/conformance-results}" -baseline_file="${2:-${script_dir}/expected-failures.yml}" -upstream_file="${3:-${script_dir}/upstream-fixture-failures.yml}" -suite_dir="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" -spec_version="${MCP_CONFORMANCE_SPEC_VERSION:-2026-07-28}" -requirements_file="${suite_dir}/requirements/${spec_version}.yaml" - -for command in awk cmp cp cut find grep jq sed sort wc; do - if ! command -v "${command}" > /dev/null 2>&1; then - echo "Required command not found: ${command}" >&2 - exit 2 - fi -done - -for required_file in "${baseline_file}" "${upstream_file}" "${requirements_file}"; do - if [ ! -f "${required_file}" ]; then - echo "Required conformance file not found: ${required_file}" >&2 - exit 2 - fi -done - -color_mode="${MCP_CONFORMANCE_COLOR:-${CARGO_TERM_COLOR:-auto}}" -case "${color_mode}" in - always) - use_color=true - ;; - never) - use_color=false - ;; - auto) - if [ -t 1 ] && [ "${TERM:-}" != "dumb" ] && [ -z "${NO_COLOR:-}" ]; then - use_color=true - else - use_color=false - fi - ;; - *) - echo "MCP_CONFORMANCE_COLOR must be auto, always, or never; got: ${color_mode}" >&2 - exit 2 - ;; -esac - -if [ -n "${NO_COLOR:-}" ]; then - use_color=false -fi - -if ${use_color}; then - bold=$'\033[1m' - dim=$'\033[2m' - red=$'\033[31m' - green=$'\033[32m' - yellow=$'\033[33m' - cyan=$'\033[36m' - reset=$'\033[0m' -else - bold="" - dim="" - red="" - green="" - yellow="" - cyan="" - reset="" -fi - -state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-baseline-diff.XXXXXX")" -actual_checks="${state_dir}/actual-checks.tsv" -actual_findings="${state_dir}/actual-findings.tsv" -actual_keys="${state_dir}/actual-keys.txt" -baseline_entries="${state_dir}/baseline-entries.txt" -upstream_entries="${state_dir}/upstream-entries.txt" -scored_scenarios="${state_dir}/scored-scenarios.txt" -executed_scenarios="${state_dir}/executed-scenarios.txt" -owned_findings="${state_dir}/owned-findings.txt" -expected_entries="${state_dir}/expected-entries.txt" -unexpected_entries="${state_dir}/unexpected-entries.txt" -stale_entries="${state_dir}/stale-entries.txt" -upstream_matches="${state_dir}/upstream-matches.txt" -passing_checks="${state_dir}/passing-checks.txt" -skipped_checks="${state_dir}/skipped-checks.txt" -baseline_candidate="${state_dir}/expected-failures.yml" - -cleanup() { - rm -f -- \ - "${actual_checks}" \ - "${actual_findings}" \ - "${actual_keys}" \ - "${baseline_entries}" \ - "${upstream_entries}" \ - "${scored_scenarios}" \ - "${executed_scenarios}" \ - "${owned_findings}" \ - "${expected_entries}" \ - "${unexpected_entries}" \ - "${stale_entries}" \ - "${upstream_matches}" \ - "${passing_checks}" \ - "${skipped_checks}" \ - "${baseline_candidate}" - rmdir -- "${state_dir}" -} -trap cleanup EXIT INT TERM - -read_baseline() { - awk ' - /^[[:space:]]*-[[:space:]]+/ { - line = $0 - sub(/^[[:space:]]*-[[:space:]]+/, "", line) - sub(/[[:space:]]+#.*$/, "", line) - print line - } - ' "$1" | LC_ALL=C sort -u -} - -line_count() { - wc -l < "$1" | tr -d '[:space:]' -} - -print_row() { - local color="$1" - local label="$2" - local message="$3" - printf ' %b%10s%b %s\n' "${color}${bold}" "${label}" "${reset}" "${message}" -} - -emit_error() { - local title="$1" - local message="$2" - if [ "${GITHUB_ACTIONS:-}" = "true" ]; then - echo "::error title=${title}::${message}" - fi -} - -write_missing_results_summary() { - local message="$1" - print_row "${red}" "ERROR" "${message}" - emit_error "Conformance results missing" "${message}" - if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then - printf '## MCP %s conformance\n\n❌ %s\n' "${spec_version}" "${message}" \ - >> "${GITHUB_STEP_SUMMARY}" - fi -} - -awk ' - /^server:$/ { in_server = 1; next } - in_server && /^[^[:space:]]/ { exit } - in_server && /^[[:space:]]*-[[:space:]]+/ { - line = $0 - sub(/^[[:space:]]*-[[:space:]]+/, "", line) - print line - } -' "${requirements_file}" | LC_ALL=C sort -u > "${scored_scenarios}" - -read_baseline "${baseline_file}" > "${baseline_entries}" -read_baseline "${upstream_file}" > "${upstream_entries}" - -printf '\n%bMCP conformance%b %b(%s)%b\n' "${bold}" "${reset}" "${dim}" "${spec_version}" "${reset}" - -if [ ! -d "${results_dir}" ]; then - write_missing_results_summary "No results directory: ${results_dir}" - exit 2 -fi - -: > "${actual_checks}" -: > "${executed_scenarios}" -while IFS= read -r -d '' checks_file; do - result_name="$(basename -- "$(dirname -- "${checks_file}")")" - if [[ ! "${result_name}" =~ ^server-(.*)-[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}-[0-9]{3}Z$ ]]; then - echo "Skipping unrecognized result directory: ${result_name}" >&2 - continue - fi - scenario="${BASH_REMATCH[1]}" - - if ! grep --fixed-strings --line-regexp --quiet -- "${scenario}" "${scored_scenarios}"; then - continue - fi - - echo "${scenario}" >> "${executed_scenarios}" - jq --raw-output --arg scenario "${scenario}" ' - def severity($status): - if $status == "FAILURE" then 3 - elif $status == "WARNING" then 2 - elif $status == "SUCCESS" then 1 - else 0 - end; - - reduce (.[] | select(.status != "INFO")) as $check - ({}; - ($check.id) as $id | - if .[$id] == null or severity($check.status) >= severity(.[$id].status) - then .[$id] = $check - else . - end) | - to_entries[] | - [($scenario + ":" + .key), .value.status, (.value.errorMessage // "")] | - @tsv - ' "${checks_file}" >> "${actual_checks}" -done < <(find "${results_dir}" -type f -name checks.json -print0) - -LC_ALL=C sort -u -o "${executed_scenarios}" "${executed_scenarios}" -if [ ! -s "${executed_scenarios}" ]; then - write_missing_results_summary "No scored conformance results were found in ${results_dir}" - exit 2 -fi - -LC_ALL=C sort -u -o "${actual_checks}" "${actual_checks}" -awk -F '\t' '$2 == "FAILURE" || $2 == "WARNING"' "${actual_checks}" > "${actual_findings}" -cut -f 1 "${actual_findings}" | LC_ALL=C sort -u > "${actual_keys}" -awk -F '\t' '$2 == "SUCCESS" { print $1 }' "${actual_checks}" | LC_ALL=C sort -u > "${passing_checks}" -awk -F '\t' '$2 == "SKIPPED" { print $1 }' "${actual_checks}" | LC_ALL=C sort -u > "${skipped_checks}" - -# Pinned fixture findings are informational: they neither satisfy the dataplane -# baseline nor count as unexpected dataplane failures. -awk -v upstream_file="${upstream_entries}" ' - FILENAME == upstream_file { - upstream[$1] = 1 - next - } - { - scenario = $1 - sub(/:.*/, "", scenario) - if (($1 in upstream) || (scenario in upstream)) print $1 - } -' "${upstream_entries}" "${actual_keys}" | LC_ALL=C sort -u > "${upstream_matches}" - -awk -v upstream_file="${upstream_matches}" ' - FILENAME == upstream_file { upstream[$1] = 1; next } - !($1 in upstream) { print $1 } -' "${upstream_matches}" "${actual_keys}" > "${owned_findings}" - -bless_changed=false -if ${bless}; then - { - # shellcheck disable=SC2016 # Backticks are literal Markdown. - echo '# Generated by `make conformance-bless` from scored dataplane findings.' - echo '# Pinned fixture findings are excluded; see upstream-fixture-failures.yml.' - if [ -s "${owned_findings}" ]; then - echo 'server:' - sed 's/^/ - /' "${owned_findings}" - else - echo 'server: []' - fi - } > "${baseline_candidate}" - - if ! cmp --silent "${owned_findings}" "${baseline_entries}"; then - cp "${baseline_candidate}" "${baseline_file}" - bless_changed=true - fi - read_baseline "${baseline_file}" > "${baseline_entries}" -fi - -# Match actual owned findings against exact or whole-scenario baseline entries. -awk -v baseline_file="${baseline_entries}" ' - FILENAME == baseline_file { - baseline[$1] = 1 - if (index($1, ":") == 0) whole[$1] = 1 - next - } - { - scenario = $1 - sub(/:.*/, "", scenario) - if (scenario in whole) matched[scenario] = 1 - else if ($1 in baseline) matched[$1] = 1 - } - END { - for (entry in matched) print entry - } -' "${baseline_entries}" "${owned_findings}" | LC_ALL=C sort -u > "${expected_entries}" - -awk -v baseline_file="${baseline_entries}" ' - FILENAME == baseline_file { - baseline[$1] = 1 - if (index($1, ":") == 0) whole[$1] = 1 - next - } - { - scenario = $1 - sub(/:.*/, "", scenario) - if (!(($1 in baseline) || (scenario in whole))) print $1 - } -' "${baseline_entries}" "${owned_findings}" > "${unexpected_entries}" - -# An exact baseline entry is stale only after a demonstrated SUCCESS. Missing -# and SKIPPED checks carry no pass signal. A whole-scenario entry is stale when -# the scenario ran without any dataplane-owned finding. -awk -F '\t' \ - -v checks_file="${actual_checks}" \ - -v scenarios_file="${executed_scenarios}" \ - -v findings_file="${owned_findings}" ' - FILENAME == checks_file { status[$1] = $2; next } - FILENAME == scenarios_file { executed[$1] = 1; next } - FILENAME == findings_file { - scenario = $1 - sub(/:.*/, "", scenario) - failed[scenario] = 1 - next - } - index($1, ":") == 0 { - if (($1 in executed) && !($1 in failed)) print $1 - next - } - status[$1] == "SUCCESS" { print $1 } - ' "${actual_checks}" "${executed_scenarios}" "${owned_findings}" "${baseline_entries}" \ - | LC_ALL=C sort -u > "${stale_entries}" - -pass_count="$(line_count "${passing_checks}")" -skip_count="$(line_count "${skipped_checks}")" -expected_count="$(line_count "${expected_entries}")" -unexpected_count="$(line_count "${unexpected_entries}")" -stale_count="$(line_count "${stale_entries}")" -upstream_count="$(line_count "${upstream_matches}")" - -print_row "${green}" "PASS" "${pass_count} scored checks passed" - -while IFS= read -r key; do - [ -n "${key}" ] || continue - print_row "${yellow}" "XFAIL" "${key} ${dim}(expected failure reproduced)${reset}" -done < "${expected_entries}" - -while IFS= read -r key; do - [ -n "${key}" ] || continue - print_row "${cyan}" "UPSTREAM" "${key} ${dim}(ignored pinned-fixture finding)${reset}" -done < "${upstream_matches}" - -while IFS= read -r key; do - [ -n "${key}" ] || continue - status="$(awk -F '\t' -v key="${key}" '$1 == key { print $2; exit }' "${actual_findings}")" - print_row "${red}" "FAIL" "${key} ${dim}(expected PASS, got ${status})${reset}" - emit_error "Expected conformance pass failed" "${key}" -done < "${unexpected_entries}" - -while IFS= read -r key; do - [ -n "${key}" ] || continue - print_row "${red}" "XPASS" "${key} ${dim}(expected FAILURE, got PASS)${reset}" - emit_error "Expected conformance failure passed" "${key}" -done < "${stale_entries}" - -if [ "${skip_count}" -gt 0 ]; then - print_row "${dim}" "SKIP" "${skip_count} scored checks skipped" -fi - -if ${bless}; then - if ${bless_changed}; then - print_row "${green}" "BLESS" "updated ${baseline_file}" - else - print_row "${green}" "BLESS" "${baseline_file} was already current" - fi -fi - -printf '\n%bSummary%b: %s passed, %s expected failures, %s upstream findings ignored, %s failed, %s unexpected passes\n' \ - "${bold}" "${reset}" \ - "${pass_count}" "${expected_count}" "${upstream_count}" "${unexpected_count}" "${stale_count}" - -if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then - { - echo "## MCP ${spec_version} conformance" - echo - echo '| Outcome | Count |' - echo '| --- | ---: |' - echo "| Scored checks passed | ${pass_count} |" - echo "| Expected failures reproduced | ${expected_count} |" - echo "| Pinned fixture findings ignored | ${upstream_count} |" - echo "| Expected pass, got failure | ${unexpected_count} |" - echo "| Expected failure, got pass | ${stale_count} |" - echo "| Skipped checks | ${skip_count} |" - - if [ "${unexpected_count}" -gt 0 ]; then - echo - echo '### Expected pass, got failure' - while IFS= read -r key; do - [ -n "${key}" ] || continue - status="$(awk -F '\t' -v key="${key}" '$1 == key { print $2; exit }' "${actual_findings}")" - echo "- \`${key}\` — ${status}" - done < "${unexpected_entries}" - fi - - if [ "${stale_count}" -gt 0 ]; then - echo - echo '### Expected failure, got pass' - while IFS= read -r key; do - [ -n "${key}" ] || continue - echo "- \`${key}\`" - done < "${stale_entries}" - fi - - echo - if ${bless}; then - echo "✅ Expected-failure baseline updated with ${expected_count} dataplane findings." - elif [ "${unexpected_count}" -eq 0 ] && [ "${stale_count}" -eq 0 ]; then - echo '✅ Actual dataplane findings match the expected-failure baseline.' - else - echo '❌ Actual dataplane findings do not match the expected-failure baseline.' - fi - } >> "${GITHUB_STEP_SUMMARY}" -fi - -if ${bless}; then - exit 0 -fi - -if [ "${unexpected_count}" -gt 0 ] || [ "${stale_count}" -gt 0 ]; then - exit 1 -fi diff --git a/tests/conformance/resolve-control-plane-image.sh b/tests/conformance/resolve-control-plane-image.sh deleted file mode 100755 index c1b87cd7..00000000 --- a/tests/conformance/resolve-control-plane-image.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -repository="IBM/mcp-context-forge" -image_repository="ghcr.io/ibm/mcp-context-forge" -api_url="https://api.github.com/repos/${repository}/commits?sha=main&per_page=100" -curl_args=( - --fail - --silent - --show-error - --retry 3 - --retry-all-errors - --header "Accept: application/vnd.github+json" - --header "X-GitHub-Api-Version: 2022-11-28" -) -if [ -n "${GITHUB_TOKEN:-}" ]; then - curl_args+=(--header "Authorization: Bearer ${GITHUB_TOKEN}") -fi - -commit_shas="$(curl "${curl_args[@]}" "${api_url}" | jq --exit-status --raw-output \ - '.[] | .sha | select(test("^[0-9a-f]{40}$"))')" - -while IFS= read -r commit_sha; do - image="${image_repository}:${commit_sha}" - if docker manifest inspect "${image}" > /dev/null 2>&1; then - echo "Resolved latest control-plane main image: ${image}" >&2 - echo "${image}" - exit 0 - fi -done <<< "${commit_shas}" - -echo "No published control-plane image found in the latest 100 main commits" >&2 -exit 1 diff --git a/tests/conformance/run-client-conformance.sh b/tests/conformance/run-client-conformance.sh deleted file mode 100755 index 289c2d41..00000000 --- a/tests/conformance/run-client-conformance.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -: "${MCP_CONFORMANCE_SPEC_VERSION:?MCP_CONFORMANCE_SPEC_VERSION must be set}" -: "${MCP_CONFORMANCE_SUBJECT:?MCP_CONFORMANCE_SUBJECT must be set}" - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(cd -- "${script_dir}/../.." && pwd)" -suite_dir="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" -results_root="${MCP_CONFORMANCE_RESULTS_DIR:-${repo_root}/conformance-results}" -results_dir="${MCP_CONFORMANCE_CLIENT_RESULTS_DIR:-${results_root}/client}" -compose_file="${script_dir}/docker-compose.yml" -baseline_file="${script_dir}/client-expected-failures.yml" -client_command="${script_dir}/client-under-test.sh" -scenarios=(tools_call request-metadata http-standard-headers http-custom-headers) - -mkdir -p "${results_dir}" - -# The control-plane publisher would overwrite the scenario-specific Redis -# config, and its own backend probes would contaminate the client observations. -docker compose -f "${compose_file}" stop control-plane > /dev/null - -status=0 -for scenario in "${scenarios[@]}"; do - set +e - ( - cd "${suite_dir}" - npm start -- \ - client \ - --command "${client_command}" \ - --scenario "${scenario}" \ - --spec-version "${MCP_CONFORMANCE_SPEC_VERSION}" \ - --expected-failures "${baseline_file}" \ - --timeout 60000 \ - --output-dir "${results_dir}" - ) - scenario_status="$?" - set -e - if [ "${scenario_status}" -ne 0 ]; then - status=1 - fi -done - -exit "${status}" diff --git a/tests/conformance/run-conformance.sh b/tests/conformance/run-conformance.sh deleted file mode 100755 index 93a8f740..00000000 --- a/tests/conformance/run-conformance.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -: "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}" -: "${MCP_CONFORMANCE_SERVER_ID:?MCP_CONFORMANCE_SERVER_ID must be set}" -: "${MCP_CONFORMANCE_SPEC_VERSION:?MCP_CONFORMANCE_SPEC_VERSION must be set}" - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(cd -- "${script_dir}/../.." && pwd)" -suite_dir="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" -conformance_port="${MCP_CONFORMANCE_PORT:-8080}" -results_root="${MCP_CONFORMANCE_RESULTS_DIR:-${repo_root}/conformance-results}" -results_dir="${MCP_CONFORMANCE_SERVER_RESULTS_DIR:-${results_root}/server}" - -mkdir -p "${results_dir}" - -set +e -( - cd "${suite_dir}" - npm start -- \ - server \ - --url "http://127.0.0.1:${conformance_port}/servers/${MCP_CONFORMANCE_SERVER_ID}/mcp" \ - --requirements "${MCP_CONFORMANCE_SPEC_VERSION}" \ - --expected-failures "${script_dir}/expected-failures.yml" \ - --output-dir "${results_dir}" -) -runner_status="$?" -set -e - -echo "status=${runner_status}" >> "${GITHUB_OUTPUT}" diff --git a/tests/conformance/run-local.sh b/tests/conformance/run-local.sh deleted file mode 100755 index ea26ab0e..00000000 --- a/tests/conformance/run-local.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(cd -- "${script_dir}/../.." && pwd)" -compose_file="${script_dir}/docker-compose.yml" - -export MCP_CONFORMANCE_VERSION="${MCP_CONFORMANCE_VERSION:-0.2.0-alpha.11}" -export MCP_CONFORMANCE_SOURCE_SHA="${MCP_CONFORMANCE_SOURCE_SHA:-c321dd32035556e6769d3724a8ee97d87c3faaac}" -export MCP_CONFORMANCE_SPEC_VERSION="${MCP_CONFORMANCE_SPEC_VERSION:-2026-07-28}" -export MCP_CONFORMANCE_SERVER_ID="${MCP_CONFORMANCE_SERVER_ID:-3f33286667d34b65a31c3bafd30e4c21}" -export MCP_CONFORMANCE_SUITE_DIR="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" -export CF_DATAPLANE_IMAGE="${CF_DATAPLANE_IMAGE:-contextforge-data-plane:conformance}" -export MCP_CONFORMANCE_COLOR="${MCP_CONFORMANCE_COLOR:-auto}" - -for command in curl docker git jq node npm; do - if ! command -v "${command}" > /dev/null 2>&1; then - echo "Required command not found: ${command}" >&2 - exit 1 - fi -done -docker compose version > /dev/null - -if [ -z "${CF_CONTROLPLANE_IMAGE:-}" ]; then - CF_CONTROLPLANE_IMAGE="$("${script_dir}/resolve-control-plane-image.sh")" - export CF_CONTROLPLANE_IMAGE -fi - -if [ -e "${MCP_CONFORMANCE_SUITE_DIR}" ] && [ ! -d "${MCP_CONFORMANCE_SUITE_DIR}/.git" ]; then - echo "MCP_CONFORMANCE_SUITE_DIR is not a git checkout: ${MCP_CONFORMANCE_SUITE_DIR}" >&2 - exit 1 -fi - -if [ ! -d "${MCP_CONFORMANCE_SUITE_DIR}/.git" ]; then - echo "Checking out the official conformance suite." - git clone --filter=blob:none \ - https://github.com/modelcontextprotocol/conformance.git \ - "${MCP_CONFORMANCE_SUITE_DIR}" - git -C "${MCP_CONFORMANCE_SUITE_DIR}" checkout --detach "${MCP_CONFORMANCE_SOURCE_SHA}" -fi - -suite_sha="$(git -C "${MCP_CONFORMANCE_SUITE_DIR}" rev-parse HEAD)" -if [ "${suite_sha}" != "${MCP_CONFORMANCE_SOURCE_SHA}" ]; then - echo "Conformance checkout is at ${suite_sha}; expected ${MCP_CONFORMANCE_SOURCE_SHA}." >&2 - echo "Use a checkout at the pinned commit or set MCP_CONFORMANCE_SUITE_DIR." >&2 - exit 1 -fi - -( - echo "Installing official conformance dependencies." - cd "${MCP_CONFORMANCE_SUITE_DIR}" - test "$(node -p "require('./package.json').version")" = "${MCP_CONFORMANCE_VERSION}" - npm ci --ignore-scripts -) - -state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-conformance.XXXXXX")" -export GITHUB_ENV="${state_dir}/github-env" -export GITHUB_OUTPUT="${state_dir}/github-output" -touch "${GITHUB_ENV}" "${GITHUB_OUTPUT}" -mkdir -p "${repo_root}/conformance-results" -export MCP_CONFORMANCE_RESULTS_DIR -MCP_CONFORMANCE_RESULTS_DIR="$(mktemp -d "${repo_root}/conformance-results/run.XXXXXX")" - -# shellcheck disable=SC2329 # Invoked by the trap below. -cleanup() { - local status="$?" - trap - EXIT INT TERM - if [ "${status}" -ne 0 ]; then - echo "Conformance run failed; printing live stack logs." >&2 - MCP_CONFORMANCE_TOKEN=diagnostics-only \ - docker compose -f "${compose_file}" logs --no-color || true - if [ -f "${repo_root}/conformance-logs/reference-server.log" ]; then - echo "Official fixture log:" >&2 - sed -n '1,240p' "${repo_root}/conformance-logs/reference-server.log" >&2 - fi - fi - MCP_CONFORMANCE_TOKEN="${MCP_CONFORMANCE_TOKEN:-cleanup-only}" \ - "${script_dir}/stop-live-stack.sh" || true - rm -f -- "${GITHUB_ENV}" "${GITHUB_OUTPUT}" - rmdir -- "${state_dir}" - exit "${status}" -} -trap cleanup EXIT INT TERM - -MCP_CONFORMANCE_TOKEN=pull-only \ - docker compose -f "${compose_file}" pull redis fixture-proxy control-plane nginx -echo "Starting the fixture and control plane." -MCP_CONFORMANCE_TOKEN=bootstrap-only \ - "${script_dir}/start-fixture-and-control-plane.sh" -echo "Registering the fixture through the control plane." -MCP_CONFORMANCE_TOKEN=bootstrap-only \ - "${script_dir}/register-fixture.sh" - -set -a -# shellcheck disable=SC1090 -source "${GITHUB_ENV}" -set +a - -echo "Starting the dataplane and nginx." -"${script_dir}/start-dataplane-and-nginx.sh" -echo "Running MCP ${MCP_CONFORMANCE_SPEC_VERSION} conformance." -"${script_dir}/run-conformance.sh" - -set +e -echo "Running scoped MCP ${MCP_CONFORMANCE_SPEC_VERSION} client conformance." -"${script_dir}/run-client-conformance.sh" -client_status="$?" -set -e - -runner_status="$(sed -n 's/^status=//p' "${GITHUB_OUTPUT}" | tail -n 1)" -if [ -z "${runner_status}" ]; then - echo "Conformance runner did not report a status." >&2 - exit 1 -fi - -set +e -if [ "${MCP_CONFORMANCE_BLESS:-false}" = "true" ]; then - "${script_dir}/report-baseline-diff.sh" --bless "${MCP_CONFORMANCE_RESULTS_DIR}/server" - report_status="$?" - "${script_dir}/bless-client-baseline.sh" \ - "${MCP_CONFORMANCE_RESULTS_DIR}/client" \ - "${script_dir}/client-expected-failures.yml" - client_baseline_status="$?" - if [ "${client_baseline_status}" -eq 0 ]; then - client_status=0 - fi -else - "${script_dir}/report-baseline-diff.sh" "${MCP_CONFORMANCE_RESULTS_DIR}/server" - report_status="$?" - client_baseline_status=0 -fi -set -e - -if [ "${runner_status}" -ne 0 ] && [ "${report_status}" -eq 0 ]; then - echo "Official runner status ${runner_status} contained no dataplane baseline mismatch." -fi -if [ "${client_baseline_status}" -ne 0 ]; then - exit "${client_baseline_status}" -fi -if [ "${client_status}" -ne 0 ]; then - exit "${client_status}" -fi -exit "${report_status}" diff --git a/tests/conformance/start-dataplane-and-nginx.sh b/tests/conformance/start-dataplane-and-nginx.sh deleted file mode 100755 index c2a793e2..00000000 --- a/tests/conformance/start-dataplane-and-nginx.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -: "${MCP_CONFORMANCE_SERVER_ID:?MCP_CONFORMANCE_SERVER_ID must be set}" - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(cd -- "${script_dir}/../.." && pwd)" -log_dir="${repo_root}/conformance-logs" -conformance_port="${MCP_CONFORMANCE_PORT:-8080}" - -docker compose -f "${script_dir}/docker-compose.yml" \ - up -d --wait data-plane nginx - -endpoint="http://127.0.0.1:${conformance_port}/servers/${MCP_CONFORMANCE_SERVER_ID}/mcp" -request='{ - "jsonrpc": "2.0", - "id": 1, - "method": "server/discover", - "params": { - "_meta": { - "io.modelcontextprotocol/protocolVersion": "2026-07-28", - "io.modelcontextprotocol/clientInfo": { - "name": "ci-route-probe", - "version": "1.0.0" - }, - "io.modelcontextprotocol/clientCapabilities": {} - } - } -}' - -for _ in $(seq 1 120); do - curl --silent --show-error \ - --dump-header "${log_dir}/route-probe-headers.txt" \ - --output "${log_dir}/route-probe-body.txt" \ - --request POST \ - --header 'Content-Type: application/json' \ - --header 'Accept: application/json, text/event-stream' \ - --header 'MCP-Protocol-Version: 2026-07-28' \ - --header 'MCP-Method: server/discover' \ - --data "${request}" \ - "${endpoint}" || true - if grep --ignore-case --quiet '^X-CF-Conformance-Backend: dataplane' \ - "${log_dir}/route-probe-headers.txt" \ - && sed -n 's/^data: //p' "${log_dir}/route-probe-body.txt" \ - | jq --exit-status \ - '.result.supportedVersions | index("2026-07-28") != null' \ - > /dev/null 2>&1; then - exit 0 - fi - sleep 0.5 -done - -echo "Modern MCP route did not reach the dataplane through nginx" >&2 -cat "${log_dir}/route-probe-headers.txt" >&2 -cat "${log_dir}/route-probe-body.txt" >&2 -exit 1 diff --git a/tests/conformance/start-fixture-and-control-plane.sh b/tests/conformance/start-fixture-and-control-plane.sh deleted file mode 100755 index 667d8f75..00000000 --- a/tests/conformance/start-fixture-and-control-plane.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(cd -- "${script_dir}/../.." && pwd)" -suite_dir="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" -log_dir="${repo_root}/conformance-logs" - -mkdir -p "${log_dir}" "${repo_root}/conformance-results" - -( - cd "${suite_dir}" - PORT=3000 npm exec -- tsx examples/servers/typescript/everything-server.ts -) > "${log_dir}/reference-server.log" 2>&1 & -echo "$!" > "${log_dir}/reference-server.pid" - -docker compose -f "${script_dir}/docker-compose.yml" \ - up -d --wait redis fixture-proxy control-plane - -for _ in $(seq 1 120); do - if curl --silent --output /dev/null http://127.0.0.1:3000/mcp; then - exit 0 - fi - sleep 0.25 -done - -echo "Timed out waiting for the official conformance fixture" >&2 -exit 1 diff --git a/tests/conformance/stop-live-stack.sh b/tests/conformance/stop-live-stack.sh deleted file mode 100755 index e88cf942..00000000 --- a/tests/conformance/stop-live-stack.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(cd -- "${script_dir}/../.." && pwd)" -pid_file="${repo_root}/conformance-logs/reference-server.pid" - -if [ -f "${pid_file}" ]; then - kill "$(cat "${pid_file}")" 2>/dev/null || true -fi - -docker compose -f "${script_dir}/docker-compose.yml" \ - down --volumes --remove-orphans diff --git a/tests/conformance/upstream-fixture-failures.yml b/tests/conformance/upstream-fixture-failures.yml deleted file mode 100644 index 08239bda..00000000 --- a/tests/conformance/upstream-fixture-failures.yml +++ /dev/null @@ -1,14 +0,0 @@ -# The pinned alpha.11 JavaScript fixture reports 129 passed and 54 failed when -# the official runner targets it directly at MCP 2026-07-28, plus one scored -# warning. Of those failures, 47 belong to extension or pending scenarios and -# are already unscored by the requirements file. These are its seven scored -# failures and one warning. Update this file with MCP_CONFORMANCE_SOURCE_SHA. -server: - - tools-call-simple-text:wire-schema-valid - - tools-call-image:wire-schema-valid - - tools-call-audio:wire-schema-valid - - tools-call-embedded-resource:wire-schema-valid - - tools-call-mixed-content:wire-schema-valid - - tools-call-error:wire-schema-valid - - tools-call-with-progress:wire-schema-valid - - input-required-result-validate-input:sep-2322-validate-input-responses diff --git a/tests/conformance/write_client_config.py b/tests/conformance/write_client_config.py deleted file mode 100755 index b31e8b11..00000000 --- a/tests/conformance/write_client_config.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 -"""Publish an isolated upstream-client conformance route for the dataplane.""" - -from __future__ import annotations - -import argparse -import json -import os -from urllib.parse import urlparse - -import msgpack -import redis - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("subject") - parser.add_argument("virtual_host_id") - parser.add_argument("backend_url") - parser.add_argument("tool_names_json") - return parser.parse_args() - - -def main() -> None: - args = parse_args() - redis_url = os.environ.get("REDIS_URL") - if not redis_url: - raise SystemExit("REDIS_URL is required") - - parsed_url = urlparse(args.backend_url) - if parsed_url.scheme not in {"http", "https"} or not parsed_url.hostname: - raise SystemExit("backend_url must be an absolute HTTP(S) URL") - - tool_names = json.loads(args.tool_names_json) - if ( - not isinstance(tool_names, list) - or not tool_names - or not all(isinstance(name, str) and name for name in tool_names) - ): - raise SystemExit("tool_names_json must be a non-empty JSON string array") - - backend_name = "conformance-backend" - config = { - "virtual_hosts": { - args.virtual_host_id: { - "backends": { - backend_name: { - "name": backend_name, - "url": args.backend_url, - "passthrough_headers": [], - "add_headers": {}, - "remove_headers": [], - "allowed_tool_names": tool_names, - "tool_schemas": {name: {} for name in tool_names}, - "tool_name_aliases": {}, - "allowed_resource_names": [], - "allowed_prompt_names": [], - } - } - } - } - } - - key = msgpack.dumps(("UserConfig", args.subject), use_bin_type=True) - value = msgpack.dumps(config, use_bin_type=True) - client = redis.Redis.from_url(redis_url, decode_responses=False) - client.set(key, value, ex=600) - - -if __name__ == "__main__": - main() From 574240fa5e56ebc1cf8dfa258cf98e8ebe4b9ad0 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Mon, 31 Aug 2026 17:02:51 +0100 Subject: [PATCH 02/11] ci: isolate devtools workspace build Signed-off-by: lucarlig --- .github/workflows/ci.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70b90020..2b632845 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,10 +108,13 @@ jobs: path: target/cf-integration-src persist-credentials: false + - name: Isolate cf-integration from the data-plane workspace + run: cp -R target/cf-integration-src "${RUNNER_TEMP}/cf-integration-src" + - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2.9.1 with: - workspaces: target/cf-integration-src -> .integration/cargo-target + workspaces: ${{ runner.temp }}/cf-integration-src -> .integration/cargo-target - name: Install Node.js 22 uses: actions/setup-node@v6.5.0 @@ -119,12 +122,12 @@ jobs: node-version: 22 - name: Build cf-integration - working-directory: target/cf-integration-src + working-directory: ${{ runner.temp }}/cf-integration-src run: cargo build --release --locked --bin cf-integration - name: Run strict modern conformance env: - CF_INTEGRATION: ${{ github.workspace }}/target/cf-integration-src/.integration/cargo-target/release/cf-integration + CF_INTEGRATION: ${{ runner.temp }}/cf-integration-src/.integration/cargo-target/release/cf-integration run: make conformance - name: Upload conformance diagnostics From 81118cb28370caedfa87ebdc7146cfb15273158c Mon Sep 17 00:00:00 2001 From: lucarlig Date: Mon, 31 Aug 2026 17:12:09 +0100 Subject: [PATCH 03/11] ci: pin portable conformance runner Signed-off-by: lucarlig --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b632845..44a61c19 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,7 +104,7 @@ jobs: uses: actions/checkout@v6.0.2 with: repository: contextforge-org/contextforge-dev-tools - ref: a056b269d28c25702339c81f35afcce33649beb7 + ref: 0864bf5ad5022070b62ea6e063c502f199dfe201 path: target/cf-integration-src persist-credentials: false From fb548387c52286a23bc8edaa8ac91861a64e2d20 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Mon, 31 Aug 2026 17:44:35 +0100 Subject: [PATCH 04/11] ci: install published conformance binary Signed-off-by: lucarlig --- .github/workflows/ci.yml | 21 ++++----------------- Makefile | 4 ++-- _context/wiki/getting-started.md | 4 ++-- _context/wiki/performance.md | 2 +- _context/wiki/testing.md | 10 +++++----- 5 files changed, 14 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44a61c19..d096b0e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,34 +100,21 @@ jobs: with: fetch-depth: 0 - - name: Check out cf-integration - uses: actions/checkout@v6.0.2 - with: - repository: contextforge-org/contextforge-dev-tools - ref: 0864bf5ad5022070b62ea6e063c502f199dfe201 - path: target/cf-integration-src - persist-credentials: false - - - name: Isolate cf-integration from the data-plane workspace - run: cp -R target/cf-integration-src "${RUNNER_TEMP}/cf-integration-src" - - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2.9.1 + - uses: taiki-e/install-action@v2.75.27 with: - workspaces: ${{ runner.temp }}/cf-integration-src -> .integration/cargo-target + tool: cargo-binstall - name: Install Node.js 22 uses: actions/setup-node@v6.5.0 with: node-version: 22 - - name: Build cf-integration - working-directory: ${{ runner.temp }}/cf-integration-src - run: cargo build --release --locked --bin cf-integration + - name: Install cf-integration 0.1.0 binary + run: cargo binstall cf-integration@0.1.0 --no-confirm - name: Run strict modern conformance - env: - CF_INTEGRATION: ${{ runner.temp }}/cf-integration-src/.integration/cargo-target/release/cf-integration run: make conformance - name: Upload conformance diagnostics diff --git a/Makefile b/Makefile index d2aba0f0..240dc62d 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ compose-down: ## Tear down the stack conformance: ## Run strict modern MCP conformance against the committed data-plane HEAD @if ! command -v "$(CF_INTEGRATION)" >/dev/null 2>&1; then \ - echo "cf-integration not found: set CF_INTEGRATION to the built CLI path."; \ + echo "cf-integration not found: install its published binary with cargo binstall or set CF_INTEGRATION to its path."; \ exit 1; \ fi @if [ -n "$$(git -C "$(CF_DATAPLANE_REPO)" status --porcelain --untracked-files=no)" ]; then \ @@ -50,7 +50,7 @@ conformance: ## Run strict modern MCP conformance against the committed data-pla conformance-bless: ## Run strict modern conformance and atomically update its baselines @if ! command -v "$(CF_INTEGRATION)" >/dev/null 2>&1; then \ - echo "cf-integration not found: set CF_INTEGRATION to the built CLI path."; \ + echo "cf-integration not found: install its published binary with cargo binstall or set CF_INTEGRATION to its path."; \ exit 1; \ fi @if [ -n "$$(git -C "$(CF_DATAPLANE_REPO)" status --porcelain --untracked-files=no)" ]; then \ diff --git a/_context/wiki/getting-started.md b/_context/wiki/getting-started.md index 979fa444..47983e2f 100644 --- a/_context/wiki/getting-started.md +++ b/_context/wiki/getting-started.md @@ -29,8 +29,8 @@ Teardown: `make compose-down` (stops containers; volumes kept). ## cf-integration Conformance ```bash -CF_INTEGRATION=../contextforge-dev-tools/.integration/cargo-target/release/cf-integration \ - make conformance +cargo binstall cf-integration@0.1.0 --no-confirm +make conformance ``` This runs the modern client and modern server eras through the committed diff --git a/_context/wiki/performance.md b/_context/wiki/performance.md index 700584f8..fe782bf0 100644 --- a/_context/wiki/performance.md +++ b/_context/wiki/performance.md @@ -3,7 +3,7 @@ ## Full-Stack Load (Locust via cf-integration) Performance testing uses the control-plane Locust suite through -[`cf-integration`](https://github.com/contextforge-org/contextforge-dev-tools). +[`cf-integration`](https://crates.io/crates/cf-integration). It measures the nginx → external dataplane → backend request path while the ContextForge control plane publishes configuration. diff --git a/_context/wiki/testing.md b/_context/wiki/testing.md index c4139416..da2cda61 100644 --- a/_context/wiki/testing.md +++ b/_context/wiki/testing.md @@ -36,7 +36,7 @@ These run in `cargo nextest run` with no Docker dependencies. ## MCP Conformance -[`cf-integration`](https://github.com/contextforge-org/contextforge-dev-tools) +[`cf-integration`](https://crates.io/crates/cf-integration) owns the official fixture, control-plane registration, Compose topology, server and client runners, result rendering, and transactional baseline handling. This repository keeps only the CI invocation, Make targets, and expected findings. @@ -46,15 +46,16 @@ external dataplane. Selecting that lane also runs the fixture-direct server leg and the scoped external-dataplane client leg: ```bash +cargo binstall cf-integration@0.1.0 --no-confirm make conformance ``` The Make target tests the committed data-plane `HEAD`. It rejects tracked uncommitted changes because the CLI clones the selected repository and commit -into `.integration/`. To use a locally built CLI from a devtools branch: +into `.integration/`. To use another local CLI binary: ```bash -CF_INTEGRATION=../contextforge-dev-tools/.integration/cargo-target/release/cf-integration \ +CF_INTEGRATION=/path/to/cf-integration \ make conformance ``` @@ -62,8 +63,7 @@ Update every selected baseline atomically only after all operational work and baseline evaluation succeeds: ```bash -CF_INTEGRATION=../contextforge-dev-tools/.integration/cargo-target/release/cf-integration \ - make conformance-bless +make conformance-bless ``` Baselines are partitioned beneath From 929ca2ccc5b8151b887d511efee2d7ff07daf5ab Mon Sep 17 00:00:00 2001 From: lucarlig Date: Tue, 1 Sep 2026 08:44:39 +0100 Subject: [PATCH 05/11] ci: run conformance on demand Signed-off-by: lucarlig --- .github/workflows/ci.yml | 40 ----------------------- .github/workflows/conformance.yml | 54 +++++++++++++++++++++++++++++++ _context/wiki/testing.md | 6 ++-- 3 files changed, 57 insertions(+), 43 deletions(-) create mode 100644 .github/workflows/conformance.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d096b0e2..910c4fad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,43 +88,3 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2.9.1 - run: cargo bench --locked --workspace --no-run - - conformance: - name: conformance - needs: build - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Check out data plane - uses: actions/checkout@v6.0.2 - with: - fetch-depth: 0 - - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2.9.1 - - uses: taiki-e/install-action@v2.75.27 - with: - tool: cargo-binstall - - - name: Install Node.js 22 - uses: actions/setup-node@v6.5.0 - with: - node-version: 22 - - - name: Install cf-integration 0.1.0 binary - run: cargo binstall cf-integration@0.1.0 --no-confirm - - - name: Run strict modern conformance - run: make conformance - - - name: Upload conformance diagnostics - if: failure() - uses: actions/upload-artifact@v7.0.1 - with: - name: conformance-diagnostics-${{ github.run_id }}-${{ github.run_attempt }} - path: | - .integration/conformance/ - .integration/reports/ - if-no-files-found: warn - include-hidden-files: true - retention-days: 7 diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 00000000..9b0fb539 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,54 @@ +name: Conformance + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: conformance-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + +jobs: + conformance: + name: conformance + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Check out data plane + uses: actions/checkout@v6.0.2 + with: + fetch-depth: 0 + + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2.9.1 + - uses: taiki-e/install-action@v2.75.27 + with: + tool: cargo-binstall + + - name: Install Node.js 22 + uses: actions/setup-node@v6.5.0 + with: + node-version: 22 + + - name: Install cf-integration 0.1.0 binary + run: cargo binstall cf-integration@0.1.0 --no-confirm + + - name: Run strict modern conformance + run: make conformance + + - name: Upload conformance diagnostics + if: failure() + uses: actions/upload-artifact@v7.0.1 + with: + name: conformance-diagnostics-${{ github.run_id }}-${{ github.run_attempt }} + path: | + .integration/conformance/ + .integration/reports/ + if-no-files-found: warn + include-hidden-files: true + retention-days: 7 diff --git a/_context/wiki/testing.md b/_context/wiki/testing.md index da2cda61..7b251305 100644 --- a/_context/wiki/testing.md +++ b/_context/wiki/testing.md @@ -41,9 +41,9 @@ owns the official fixture, control-plane registration, Compose topology, server and client runners, result rendering, and transactional baseline handling. This repository keeps only the CI invocation, Make targets, and expected findings. -The blocking CI job runs the modern client and modern server eras through the -external dataplane. Selecting that lane also runs the fixture-direct server -leg and the scoped external-dataplane client leg: +The manually dispatched **Conformance** Actions workflow runs the modern client +and modern server eras through the external dataplane. Selecting that lane also +runs the fixture-direct server leg and the scoped external-dataplane client leg: ```bash cargo binstall cf-integration@0.1.0 --no-confirm From 31f38f8270a8c03d03ba207eef4e1b08a6f52d0d Mon Sep 17 00:00:00 2001 From: lucarlig Date: Tue, 1 Sep 2026 09:21:42 +0100 Subject: [PATCH 06/11] ci: trigger conformance from pull request label Signed-off-by: lucarlig --- .github/workflows/conformance.yml | 4 +++- _context/wiki/testing.md | 7 ++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 9b0fb539..94d59bdc 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -1,7 +1,8 @@ name: Conformance on: - workflow_dispatch: + pull_request: + types: [labeled] permissions: contents: read @@ -16,6 +17,7 @@ env: jobs: conformance: name: conformance + if: github.event.label.name == 'run-conformance' runs-on: ubuntu-latest timeout-minutes: 60 steps: diff --git a/_context/wiki/testing.md b/_context/wiki/testing.md index 7b251305..d58b5e56 100644 --- a/_context/wiki/testing.md +++ b/_context/wiki/testing.md @@ -41,9 +41,10 @@ owns the official fixture, control-plane registration, Compose topology, server and client runners, result rendering, and transactional baseline handling. This repository keeps only the CI invocation, Make targets, and expected findings. -The manually dispatched **Conformance** Actions workflow runs the modern client -and modern server eras through the external dataplane. Selecting that lane also -runs the fixture-direct server leg and the scoped external-dataplane client leg: +Apply the `run-conformance` label to a pull request to run the **Conformance** +Actions workflow. It runs the modern client and modern server eras through the +external dataplane. Selecting that lane also runs the fixture-direct server leg +and the scoped external-dataplane client leg: ```bash cargo binstall cf-integration@0.1.0 --no-confirm From f51e2ecc17db06e0671f113a0936228653cc3f11 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Tue, 1 Sep 2026 09:50:49 +0100 Subject: [PATCH 07/11] ci: trigger conformance from pull request comments Signed-off-by: lucarlig --- .github/workflows/conformance.yml | 61 ++++++++++++++++++++++++++++++- _context/wiki/testing.md | 10 +++-- 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 94d59bdc..381b50a6 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -1,23 +1,50 @@ name: Conformance +run-name: Conformance for PR #${{ github.event.pull_request.number || github.event.issue.number }} on: + # Keep the label trigger until this workflow reaches the default branch. pull_request: types: [labeled] + issue_comment: + types: [created] permissions: contents: read concurrency: - group: conformance-${{ github.ref }} + group: conformance-pr-${{ github.event.pull_request.number || github.event.issue.number }} cancel-in-progress: true env: CARGO_TERM_COLOR: always jobs: + acknowledge: + name: acknowledge command + if: >- + github.event_name == 'issue_comment' && + github.event.issue.pull_request && + github.event.comment.body == '/conformance' && + contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association) + permissions: + issues: write + runs-on: ubuntu-latest + steps: + - name: Acknowledge the command + env: + COMMENT_ID: ${{ github.event.comment.id }} + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + run: gh api --method POST "repos/$REPOSITORY/issues/comments/$COMMENT_ID/reactions" -f content=eyes + conformance: name: conformance - if: github.event.label.name == 'run-conformance' + if: >- + (github.event_name == 'pull_request' && github.event.label.name == 'run-conformance') || + (github.event_name == 'issue_comment' && + github.event.issue.pull_request && + github.event.comment.body == '/conformance' && + contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association)) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -25,6 +52,8 @@ jobs: uses: actions/checkout@v6.0.2 with: fetch-depth: 0 + persist-credentials: false + ref: refs/pull/${{ github.event.pull_request.number || github.event.issue.number }}/merge - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2.9.1 @@ -54,3 +83,31 @@ jobs: if-no-files-found: warn include-hidden-files: true retention-days: 7 + + report: + name: report conformance result + if: >- + always() && + github.event_name == 'issue_comment' && + needs.conformance.result != 'skipped' + needs: conformance + permissions: + issues: write + runs-on: ubuntu-latest + steps: + - name: Report the result + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.issue.number }} + REPOSITORY: ${{ github.repository }} + RESULT: ${{ needs.conformance.result }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + case "$RESULT" in + success) marker="✅" ;; + failure) marker="❌" ;; + cancelled) marker="⚪" ;; + *) marker="⚠️" ;; + esac + body="$marker Conformance **$RESULT** — [view workflow run]($RUN_URL)." + gh api --method POST "repos/$REPOSITORY/issues/$PR_NUMBER/comments" -f body="$body" diff --git a/_context/wiki/testing.md b/_context/wiki/testing.md index d58b5e56..cf17c7d9 100644 --- a/_context/wiki/testing.md +++ b/_context/wiki/testing.md @@ -41,10 +41,12 @@ owns the official fixture, control-plane registration, Compose topology, server and client runners, result rendering, and transactional baseline handling. This repository keeps only the CI invocation, Make targets, and expected findings. -Apply the `run-conformance` label to a pull request to run the **Conformance** -Actions workflow. It runs the modern client and modern server eras through the -external dataplane. Selecting that lane also runs the fixture-direct server leg -and the scoped external-dataplane client leg: +Comment exactly `/conformance` on a pull request to run the **Conformance** +Actions workflow. Only repository owners, members, and collaborators can start +it. The workflow acknowledges the command, tests the pull request merge commit, +and reports the final result back to the pull request. It runs the modern client +and modern server eras through the external dataplane. Selecting that lane also +runs the fixture-direct server leg and the scoped external-dataplane client leg: ```bash cargo binstall cf-integration@0.1.0 --no-confirm From 82b4d291c7c75b98f3e5e9dd2aef850b73014556 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Tue, 1 Sep 2026 10:42:58 +0100 Subject: [PATCH 08/11] ci: reuse the data-plane build for conformance Signed-off-by: lucarlig --- .github/workflows/ci.yml | 17 ++++++++++ .github/workflows/conformance.yml | 53 +++++++++++++++++++++++++++++-- docker/Dockerfile | 3 ++ 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 910c4fad..73cf162b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,23 @@ jobs: - name: Build the workspace run: cargo build --locked --workspace --all-features + - name: Prepare the conformance binary artifact + run: | + mkdir -p target/conformance + cp target/debug/contextforge-data-plane target/conformance/contextforge-data-plane + strip target/conformance/contextforge-data-plane + git rev-parse HEAD > target/conformance/source-revision + + - name: Upload the conformance binary + uses: actions/upload-artifact@v7.0.1 + with: + name: contextforge-data-plane-conformance + path: | + target/conformance/contextforge-data-plane + target/conformance/source-revision + if-no-files-found: error + retention-days: 1 + bench: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 381b50a6..65b1be79 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -9,6 +9,7 @@ on: types: [created] permissions: + actions: read contents: read concurrency: @@ -55,12 +56,56 @@ jobs: persist-credentials: false ref: refs/pull/${{ github.event.pull_request.number || github.event.issue.number }}/merge - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2.9.1 - uses: taiki-e/install-action@v2.75.27 with: tool: cargo-binstall + - name: Find the matching CI build + id: ci-build + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }} + REPOSITORY: ${{ github.repository }} + run: | + head_revision=$(gh api "repos/$REPOSITORY/pulls/$PR_NUMBER" --jq '.head.sha') + for attempt in $(seq 1 60); do + run_id=$(gh api \ + "repos/$REPOSITORY/actions/workflows/ci.yml/runs?event=pull_request&head_sha=$head_revision&per_page=10" \ + --jq '.workflow_runs[0].id // empty') + if [ -n "$run_id" ]; then + artifact_count=$(gh api \ + "repos/$REPOSITORY/actions/runs/$run_id/artifacts?name=contextforge-data-plane-conformance" \ + --jq '.total_count') + if [ "$artifact_count" -gt 0 ]; then + echo "run-id=$run_id" >> "$GITHUB_OUTPUT" + exit 0 + fi + fi + echo "Waiting for the CI build artifact for $head_revision ($attempt/60)" + sleep 10 + done + echo "No conformance binary artifact was produced by CI for $head_revision" >&2 + exit 1 + + - name: Download the matching data-plane binary + uses: actions/download-artifact@v8.0.1 + with: + name: contextforge-data-plane-conformance + path: target/conformance + run-id: ${{ steps.ci-build.outputs.run-id }} + github-token: ${{ github.token }} + + - name: Verify and package the data-plane image + run: | + revision=$(git rev-parse HEAD) + test "$(cat target/conformance/source-revision)" = "$revision" + chmod +x target/conformance/contextforge-data-plane + docker buildx build --load \ + --target conformance-prebuilt \ + --build-context prebuilt=target/conformance \ + --tag contextforge-data-plane:conformance \ + --file docker/Dockerfile . + - name: Install Node.js 22 uses: actions/setup-node@v6.5.0 with: @@ -70,6 +115,10 @@ jobs: run: cargo binstall cf-integration@0.1.0 --no-confirm - name: Run strict modern conformance + env: + CF_DATAPLANE_IMAGE: contextforge-data-plane:conformance + CF_DATAPLANE_PULL_POLICY: never + CF_DATAPLANE_REF: "" run: make conformance - name: Upload conformance diagnostics diff --git a/docker/Dockerfile b/docker/Dockerfile index 4be00c22..7f8a9b6f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -26,5 +26,8 @@ LABEL org.opencontainers.image.source=https://github.com/contextforge-org/contex LABEL org.opencontainers.image.description="contextforge-data-plane - open source experimental data plane for ContextForge." ENTRYPOINT ["/contextforge-data-plane"] +FROM runtime AS conformance-prebuilt +COPY --from=prebuilt /contextforge-data-plane /contextforge-data-plane + FROM runtime COPY --from=builder /app/target/release/contextforge-data-plane /contextforge-data-plane From a2ce4f16b83591ffb3cebb56e6258834516226b1 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Tue, 1 Sep 2026 11:18:02 +0100 Subject: [PATCH 09/11] ci: delegate artifact preparation to cf-integration Signed-off-by: lucarlig --- .github/workflows/ci.yml | 14 +++----- .github/workflows/conformance.yml | 55 +++++-------------------------- 2 files changed, 13 insertions(+), 56 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73cf162b..450c945f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,20 +81,14 @@ jobs: - name: Build the workspace run: cargo build --locked --workspace --all-features - - name: Prepare the conformance binary artifact - run: | - mkdir -p target/conformance - cp target/debug/contextforge-data-plane target/conformance/contextforge-data-plane - strip target/conformance/contextforge-data-plane - git rev-parse HEAD > target/conformance/source-revision + - name: Strip the conformance binary + run: strip target/debug/contextforge-data-plane - name: Upload the conformance binary uses: actions/upload-artifact@v7.0.1 with: - name: contextforge-data-plane-conformance - path: | - target/conformance/contextforge-data-plane - target/conformance/source-revision + name: contextforge-data-plane-conformance-${{ github.sha }} + path: target/debug/contextforge-data-plane if-no-files-found: error retention-days: 1 diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 65b1be79..d55267bb 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -60,60 +60,23 @@ jobs: with: tool: cargo-binstall - - name: Find the matching CI build - id: ci-build + - name: Install cf-integration 0.2.0 binary + run: cargo binstall cf-integration@0.2.0 --no-confirm + + - name: Prepare the exact data-plane image env: GH_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }} - REPOSITORY: ${{ github.repository }} - run: | - head_revision=$(gh api "repos/$REPOSITORY/pulls/$PR_NUMBER" --jq '.head.sha') - for attempt in $(seq 1 60); do - run_id=$(gh api \ - "repos/$REPOSITORY/actions/workflows/ci.yml/runs?event=pull_request&head_sha=$head_revision&per_page=10" \ - --jq '.workflow_runs[0].id // empty') - if [ -n "$run_id" ]; then - artifact_count=$(gh api \ - "repos/$REPOSITORY/actions/runs/$run_id/artifacts?name=contextforge-data-plane-conformance" \ - --jq '.total_count') - if [ "$artifact_count" -gt 0 ]; then - echo "run-id=$run_id" >> "$GITHUB_OUTPUT" - exit 0 - fi - fi - echo "Waiting for the CI build artifact for $head_revision ($attempt/60)" - sleep 10 - done - echo "No conformance binary artifact was produced by CI for $head_revision" >&2 - exit 1 - - - name: Download the matching data-plane binary - uses: actions/download-artifact@v8.0.1 - with: - name: contextforge-data-plane-conformance - path: target/conformance - run-id: ${{ steps.ci-build.outputs.run-id }} - github-token: ${{ github.token }} - - - name: Verify and package the data-plane image - run: | - revision=$(git rev-parse HEAD) - test "$(cat target/conformance/source-revision)" = "$revision" - chmod +x target/conformance/contextforge-data-plane - docker buildx build --load \ - --target conformance-prebuilt \ - --build-context prebuilt=target/conformance \ - --tag contextforge-data-plane:conformance \ - --file docker/Dockerfile . + run: >- + cf-integration ci prepare-image + --artifact contextforge-data-plane-conformance + --binary contextforge-data-plane + --image contextforge-data-plane:conformance - name: Install Node.js 22 uses: actions/setup-node@v6.5.0 with: node-version: 22 - - name: Install cf-integration 0.1.0 binary - run: cargo binstall cf-integration@0.1.0 --no-confirm - - name: Run strict modern conformance env: CF_DATAPLANE_IMAGE: contextforge-data-plane:conformance From 72cd68472de6c46ba4842bc864a649a5f820eb6e Mon Sep 17 00:00:00 2001 From: lucarlig Date: Tue, 1 Sep 2026 13:07:24 +0100 Subject: [PATCH 10/11] chore: ignore conformance output directories Signed-off-by: lucarlig --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 1f66ef2f..63bf79d6 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,9 @@ target contextforge-data-plane.log.* # Generated by cf-integration .integration/ +/.conformance-suite/ +/conformance-logs/ +/conformance-results/ # Generated by mdBook (wiki) _context/wiki/book/ From 2a7dab7195de617a9d490bda07878a506ea9a054 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Tue, 1 Sep 2026 13:08:33 +0100 Subject: [PATCH 11/11] ci: remove conformance label trigger Signed-off-by: lucarlig --- .github/workflows/conformance.yml | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index d55267bb..6bae224b 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -1,10 +1,7 @@ name: Conformance -run-name: Conformance for PR #${{ github.event.pull_request.number || github.event.issue.number }} +run-name: Conformance for PR #${{ github.event.issue.number }} on: - # Keep the label trigger until this workflow reaches the default branch. - pull_request: - types: [labeled] issue_comment: types: [created] @@ -13,7 +10,7 @@ permissions: contents: read concurrency: - group: conformance-pr-${{ github.event.pull_request.number || github.event.issue.number }} + group: conformance-pr-${{ github.event.issue.number }} cancel-in-progress: true env: @@ -23,7 +20,6 @@ jobs: acknowledge: name: acknowledge command if: >- - github.event_name == 'issue_comment' && github.event.issue.pull_request && github.event.comment.body == '/conformance' && contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association) @@ -41,11 +37,9 @@ jobs: conformance: name: conformance if: >- - (github.event_name == 'pull_request' && github.event.label.name == 'run-conformance') || - (github.event_name == 'issue_comment' && - github.event.issue.pull_request && - github.event.comment.body == '/conformance' && - contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association)) + github.event.issue.pull_request && + github.event.comment.body == '/conformance' && + contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association) runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -54,7 +48,7 @@ jobs: with: fetch-depth: 0 persist-credentials: false - ref: refs/pull/${{ github.event.pull_request.number || github.event.issue.number }}/merge + ref: refs/pull/${{ github.event.issue.number }}/merge - uses: taiki-e/install-action@v2.75.27 with: @@ -100,7 +94,6 @@ jobs: name: report conformance result if: >- always() && - github.event_name == 'issue_comment' && needs.conformance.result != 'skipped' needs: conformance permissions: