conformance/.github/workflows/traceability.yml at main · modelcontextprotocol/conformance · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
name: Refresh SEP traceability manifest

# Regenerates src/seps/traceability.json by running the conformance suite against
# the reference SDK and recording which check IDs were emitted, then opens a PR
# with the diff. NOT a PR gate — runs on demand / on a schedule and proposes an
# update for review. plan.modelcontextprotocol.io reads the committed file from
# main.
#
# Depends on the `conformance sdk` subcommand (#277), which clones+builds the SDK
# and runs the client+server suites. The `run` job executes third-party SDK code,
# so it has NO repo write token (read-only perms, persist-credentials: false) and
# only uploads results as an artifact; the separate `propose` job holds the
# write/PR permissions and never executes SDK code.

on:
  workflow_dispatch:
    inputs:
      sdk:
        description: 'SDK ref to run against (e.g. typescript-sdk@<sha>)'
        default: 'typescript-sdk@main'
  schedule:
    - cron: '0 6 * * 1' # Weekly, Monday 06:00 UTC.

concurrency:
  group: traceability-refresh
  cancel-in-progress: true

jobs:
  run:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    env:
      SDK_REF: ${{ inputs.sdk || 'typescript-sdk@main' }}
    steps:
      - uses: actions/checkout@v6
        with:
          persist-credentials: false # no git token while SDK code runs

      - uses: actions/setup-node@v6
        with:
          node-version: 24
          cache: npm

      # The SDK's own build (e.g. typescript-sdk uses `pnpm install && pnpm run
      # build:all`) needs pnpm on PATH; corepack provides it.
      - run: corepack enable

      - run: npm ci
      - run: npm run build

      - name: Run conformance suites against the reference SDK
        # `sdk` requires --mode client|server; run both into the same results dir
        # (the second reuses the cached checkout + build via --skip-build).
        # `|| true`: the manifest only needs the emitted check IDs (written
        # regardless of pass/fail), so SDK conformance failures must not fail
        # this step. The "no results produced" guard below is the real safety net.
        run: |
          node dist/index.js sdk "$SDK_REF" --mode client --suite all -o results || true
          node dist/index.js sdk "$SDK_REF" --mode server --suite all --skip-build -o results || true

      - name: Fail if no results were produced
        run: |
          if [ -z "$(find results -name checks.json -print -quit 2>/dev/null)" ]; then
            echo "No checks.json produced — the suite run failed; not proposing a manifest."
            exit 1
          fi

      - uses: actions/upload-artifact@v4
        with:
          name: conformance-results
          path: results
          retention-days: 7

  propose:
    needs: run
    runs-on: ubuntu-latest
    # Requires the repo/org setting "Allow GitHub Actions to create and approve
    # pull requests" to be enabled, otherwise `gh pr create` fails.
    permissions:
      contents: write
      pull-requests: write
    env:
      SDK_REF: ${{ inputs.sdk || 'typescript-sdk@main' }}
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-node@v6
        with:
          node-version: 24
          cache: npm
      - run: npm ci
      - run: npm run build

      - uses: actions/download-artifact@v4
        with:
          name: conformance-results
          path: results

      - name: Regenerate manifest
        run: |
          set -euo pipefail
          # Record the resolved sha (stable per SDK commit) so the manifest's
          # `source` only changes when the SDK actually advances — no per-run noise.
          ref="${SDK_REF#*@}"
          sha="$(git ls-remote https://github.com/modelcontextprotocol/typescript-sdk.git "$ref" | cut -f1)"
          node dist/index.js traceability --results results \
            --source "typescript-sdk@${sha:0:12}"

      - name: Open/update the rolling refresh PR
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail
          if git diff --quiet -- src/seps/traceability.json; then
            echo "traceability.json unchanged"
            exit 0
          fi
          # One rolling branch/PR, force-updated each run, so the schedule does
          # not accrue a new PR every week.
          branch="traceability-refresh"
          git config user.name 'github-actions[bot]'
          git config user.email 'github-actions[bot]@users.noreply.github.com'
          git checkout -B "$branch"
          git add src/seps/traceability.json
          git commit -m "chore: refresh SEP traceability manifest ($SDK_REF)"
          git push --force origin "$branch"
          gh pr view "$branch" >/dev/null 2>&1 || gh pr create \
            --head "$branch" \
            --title 'chore: refresh SEP traceability manifest' \
            --body 'Automated refresh from a conformance run against the reference SDK. Review the coverage diff before merging.'