From b900defa8c3cc9d2d066ce41737cf344bb3511a6 Mon Sep 17 00:00:00 2001 From: Andriy Svyryd Date: Fri, 18 Sep 2026 16:51:33 -0700 Subject: [PATCH] Give the eval workflow permissions to comment on PRs Skip checks if eval harness doesn't exist --- .github/workflows/agent-harness-coverage.yml | 20 ++++++++++ .../workflows/agent-harness-evaluation.yml | 37 ++++++++++++++----- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/.github/workflows/agent-harness-coverage.yml b/.github/workflows/agent-harness-coverage.yml index 1340405ab53..8fe38b433aa 100644 --- a/.github/workflows/agent-harness-coverage.yml +++ b/.github/workflows/agent-harness-coverage.yml @@ -20,7 +20,22 @@ jobs: fetch-depth: 0 persist-credentials: false + - name: Detect harness on target branch + id: harness + shell: bash + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + git cat-file -e "$BASE_SHA:eng/harness-evaluation" 2>/dev/null \ + && echo "exists=true" >> "$GITHUB_OUTPUT" \ + || true + + - name: Harness not applicable + if: steps.harness.outputs.exists != 'true' + run: echo "No eval harness exists on the target branch." + - name: Restore repository SDK + if: steps.harness.outputs.exists == 'true' shell: bash run: | ./restore.sh @@ -29,6 +44,7 @@ jobs: echo "$PWD/.dotnet" >> "$GITHUB_PATH" - name: Set up Node.js + if: steps.harness.outputs.exists == 'true' uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 with: node-version: 22.12 @@ -36,17 +52,21 @@ jobs: cache-dependency-path: eng/harness-evaluation/package-lock.json - name: Install evaluation tools + if: steps.harness.outputs.exists == 'true' working-directory: eng/harness-evaluation run: npm ci - name: Audit evaluation dependencies + if: steps.harness.outputs.exists == 'true' working-directory: eng/harness-evaluation run: npm audit --audit-level=moderate - name: Test evaluation harness + if: steps.harness.outputs.exists == 'true' working-directory: eng/harness-evaluation run: npm test - name: Validate customization coverage and lint skills and evaluations + if: steps.harness.outputs.exists == 'true' working-directory: eng/harness-evaluation run: npm run lint diff --git a/.github/workflows/agent-harness-evaluation.yml b/.github/workflows/agent-harness-evaluation.yml index 113c52ccde7..e06a44a25af 100644 --- a/.github/workflows/agent-harness-evaluation.yml +++ b/.github/workflows/agent-harness-evaluation.yml @@ -34,7 +34,7 @@ jobs: permissions: contents: read issues: write - pull-requests: read + pull-requests: write outputs: base_sha: ${{ steps.pr.outputs.base_sha }} head_sha: ${{ steps.pr.outputs.target_sha }} @@ -45,6 +45,7 @@ jobs: author: ${{ steps.pr.outputs.author }} matrix: ${{ steps.pr.outputs.matrix }} has_entries: ${{ steps.pr.outputs.has_entries }} + harness_exists: ${{ steps.pr.outputs.harness_exists }} components: ${{ steps.pr.outputs.components }} steps: - name: Resolve pull request and authorization @@ -123,6 +124,23 @@ jobs: ? fromBranch && (manual || automatic) : manual || (automatic && ['admin', 'maintain', 'write'].includes(permission))); + const evaluationRoot = 'eng/harness-evaluation'; + try { + await github.rest.repos.getContent({ + owner: context.repo.owner, + repo: context.repo.repo, + path: evaluationRoot, + ref: mergedIntoDefault ? targetSha : pull.base.sha + }); + } catch (error) { + if (error.status !== 404) throw error; + core.setOutput('matrix', JSON.stringify({ include: [] })); + core.setOutput('has_entries', 'false'); + core.setOutput('harness_exists', 'false'); + core.notice('Agent harness evaluation is not available on the target branch.'); + return; + } + const { data: trustedCommit } = await github.rest.repos.getCommit({ owner: context.repo.owner, repo: context.repo.repo, @@ -143,7 +161,9 @@ jobs: throw new Error(`The repository tree for ${targetSha} was truncated.`); } - const evaluationRoot = 'eng/harness-evaluation'; + const paths = tree.tree + .filter(entry => entry.type === 'blob') + .map(entry => entry.path); const componentPatterns = [ { directory: '.github/instructions', suffixes: ['.instructions.md'], kind: 'instruction', evalDirectory: 'instructions' }, { directory: '.github/agents', suffixes: ['.agent.md', '.md'], kind: 'agent', evalDirectory: 'agents' }, @@ -156,9 +176,6 @@ jobs: source: '.github/copilot-instructions.md', eval: `${evaluationRoot}/instructions/copilot-instructions/eval.yaml` }]; - const paths = tree.tree - .filter(entry => entry.type === 'blob') - .map(entry => entry.path); for (const path of paths) { const skill = path.match(/^\.agents\/skills\/([^/]+)\/SKILL\.md$/); if (skill) { @@ -283,6 +300,7 @@ jobs: core.setOutput('author', author); core.setOutput('matrix', JSON.stringify(matrix)); core.setOutput('has_entries', String(selected.length > 0)); + core.setOutput('harness_exists', 'true'); core.setOutput('components', JSON.stringify(selected)); await core.summary .addHeading('Agent Harness Evaluation') @@ -292,9 +310,10 @@ jobs: - name: Update pull request metadata if: >- - github.event.action != 'closed' || - (github.event.pull_request.merged == true && - github.event.pull_request.base.ref == github.event.repository.default_branch) + steps.pr.outputs.harness_exists == 'true' && + (github.event.action != 'closed' || + (github.event.pull_request.merged == true && + github.event.pull_request.base.ref == github.event.repository.default_branch)) uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 env: CAN_USE_SECRETS: ${{ steps.pr.outputs.can_use_secrets }} @@ -416,7 +435,7 @@ jobs: contents: read copilot-requests: write issues: write - pull-requests: read + pull-requests: write env: COMPONENT: ${{ matrix.id }} CANDIDATE_ROOT: candidate