Give the eval workflow permissions to comment on PRs

.github/workflows/agent-harness-coverage.yml

@@ -20,7 +20,22 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
 
+      - name: Detect harness on target branch
+        id: harness
+        shell: bash
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          git cat-file -e "$BASE_SHA:eng/harness-evaluation" 2>/dev/null \
+            && echo "exists=true" >> "$GITHUB_OUTPUT" \
+            || true
+
+      - name: Harness not applicable
+        if: steps.harness.outputs.exists != 'true'
+        run: echo "No eval harness exists on the target branch."
+
       - name: Restore repository SDK
+        if: steps.harness.outputs.exists == 'true'
         shell: bash
         run: |
           ./restore.sh
@@ -29,24 +44,29 @@ jobs:
           echo "$PWD/.dotnet" >> "$GITHUB_PATH"
 
       - name: Set up Node.js
+        if: steps.harness.outputs.exists == 'true'
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
         with:
           node-version: 22.12
           cache: npm
           cache-dependency-path: eng/harness-evaluation/package-lock.json
 
       - name: Install evaluation tools
+        if: steps.harness.outputs.exists == 'true'
         working-directory: eng/harness-evaluation
         run: npm ci
 
       - name: Audit evaluation dependencies
+        if: steps.harness.outputs.exists == 'true'
         working-directory: eng/harness-evaluation
         run: npm audit --audit-level=moderate
 
       - name: Test evaluation harness
+        if: steps.harness.outputs.exists == 'true'
         working-directory: eng/harness-evaluation
         run: npm test
 
       - name: Validate customization coverage and lint skills and evaluations
+        if: steps.harness.outputs.exists == 'true'
         working-directory: eng/harness-evaluation
         run: npm run lint

.github/workflows/agent-harness-evaluation.yml

@@ -34,7 +34,7 @@ jobs:
     permissions:
       contents: read
       issues: write
-      pull-requests: read
+      pull-requests: write
     outputs:
       base_sha: ${{ steps.pr.outputs.base_sha }}
       head_sha: ${{ steps.pr.outputs.target_sha }}
@@ -45,6 +45,7 @@ jobs:
       author: ${{ steps.pr.outputs.author }}
       matrix: ${{ steps.pr.outputs.matrix }}
       has_entries: ${{ steps.pr.outputs.has_entries }}
+      harness_exists: ${{ steps.pr.outputs.harness_exists }}
       components: ${{ steps.pr.outputs.components }}
     steps:
       - name: Resolve pull request and authorization
@@ -123,6 +124,23 @@ jobs:
               ? fromBranch && (manual || automatic)
               : manual || (automatic && ['admin', 'maintain', 'write'].includes(permission)));
 
+            const evaluationRoot = 'eng/harness-evaluation';
+            try {
+              await github.rest.repos.getContent({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                path: evaluationRoot,
+                ref: mergedIntoDefault ? targetSha : pull.base.sha
+              });
+            } catch (error) {
+              if (error.status !== 404) throw error;
+              core.setOutput('matrix', JSON.stringify({ include: [] }));
+              core.setOutput('has_entries', 'false');
+              core.setOutput('harness_exists', 'false');
+              core.notice('Agent harness evaluation is not available on the target branch.');
+              return;
+            }
+
             const { data: trustedCommit } = await github.rest.repos.getCommit({
               owner: context.repo.owner,
               repo: context.repo.repo,
@@ -143,7 +161,9 @@ jobs:
               throw new Error(`The repository tree for ${targetSha} was truncated.`);
             }
 
-            const evaluationRoot = 'eng/harness-evaluation';
+            const paths = tree.tree
+              .filter(entry => entry.type === 'blob')
+              .map(entry => entry.path);
             const componentPatterns = [
               { directory: '.github/instructions', suffixes: ['.instructions.md'], kind: 'instruction', evalDirectory: 'instructions' },
               { directory: '.github/agents', suffixes: ['.agent.md', '.md'], kind: 'agent', evalDirectory: 'agents' },
@@ -156,9 +176,6 @@ jobs:
               source: '.github/copilot-instructions.md',
               eval: `${evaluationRoot}/instructions/copilot-instructions/eval.yaml`
             }];
-            const paths = tree.tree
-              .filter(entry => entry.type === 'blob')
-              .map(entry => entry.path);
             for (const path of paths) {
               const skill = path.match(/^\.agents\/skills\/([^/]+)\/SKILL\.md$/);
               if (skill) {
@@ -283,6 +300,7 @@ jobs:
             core.setOutput('author', author);
             core.setOutput('matrix', JSON.stringify(matrix));
             core.setOutput('has_entries', String(selected.length > 0));
+            core.setOutput('harness_exists', 'true');
             core.setOutput('components', JSON.stringify(selected));
             await core.summary
               .addHeading('Agent Harness Evaluation')
@@ -292,9 +310,10 @@ jobs:
 
       - name: Update pull request metadata
         if: >-
-          github.event.action != 'closed' ||
-          (github.event.pull_request.merged == true &&
-           github.event.pull_request.base.ref == github.event.repository.default_branch)
+          steps.pr.outputs.harness_exists == 'true' &&
+          (github.event.action != 'closed' ||
+           (github.event.pull_request.merged == true &&
+            github.event.pull_request.base.ref == github.event.repository.default_branch))
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3
         env:
           CAN_USE_SECRETS: ${{ steps.pr.outputs.can_use_secrets }}
@@ -416,7 +435,7 @@ jobs:
       contents: read
       copilot-requests: write
       issues: write
-      pull-requests: read
+      pull-requests: write
     env:
       COMPONENT: ${{ matrix.id }}
       CANDIDATE_ROOT: candidate