Add LLM-as-judge evaluation framework for Docker skills

.github/workflows/skill-evals.yml

@@ -1,23 +1,24 @@
 name: Live skill evals
 
-# Runs skills/*/evals/*.yaml with Claude Code and/or Codex against real CLIs.
-# Calls a live model and may need tool-specific authentication, so it stays
-# out of offline `task ci`.
+# Runs skill evaluations using LLM-as-judge against Claude Code and/or Codex.
+# Evaluations are defined in skills/<skill>/evals/<skill>.md markdown files.
+# Each skill runs as a separate job for parallelism and targeted re-runs.
 on:
   pull_request:
     paths:
       - ".claude-plugin/**"
       - ".codex-plugin/**"
       - ".agents/**"
       - "skills/**"
-      - "scripts/skill_eval.py"
+      - "scripts/eval_judge.py"
+      - "scripts/test_eval_judge.py"
       - ".github/workflows/skill-evals.yml"
   schedule:
     - cron: "17 6 * * 1"
   workflow_dispatch:
     inputs:
       skill:
-        description: "Only run scenarios for this skill id (empty: all)"
+        description: "Only run evals for this skill id (empty: all)"
         required: false
         default: ""
       agent:
@@ -34,7 +35,7 @@ on:
         required: false
         default: ""
       repeat:
-        description: "Runs per scenario"
+        description: "Runs per prompt"
         required: false
         default: "1"
 
@@ -46,6 +47,75 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  # Determine which skills to evaluate based on changed files
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      skills: ${{ steps.detect.outputs.skills }}
+      run_all: ${{ steps.detect.outputs.run_all }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Detect changed skills
+        id: detect
+        run: |
+          # All skills except docker-destructive-guardrails (too flaky for CI)
+          ALL_SKILLS='["docker-agent-config","docker-agent-deploy","docker-agent-run","docker-build-strategies","docker-compose-patterns","docker-docs","docker-project-foundations","docker-sandboxes-env","docker-sandboxes-kits","docker-sandboxes-lifecycle","docker-sandboxes-network-credentials"]'
+
+          # If workflow_dispatch with specific skill, use that
+          if [ -n "${{ inputs.skill }}" ]; then
+            echo "skills=[\"${{ inputs.skill }}\"]" >> $GITHUB_OUTPUT
+            echo "run_all=false" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+
+          # If scheduled or workflow_dispatch without skill, run all skills
+          if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "run_all=true" >> $GITHUB_OUTPUT
+            echo "skills=$ALL_SKILLS" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+
+          # Get changed files
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            BASE_SHA=${{ github.event.pull_request.base.sha }}
+            HEAD_SHA=${{ github.event.pull_request.head.sha }}
+          else
+            BASE_SHA=${{ github.event.before }}
+            HEAD_SHA=${{ github.sha }}
+          fi
+
+          CHANGED_FILES=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA" 2>/dev/null || echo "")
+
+          # Check if framework files changed
+          FRAMEWORK_CHANGED=false
+          if echo "$CHANGED_FILES" | grep -qE '^(scripts/eval_judge\.py|\.github/workflows/skill-evals\.yml|\.claude-plugin/|\.codex-plugin/|\.agents/)'; then
+            FRAMEWORK_CHANGED=true
+          fi
+
+          if [ "$FRAMEWORK_CHANGED" = "true" ]; then
+            echo "run_all=true" >> $GITHUB_OUTPUT
+            echo "skills=$ALL_SKILLS" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+
+          # Extract specific skills that changed
+          SKILLS=$(echo "$CHANGED_FILES" | grep -oE '^skills/[^/]+' | sed 's|skills/||' | sort -u | grep -v '^docker-destructive-guardrails$' || true)
+
+          if [ -z "$SKILLS" ]; then
+            echo "No skill changes detected"
+            echo "skills=[]" >> $GITHUB_OUTPUT
+            echo "run_all=false" >> $GITHUB_OUTPUT
+          else
+            # Convert to JSON array
+            SKILLS_JSON=$(echo "$SKILLS" | jq -R -s -c 'split("\n") | map(select(length > 0))')
+            echo "skills=$SKILLS_JSON" >> $GITHUB_OUTPUT
+            echo "run_all=false" >> $GITHUB_OUTPUT
+          fi
+
   plugin-validate:
     runs-on: ubuntu-latest
     timeout-minutes: 5
@@ -64,37 +134,43 @@ jobs:
       - name: Validate plugin manifest
         run: claude plugin validate .
 
-  scenarios-claude:
-    if: ${{ inputs.agent == 'claude' || inputs.agent == 'both' || inputs.agent == '' }}
+  # Claude evals - one job per skill
+  eval-claude:
+    needs: detect-changes
+    if: |
+      (inputs.agent == 'claude' || inputs.agent == 'both' || inputs.agent == '') &&
+      needs.detect-changes.outputs.skills != '[]'
     runs-on: ubuntu-24.04
-    timeout-minutes: 45
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        skill: ${{ fromJson(needs.detect-changes.outputs.skills) }}
     env:
       CLAUDE_CODE_VERSION: "2.1.316"
       SBX_VERSION: "0.50.1"
-      SKILL_EVAL_AGENT: "claude"
-      SKILL_EVAL_MODEL: ${{ inputs.model || 'sonnet' }}
-      SKILL_EVAL_REPEAT: ${{ inputs.repeat || '1' }}
-      SKILL_EVAL_MIN_PASS_RATE: "1.0"
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
 
+      # Only Docker Sandboxes skills need the sbx CLI at eval time
       - name: Install Docker Sandboxes (sbx)
+        if: startsWith(matrix.skill, 'docker-sandboxes-')
         run: |
           curl -fsSL "https://github.com/docker/sbx-releases/releases/download/v${SBX_VERSION}/DockerSandboxes-linux-amd64-ubuntu2404.deb" -o sbx.deb
+          sudo apt-get update -q
           sudo apt-get install -y ./sbx.deb
           sudo usermod -aG kvm "$USER"
           rm sbx.deb
           sbx version
 
       - name: Setup sbx (login and policy)
+        if: startsWith(matrix.skill, 'docker-sandboxes-')
         env:
           DOCKERPUBLICBOT_READ_PAT: ${{ secrets.DOCKERPUBLICBOT_READ_PAT }}
           DOCKERPUBLICBOT_USERNAME: ${{ vars.DOCKERPUBLICBOT_USERNAME }}
         run: |
-          # sg kvm runs commands in kvm group context (usermod doesn't affect current shell)
-          # Login starts the daemon and authenticates for pulling sandbox images
           sg kvm -c 'echo "$DOCKERPUBLICBOT_READ_PAT" | sbx login --username "$DOCKERPUBLICBOT_USERNAME" --password-stdin'
           sg kvm -c 'sbx policy init balanced'
 
@@ -109,72 +185,80 @@ jobs:
           python3 -m venv .venv
           .venv/bin/pip install -q --require-hashes -r scripts/requirements.txt
 
-      - name: Run live skill scenarios (Claude)
+      - name: Run eval for ${{ matrix.skill }} (Claude)
         env:
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          INPUT_SKILL: ${{ inputs.skill }}
+          INPUT_MODEL: ${{ inputs.model || 'sonnet' }}
+          INPUT_REPEAT: ${{ inputs.repeat || '1' }}
         run: |
-          set +e
-          .venv/bin/python scripts/skill_eval.py --agent claude ${INPUT_SKILL:+--skill "$INPUT_SKILL"}
-          exit_code=$?
-          # Debug output
-          echo "=== Listing .skill-evals ==="
-          ls -laR .skill-evals/ 2>&1 | head -50 || echo "No .skill-evals directory found"
-          echo "=== report.json ==="
-          cat .skill-evals/*/docker-sandboxes-lifecycle/mount-ls-delete/1/report.json 2>/dev/null | head -200 || true
-          echo "=== transcript.jsonl (last 50 lines) ==="
-          tail -50 .skill-evals/*/docker-sandboxes-lifecycle/mount-ls-delete/1/transcript.jsonl 2>/dev/null || true
-          exit $exit_code
+          # Run eval with one automatic retry on failure
+          if .venv/bin/python scripts/eval_judge.py \
+            --agent claude \
+            --model "$INPUT_MODEL" \
+            --repeat "$INPUT_REPEAT" \
+            --skill "${{ matrix.skill }}"; then
+            echo "Eval passed on first attempt"
+          else
+            echo "::warning::First attempt failed, retrying..."
+            .venv/bin/python scripts/eval_judge.py \
+              --agent claude \
+              --model "$INPUT_MODEL" \
+              --repeat "$INPUT_REPEAT" \
+              --skill "${{ matrix.skill }}"
+          fi
 
-      - name: Upload transcripts and call logs (Claude)
+      - name: Upload eval artifacts
         if: always()
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
-          name: skill-evals-claude
-          path: .skill-evals/
+          name: eval-claude-${{ matrix.skill }}
+          path: .eval-judge/
           if-no-files-found: ignore
           retention-days: 14
 
       - name: Cleanup sandboxes
-        if: always()
+        if: always() && startsWith(matrix.skill, 'docker-sandboxes-')
         run: |
           sbx ls -q 2>/dev/null | grep "^eval-" | xargs -r sbx rm --force || true
           sbx daemon stop || true
 
-  scenarios-codex:
-    # Codex evals use Docker AI Gateway with Docker Hub authentication
-    if: ${{ inputs.agent == 'codex' || inputs.agent == 'both' || inputs.agent == '' }}
+  # Codex evals - one job per skill
+  eval-codex:
+    needs: detect-changes
+    if: |
+      (inputs.agent == 'codex' || inputs.agent == 'both' || inputs.agent == '') &&
+      needs.detect-changes.outputs.skills != '[]'
     runs-on: ubuntu-24.04
-    timeout-minutes: 45
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        skill: ${{ fromJson(needs.detect-changes.outputs.skills) }}
     env:
       CODEX_VERSION: "0.166.0"
       SBX_VERSION: "0.50.1"
-      SKILL_EVAL_AGENT: "codex"
-      # Don't specify a model by default - let Codex use the gateway's default
-      # (explicitly specifying gpt-4o causes flaky behavior where the model refuses to execute commands)
-      SKILL_EVAL_MODEL: ${{ inputs.model || '' }}
-      SKILL_EVAL_REPEAT: ${{ inputs.repeat || '1' }}
-      SKILL_EVAL_MIN_PASS_RATE: "1.0"
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
 
+      # Only Docker Sandboxes skills need the sbx CLI at eval time
       - name: Install Docker Sandboxes (sbx)
+        if: startsWith(matrix.skill, 'docker-sandboxes-')
         run: |
           curl -fsSL "https://github.com/docker/sbx-releases/releases/download/v${SBX_VERSION}/DockerSandboxes-linux-amd64-ubuntu2404.deb" -o sbx.deb
+          sudo apt-get update -q
           sudo apt-get install -y ./sbx.deb
           sudo usermod -aG kvm "$USER"
           rm sbx.deb
           sbx version
 
       - name: Setup sbx (login and policy)
+        if: startsWith(matrix.skill, 'docker-sandboxes-')
         env:
           DOCKERPUBLICBOT_READ_PAT: ${{ secrets.DOCKERPUBLICBOT_READ_PAT }}
           DOCKERPUBLICBOT_USERNAME: ${{ vars.DOCKERPUBLICBOT_USERNAME }}
         run: |
-          # sg kvm runs commands in kvm group context (usermod doesn't affect current shell)
-          # Login starts the daemon and authenticates for pulling sandbox images
           sg kvm -c 'echo "$DOCKERPUBLICBOT_READ_PAT" | sbx login --username "$DOCKERPUBLICBOT_USERNAME" --password-stdin'
           sg kvm -c 'sbx policy init balanced'
 
@@ -188,7 +272,6 @@ jobs:
           DOCKERPUBLICBOT_READ_PAT: ${{ secrets.DOCKERPUBLICBOT_READ_PAT }}
           DOCKERPUBLICBOT_USERNAME: ${{ vars.DOCKERPUBLICBOT_USERNAME }}
         run: |
-          # Get JWT token from Docker Hub
           JWT=$(curl -s -X POST "https://hub.docker.com/v2/users/login" \
             -H "Content-Type: application/json" \
             -d "{\"username\":\"${DOCKERPUBLICBOT_USERNAME}\",\"password\":\"${DOCKERPUBLICBOT_READ_PAT}\"}" \
@@ -199,10 +282,7 @@ jobs:
             exit 1
           fi
 
-          # Create Codex config directory
           mkdir -p ~/.codex
-
-          # Configure Codex to use Docker AI Gateway
           cat > ~/.codex/config.toml << EOF
           model_provider = "docker-gateway"
 
@@ -214,50 +294,47 @@ jobs:
           requires_openai_auth = false
           EOF
 
-          # Export token for later steps
           echo "DOCKER_GATEWAY_TOKEN=${JWT}" >> $GITHUB_ENV
           echo "::add-mask::${JWT}"
-
-          echo "Codex configured to use Docker AI Gateway"
 
       - name: Install Python dependencies
         run: |
           python3 -m venv .venv
           .venv/bin/pip install -q --require-hashes -r scripts/requirements.txt
 
-      - name: Run live skill scenarios (Codex)
+      - name: Run eval for ${{ matrix.skill }} (Codex)
         env:
-          INPUT_SKILL: ${{ inputs.skill }}
-          INPUT_MODEL: ${{ inputs.model }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          INPUT_MODEL: ${{ inputs.model || 'gpt-6-luna' }}
+          INPUT_REPEAT: ${{ inputs.repeat || '1' }}
         run: |
-          set +e
-          # Only pass --model if explicitly specified; gateway default works better than explicit gpt-4o
-          model_arg=""
-          if [ -n "$INPUT_MODEL" ]; then
-            model_arg="--model $INPUT_MODEL"
+          # Run eval with one automatic retry on failure
+          if .venv/bin/python scripts/eval_judge.py \
+            --agent codex \
+            --model "$INPUT_MODEL" \
+            --repeat "$INPUT_REPEAT" \
+            --skill "${{ matrix.skill }}"; then
+            echo "Eval passed on first attempt"
+          else
+            echo "::warning::First attempt failed, retrying..."
+            .venv/bin/python scripts/eval_judge.py \
+              --agent codex \
+              --model "$INPUT_MODEL" \
+              --repeat "$INPUT_REPEAT" \
+              --skill "${{ matrix.skill }}"
           fi
-          .venv/bin/python scripts/skill_eval.py --agent codex $model_arg ${INPUT_SKILL:+--skill "$INPUT_SKILL"}
-          exit_code=$?
-          # Debug output
-          echo "=== Listing .skill-evals ==="
-          ls -laR .skill-evals/ 2>&1 | head -50 || echo "No .skill-evals directory found"
-          echo "=== report.json ==="
-          cat .skill-evals/*/docker-sandboxes-lifecycle/mount-ls-delete/1/report.json 2>/dev/null | head -200 || true
-          echo "=== transcript.jsonl (last 50 lines) ==="
-          tail -50 .skill-evals/*/docker-sandboxes-lifecycle/mount-ls-delete/1/transcript.jsonl 2>/dev/null || true
-          exit $exit_code
 
-      - name: Upload transcripts and call logs (Codex)
+      - name: Upload eval artifacts
         if: always()
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
-          name: skill-evals-codex
-          path: .skill-evals/
+          name: eval-codex-${{ matrix.skill }}
+          path: .eval-judge/
           if-no-files-found: ignore
           retention-days: 14
 
       - name: Cleanup sandboxes
-        if: always()
+        if: always() && startsWith(matrix.skill, 'docker-sandboxes-')
         run: |
           sbx ls -q 2>/dev/null | grep "^eval-" | xargs -r sbx rm --force || true
           sbx daemon stop || true

.gitignore

@@ -5,6 +5,7 @@
 .sbx/
 .data/
 .skill-evals/
+.eval-judge/
 
 # Environment, credentials, and key material
 .env

AGENTS.md

@@ -41,7 +41,7 @@ are required; validation runs in pinned container images.
   not a live model evaluation.
 - `task eval:skills`: opt-in live model evaluation. It runs scenarios in
   `skills/*/evals/` with Claude Code against the real `sbx` CLI; see
-  [evals/README.md](evals/README.md#live-skill-scenarios). It needs `claude`,
+  [evals/README.md](evals/README.md#llm-as-judge-evaluations). It needs `claude`,
   `sbx` authenticated, and optionally model credentials. Runs separately
   from offline `task`/`task ci`.
 - `task links`: check local Markdown destinations and heading anchors.