diff --git a/.github/workflows/daily-cache-strategy-analyzer.lock.yml b/.github/workflows/daily-cache-strategy-analyzer.lock.yml index 3b271e56f8c..625f9496b41 100644 --- a/.github/workflows/daily-cache-strategy-analyzer.lock.yml +++ b/.github/workflows/daily-cache-strategy-analyzer.lock.yml @@ -1080,6 +1080,7 @@ jobs: GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} GH_AW_MCP_CONFIG: ${{ runner.temp }}/gh-aw/mcp-config/config.toml GH_AW_MODEL_AGENT_CODEX: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_AGENT_CODEX || vars.GH_AW_DEFAULT_MODEL_CODEX || 'gpt-5.4' }} GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} @@ -1875,7 +1876,8 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_EVALS_QUESTIONS: '[{"id":"cache_logs_analyzed","question":"Did the agent analyze agentic workflow logs for cache misses and misconfigured caches?"},{"id":"issue_created_or_noop","question":"Was an issue created for cache problems or improvements, or was noop used when no problems were found?"}]' - GH_AW_EVALS_MODEL: "${{ needs.activation.outputs.model_size }}" + GH_AW_EVALS_MODEL: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_DETECTION_CODEX || vars.GH_AW_DEFAULT_MODEL_CODEX || 'gpt-5.4' }} GH_AW_EVALS_PHASE: setup with: script: | @@ -2010,6 +2012,7 @@ jobs: GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} GH_AW_MCP_CONFIG: ${{ runner.temp }}/gh-aw/mcp-config/config.toml GH_AW_MODEL_DETECTION_CODEX: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_DETECTION_CODEX || vars.GH_AW_DEFAULT_MODEL_CODEX || 'gpt-5.4' }} GH_AW_PHASE: detection GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_VERSION: dev @@ -2029,7 +2032,8 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_EVALS_QUESTIONS: '[{"id":"cache_logs_analyzed","question":"Did the agent analyze agentic workflow logs for cache misses and misconfigured caches?"},{"id":"issue_created_or_noop","question":"Was an issue created for cache problems or improvements, or was noop used when no problems were found?"}]' - GH_AW_EVALS_MODEL: "${{ needs.activation.outputs.model_size }}" + GH_AW_EVALS_MODEL: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_DETECTION_CODEX || vars.GH_AW_DEFAULT_MODEL_CODEX || 'gpt-5.4' }} GH_AW_EVALS_PHASE: parse GITHUB_RUN_ID: ${{ github.run_id }} with: diff --git a/.github/workflows/daily-caveman-optimizer.lock.yml b/.github/workflows/daily-caveman-optimizer.lock.yml index 676b60720bf..dbe4ac6a902 100644 --- a/.github/workflows/daily-caveman-optimizer.lock.yml +++ b/.github/workflows/daily-caveman-optimizer.lock.yml @@ -983,6 +983,7 @@ jobs: GH_AW_LLM_PROVIDER: anthropic GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} GH_AW_MCP_CONFIG: ${{ runner.temp }}/gh-aw/mcp-config/mcp-servers.json + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_AGENT_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} @@ -1762,7 +1763,8 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_EVALS_QUESTIONS: '[{"id":"instruction_file_optimized","question":"Did the agent apply caveman optimization to an instruction file in .github/aw or .github/agents?"},{"id":"pr_created_or_noop","question":"Was a PR created with concise improvements, or was noop used when no improvements were needed?"}]' - GH_AW_EVALS_MODEL: "${{ needs.activation.outputs.model_size }}" + GH_AW_EVALS_MODEL: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_DETECTION_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} GH_AW_EVALS_PHASE: setup with: script: | @@ -1843,6 +1845,7 @@ jobs: DISABLE_TELEMETRY: 1 GH_AW_LLM_PROVIDER: anthropic GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_DETECTION_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} GH_AW_PHASE: detection GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_VERSION: dev @@ -1863,7 +1866,8 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_EVALS_QUESTIONS: '[{"id":"instruction_file_optimized","question":"Did the agent apply caveman optimization to an instruction file in .github/aw or .github/agents?"},{"id":"pr_created_or_noop","question":"Was a PR created with concise improvements, or was noop used when no improvements were needed?"}]' - GH_AW_EVALS_MODEL: "${{ needs.activation.outputs.model_size }}" + GH_AW_EVALS_MODEL: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_DETECTION_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} GH_AW_EVALS_PHASE: parse GITHUB_RUN_ID: ${{ github.run_id }} with: diff --git a/.github/workflows/daily-doc-healer.lock.yml b/.github/workflows/daily-doc-healer.lock.yml index b792f7b71ad..4543496fcc4 100644 --- a/.github/workflows/daily-doc-healer.lock.yml +++ b/.github/workflows/daily-doc-healer.lock.yml @@ -1087,6 +1087,7 @@ jobs: GH_AW_LLM_PROVIDER: anthropic GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} GH_AW_MCP_CONFIG: ${{ runner.temp }}/gh-aw/mcp-config/mcp-servers.json + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_AGENT_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} @@ -1869,7 +1870,8 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_EVALS_QUESTIONS: '[{"id":"gaps-confirmed","question":"Did the workflow identify at least one confirmed documentation gap to fix, or correctly conclude that no actionable gap remained?"},{"id":"pr-issue-or-noop","question":"Was a documentation pull request or issue created for confirmed gaps, or was noop used appropriately when nothing required action?"}]' - GH_AW_EVALS_MODEL: "${{ needs.activation.outputs.model_size }}" + GH_AW_EVALS_MODEL: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_DETECTION_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} GH_AW_EVALS_PHASE: setup with: script: | @@ -1950,6 +1952,7 @@ jobs: DISABLE_TELEMETRY: 1 GH_AW_LLM_PROVIDER: anthropic GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_DETECTION_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} GH_AW_PHASE: detection GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_VERSION: dev @@ -1970,7 +1973,8 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_EVALS_QUESTIONS: '[{"id":"gaps-confirmed","question":"Did the workflow identify at least one confirmed documentation gap to fix, or correctly conclude that no actionable gap remained?"},{"id":"pr-issue-or-noop","question":"Was a documentation pull request or issue created for confirmed gaps, or was noop used appropriately when nothing required action?"}]' - GH_AW_EVALS_MODEL: "${{ needs.activation.outputs.model_size }}" + GH_AW_EVALS_MODEL: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_DETECTION_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} GH_AW_EVALS_PHASE: parse GITHUB_RUN_ID: ${{ github.run_id }} with: diff --git a/.github/workflows/test-quality-sentinel.lock.yml b/.github/workflows/test-quality-sentinel.lock.yml index fa095fd4668..40a34231807 100644 --- a/.github/workflows/test-quality-sentinel.lock.yml +++ b/.github/workflows/test-quality-sentinel.lock.yml @@ -985,6 +985,7 @@ jobs: GH_AW_LLM_PROVIDER: github GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }} GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} diff --git a/actions/setup/js/claude_harness.cjs b/actions/setup/js/claude_harness.cjs index e84e02b496a..33be2194615 100644 --- a/actions/setup/js/claude_harness.cjs +++ b/actions/setup/js/claude_harness.cjs @@ -53,6 +53,7 @@ const { emitMissingToolPermissionIssue, hasExpectedSafeOutputs, hasNoopInSafeOut const { countPermissionDeniedIssues, hasNumerousPermissionDeniedIssues, extractDeniedCommands, buildMissingToolPermissionIssuePayload } = require("./permission_denied_helpers.cjs"); const { detectNonRetryableHarnessGuard, buildSoftTimeoutGuard, emitSoftTimeoutSignal, isAuthenticationFailedError } = require("./harness_retry_guard.cjs"); const { MODEL_NOT_SUPPORTED_PATTERN: INVALID_MODEL_ERROR_PATTERN } = require("./detect_agent_errors.cjs"); +const { applyModelFallback } = require("./model_fallback.cjs"); // Pattern to detect Anthropic API overload errors (HTTP 529). // Matches "overloaded_error" from the Anthropic error type field, and the @@ -318,6 +319,7 @@ function stripContinueArgs(args) { */ async function buildClaudeChildEnv() { const childEnv = { ...process.env }; + applyModelFallback(childEnv, "ANTHROPIC_MODEL", log); const provider = normalizeReflectProviderName(process.env.GH_AW_LLM_PROVIDER, "anthropic"); try { const raw = fs.readFileSync(AWF_REFLECT_OUTPUT_PATH, "utf8"); @@ -611,6 +613,7 @@ if (typeof module !== "undefined" && module.exports) { hasExpectedSafeOutputs, resolveRetryConfig, resolveStartupRetryLimit, + applyModelFallback, }; } diff --git a/actions/setup/js/codex_harness.cjs b/actions/setup/js/codex_harness.cjs index 92f97cc9d84..81fba7e97f9 100644 --- a/actions/setup/js/codex_harness.cjs +++ b/actions/setup/js/codex_harness.cjs @@ -54,6 +54,7 @@ const { countPermissionDeniedIssues, hasNumerousPermissionDeniedIssues, extractD const { detectNonRetryableHarnessGuard, buildSoftTimeoutGuard, emitSoftTimeoutSignal, isAuthenticationFailedError } = require("./harness_retry_guard.cjs"); const { MODEL_NOT_SUPPORTED_PATTERN: INVALID_MODEL_ERROR_PATTERN } = require("./detect_agent_errors.cjs"); const { resolveRetryConfig } = require("./harness_retry_config.cjs"); +const { applyModelFallback, injectModelFlagAfterExec } = require("./model_fallback.cjs"); // Pattern to detect OpenAI rate-limit errors. // Matches the JSON error type field ("rate_limit_exceeded"), the HTTP status code @@ -221,6 +222,16 @@ function injectJsonFlag(args) { return ["exec", "--json", ...args.slice(1)]; } +function getCodexModelEnvVar(env = process.env) { + if ("GH_AW_MODEL_DETECTION_CODEX" in env) { + return "GH_AW_MODEL_DETECTION_CODEX"; + } + if ("GH_AW_MODEL_AGENT_CODEX" in env) { + return "GH_AW_MODEL_AGENT_CODEX"; + } + return ""; +} + /** * Build child process environment for Codex execution. * Preserve API keys captured at harness startup, even if the parent environment @@ -425,6 +436,10 @@ async function main() { process.exit(1); } + const codexModelEnvVar = getCodexModelEnvVar(process.env); + const resolvedModel = codexModelEnvVar ? applyModelFallback(process.env, codexModelEnvVar, log) : ""; + resolvedArgs = injectModelFlagAfterExec(resolvedArgs, resolvedModel); + // Safe arg list for logging: when --prompt-file was present, the last element of // resolvedArgs is the resolved prompt content. Replace it with a placeholder so that // task instructions are never written to stderr or captured in agent logs. @@ -641,6 +656,9 @@ if (typeof module !== "undefined" && module.exports) { hasNoopInSafeOutputs, hasExpectedSafeOutputs, resolveRetryConfig, + applyModelFallback, + injectModelFlagAfterExec, + getCodexModelEnvVar, }; } diff --git a/actions/setup/js/copilot_harness.cjs b/actions/setup/js/copilot_harness.cjs index 4ed31e07d16..d29e70d4d44 100644 --- a/actions/setup/js/copilot_harness.cjs +++ b/actions/setup/js/copilot_harness.cjs @@ -65,6 +65,7 @@ const { runSafeOutputsCLI, buildMissingToolAlternatives, emitMissingToolPermissi const { countPermissionDeniedIssues, hasNumerousPermissionDeniedIssues, extractDeniedCommands, buildMissingToolPermissionIssuePayload } = require("./permission_denied_helpers.cjs"); const { detectNonRetryableHarnessGuard, buildSoftTimeoutGuard, emitSoftTimeoutSignal, isAuthenticationFailedError: isCommonAuthenticationFailedError } = require("./harness_retry_guard.cjs"); const { isCAPIQuotaExceededError } = require("./detect_agent_errors.cjs"); +const { applyModelFallback } = require("./model_fallback.cjs"); const { loadModelsJson } = require("./model_costs.cjs"); const { resolveConfiguredCopilotModel } = require("./resolve_model_alias.cjs"); @@ -950,6 +951,7 @@ async function main() { } } + applyModelFallback(process.env, "COPILOT_MODEL", log); applyCopilotModelAliasResolution({ awfReflectData, logger: log }); applyCopilotWireAPI({ modelsJson: loadModelsJson(), logger: log }); @@ -1456,6 +1458,7 @@ if (typeof module !== "undefined" && module.exports) { parseCopilotSDKServerArgsFromEnv, isCAPIQuotaExceededError, hasTerminalSafeOutput, + applyModelFallback, applyCopilotModelAliasResolution, applyCopilotWireAPI, loadAwfConfigData, diff --git a/actions/setup/js/copilot_sdk_driver.cjs b/actions/setup/js/copilot_sdk_driver.cjs index 2e609e0462a..9716aaf3ffb 100644 --- a/actions/setup/js/copilot_sdk_driver.cjs +++ b/actions/setup/js/copilot_sdk_driver.cjs @@ -30,6 +30,7 @@ const fs = require("fs"); const { runWithCopilotSDK, extractPromptFromArgs } = require("./copilot_sdk_session.cjs"); const { parsePermissionConfigFromServerArgs } = require("./copilot_sdk_permissions.cjs"); const { parseMultiProviderJson } = require("./copilot_sdk_multi_provider.cjs"); +const { applyModelFallback } = require("./model_fallback.cjs"); // Re-export the session and permission helpers so that existing callers that // require("./copilot_sdk_driver.cjs") (e.g. copilot_harness.cjs) continue to work. @@ -103,7 +104,7 @@ async function main() { const providers = multiProviderConfig.providers; /** @type {import("@github/copilot-sdk").ProviderModelConfig[]} */ const sdkModels = multiProviderConfig.models; - let model = process.env.COPILOT_MODEL || multiProviderConfig.model || undefined; + let model = applyModelFallback(process.env, "COPILOT_MODEL", log) || multiProviderConfig.model || undefined; log(`multi-provider mode: ${providers.length} providers, ${sdkModels.length} models, model=${model ?? "(env)"}`); for (const p of providers) { log(` provider: name=${p.name} type=${p.type} baseUrl=${p.baseUrl}${p.wireApi ? ` wireApi=${p.wireApi}` : ""}`); diff --git a/actions/setup/js/copilot_sdk_session.cjs b/actions/setup/js/copilot_sdk_session.cjs index 30c3a6251e9..1c0172ddaf1 100644 --- a/actions/setup/js/copilot_sdk_session.cjs +++ b/actions/setup/js/copilot_sdk_session.cjs @@ -36,6 +36,7 @@ const fs = require("fs"); const path = require("path"); const os = require("os"); const { buildCopilotSDKPermissionHandler, getEnvPositiveIntOrDefault, parseMaxToolDenialsLimit, MAX_TOOL_DENIALS_DEFAULT } = require("./copilot_sdk_permissions.cjs"); +const { resolveModelWithFallback } = require("./model_fallback.cjs"); const { extractShellCommandFromToolData } = require("./tool_call_details.cjs"); // Default timeout for a single sendAndWait call: 10 minutes. @@ -254,7 +255,7 @@ async function runWithCopilotSDK({ sdkUri, prompt, logger, attempt = 0, model, c // Build session config using the multi-provider surface. /** @type {import("@github/copilot-sdk").SessionConfig} */ const sessionConfig = { - model: model || process.env.COPILOT_MODEL || undefined, + model: model || resolveModelWithFallback(process.env, "COPILOT_MODEL") || undefined, providers, models: providerModels, onPermissionRequest, diff --git a/actions/setup/js/model_fallback.cjs b/actions/setup/js/model_fallback.cjs new file mode 100644 index 00000000000..70c1f205ff2 --- /dev/null +++ b/actions/setup/js/model_fallback.cjs @@ -0,0 +1,48 @@ +"use strict"; + +const MODEL_FALLBACK_ENV_VAR = "GH_AW_MODEL_FALLBACK"; + +function readTrimmedEnv(env, name) { + return typeof env?.[name] === "string" ? env[name].trim() : ""; +} + +function resolveModelWithFallback(env, primaryEnvVar) { + return readTrimmedEnv(env, primaryEnvVar) || readTrimmedEnv(env, MODEL_FALLBACK_ENV_VAR); +} + +/** + * @param {NodeJS.ProcessEnv} env + * @param {string} primaryEnvVar + * @param {(message: string) => void} [logger] + * @returns {string} + */ +function applyModelFallback(env, primaryEnvVar, logger = () => {}) { + const primary = readTrimmedEnv(env, primaryEnvVar); + if (primary) { + return primary; + } + const fallback = readTrimmedEnv(env, MODEL_FALLBACK_ENV_VAR); + if (fallback) { + env[primaryEnvVar] = fallback; + logger(`applied ${MODEL_FALLBACK_ENV_VAR} to ${primaryEnvVar}`); + } + return fallback; +} + +function injectModelFlagAfterExec(args, model) { + if (!model || args.includes("--model")) { + return args; + } + const execIndex = args.indexOf("exec"); + if (execIndex === -1) { + return [...args, "--model", model]; + } + return [...args.slice(0, execIndex + 1), "--model", model, ...args.slice(execIndex + 1)]; +} + +module.exports = { + MODEL_FALLBACK_ENV_VAR, + resolveModelWithFallback, + applyModelFallback, + injectModelFlagAfterExec, +}; diff --git a/actions/setup/js/model_fallback.test.js b/actions/setup/js/model_fallback.test.js new file mode 100644 index 00000000000..ce524c81572 --- /dev/null +++ b/actions/setup/js/model_fallback.test.js @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { createRequire } from "module"; + +const require = createRequire(import.meta.url); +const { MODEL_FALLBACK_ENV_VAR, resolveModelWithFallback, applyModelFallback, injectModelFlagAfterExec } = require("./model_fallback.cjs"); + +describe("model_fallback.cjs", () => { + it("prefers the primary model env var when it is non-empty", () => { + expect(resolveModelWithFallback({ COPILOT_MODEL: "gpt-5", [MODEL_FALLBACK_ENV_VAR]: "claude-sonnet-4.6" }, "COPILOT_MODEL")).toBe("gpt-5"); + }); + + it("uses the fallback model when the primary resolves empty", () => { + const env = { COPILOT_MODEL: "", [MODEL_FALLBACK_ENV_VAR]: "claude-sonnet-4.6" }; + expect(applyModelFallback(env, "COPILOT_MODEL")).toBe("claude-sonnet-4.6"); + expect(env.COPILOT_MODEL).toBe("claude-sonnet-4.6"); + }); + + it("injects --model immediately after exec when needed", () => { + expect(injectModelFlagAfterExec(["exec", "--json", "prompt"], "gpt-5")).toEqual(["exec", "--model", "gpt-5", "--json", "prompt"]); + }); + + it("does not inject --model when already present", () => { + expect(injectModelFlagAfterExec(["exec", "--model", "gpt-5", "--json"], "claude-sonnet-4.6")).toEqual(["exec", "--model", "gpt-5", "--json"]); + }); +}); diff --git a/actions/setup/js/run_evals.cjs b/actions/setup/js/run_evals.cjs index f41af3a012e..75f2627a205 100644 --- a/actions/setup/js/run_evals.cjs +++ b/actions/setup/js/run_evals.cjs @@ -35,6 +35,7 @@ const path = require("path"); const { ERR_VALIDATION } = require("./error_codes.cjs"); const { EVALS_OUTPUT_PATH } = require("./evals_constants.cjs"); +const { resolveModelWithFallback } = require("./model_fallback.cjs"); const EVALS_DIR = "/tmp/gh-aw/evals"; const EVALS_LOG_PATH = "/tmp/gh-aw/evals/evals.log"; @@ -118,7 +119,7 @@ async function setupMain() { */ async function parseMain() { const questionsRaw = process.env.GH_AW_EVALS_QUESTIONS; - const model = process.env.GH_AW_EVALS_MODEL || ""; + const model = resolveModelWithFallback(process.env, "GH_AW_EVALS_MODEL") || ""; const runID = process.env.GITHUB_RUN_ID || "unknown"; /** @type {Array<{id: string, question: string}>} */ diff --git a/actions/setup/js/run_evals.test.cjs b/actions/setup/js/run_evals.test.cjs index 7bbf2c4ea9b..52825d72464 100644 --- a/actions/setup/js/run_evals.test.cjs +++ b/actions/setup/js/run_evals.test.cjs @@ -6,6 +6,7 @@ const EVALS_DIR = "/tmp/gh-aw/evals"; const EVALS_LOG_PATH = `${EVALS_DIR}/evals.log`; const EVALS_OUTPUT_PATH = "/tmp/gh-aw/evals.jsonl"; const require = createRequire(import.meta.url); +const { MODEL_FALLBACK_ENV_VAR } = require("./model_fallback.cjs"); const { parseMain, extractAssistantTextFromJsonlLog } = require("./run_evals.cjs"); const mockCore = { @@ -75,6 +76,19 @@ describe("run_evals.cjs", () => { expect(JSON.parse(line).runid).toBe("unknown"); }); + it("uses GH_AW_MODEL_FALLBACK when GH_AW_EVALS_MODEL resolves empty", async () => { + vi.stubEnv("GH_AW_EVALS_QUESTIONS", JSON.stringify([{ id: "labels-applied", question: "Did labels get applied?" }])); + vi.stubEnv("GH_AW_EVALS_MODEL", ""); + vi.stubEnv(MODEL_FALLBACK_ENV_VAR, "claude-sonnet-4.6"); + vi.stubEnv("GITHUB_RUN_ID", "123456789"); + fs.writeFileSync(EVALS_LOG_PATH, "labels-applied: YES\n", "utf8"); + + await parseMain(); + + const [line] = fs.readFileSync(EVALS_OUTPUT_PATH, "utf8").trim().split("\n"); + expect(JSON.parse(line).model).toBe("claude-sonnet-4.6"); + }); + it("parses answers from Pi v3 JSONL turn_end events (positional format)", async () => { vi.stubEnv( "GH_AW_EVALS_QUESTIONS", diff --git a/pkg/constants/engine_constants.go b/pkg/constants/engine_constants.go index fdc66a30b7c..ef63b38a4d2 100644 --- a/pkg/constants/engine_constants.go +++ b/pkg/constants/engine_constants.go @@ -243,6 +243,11 @@ const ( // EnvVarModelAgentPi configures the default Pi model for agent execution EnvVarModelAgentPi = "GH_AW_MODEL_AGENT_PI" + // EnvVarModelFallback carries the standard org/enterprise/built-in model fallback + // expression for the current engine step. Runtime JavaScript harnesses can use this + // when a configured model expression resolves to an empty string. + EnvVarModelFallback = "GH_AW_MODEL_FALLBACK" + // CopilotCLIModelEnvVar is the native environment variable name supported by the Copilot CLI // for selecting the model. Setting this env var is equivalent to passing --model to the CLI. CopilotCLIModelEnvVar = "COPILOT_MODEL" diff --git a/pkg/workflow/claude_engine.go b/pkg/workflow/claude_engine.go index 8cf2070a4e7..0a361135e0a 100644 --- a/pkg/workflow/claude_engine.go +++ b/pkg/workflow/claude_engine.go @@ -474,6 +474,16 @@ func (e *ClaudeEngine) GetExecutionSteps(workflowData *WorkflowData, logFile str // When model is not configured, fall back to GH_AW_MODEL_AGENT/DETECTION_CLAUDE so users // can set a default via GitHub Actions variables. if modelConfigured { + isDetectionJob := workflowData.SafeOutputs == nil + var claudeModelVar string + if isDetectionJob { + claudeModelVar = constants.EnvVarModelDetectionClaude + } else { + claudeModelVar = constants.EnvVarModelAgentClaude + } + if containsExpression(workflowData.Model) { + env[constants.EnvVarModelFallback] = compilerenv.BuildModelOverrideExpressionEmptyFallback(claudeModelVar, compilerenv.DefaultModelClaude) + } claudeLog.Printf("Setting %s env var for model: %s", constants.ClaudeCLIModelEnvVar, workflowData.Model) env[constants.ClaudeCLIModelEnvVar] = workflowData.Model } else { diff --git a/pkg/workflow/codex_engine.go b/pkg/workflow/codex_engine.go index 9f53295235a..8876100600b 100644 --- a/pkg/workflow/codex_engine.go +++ b/pkg/workflow/codex_engine.go @@ -487,6 +487,9 @@ mkdir -p "$CODEX_HOME/logs" // When model is configured (static or GitHub Actions expression), set the env var directly. // When not configured, use the GitHub variable fallback so users can set a default. if modelConfigured { + if containsExpression(workflowData.Model) { + env[constants.EnvVarModelFallback] = compilerenv.BuildModelOverrideExpression(modelEnvVar, compilerenv.DefaultModelCodex, constants.CodexDefaultModel) + } codexEngineLog.Printf("Setting %s env var for model: %s", modelEnvVar, workflowData.Model) env[modelEnvVar] = workflowData.Model } else { diff --git a/pkg/workflow/copilot_engine_execution.go b/pkg/workflow/copilot_engine_execution.go index 83dac99da39..005fcfede45 100644 --- a/pkg/workflow/copilot_engine_execution.go +++ b/pkg/workflow/copilot_engine_execution.go @@ -588,6 +588,9 @@ func (e *CopilotEngine) addCopilotModelEnv(env map[string]string, workflowData * // The model is always passed via the native COPILOT_MODEL env var, which the Copilot CLI reads directly. // When model is not configured, map the GitHub org variable to COPILOT_MODEL so users can set a default. if modelConfigured { + if containsExpression(workflowData.Model) { + env[constants.EnvVarModelFallback] = compilerenv.BuildModelOverrideExpression(modelEnvVar, compilerenv.DefaultModelCopilot, constants.CopilotBYOKDefaultModel) + } copilotExecLog.Printf("Setting %s env var for model: %s", constants.CopilotCLIModelEnvVar, workflowData.Model) env[constants.CopilotCLIModelEnvVar] = workflowData.Model return diff --git a/pkg/workflow/evals_steps.go b/pkg/workflow/evals_steps.go index 82cacd4af60..63a8f98c505 100644 --- a/pkg/workflow/evals_steps.go +++ b/pkg/workflow/evals_steps.go @@ -121,11 +121,13 @@ await main();` fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data)), " env:\n", fmt.Sprintf(" GH_AW_EVALS_QUESTIONS: '%s'\n", escapeYAMLSingleQuoted(questionsJSON)), - fmt.Sprintf(" GH_AW_EVALS_MODEL: %q\n", model), + } + steps = appendEvalsModelEnvLines(steps, c.getEvalsEngineID(data), model) + steps = append(steps, " GH_AW_EVALS_PHASE: setup\n", " with:\n", " script: |\n", - } + ) steps = append(steps, FormatJavaScriptForYAML(script)...) return steps } @@ -277,12 +279,14 @@ await main();` fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data)), " env:\n", fmt.Sprintf(" GH_AW_EVALS_QUESTIONS: '%s'\n", escapeYAMLSingleQuoted(questionsJSON)), - fmt.Sprintf(" GH_AW_EVALS_MODEL: %q\n", model), + } + steps = appendEvalsModelEnvLines(steps, c.getEvalsEngineID(data), model) + steps = append(steps, " GH_AW_EVALS_PHASE: parse\n", " GITHUB_RUN_ID: ${{ github.run_id }}\n", " with:\n", " script: |\n", - } + ) steps = append(steps, FormatJavaScriptForYAML(script)...) return steps } @@ -408,6 +412,35 @@ func (c *Compiler) resolveEvalsExecutionModel(data *WorkflowData) string { return model } +func appendEvalsModelEnvLines(steps []string, engineID, model string) []string { + hasExpression := containsExpression(model) + modelValue := fmt.Sprintf("%q", model) + if hasExpression { + modelValue = model + } + steps = append(steps, fmt.Sprintf(" GH_AW_EVALS_MODEL: %s\n", modelValue)) + if !hasExpression { + return steps + } + if fallback := buildEvalsModelFallbackExpression(engineID); fallback != "" { + steps = append(steps, fmt.Sprintf(" %s: %s\n", constants.EnvVarModelFallback, fallback)) + } + return steps +} + +func buildEvalsModelFallbackExpression(engineID string) string { + switch engineID { + case string(constants.CopilotEngine): + return compilerenv.BuildModelOverrideExpression(constants.EnvVarModelDetectionCopilot, compilerenv.DefaultModelCopilot, constants.CopilotBYOKDefaultModel) + case string(constants.ClaudeEngine): + return compilerenv.BuildModelOverrideExpressionEmptyFallback(constants.EnvVarModelDetectionClaude, compilerenv.DefaultModelClaude) + case string(constants.CodexEngine): + return compilerenv.BuildModelOverrideExpression(constants.EnvVarModelDetectionCodex, compilerenv.DefaultModelCodex, constants.CodexDefaultModel) + default: + return "" + } +} + func (c *Compiler) collectEvalsSecretReferences(data *WorkflowData) []string { if data.Evals == nil { return nil diff --git a/pkg/workflow/evals_steps_test.go b/pkg/workflow/evals_steps_test.go index a834aaaab2b..63711ca310a 100644 --- a/pkg/workflow/evals_steps_test.go +++ b/pkg/workflow/evals_steps_test.go @@ -3,10 +3,12 @@ package workflow import ( + "fmt" "strings" "testing" "github.com/github/gh-aw/pkg/constants" + "github.com/github/gh-aw/pkg/workflow/compilerenv" ) // TestBuildEvalsEngineStepsArcDindTopology verifies that the evals job @@ -198,6 +200,38 @@ func TestBuildParseEvalsResultsStepUsesResolvedExecutionModel(t *testing.T) { } } +func TestBuildParseEvalsResultsStepUsesExpressionModelAndFallbackEnv(t *testing.T) { + compiler := NewCompiler() + + data := &WorkflowData{ + AI: "copilot", + Model: "${{ inputs.model }}", + EngineConfig: &EngineConfig{ + ID: "copilot", + }, + Evals: &EvalsConfig{ + Questions: []EvalDefinition{ + {ID: "builds", Question: "Does it build?"}, + }, + }, + } + + steps := strings.Join(compiler.buildParseEvalsResultsStep(data), "") + if !strings.Contains(steps, `GH_AW_EVALS_MODEL: ${{ inputs.model }}`) { + t.Errorf("expected parse step to preserve expression-backed evals model; got:\n%s", steps) + } + expectedFallbackEnvLine := fmt.Sprintf( + "%s: ${{ vars.%s || vars.%s || '%s' }}", + constants.EnvVarModelFallback, + constants.EnvVarModelDetectionCopilot, + compilerenv.DefaultModelCopilot, + constants.CopilotBYOKDefaultModel, + ) + if !strings.Contains(steps, expectedFallbackEnvLine) { + t.Errorf("expected parse step to expose evals model fallback env %q; got:\n%s", expectedFallbackEnvLine, steps) + } +} + // TestBuildEvalsJobStepsCodexNoDuplicateContainerDownload verifies that a codex // workflow with evals does not generate a duplicate "Download container images" step. // Regression: buildEvalsJobSteps previously called buildPullAWFContainersStep diff --git a/pkg/workflow/model_env_vars_test.go b/pkg/workflow/model_env_vars_test.go index 9a048d657c2..21309bd260e 100644 --- a/pkg/workflow/model_env_vars_test.go +++ b/pkg/workflow/model_env_vars_test.go @@ -212,6 +212,9 @@ func TestExplicitModelConfigOverridesEnvVar(t *testing.T) { if strings.Contains(stepsContent, constants.EnvVarModelAgentCopilot+":") { t.Errorf("Fallback env var %s should not be present when model is explicitly configured", constants.EnvVarModelAgentCopilot) } + if strings.Contains(stepsContent, constants.EnvVarModelFallback+":") { + t.Errorf("Fallback env var %s should not be present for a literal configured model", constants.EnvVarModelFallback) + } // The model should be passed via the native COPILOT_MODEL env var (not via --model flag) expectedEnvLine := constants.CopilotCLIModelEnvVar + ": gpt-4" @@ -305,40 +308,54 @@ func TestExpressionModelUsesEnvVar(t *testing.T) { name string engine string model string - expectedEnvVar string - expectedEnvVal string + expectedModelEnvVar string + expectedModelEnvVal string + expectedFallbackVal string expectShellExpansion bool // whether command should use ${VAR:+ --model "$VAR"} }{ { - name: "Copilot agent with inputs.model expression uses native COPILOT_MODEL", + name: "Copilot agent keeps pure expression model and adds JS fallback env", engine: "copilot", model: "${{ inputs.model }}", - expectedEnvVar: constants.CopilotCLIModelEnvVar, - expectedEnvVal: "${{ inputs.model }}", + expectedModelEnvVar: constants.CopilotCLIModelEnvVar, + expectedModelEnvVal: "${{ inputs.model }}", + expectedFallbackVal: "${{ vars." + constants.EnvVarModelAgentCopilot + " || vars." + compilerenv.DefaultModelCopilot + " || '" + constants.CopilotBYOKDefaultModel + "' }}", expectShellExpansion: false, // Copilot reads COPILOT_MODEL natively, no shell expansion needed }, { - name: "Copilot agent with vars.model expression uses native COPILOT_MODEL", + name: "Copilot agent keeps composite expression model and adds JS fallback env", engine: "copilot", - model: "${{ vars.MY_MODEL }}", - expectedEnvVar: constants.CopilotCLIModelEnvVar, - expectedEnvVal: "${{ vars.MY_MODEL }}", + model: "${{ inputs.provider }}/${{ inputs.model }}", + expectedModelEnvVar: constants.CopilotCLIModelEnvVar, + expectedModelEnvVal: "${{ inputs.provider }}/${{ inputs.model }}", + expectedFallbackVal: "${{ vars." + constants.EnvVarModelAgentCopilot + " || vars." + compilerenv.DefaultModelCopilot + " || '" + constants.CopilotBYOKDefaultModel + "' }}", expectShellExpansion: false, }, { - name: "Claude agent with inputs.model expression uses native ANTHROPIC_MODEL", + name: "Claude agent keeps expression model and adds JS fallback env", engine: "claude", model: "${{ inputs.model }}", - expectedEnvVar: constants.ClaudeCLIModelEnvVar, - expectedEnvVal: "${{ inputs.model }}", + expectedModelEnvVar: constants.ClaudeCLIModelEnvVar, + expectedModelEnvVal: "${{ inputs.model }}", + expectedFallbackVal: "${{ vars." + constants.EnvVarModelAgentClaude + " || vars." + compilerenv.DefaultModelClaude + " || '' }}", expectShellExpansion: false, // Claude reads ANTHROPIC_MODEL natively, no shell expansion needed }, { - name: "Codex agent with inputs.model expression", + name: "Claude agent keeps composite expression model and adds JS fallback env", + engine: "claude", + model: "${{ inputs.provider }}/${{ inputs.model }}", + expectedModelEnvVar: constants.ClaudeCLIModelEnvVar, + expectedModelEnvVal: "${{ inputs.provider }}/${{ inputs.model }}", + expectedFallbackVal: "${{ vars." + constants.EnvVarModelAgentClaude + " || vars." + compilerenv.DefaultModelClaude + " || '' }}", + expectShellExpansion: false, + }, + { + name: "Codex agent keeps composite expression model and adds JS fallback env", engine: "codex", - model: "${{ inputs.model }}", - expectedEnvVar: constants.EnvVarModelAgentCodex, - expectedEnvVal: "${{ inputs.model }}", + model: "${{ inputs.provider }}/${{ inputs.model }}", + expectedModelEnvVar: constants.EnvVarModelAgentCodex, + expectedModelEnvVal: "${{ inputs.provider }}/${{ inputs.model }}", + expectedFallbackVal: "${{ vars." + constants.EnvVarModelAgentCodex + " || vars." + compilerenv.DefaultModelCodex + " || '" + constants.CodexDefaultModel + "' }}", expectShellExpansion: true, // Codex has no native model env var, uses shell expansion }, } @@ -381,18 +398,22 @@ func TestExpressionModelUsesEnvVar(t *testing.T) { } // The env var must be set to the expression value - expectedEnvLine := tt.expectedEnvVar + ": " + tt.expectedEnvVal - if !strings.Contains(stepsContent, expectedEnvLine) { - t.Errorf("Expected env line '%s' not found in steps:\n%s", expectedEnvLine, stepsContent) + expectedModelEnvLine := tt.expectedModelEnvVar + ": " + tt.expectedModelEnvVal + if !strings.Contains(stepsContent, expectedModelEnvLine) { + t.Errorf("Expected env line '%s' not found in steps:\n%s", expectedModelEnvLine, stepsContent) + } + expectedFallbackEnvLine := constants.EnvVarModelFallback + ": " + tt.expectedFallbackVal + if !strings.Contains(stepsContent, expectedFallbackEnvLine) { + t.Errorf("Expected fallback env line '%s' not found in steps:\n%s", expectedFallbackEnvLine, stepsContent) } // Check shell expansion expectation - shellExpansionPattern := "${" + tt.expectedEnvVar + ":+" + shellExpansionPattern := "${" + tt.expectedModelEnvVar + ":+" hasShellExpansion := strings.Contains(stepsContent, shellExpansionPattern) if tt.expectShellExpansion && !hasShellExpansion { - t.Errorf("Expected conditional env var usage '${%s:+' not found in steps:\n%s", tt.expectedEnvVar, stepsContent) + t.Errorf("Expected conditional env var usage '${%s:+' not found in steps:\n%s", tt.expectedModelEnvVar, stepsContent) } else if !tt.expectShellExpansion && hasShellExpansion { - t.Errorf("Unexpected conditional env var usage '${%s:+' found in steps (should use native env var):\n%s", tt.expectedEnvVar, stepsContent) + t.Errorf("Unexpected conditional env var usage '${%s:+' found in steps (should use native env var):\n%s", tt.expectedModelEnvVar, stepsContent) } }) } @@ -430,10 +451,14 @@ func TestExpressionModelDetectionJobUsesEnvVar(t *testing.T) { } stepsContent := stepsStr.String() - // Detection job for Copilot should use COPILOT_MODEL (native CLI env var) - expectedEnvLine := constants.CopilotCLIModelEnvVar + ": ${{ inputs.model }}" - if !strings.Contains(stepsContent, expectedEnvLine) { - t.Errorf("Expected env line '%s' not found in steps:\n%s", expectedEnvLine, stepsContent) + // Detection job for Copilot should keep the configured expression and expose the runtime fallback. + expectedModelEnvLine := constants.CopilotCLIModelEnvVar + ": ${{ inputs.model }}" + if !strings.Contains(stepsContent, expectedModelEnvLine) { + t.Errorf("Expected env line '%s' not found in steps:\n%s", expectedModelEnvLine, stepsContent) + } + expectedFallbackEnvLine := constants.EnvVarModelFallback + ": ${{ vars." + constants.EnvVarModelDetectionCopilot + " || vars." + compilerenv.DefaultModelCopilot + " || '" + constants.CopilotBYOKDefaultModel + "' }}" + if !strings.Contains(stepsContent, expectedFallbackEnvLine) { + t.Errorf("Expected fallback env line '%s' not found in steps:\n%s", expectedFallbackEnvLine, stepsContent) } // Must not embed expression directly in shell command