From 18ae7a08c2d7da2057bcab45d29ba5cc72c7afab Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:12:36 +0000 Subject: [PATCH 1/7] Initial plan From 375642dd1b02f4f069da1528829e34030a4a5fe8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:46:39 +0000 Subject: [PATCH 2/7] Treat expression-backed engine.model as omitted when empty When engine.model or model is a pure GitHub Actions expression (e.g., ${{ vars.MY_MODEL }}), it may resolve to an empty string at runtime if the variable is unset. Previously this empty value was propagated directly to the engine, causing failures. This change wraps pure expression-backed model values with the normal fallback chain for Copilot, Claude, and Codex engines, so that an empty resolution falls back to org/enterprise variables and the built-in default. For Copilot: ${{ vars.MY_MODEL || vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }} For Claude: ${{ inputs.model || vars.GH_AW_MODEL_AGENT_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} For Codex: ${{ inputs.model || vars.GH_AW_MODEL_AGENT_CODEX || vars.GH_AW_DEFAULT_MODEL_CODEX || 'gpt-5.4' }} Changes: - pkg/workflow/compilerenv/manager.go: add BuildModelOverrideExpressionWithUserOverride - pkg/workflow/expression_patterns.go: add extractExpressionInner helper - pkg/workflow/copilot_engine_execution.go: wrap expressions in addCopilotModelEnv - pkg/workflow/claude_engine.go: wrap expressions in model env assignment - pkg/workflow/codex_engine.go: wrap expressions in model env assignment - Updated/added tests - Recompiled 4 internal workflows using expression-backed models Closes #46499 Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../daily-cache-strategy-analyzer.lock.yml | 4 ++-- .../daily-caveman-optimizer.lock.yml | 4 ++-- .github/workflows/daily-doc-healer.lock.yml | 4 ++-- .../workflows/test-quality-sentinel.lock.yml | 2 +- pkg/workflow/claude_engine.go | 20 ++++++++++++++++-- pkg/workflow/codex_engine.go | 14 +++++++++++-- pkg/workflow/compilerenv/manager.go | 11 ++++++++++ pkg/workflow/compilerenv/manager_test.go | 21 +++++++++++++++++++ pkg/workflow/copilot_engine_execution.go | 11 ++++++++++ pkg/workflow/expression_patterns.go | 13 ++++++++++++ pkg/workflow/model_env_vars_test.go | 20 +++++++++--------- 11 files changed, 103 insertions(+), 21 deletions(-) diff --git a/.github/workflows/daily-cache-strategy-analyzer.lock.yml b/.github/workflows/daily-cache-strategy-analyzer.lock.yml index 3b271e56f8c..b2760e151c1 100644 --- a/.github/workflows/daily-cache-strategy-analyzer.lock.yml +++ b/.github/workflows/daily-cache-strategy-analyzer.lock.yml @@ -1079,7 +1079,7 @@ jobs: CODEX_HOME: /tmp/gh-aw/mcp-config GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} GH_AW_MCP_CONFIG: ${{ runner.temp }}/gh-aw/mcp-config/config.toml - GH_AW_MODEL_AGENT_CODEX: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_AGENT_CODEX: ${{ needs.activation.outputs.model_size || vars.GH_AW_MODEL_AGENT_CODEX || vars.GH_AW_DEFAULT_MODEL_CODEX || 'gpt-5.4' }} GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} @@ -2009,7 +2009,7 @@ jobs: CODEX_HOME: /tmp/gh-aw/mcp-config GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} GH_AW_MCP_CONFIG: ${{ runner.temp }}/gh-aw/mcp-config/config.toml - GH_AW_MODEL_DETECTION_CODEX: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_DETECTION_CODEX: ${{ needs.activation.outputs.model_size || vars.GH_AW_MODEL_DETECTION_CODEX || vars.GH_AW_DEFAULT_MODEL_CODEX || 'gpt-5.4' }} GH_AW_PHASE: detection GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_VERSION: dev diff --git a/.github/workflows/daily-caveman-optimizer.lock.yml b/.github/workflows/daily-caveman-optimizer.lock.yml index 676b60720bf..21935b20184 100644 --- a/.github/workflows/daily-caveman-optimizer.lock.yml +++ b/.github/workflows/daily-caveman-optimizer.lock.yml @@ -973,7 +973,7 @@ jobs: -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/claude_harness.cjs claude --print --no-chrome --allowed-tools '\''Bash,BashOutput,Edit,Edit(/tmp/*),Edit(/tmp/gh-aw/agent/*),Edit(/tmp/gh-aw/cache-memory/*),ExitPlanMode,Glob,Grep,KillBash,LS,MultiEdit,MultiEdit(/tmp/*),MultiEdit(/tmp/gh-aw/agent/*),MultiEdit(/tmp/gh-aw/cache-memory/*),NotebookEdit,NotebookRead,Read,Read(/tmp/*),Read(/tmp/gh-aw/agent/*),Read(/tmp/gh-aw/cache-memory/*),Task,TodoWrite,Write,Write(/tmp/*),Write(/tmp/gh-aw/agent/*),Write(/tmp/gh-aw/cache-memory/*),mcp__github__actions_get,mcp__github__actions_list,mcp__github__get_code_scanning_alert,mcp__github__get_commit,mcp__github__get_dependabot_alert,mcp__github__get_discussion,mcp__github__get_discussion_comments,mcp__github__get_file_contents,mcp__github__get_job_logs,mcp__github__get_label,mcp__github__get_latest_release,mcp__github__get_me,mcp__github__get_notification_details,mcp__github__get_pull_request,mcp__github__get_pull_request_comments,mcp__github__get_pull_request_diff,mcp__github__get_pull_request_files,mcp__github__get_pull_request_review_comments,mcp__github__get_pull_request_reviews,mcp__github__get_pull_request_status,mcp__github__get_release_by_tag,mcp__github__get_secret_scanning_alert,mcp__github__get_tag,mcp__github__issue_read,mcp__github__list_branches,mcp__github__list_code_scanning_alerts,mcp__github__list_commits,mcp__github__list_dependabot_alerts,mcp__github__list_discussion_categories,mcp__github__list_discussions,mcp__github__list_issue_types,mcp__github__list_issues,mcp__github__list_label,mcp__github__list_notifications,mcp__github__list_pull_requests,mcp__github__list_releases,mcp__github__list_secret_scanning_alerts,mcp__github__list_starred_repositories,mcp__github__list_tags,mcp__github__pull_request_read,mcp__github__search_code,mcp__github__search_issues,mcp__github__search_orgs,mcp__github__search_pull_requests,mcp__github__search_repositories,mcp__github__search_users,mcp__safeoutputs'\'' --debug-file /tmp/gh-aw/agent-stdio.log --verbose --permission-mode acceptEdits --output-format stream-json --mcp-config "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - ANTHROPIC_MODEL: ${{ needs.activation.outputs.model_size }} + ANTHROPIC_MODEL: ${{ needs.activation.outputs.model_size || vars.GH_AW_MODEL_AGENT_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} BASH_DEFAULT_TIMEOUT_MS: 60000 BASH_MAX_TIMEOUT_MS: 60000 CLAUDE_CODE_DISABLE_FAST_MODE: 1 @@ -1834,7 +1834,7 @@ jobs: -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/claude_harness.cjs claude --print --no-chrome --allowed-tools '\''Bash,BashOutput,Edit(/tmp/*),ExitPlanMode,Glob,Grep,KillBash,LS,MultiEdit(/tmp/*),NotebookRead,Read,Read(/tmp/*),Task,TodoWrite,Write(/tmp/*)'\'' --debug-file /tmp/gh-aw/evals/evals.log --verbose --permission-mode acceptEdits --output-format stream-json --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/evals/evals.log env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - ANTHROPIC_MODEL: ${{ needs.activation.outputs.model_size }} + ANTHROPIC_MODEL: ${{ needs.activation.outputs.model_size || vars.GH_AW_MODEL_DETECTION_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} BASH_DEFAULT_TIMEOUT_MS: 60000 BASH_MAX_TIMEOUT_MS: 60000 CLAUDE_CODE_DISABLE_FAST_MODE: 1 diff --git a/.github/workflows/daily-doc-healer.lock.yml b/.github/workflows/daily-doc-healer.lock.yml index b792f7b71ad..cf92f5de964 100644 --- a/.github/workflows/daily-doc-healer.lock.yml +++ b/.github/workflows/daily-doc-healer.lock.yml @@ -1077,7 +1077,7 @@ jobs: -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/claude_harness.cjs claude --print --no-chrome --allowed-tools '\''Bash(cat .github/workflows/daily-doc-updater.md),Bash(cat /tmp/gh-aw/cache-memory/),Bash(cat > /tmp/gh-aw/cache-memory/),Bash(cat),Bash(date),Bash(echo),Bash(find docs -name "*.md" -o -name "*.mdx"),Bash(gh:*),Bash(git add:*),Bash(git branch:*),Bash(git checkout:*),Bash(git commit:*),Bash(git diff:*),Bash(git log:*),Bash(git merge:*),Bash(git rm:*),Bash(git show:*),Bash(git status),Bash(git switch:*),Bash(grep),Bash(grep:*),Bash(head),Bash(ls),Bash(mkdir -p /tmp/gh-aw/cache-memory/),Bash(mv /tmp/gh-aw/cache-memory/),Bash(printf),Bash(pwd),Bash(safeoutputs:*),Bash(sort),Bash(tail),Bash(uniq),Bash(wc),Bash(yq),BashOutput,Edit,Edit(/tmp/*),Edit(/tmp/gh-aw/agent/*),Edit(/tmp/gh-aw/cache-memory/*),ExitPlanMode,Glob,Grep,KillBash,LS,MultiEdit,MultiEdit(/tmp/*),MultiEdit(/tmp/gh-aw/agent/*),MultiEdit(/tmp/gh-aw/cache-memory/*),NotebookEdit,NotebookRead,Read,Read(/tmp/*),Read(/tmp/gh-aw/agent/*),Read(/tmp/gh-aw/cache-memory/*),Task,TodoWrite,Write,Write(/tmp/*),Write(/tmp/gh-aw/agent/*),Write(/tmp/gh-aw/cache-memory/*),mcp__github__actions_get,mcp__github__actions_list,mcp__github__get_code_scanning_alert,mcp__github__get_commit,mcp__github__get_dependabot_alert,mcp__github__get_discussion,mcp__github__get_discussion_comments,mcp__github__get_file_contents,mcp__github__get_job_logs,mcp__github__get_label,mcp__github__get_latest_release,mcp__github__get_me,mcp__github__get_notification_details,mcp__github__get_pull_request,mcp__github__get_pull_request_comments,mcp__github__get_pull_request_diff,mcp__github__get_pull_request_files,mcp__github__get_pull_request_review_comments,mcp__github__get_pull_request_reviews,mcp__github__get_pull_request_status,mcp__github__get_release_by_tag,mcp__github__get_secret_scanning_alert,mcp__github__get_tag,mcp__github__issue_read,mcp__github__list_branches,mcp__github__list_code_scanning_alerts,mcp__github__list_commits,mcp__github__list_dependabot_alerts,mcp__github__list_discussion_categories,mcp__github__list_discussions,mcp__github__list_issue_types,mcp__github__list_issues,mcp__github__list_label,mcp__github__list_notifications,mcp__github__list_pull_requests,mcp__github__list_releases,mcp__github__list_secret_scanning_alerts,mcp__github__list_starred_repositories,mcp__github__list_tags,mcp__github__pull_request_read,mcp__github__search_code,mcp__github__search_issues,mcp__github__search_orgs,mcp__github__search_pull_requests,mcp__github__search_repositories,mcp__github__search_users,mcp__safeoutputs'\'' --debug-file /tmp/gh-aw/agent-stdio.log --verbose --permission-mode acceptEdits --output-format stream-json --mcp-config "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - ANTHROPIC_MODEL: ${{ needs.activation.outputs.model_size }} + ANTHROPIC_MODEL: ${{ needs.activation.outputs.model_size || vars.GH_AW_MODEL_AGENT_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} BASH_DEFAULT_TIMEOUT_MS: 60000 BASH_MAX_TIMEOUT_MS: 60000 CLAUDE_CODE_DISABLE_FAST_MODE: 1 @@ -1941,7 +1941,7 @@ jobs: -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/claude_harness.cjs claude --print --no-chrome --allowed-tools '\''Bash,BashOutput,Edit(/tmp/*),ExitPlanMode,Glob,Grep,KillBash,LS,MultiEdit(/tmp/*),NotebookRead,Read,Read(/tmp/*),Task,TodoWrite,Write(/tmp/*)'\'' --debug-file /tmp/gh-aw/evals/evals.log --verbose --permission-mode acceptEdits --output-format stream-json --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/evals/evals.log env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - ANTHROPIC_MODEL: ${{ needs.activation.outputs.model_size }} + ANTHROPIC_MODEL: ${{ needs.activation.outputs.model_size || vars.GH_AW_MODEL_DETECTION_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} BASH_DEFAULT_TIMEOUT_MS: 60000 BASH_MAX_TIMEOUT_MS: 60000 CLAUDE_CODE_DISABLE_FAST_MODE: 1 diff --git a/.github/workflows/test-quality-sentinel.lock.yml b/.github/workflows/test-quality-sentinel.lock.yml index c05058634e1..742167688d3 100644 --- a/.github/workflows/test-quality-sentinel.lock.yml +++ b/.github/workflows/test-quality-sentinel.lock.yml @@ -980,7 +980,7 @@ jobs: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ github.token }} - COPILOT_MODEL: ${{ needs.activation.outputs.model_size }} + COPILOT_MODEL: ${{ needs.activation.outputs.model_size || vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }} GH_AW_LLM_PROVIDER: github GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} diff --git a/pkg/workflow/claude_engine.go b/pkg/workflow/claude_engine.go index 8cf2070a4e7..3c7965e8998 100644 --- a/pkg/workflow/claude_engine.go +++ b/pkg/workflow/claude_engine.go @@ -474,8 +474,24 @@ func (e *ClaudeEngine) GetExecutionSteps(workflowData *WorkflowData, logFile str // When model is not configured, fall back to GH_AW_MODEL_AGENT/DETECTION_CLAUDE so users // can set a default via GitHub Actions variables. if modelConfigured { - claudeLog.Printf("Setting %s env var for model: %s", constants.ClaudeCLIModelEnvVar, workflowData.Model) - env[constants.ClaudeCLIModelEnvVar] = workflowData.Model + // When model is a pure GitHub Actions expression it may resolve to an empty string at + // runtime (e.g. ${{ vars.MY_MODEL }} when the variable is not set). Wrap it with the + // normal org/enterprise fallback chain so an empty resolution is treated as omitted. + if isExpression(workflowData.Model) { + innerExpr := extractExpressionInner(workflowData.Model) + isDetectionJob := workflowData.SafeOutputs == nil + var claudeModelVar string + if isDetectionJob { + claudeModelVar = constants.EnvVarModelDetectionClaude + } else { + claudeModelVar = constants.EnvVarModelAgentClaude + } + claudeLog.Printf("Setting %s env var for expression-backed model", constants.ClaudeCLIModelEnvVar) + env[constants.ClaudeCLIModelEnvVar] = compilerenv.BuildModelOverrideExpressionWithUserOverride(innerExpr, claudeModelVar, compilerenv.DefaultModelClaude, "") + } else { + claudeLog.Printf("Setting %s env var for model: %s", constants.ClaudeCLIModelEnvVar, workflowData.Model) + env[constants.ClaudeCLIModelEnvVar] = workflowData.Model + } } else { // No model configured - use fallback GitHub variable with shell expansion isDetectionJob := workflowData.SafeOutputs == nil diff --git a/pkg/workflow/codex_engine.go b/pkg/workflow/codex_engine.go index 9f53295235a..a4b25d37c88 100644 --- a/pkg/workflow/codex_engine.go +++ b/pkg/workflow/codex_engine.go @@ -487,8 +487,18 @@ mkdir -p "$CODEX_HOME/logs" // When model is configured (static or GitHub Actions expression), set the env var directly. // When not configured, use the GitHub variable fallback so users can set a default. if modelConfigured { - codexEngineLog.Printf("Setting %s env var for model: %s", modelEnvVar, workflowData.Model) - env[modelEnvVar] = workflowData.Model + // When model is a pure GitHub Actions expression it may resolve to an empty string at + // runtime (e.g. ${{ vars.MY_MODEL }} when the variable is not set). Wrap it with the + // normal org/enterprise/built-in fallback chain so an empty resolution is treated as omitted. + if isExpression(workflowData.Model) { + innerExpr := extractExpressionInner(workflowData.Model) + expr := compilerenv.BuildModelOverrideExpressionWithUserOverride(innerExpr, modelEnvVar, compilerenv.DefaultModelCodex, constants.CodexDefaultModel) + codexEngineLog.Printf("Setting %s env var for expression-backed model: %s", modelEnvVar, expr) + env[modelEnvVar] = expr + } else { + codexEngineLog.Printf("Setting %s env var for model: %s", modelEnvVar, workflowData.Model) + env[modelEnvVar] = workflowData.Model + } } else { env[modelEnvVar] = compilerenv.BuildModelOverrideExpression(modelEnvVar, compilerenv.DefaultModelCodex, constants.CodexDefaultModel) } diff --git a/pkg/workflow/compilerenv/manager.go b/pkg/workflow/compilerenv/manager.go index 928aa329f66..5ac45ac53cc 100644 --- a/pkg/workflow/compilerenv/manager.go +++ b/pkg/workflow/compilerenv/manager.go @@ -229,6 +229,17 @@ func BuildModelOverrideExpressionEmptyFallback(primaryVar, enterpriseDefaultVar return fmt.Sprintf("${{ vars.%s || vars.%s || '' }}", primaryVar, enterpriseDefaultVar) } +// BuildModelOverrideExpressionWithUserOverride builds a vars expression that places a +// user-supplied runtime expression first, followed by the normal org/enterprise/built-in +// fallback chain. Use this when engine.model or model is a GitHub Actions expression that +// may resolve to an empty string (e.g. ${{ vars.MY_MODEL }} when the variable is unset). +// userExprInner is the raw content between "${{" and "}}" of the user expression. +// When builtinFallback is empty the expression ends with || ”. +func BuildModelOverrideExpressionWithUserOverride(userExprInner, primaryVar, enterpriseDefaultVar, builtinFallback string) string { + escaped := strings.ReplaceAll(builtinFallback, "'", "''") + return fmt.Sprintf("${{ %s || vars.%s || vars.%s || '%s' }}", userExprInner, primaryVar, enterpriseDefaultVar, escaped) +} + // ResolvePolicyModelsAllowed returns configured allowed model policy entries. // When the env var is unset/empty, ok=false and callers should use frontmatter policy. func (m *Manager) ResolvePolicyModelsAllowed() ([]string, bool) { diff --git a/pkg/workflow/compilerenv/manager_test.go b/pkg/workflow/compilerenv/manager_test.go index aad8a4a10ae..3072ce7219e 100644 --- a/pkg/workflow/compilerenv/manager_test.go +++ b/pkg/workflow/compilerenv/manager_test.go @@ -145,6 +145,27 @@ func TestBuildModelOverrideExpression(t *testing.T) { ) } +func TestBuildModelOverrideExpressionWithUserOverride(t *testing.T) { + // With built-in fallback: user expression is first, then org/enterprise/built-in chain. + assert.Equal( + t, + "${{ vars.MY_MODEL || vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }}", + BuildModelOverrideExpressionWithUserOverride("vars.MY_MODEL", "GH_AW_MODEL_AGENT_COPILOT", "GH_AW_DEFAULT_MODEL_COPILOT", "claude-sonnet-4.6"), + ) + // Without built-in fallback (empty string): ends with || ''. + assert.Equal( + t, + "${{ inputs.model || vars.GH_AW_MODEL_AGENT_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }}", + BuildModelOverrideExpressionWithUserOverride("inputs.model", "GH_AW_MODEL_AGENT_CLAUDE", "GH_AW_DEFAULT_MODEL_CLAUDE", ""), + ) + // Single-quotes in fallback are escaped. + assert.Equal( + t, + "${{ vars.M || vars.PRIMARY || vars.ENTERPRISE || 'it''s default' }}", + BuildModelOverrideExpressionWithUserOverride("vars.M", "PRIMARY", "ENTERPRISE", "it's default"), + ) +} + func TestResolveDefaultMaxTurns(t *testing.T) { t.Run("unset uses fallback", func(t *testing.T) { t.Setenv(DefaultMaxTurns, "") diff --git a/pkg/workflow/copilot_engine_execution.go b/pkg/workflow/copilot_engine_execution.go index 83dac99da39..2be5f6e4468 100644 --- a/pkg/workflow/copilot_engine_execution.go +++ b/pkg/workflow/copilot_engine_execution.go @@ -588,6 +588,17 @@ func (e *CopilotEngine) addCopilotModelEnv(env map[string]string, workflowData * // The model is always passed via the native COPILOT_MODEL env var, which the Copilot CLI reads directly. // When model is not configured, map the GitHub org variable to COPILOT_MODEL so users can set a default. if modelConfigured { + // When model is a pure GitHub Actions expression it may resolve to an empty string at + // runtime (e.g. ${{ vars.MY_MODEL }} when the variable is not set). Wrap it with the + // normal org/enterprise/built-in fallback chain so an empty resolution is treated as + // omitted rather than propagated as an empty model value. + if isExpression(workflowData.Model) { + innerExpr := extractExpressionInner(workflowData.Model) + expr := compilerenv.BuildModelOverrideExpressionWithUserOverride(innerExpr, modelEnvVar, compilerenv.DefaultModelCopilot, constants.CopilotBYOKDefaultModel) + copilotExecLog.Printf("Setting %s env var for expression-backed model: %s", constants.CopilotCLIModelEnvVar, expr) + env[constants.CopilotCLIModelEnvVar] = expr + return + } copilotExecLog.Printf("Setting %s env var for model: %s", constants.CopilotCLIModelEnvVar, workflowData.Model) env[constants.CopilotCLIModelEnvVar] = workflowData.Model return diff --git a/pkg/workflow/expression_patterns.go b/pkg/workflow/expression_patterns.go index 86c338e0372..00dc2f9511b 100644 --- a/pkg/workflow/expression_patterns.go +++ b/pkg/workflow/expression_patterns.go @@ -85,6 +85,19 @@ func isExpression(s string) bool { return strings.HasPrefix(s, "${{") && strings.HasSuffix(s, "}}") } +// extractExpressionInner extracts and trims the raw content between "${{" and "}}" +// from a pure GitHub Actions expression. The caller should first verify that +// isExpression(s) is true. Returns an empty string if s is too short to contain +// valid inner content. +func extractExpressionInner(s string) string { + const prefixLen = 3 // len("${{") + const suffixLen = 2 // len("}}") + if len(s) < prefixLen+suffixLen { + return "" + } + return strings.TrimSpace(s[prefixLen : len(s)-suffixLen]) +} + // Core Expression Patterns var ( // ExpressionPattern matches GitHub Actions expressions: ${{ ... }} diff --git a/pkg/workflow/model_env_vars_test.go b/pkg/workflow/model_env_vars_test.go index 9a048d657c2..1abeeee80cf 100644 --- a/pkg/workflow/model_env_vars_test.go +++ b/pkg/workflow/model_env_vars_test.go @@ -310,35 +310,35 @@ func TestExpressionModelUsesEnvVar(t *testing.T) { expectShellExpansion bool // whether command should use ${VAR:+ --model "$VAR"} }{ { - name: "Copilot agent with inputs.model expression uses native COPILOT_MODEL", + name: "Copilot agent with inputs.model expression uses native COPILOT_MODEL with fallback", engine: "copilot", model: "${{ inputs.model }}", expectedEnvVar: constants.CopilotCLIModelEnvVar, - expectedEnvVal: "${{ inputs.model }}", + expectedEnvVal: "${{ inputs.model || vars." + constants.EnvVarModelAgentCopilot + " || vars." + compilerenv.DefaultModelCopilot + " || '" + constants.CopilotBYOKDefaultModel + "' }}", expectShellExpansion: false, // Copilot reads COPILOT_MODEL natively, no shell expansion needed }, { - name: "Copilot agent with vars.model expression uses native COPILOT_MODEL", + name: "Copilot agent with vars.model expression uses native COPILOT_MODEL with fallback", engine: "copilot", model: "${{ vars.MY_MODEL }}", expectedEnvVar: constants.CopilotCLIModelEnvVar, - expectedEnvVal: "${{ vars.MY_MODEL }}", + expectedEnvVal: "${{ vars.MY_MODEL || vars." + constants.EnvVarModelAgentCopilot + " || vars." + compilerenv.DefaultModelCopilot + " || '" + constants.CopilotBYOKDefaultModel + "' }}", expectShellExpansion: false, }, { - name: "Claude agent with inputs.model expression uses native ANTHROPIC_MODEL", + name: "Claude agent with inputs.model expression uses native ANTHROPIC_MODEL with fallback", engine: "claude", model: "${{ inputs.model }}", expectedEnvVar: constants.ClaudeCLIModelEnvVar, - expectedEnvVal: "${{ inputs.model }}", + expectedEnvVal: "${{ inputs.model || vars." + constants.EnvVarModelAgentClaude + " || vars." + compilerenv.DefaultModelClaude + " || '' }}", expectShellExpansion: false, // Claude reads ANTHROPIC_MODEL natively, no shell expansion needed }, { - name: "Codex agent with inputs.model expression", + name: "Codex agent with inputs.model expression uses fallback chain", engine: "codex", model: "${{ inputs.model }}", expectedEnvVar: constants.EnvVarModelAgentCodex, - expectedEnvVal: "${{ inputs.model }}", + expectedEnvVal: "${{ inputs.model || vars." + constants.EnvVarModelAgentCodex + " || vars." + compilerenv.DefaultModelCodex + " || '" + constants.CodexDefaultModel + "' }}", expectShellExpansion: true, // Codex has no native model env var, uses shell expansion }, } @@ -430,8 +430,8 @@ func TestExpressionModelDetectionJobUsesEnvVar(t *testing.T) { } stepsContent := stepsStr.String() - // Detection job for Copilot should use COPILOT_MODEL (native CLI env var) - expectedEnvLine := constants.CopilotCLIModelEnvVar + ": ${{ inputs.model }}" + // Detection job for Copilot should use COPILOT_MODEL (native CLI env var) with fallback + expectedEnvLine := constants.CopilotCLIModelEnvVar + ": ${{ inputs.model || vars." + constants.EnvVarModelDetectionCopilot + " || vars." + compilerenv.DefaultModelCopilot + " || '" + constants.CopilotBYOKDefaultModel + "' }}" if !strings.Contains(stepsContent, expectedEnvLine) { t.Errorf("Expected env line '%s' not found in steps:\n%s", expectedEnvLine, stepsContent) } From 0ce2115bd9e327f0534e44e1cea819af88c5e2d2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:20:32 +0000 Subject: [PATCH 3/7] Handle expression model fallback in JS runtime Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../daily-cache-strategy-analyzer.lock.yml | 6 +- .../daily-caveman-optimizer.lock.yml | 6 +- .github/workflows/daily-doc-healer.lock.yml | 6 +- .../workflows/test-quality-sentinel.lock.yml | 3 +- actions/setup/js/claude_harness.cjs | 3 + actions/setup/js/codex_harness.cjs | 18 +++++ actions/setup/js/copilot_harness.cjs | 3 + actions/setup/js/copilot_sdk_driver.cjs | 3 +- actions/setup/js/copilot_sdk_session.cjs | 3 +- actions/setup/js/model_fallback.cjs | 42 ++++++++++++ actions/setup/js/model_fallback.test.cjs | 25 +++++++ pkg/constants/engine_constants.go | 5 ++ pkg/workflow/claude_engine.go | 26 +++---- pkg/workflow/codex_engine.go | 15 ++-- pkg/workflow/compilerenv/manager.go | 11 --- pkg/workflow/compilerenv/manager_test.go | 21 ------ pkg/workflow/copilot_engine_execution.go | 12 +--- pkg/workflow/expression_patterns.go | 13 ---- pkg/workflow/model_env_vars_test.go | 68 ++++++++++++------- 19 files changed, 172 insertions(+), 117 deletions(-) create mode 100644 actions/setup/js/model_fallback.cjs create mode 100644 actions/setup/js/model_fallback.test.cjs diff --git a/.github/workflows/daily-cache-strategy-analyzer.lock.yml b/.github/workflows/daily-cache-strategy-analyzer.lock.yml index b2760e151c1..e52d777b303 100644 --- a/.github/workflows/daily-cache-strategy-analyzer.lock.yml +++ b/.github/workflows/daily-cache-strategy-analyzer.lock.yml @@ -1079,7 +1079,8 @@ jobs: CODEX_HOME: /tmp/gh-aw/mcp-config GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} GH_AW_MCP_CONFIG: ${{ runner.temp }}/gh-aw/mcp-config/config.toml - GH_AW_MODEL_AGENT_CODEX: ${{ needs.activation.outputs.model_size || vars.GH_AW_MODEL_AGENT_CODEX || vars.GH_AW_DEFAULT_MODEL_CODEX || 'gpt-5.4' }} + GH_AW_MODEL_AGENT_CODEX: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_AGENT_CODEX || vars.GH_AW_DEFAULT_MODEL_CODEX || 'gpt-5.4' }} GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} @@ -2009,7 +2010,8 @@ jobs: CODEX_HOME: /tmp/gh-aw/mcp-config GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} GH_AW_MCP_CONFIG: ${{ runner.temp }}/gh-aw/mcp-config/config.toml - GH_AW_MODEL_DETECTION_CODEX: ${{ needs.activation.outputs.model_size || vars.GH_AW_MODEL_DETECTION_CODEX || vars.GH_AW_DEFAULT_MODEL_CODEX || 'gpt-5.4' }} + GH_AW_MODEL_DETECTION_CODEX: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_DETECTION_CODEX || vars.GH_AW_DEFAULT_MODEL_CODEX || 'gpt-5.4' }} GH_AW_PHASE: detection GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_VERSION: dev diff --git a/.github/workflows/daily-caveman-optimizer.lock.yml b/.github/workflows/daily-caveman-optimizer.lock.yml index 21935b20184..8cf5fc60f5b 100644 --- a/.github/workflows/daily-caveman-optimizer.lock.yml +++ b/.github/workflows/daily-caveman-optimizer.lock.yml @@ -973,7 +973,7 @@ jobs: -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/claude_harness.cjs claude --print --no-chrome --allowed-tools '\''Bash,BashOutput,Edit,Edit(/tmp/*),Edit(/tmp/gh-aw/agent/*),Edit(/tmp/gh-aw/cache-memory/*),ExitPlanMode,Glob,Grep,KillBash,LS,MultiEdit,MultiEdit(/tmp/*),MultiEdit(/tmp/gh-aw/agent/*),MultiEdit(/tmp/gh-aw/cache-memory/*),NotebookEdit,NotebookRead,Read,Read(/tmp/*),Read(/tmp/gh-aw/agent/*),Read(/tmp/gh-aw/cache-memory/*),Task,TodoWrite,Write,Write(/tmp/*),Write(/tmp/gh-aw/agent/*),Write(/tmp/gh-aw/cache-memory/*),mcp__github__actions_get,mcp__github__actions_list,mcp__github__get_code_scanning_alert,mcp__github__get_commit,mcp__github__get_dependabot_alert,mcp__github__get_discussion,mcp__github__get_discussion_comments,mcp__github__get_file_contents,mcp__github__get_job_logs,mcp__github__get_label,mcp__github__get_latest_release,mcp__github__get_me,mcp__github__get_notification_details,mcp__github__get_pull_request,mcp__github__get_pull_request_comments,mcp__github__get_pull_request_diff,mcp__github__get_pull_request_files,mcp__github__get_pull_request_review_comments,mcp__github__get_pull_request_reviews,mcp__github__get_pull_request_status,mcp__github__get_release_by_tag,mcp__github__get_secret_scanning_alert,mcp__github__get_tag,mcp__github__issue_read,mcp__github__list_branches,mcp__github__list_code_scanning_alerts,mcp__github__list_commits,mcp__github__list_dependabot_alerts,mcp__github__list_discussion_categories,mcp__github__list_discussions,mcp__github__list_issue_types,mcp__github__list_issues,mcp__github__list_label,mcp__github__list_notifications,mcp__github__list_pull_requests,mcp__github__list_releases,mcp__github__list_secret_scanning_alerts,mcp__github__list_starred_repositories,mcp__github__list_tags,mcp__github__pull_request_read,mcp__github__search_code,mcp__github__search_issues,mcp__github__search_orgs,mcp__github__search_pull_requests,mcp__github__search_repositories,mcp__github__search_users,mcp__safeoutputs'\'' --debug-file /tmp/gh-aw/agent-stdio.log --verbose --permission-mode acceptEdits --output-format stream-json --mcp-config "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - ANTHROPIC_MODEL: ${{ needs.activation.outputs.model_size || vars.GH_AW_MODEL_AGENT_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} + ANTHROPIC_MODEL: ${{ needs.activation.outputs.model_size }} BASH_DEFAULT_TIMEOUT_MS: 60000 BASH_MAX_TIMEOUT_MS: 60000 CLAUDE_CODE_DISABLE_FAST_MODE: 1 @@ -983,6 +983,7 @@ jobs: GH_AW_LLM_PROVIDER: anthropic GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} GH_AW_MCP_CONFIG: ${{ runner.temp }}/gh-aw/mcp-config/mcp-servers.json + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_AGENT_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} @@ -1834,7 +1835,7 @@ jobs: -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/claude_harness.cjs claude --print --no-chrome --allowed-tools '\''Bash,BashOutput,Edit(/tmp/*),ExitPlanMode,Glob,Grep,KillBash,LS,MultiEdit(/tmp/*),NotebookRead,Read,Read(/tmp/*),Task,TodoWrite,Write(/tmp/*)'\'' --debug-file /tmp/gh-aw/evals/evals.log --verbose --permission-mode acceptEdits --output-format stream-json --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/evals/evals.log env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - ANTHROPIC_MODEL: ${{ needs.activation.outputs.model_size || vars.GH_AW_MODEL_DETECTION_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} + ANTHROPIC_MODEL: ${{ needs.activation.outputs.model_size }} BASH_DEFAULT_TIMEOUT_MS: 60000 BASH_MAX_TIMEOUT_MS: 60000 CLAUDE_CODE_DISABLE_FAST_MODE: 1 @@ -1843,6 +1844,7 @@ jobs: DISABLE_TELEMETRY: 1 GH_AW_LLM_PROVIDER: anthropic GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_DETECTION_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} GH_AW_PHASE: detection GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_VERSION: dev diff --git a/.github/workflows/daily-doc-healer.lock.yml b/.github/workflows/daily-doc-healer.lock.yml index cf92f5de964..76b6d3bbd6f 100644 --- a/.github/workflows/daily-doc-healer.lock.yml +++ b/.github/workflows/daily-doc-healer.lock.yml @@ -1077,7 +1077,7 @@ jobs: -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/claude_harness.cjs claude --print --no-chrome --allowed-tools '\''Bash(cat .github/workflows/daily-doc-updater.md),Bash(cat /tmp/gh-aw/cache-memory/),Bash(cat > /tmp/gh-aw/cache-memory/),Bash(cat),Bash(date),Bash(echo),Bash(find docs -name "*.md" -o -name "*.mdx"),Bash(gh:*),Bash(git add:*),Bash(git branch:*),Bash(git checkout:*),Bash(git commit:*),Bash(git diff:*),Bash(git log:*),Bash(git merge:*),Bash(git rm:*),Bash(git show:*),Bash(git status),Bash(git switch:*),Bash(grep),Bash(grep:*),Bash(head),Bash(ls),Bash(mkdir -p /tmp/gh-aw/cache-memory/),Bash(mv /tmp/gh-aw/cache-memory/),Bash(printf),Bash(pwd),Bash(safeoutputs:*),Bash(sort),Bash(tail),Bash(uniq),Bash(wc),Bash(yq),BashOutput,Edit,Edit(/tmp/*),Edit(/tmp/gh-aw/agent/*),Edit(/tmp/gh-aw/cache-memory/*),ExitPlanMode,Glob,Grep,KillBash,LS,MultiEdit,MultiEdit(/tmp/*),MultiEdit(/tmp/gh-aw/agent/*),MultiEdit(/tmp/gh-aw/cache-memory/*),NotebookEdit,NotebookRead,Read,Read(/tmp/*),Read(/tmp/gh-aw/agent/*),Read(/tmp/gh-aw/cache-memory/*),Task,TodoWrite,Write,Write(/tmp/*),Write(/tmp/gh-aw/agent/*),Write(/tmp/gh-aw/cache-memory/*),mcp__github__actions_get,mcp__github__actions_list,mcp__github__get_code_scanning_alert,mcp__github__get_commit,mcp__github__get_dependabot_alert,mcp__github__get_discussion,mcp__github__get_discussion_comments,mcp__github__get_file_contents,mcp__github__get_job_logs,mcp__github__get_label,mcp__github__get_latest_release,mcp__github__get_me,mcp__github__get_notification_details,mcp__github__get_pull_request,mcp__github__get_pull_request_comments,mcp__github__get_pull_request_diff,mcp__github__get_pull_request_files,mcp__github__get_pull_request_review_comments,mcp__github__get_pull_request_reviews,mcp__github__get_pull_request_status,mcp__github__get_release_by_tag,mcp__github__get_secret_scanning_alert,mcp__github__get_tag,mcp__github__issue_read,mcp__github__list_branches,mcp__github__list_code_scanning_alerts,mcp__github__list_commits,mcp__github__list_dependabot_alerts,mcp__github__list_discussion_categories,mcp__github__list_discussions,mcp__github__list_issue_types,mcp__github__list_issues,mcp__github__list_label,mcp__github__list_notifications,mcp__github__list_pull_requests,mcp__github__list_releases,mcp__github__list_secret_scanning_alerts,mcp__github__list_starred_repositories,mcp__github__list_tags,mcp__github__pull_request_read,mcp__github__search_code,mcp__github__search_issues,mcp__github__search_orgs,mcp__github__search_pull_requests,mcp__github__search_repositories,mcp__github__search_users,mcp__safeoutputs'\'' --debug-file /tmp/gh-aw/agent-stdio.log --verbose --permission-mode acceptEdits --output-format stream-json --mcp-config "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - ANTHROPIC_MODEL: ${{ needs.activation.outputs.model_size || vars.GH_AW_MODEL_AGENT_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} + ANTHROPIC_MODEL: ${{ needs.activation.outputs.model_size }} BASH_DEFAULT_TIMEOUT_MS: 60000 BASH_MAX_TIMEOUT_MS: 60000 CLAUDE_CODE_DISABLE_FAST_MODE: 1 @@ -1087,6 +1087,7 @@ jobs: GH_AW_LLM_PROVIDER: anthropic GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} GH_AW_MCP_CONFIG: ${{ runner.temp }}/gh-aw/mcp-config/mcp-servers.json + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_AGENT_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} @@ -1941,7 +1942,7 @@ jobs: -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/claude_harness.cjs claude --print --no-chrome --allowed-tools '\''Bash,BashOutput,Edit(/tmp/*),ExitPlanMode,Glob,Grep,KillBash,LS,MultiEdit(/tmp/*),NotebookRead,Read,Read(/tmp/*),Task,TodoWrite,Write(/tmp/*)'\'' --debug-file /tmp/gh-aw/evals/evals.log --verbose --permission-mode acceptEdits --output-format stream-json --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/evals/evals.log env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - ANTHROPIC_MODEL: ${{ needs.activation.outputs.model_size || vars.GH_AW_MODEL_DETECTION_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} + ANTHROPIC_MODEL: ${{ needs.activation.outputs.model_size }} BASH_DEFAULT_TIMEOUT_MS: 60000 BASH_MAX_TIMEOUT_MS: 60000 CLAUDE_CODE_DISABLE_FAST_MODE: 1 @@ -1950,6 +1951,7 @@ jobs: DISABLE_TELEMETRY: 1 GH_AW_LLM_PROVIDER: anthropic GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_DETECTION_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} GH_AW_PHASE: detection GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_VERSION: dev diff --git a/.github/workflows/test-quality-sentinel.lock.yml b/.github/workflows/test-quality-sentinel.lock.yml index 742167688d3..407c15f3dba 100644 --- a/.github/workflows/test-quality-sentinel.lock.yml +++ b/.github/workflows/test-quality-sentinel.lock.yml @@ -980,10 +980,11 @@ jobs: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ github.token }} - COPILOT_MODEL: ${{ needs.activation.outputs.model_size || vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }} + COPILOT_MODEL: ${{ needs.activation.outputs.model_size }} GH_AW_LLM_PROVIDER: github GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }} GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} diff --git a/actions/setup/js/claude_harness.cjs b/actions/setup/js/claude_harness.cjs index e84e02b496a..33be2194615 100644 --- a/actions/setup/js/claude_harness.cjs +++ b/actions/setup/js/claude_harness.cjs @@ -53,6 +53,7 @@ const { emitMissingToolPermissionIssue, hasExpectedSafeOutputs, hasNoopInSafeOut const { countPermissionDeniedIssues, hasNumerousPermissionDeniedIssues, extractDeniedCommands, buildMissingToolPermissionIssuePayload } = require("./permission_denied_helpers.cjs"); const { detectNonRetryableHarnessGuard, buildSoftTimeoutGuard, emitSoftTimeoutSignal, isAuthenticationFailedError } = require("./harness_retry_guard.cjs"); const { MODEL_NOT_SUPPORTED_PATTERN: INVALID_MODEL_ERROR_PATTERN } = require("./detect_agent_errors.cjs"); +const { applyModelFallback } = require("./model_fallback.cjs"); // Pattern to detect Anthropic API overload errors (HTTP 529). // Matches "overloaded_error" from the Anthropic error type field, and the @@ -318,6 +319,7 @@ function stripContinueArgs(args) { */ async function buildClaudeChildEnv() { const childEnv = { ...process.env }; + applyModelFallback(childEnv, "ANTHROPIC_MODEL", log); const provider = normalizeReflectProviderName(process.env.GH_AW_LLM_PROVIDER, "anthropic"); try { const raw = fs.readFileSync(AWF_REFLECT_OUTPUT_PATH, "utf8"); @@ -611,6 +613,7 @@ if (typeof module !== "undefined" && module.exports) { hasExpectedSafeOutputs, resolveRetryConfig, resolveStartupRetryLimit, + applyModelFallback, }; } diff --git a/actions/setup/js/codex_harness.cjs b/actions/setup/js/codex_harness.cjs index 92f97cc9d84..5abe1345eb3 100644 --- a/actions/setup/js/codex_harness.cjs +++ b/actions/setup/js/codex_harness.cjs @@ -54,6 +54,7 @@ const { countPermissionDeniedIssues, hasNumerousPermissionDeniedIssues, extractD const { detectNonRetryableHarnessGuard, buildSoftTimeoutGuard, emitSoftTimeoutSignal, isAuthenticationFailedError } = require("./harness_retry_guard.cjs"); const { MODEL_NOT_SUPPORTED_PATTERN: INVALID_MODEL_ERROR_PATTERN } = require("./detect_agent_errors.cjs"); const { resolveRetryConfig } = require("./harness_retry_config.cjs"); +const { applyModelFallback, injectModelFlagAfterExec } = require("./model_fallback.cjs"); // Pattern to detect OpenAI rate-limit errors. // Matches the JSON error type field ("rate_limit_exceeded"), the HTTP status code @@ -221,6 +222,16 @@ function injectJsonFlag(args) { return ["exec", "--json", ...args.slice(1)]; } +function getCodexModelEnvVar(env = process.env) { + if (Object.prototype.hasOwnProperty.call(env, "GH_AW_MODEL_DETECTION_CODEX")) { + return "GH_AW_MODEL_DETECTION_CODEX"; + } + if (Object.prototype.hasOwnProperty.call(env, "GH_AW_MODEL_AGENT_CODEX")) { + return "GH_AW_MODEL_AGENT_CODEX"; + } + return ""; +} + /** * Build child process environment for Codex execution. * Preserve API keys captured at harness startup, even if the parent environment @@ -425,6 +436,10 @@ async function main() { process.exit(1); } + const codexModelEnvVar = getCodexModelEnvVar(process.env); + const resolvedModel = codexModelEnvVar ? applyModelFallback(process.env, codexModelEnvVar, log) : ""; + resolvedArgs = injectModelFlagAfterExec(resolvedArgs, resolvedModel); + // Safe arg list for logging: when --prompt-file was present, the last element of // resolvedArgs is the resolved prompt content. Replace it with a placeholder so that // task instructions are never written to stderr or captured in agent logs. @@ -641,6 +656,9 @@ if (typeof module !== "undefined" && module.exports) { hasNoopInSafeOutputs, hasExpectedSafeOutputs, resolveRetryConfig, + applyModelFallback, + injectModelFlagAfterExec, + getCodexModelEnvVar, }; } diff --git a/actions/setup/js/copilot_harness.cjs b/actions/setup/js/copilot_harness.cjs index 4ed31e07d16..d29e70d4d44 100644 --- a/actions/setup/js/copilot_harness.cjs +++ b/actions/setup/js/copilot_harness.cjs @@ -65,6 +65,7 @@ const { runSafeOutputsCLI, buildMissingToolAlternatives, emitMissingToolPermissi const { countPermissionDeniedIssues, hasNumerousPermissionDeniedIssues, extractDeniedCommands, buildMissingToolPermissionIssuePayload } = require("./permission_denied_helpers.cjs"); const { detectNonRetryableHarnessGuard, buildSoftTimeoutGuard, emitSoftTimeoutSignal, isAuthenticationFailedError: isCommonAuthenticationFailedError } = require("./harness_retry_guard.cjs"); const { isCAPIQuotaExceededError } = require("./detect_agent_errors.cjs"); +const { applyModelFallback } = require("./model_fallback.cjs"); const { loadModelsJson } = require("./model_costs.cjs"); const { resolveConfiguredCopilotModel } = require("./resolve_model_alias.cjs"); @@ -950,6 +951,7 @@ async function main() { } } + applyModelFallback(process.env, "COPILOT_MODEL", log); applyCopilotModelAliasResolution({ awfReflectData, logger: log }); applyCopilotWireAPI({ modelsJson: loadModelsJson(), logger: log }); @@ -1456,6 +1458,7 @@ if (typeof module !== "undefined" && module.exports) { parseCopilotSDKServerArgsFromEnv, isCAPIQuotaExceededError, hasTerminalSafeOutput, + applyModelFallback, applyCopilotModelAliasResolution, applyCopilotWireAPI, loadAwfConfigData, diff --git a/actions/setup/js/copilot_sdk_driver.cjs b/actions/setup/js/copilot_sdk_driver.cjs index 2e609e0462a..9716aaf3ffb 100644 --- a/actions/setup/js/copilot_sdk_driver.cjs +++ b/actions/setup/js/copilot_sdk_driver.cjs @@ -30,6 +30,7 @@ const fs = require("fs"); const { runWithCopilotSDK, extractPromptFromArgs } = require("./copilot_sdk_session.cjs"); const { parsePermissionConfigFromServerArgs } = require("./copilot_sdk_permissions.cjs"); const { parseMultiProviderJson } = require("./copilot_sdk_multi_provider.cjs"); +const { applyModelFallback } = require("./model_fallback.cjs"); // Re-export the session and permission helpers so that existing callers that // require("./copilot_sdk_driver.cjs") (e.g. copilot_harness.cjs) continue to work. @@ -103,7 +104,7 @@ async function main() { const providers = multiProviderConfig.providers; /** @type {import("@github/copilot-sdk").ProviderModelConfig[]} */ const sdkModels = multiProviderConfig.models; - let model = process.env.COPILOT_MODEL || multiProviderConfig.model || undefined; + let model = applyModelFallback(process.env, "COPILOT_MODEL", log) || multiProviderConfig.model || undefined; log(`multi-provider mode: ${providers.length} providers, ${sdkModels.length} models, model=${model ?? "(env)"}`); for (const p of providers) { log(` provider: name=${p.name} type=${p.type} baseUrl=${p.baseUrl}${p.wireApi ? ` wireApi=${p.wireApi}` : ""}`); diff --git a/actions/setup/js/copilot_sdk_session.cjs b/actions/setup/js/copilot_sdk_session.cjs index 30c3a6251e9..1c0172ddaf1 100644 --- a/actions/setup/js/copilot_sdk_session.cjs +++ b/actions/setup/js/copilot_sdk_session.cjs @@ -36,6 +36,7 @@ const fs = require("fs"); const path = require("path"); const os = require("os"); const { buildCopilotSDKPermissionHandler, getEnvPositiveIntOrDefault, parseMaxToolDenialsLimit, MAX_TOOL_DENIALS_DEFAULT } = require("./copilot_sdk_permissions.cjs"); +const { resolveModelWithFallback } = require("./model_fallback.cjs"); const { extractShellCommandFromToolData } = require("./tool_call_details.cjs"); // Default timeout for a single sendAndWait call: 10 minutes. @@ -254,7 +255,7 @@ async function runWithCopilotSDK({ sdkUri, prompt, logger, attempt = 0, model, c // Build session config using the multi-provider surface. /** @type {import("@github/copilot-sdk").SessionConfig} */ const sessionConfig = { - model: model || process.env.COPILOT_MODEL || undefined, + model: model || resolveModelWithFallback(process.env, "COPILOT_MODEL") || undefined, providers, models: providerModels, onPermissionRequest, diff --git a/actions/setup/js/model_fallback.cjs b/actions/setup/js/model_fallback.cjs new file mode 100644 index 00000000000..8991a8087e2 --- /dev/null +++ b/actions/setup/js/model_fallback.cjs @@ -0,0 +1,42 @@ +"use strict"; + +const MODEL_FALLBACK_ENV_VAR = "GH_AW_MODEL_FALLBACK"; + +function readTrimmedEnv(env, name) { + return typeof env?.[name] === "string" ? env[name].trim() : ""; +} + +function resolveModelWithFallback(env, primaryEnvVar) { + return readTrimmedEnv(env, primaryEnvVar) || readTrimmedEnv(env, MODEL_FALLBACK_ENV_VAR); +} + +function applyModelFallback(env, primaryEnvVar, logger = () => {}) { + const primary = readTrimmedEnv(env, primaryEnvVar); + if (primary) { + return primary; + } + const fallback = readTrimmedEnv(env, MODEL_FALLBACK_ENV_VAR); + if (fallback) { + env[primaryEnvVar] = fallback; + logger(`applied ${MODEL_FALLBACK_ENV_VAR} to ${primaryEnvVar}`); + } + return fallback; +} + +function injectModelFlagAfterExec(args, model) { + if (!model || args.includes("--model")) { + return args; + } + const execIndex = args.indexOf("exec"); + if (execIndex === -1) { + return [...args, "--model", model]; + } + return [...args.slice(0, execIndex + 1), "--model", model, ...args.slice(execIndex + 1)]; +} + +module.exports = { + MODEL_FALLBACK_ENV_VAR, + resolveModelWithFallback, + applyModelFallback, + injectModelFlagAfterExec, +}; diff --git a/actions/setup/js/model_fallback.test.cjs b/actions/setup/js/model_fallback.test.cjs new file mode 100644 index 00000000000..ce524c81572 --- /dev/null +++ b/actions/setup/js/model_fallback.test.cjs @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { createRequire } from "module"; + +const require = createRequire(import.meta.url); +const { MODEL_FALLBACK_ENV_VAR, resolveModelWithFallback, applyModelFallback, injectModelFlagAfterExec } = require("./model_fallback.cjs"); + +describe("model_fallback.cjs", () => { + it("prefers the primary model env var when it is non-empty", () => { + expect(resolveModelWithFallback({ COPILOT_MODEL: "gpt-5", [MODEL_FALLBACK_ENV_VAR]: "claude-sonnet-4.6" }, "COPILOT_MODEL")).toBe("gpt-5"); + }); + + it("uses the fallback model when the primary resolves empty", () => { + const env = { COPILOT_MODEL: "", [MODEL_FALLBACK_ENV_VAR]: "claude-sonnet-4.6" }; + expect(applyModelFallback(env, "COPILOT_MODEL")).toBe("claude-sonnet-4.6"); + expect(env.COPILOT_MODEL).toBe("claude-sonnet-4.6"); + }); + + it("injects --model immediately after exec when needed", () => { + expect(injectModelFlagAfterExec(["exec", "--json", "prompt"], "gpt-5")).toEqual(["exec", "--model", "gpt-5", "--json", "prompt"]); + }); + + it("does not inject --model when already present", () => { + expect(injectModelFlagAfterExec(["exec", "--model", "gpt-5", "--json"], "claude-sonnet-4.6")).toEqual(["exec", "--model", "gpt-5", "--json"]); + }); +}); diff --git a/pkg/constants/engine_constants.go b/pkg/constants/engine_constants.go index fdc66a30b7c..ef63b38a4d2 100644 --- a/pkg/constants/engine_constants.go +++ b/pkg/constants/engine_constants.go @@ -243,6 +243,11 @@ const ( // EnvVarModelAgentPi configures the default Pi model for agent execution EnvVarModelAgentPi = "GH_AW_MODEL_AGENT_PI" + // EnvVarModelFallback carries the standard org/enterprise/built-in model fallback + // expression for the current engine step. Runtime JavaScript harnesses can use this + // when a configured model expression resolves to an empty string. + EnvVarModelFallback = "GH_AW_MODEL_FALLBACK" + // CopilotCLIModelEnvVar is the native environment variable name supported by the Copilot CLI // for selecting the model. Setting this env var is equivalent to passing --model to the CLI. CopilotCLIModelEnvVar = "COPILOT_MODEL" diff --git a/pkg/workflow/claude_engine.go b/pkg/workflow/claude_engine.go index 3c7965e8998..0a361135e0a 100644 --- a/pkg/workflow/claude_engine.go +++ b/pkg/workflow/claude_engine.go @@ -474,24 +474,18 @@ func (e *ClaudeEngine) GetExecutionSteps(workflowData *WorkflowData, logFile str // When model is not configured, fall back to GH_AW_MODEL_AGENT/DETECTION_CLAUDE so users // can set a default via GitHub Actions variables. if modelConfigured { - // When model is a pure GitHub Actions expression it may resolve to an empty string at - // runtime (e.g. ${{ vars.MY_MODEL }} when the variable is not set). Wrap it with the - // normal org/enterprise fallback chain so an empty resolution is treated as omitted. - if isExpression(workflowData.Model) { - innerExpr := extractExpressionInner(workflowData.Model) - isDetectionJob := workflowData.SafeOutputs == nil - var claudeModelVar string - if isDetectionJob { - claudeModelVar = constants.EnvVarModelDetectionClaude - } else { - claudeModelVar = constants.EnvVarModelAgentClaude - } - claudeLog.Printf("Setting %s env var for expression-backed model", constants.ClaudeCLIModelEnvVar) - env[constants.ClaudeCLIModelEnvVar] = compilerenv.BuildModelOverrideExpressionWithUserOverride(innerExpr, claudeModelVar, compilerenv.DefaultModelClaude, "") + isDetectionJob := workflowData.SafeOutputs == nil + var claudeModelVar string + if isDetectionJob { + claudeModelVar = constants.EnvVarModelDetectionClaude } else { - claudeLog.Printf("Setting %s env var for model: %s", constants.ClaudeCLIModelEnvVar, workflowData.Model) - env[constants.ClaudeCLIModelEnvVar] = workflowData.Model + claudeModelVar = constants.EnvVarModelAgentClaude + } + if containsExpression(workflowData.Model) { + env[constants.EnvVarModelFallback] = compilerenv.BuildModelOverrideExpressionEmptyFallback(claudeModelVar, compilerenv.DefaultModelClaude) } + claudeLog.Printf("Setting %s env var for model: %s", constants.ClaudeCLIModelEnvVar, workflowData.Model) + env[constants.ClaudeCLIModelEnvVar] = workflowData.Model } else { // No model configured - use fallback GitHub variable with shell expansion isDetectionJob := workflowData.SafeOutputs == nil diff --git a/pkg/workflow/codex_engine.go b/pkg/workflow/codex_engine.go index a4b25d37c88..8876100600b 100644 --- a/pkg/workflow/codex_engine.go +++ b/pkg/workflow/codex_engine.go @@ -487,18 +487,11 @@ mkdir -p "$CODEX_HOME/logs" // When model is configured (static or GitHub Actions expression), set the env var directly. // When not configured, use the GitHub variable fallback so users can set a default. if modelConfigured { - // When model is a pure GitHub Actions expression it may resolve to an empty string at - // runtime (e.g. ${{ vars.MY_MODEL }} when the variable is not set). Wrap it with the - // normal org/enterprise/built-in fallback chain so an empty resolution is treated as omitted. - if isExpression(workflowData.Model) { - innerExpr := extractExpressionInner(workflowData.Model) - expr := compilerenv.BuildModelOverrideExpressionWithUserOverride(innerExpr, modelEnvVar, compilerenv.DefaultModelCodex, constants.CodexDefaultModel) - codexEngineLog.Printf("Setting %s env var for expression-backed model: %s", modelEnvVar, expr) - env[modelEnvVar] = expr - } else { - codexEngineLog.Printf("Setting %s env var for model: %s", modelEnvVar, workflowData.Model) - env[modelEnvVar] = workflowData.Model + if containsExpression(workflowData.Model) { + env[constants.EnvVarModelFallback] = compilerenv.BuildModelOverrideExpression(modelEnvVar, compilerenv.DefaultModelCodex, constants.CodexDefaultModel) } + codexEngineLog.Printf("Setting %s env var for model: %s", modelEnvVar, workflowData.Model) + env[modelEnvVar] = workflowData.Model } else { env[modelEnvVar] = compilerenv.BuildModelOverrideExpression(modelEnvVar, compilerenv.DefaultModelCodex, constants.CodexDefaultModel) } diff --git a/pkg/workflow/compilerenv/manager.go b/pkg/workflow/compilerenv/manager.go index 5ac45ac53cc..928aa329f66 100644 --- a/pkg/workflow/compilerenv/manager.go +++ b/pkg/workflow/compilerenv/manager.go @@ -229,17 +229,6 @@ func BuildModelOverrideExpressionEmptyFallback(primaryVar, enterpriseDefaultVar return fmt.Sprintf("${{ vars.%s || vars.%s || '' }}", primaryVar, enterpriseDefaultVar) } -// BuildModelOverrideExpressionWithUserOverride builds a vars expression that places a -// user-supplied runtime expression first, followed by the normal org/enterprise/built-in -// fallback chain. Use this when engine.model or model is a GitHub Actions expression that -// may resolve to an empty string (e.g. ${{ vars.MY_MODEL }} when the variable is unset). -// userExprInner is the raw content between "${{" and "}}" of the user expression. -// When builtinFallback is empty the expression ends with || ”. -func BuildModelOverrideExpressionWithUserOverride(userExprInner, primaryVar, enterpriseDefaultVar, builtinFallback string) string { - escaped := strings.ReplaceAll(builtinFallback, "'", "''") - return fmt.Sprintf("${{ %s || vars.%s || vars.%s || '%s' }}", userExprInner, primaryVar, enterpriseDefaultVar, escaped) -} - // ResolvePolicyModelsAllowed returns configured allowed model policy entries. // When the env var is unset/empty, ok=false and callers should use frontmatter policy. func (m *Manager) ResolvePolicyModelsAllowed() ([]string, bool) { diff --git a/pkg/workflow/compilerenv/manager_test.go b/pkg/workflow/compilerenv/manager_test.go index 3072ce7219e..aad8a4a10ae 100644 --- a/pkg/workflow/compilerenv/manager_test.go +++ b/pkg/workflow/compilerenv/manager_test.go @@ -145,27 +145,6 @@ func TestBuildModelOverrideExpression(t *testing.T) { ) } -func TestBuildModelOverrideExpressionWithUserOverride(t *testing.T) { - // With built-in fallback: user expression is first, then org/enterprise/built-in chain. - assert.Equal( - t, - "${{ vars.MY_MODEL || vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }}", - BuildModelOverrideExpressionWithUserOverride("vars.MY_MODEL", "GH_AW_MODEL_AGENT_COPILOT", "GH_AW_DEFAULT_MODEL_COPILOT", "claude-sonnet-4.6"), - ) - // Without built-in fallback (empty string): ends with || ''. - assert.Equal( - t, - "${{ inputs.model || vars.GH_AW_MODEL_AGENT_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }}", - BuildModelOverrideExpressionWithUserOverride("inputs.model", "GH_AW_MODEL_AGENT_CLAUDE", "GH_AW_DEFAULT_MODEL_CLAUDE", ""), - ) - // Single-quotes in fallback are escaped. - assert.Equal( - t, - "${{ vars.M || vars.PRIMARY || vars.ENTERPRISE || 'it''s default' }}", - BuildModelOverrideExpressionWithUserOverride("vars.M", "PRIMARY", "ENTERPRISE", "it's default"), - ) -} - func TestResolveDefaultMaxTurns(t *testing.T) { t.Run("unset uses fallback", func(t *testing.T) { t.Setenv(DefaultMaxTurns, "") diff --git a/pkg/workflow/copilot_engine_execution.go b/pkg/workflow/copilot_engine_execution.go index 2be5f6e4468..005fcfede45 100644 --- a/pkg/workflow/copilot_engine_execution.go +++ b/pkg/workflow/copilot_engine_execution.go @@ -588,16 +588,8 @@ func (e *CopilotEngine) addCopilotModelEnv(env map[string]string, workflowData * // The model is always passed via the native COPILOT_MODEL env var, which the Copilot CLI reads directly. // When model is not configured, map the GitHub org variable to COPILOT_MODEL so users can set a default. if modelConfigured { - // When model is a pure GitHub Actions expression it may resolve to an empty string at - // runtime (e.g. ${{ vars.MY_MODEL }} when the variable is not set). Wrap it with the - // normal org/enterprise/built-in fallback chain so an empty resolution is treated as - // omitted rather than propagated as an empty model value. - if isExpression(workflowData.Model) { - innerExpr := extractExpressionInner(workflowData.Model) - expr := compilerenv.BuildModelOverrideExpressionWithUserOverride(innerExpr, modelEnvVar, compilerenv.DefaultModelCopilot, constants.CopilotBYOKDefaultModel) - copilotExecLog.Printf("Setting %s env var for expression-backed model: %s", constants.CopilotCLIModelEnvVar, expr) - env[constants.CopilotCLIModelEnvVar] = expr - return + if containsExpression(workflowData.Model) { + env[constants.EnvVarModelFallback] = compilerenv.BuildModelOverrideExpression(modelEnvVar, compilerenv.DefaultModelCopilot, constants.CopilotBYOKDefaultModel) } copilotExecLog.Printf("Setting %s env var for model: %s", constants.CopilotCLIModelEnvVar, workflowData.Model) env[constants.CopilotCLIModelEnvVar] = workflowData.Model diff --git a/pkg/workflow/expression_patterns.go b/pkg/workflow/expression_patterns.go index 00dc2f9511b..86c338e0372 100644 --- a/pkg/workflow/expression_patterns.go +++ b/pkg/workflow/expression_patterns.go @@ -85,19 +85,6 @@ func isExpression(s string) bool { return strings.HasPrefix(s, "${{") && strings.HasSuffix(s, "}}") } -// extractExpressionInner extracts and trims the raw content between "${{" and "}}" -// from a pure GitHub Actions expression. The caller should first verify that -// isExpression(s) is true. Returns an empty string if s is too short to contain -// valid inner content. -func extractExpressionInner(s string) string { - const prefixLen = 3 // len("${{") - const suffixLen = 2 // len("}}") - if len(s) < prefixLen+suffixLen { - return "" - } - return strings.TrimSpace(s[prefixLen : len(s)-suffixLen]) -} - // Core Expression Patterns var ( // ExpressionPattern matches GitHub Actions expressions: ${{ ... }} diff --git a/pkg/workflow/model_env_vars_test.go b/pkg/workflow/model_env_vars_test.go index 1abeeee80cf..8c27caf3128 100644 --- a/pkg/workflow/model_env_vars_test.go +++ b/pkg/workflow/model_env_vars_test.go @@ -212,6 +212,9 @@ func TestExplicitModelConfigOverridesEnvVar(t *testing.T) { if strings.Contains(stepsContent, constants.EnvVarModelAgentCopilot+":") { t.Errorf("Fallback env var %s should not be present when model is explicitly configured", constants.EnvVarModelAgentCopilot) } + if strings.Contains(stepsContent, constants.EnvVarModelFallback+":") { + t.Errorf("Fallback env var %s should not be present for a literal configured model", constants.EnvVarModelFallback) + } // The model should be passed via the native COPILOT_MODEL env var (not via --model flag) expectedEnvLine := constants.CopilotCLIModelEnvVar + ": gpt-4" @@ -305,40 +308,45 @@ func TestExpressionModelUsesEnvVar(t *testing.T) { name string engine string model string - expectedEnvVar string - expectedEnvVal string + expectedModelEnvVar string + expectedModelEnvVal string + expectedFallbackVal string expectShellExpansion bool // whether command should use ${VAR:+ --model "$VAR"} }{ { - name: "Copilot agent with inputs.model expression uses native COPILOT_MODEL with fallback", + name: "Copilot agent keeps pure expression model and adds JS fallback env", engine: "copilot", model: "${{ inputs.model }}", - expectedEnvVar: constants.CopilotCLIModelEnvVar, - expectedEnvVal: "${{ inputs.model || vars." + constants.EnvVarModelAgentCopilot + " || vars." + compilerenv.DefaultModelCopilot + " || '" + constants.CopilotBYOKDefaultModel + "' }}", + expectedModelEnvVar: constants.CopilotCLIModelEnvVar, + expectedModelEnvVal: "${{ inputs.model }}", + expectedFallbackVal: "${{ vars." + constants.EnvVarModelAgentCopilot + " || vars." + compilerenv.DefaultModelCopilot + " || '" + constants.CopilotBYOKDefaultModel + "' }}", expectShellExpansion: false, // Copilot reads COPILOT_MODEL natively, no shell expansion needed }, { - name: "Copilot agent with vars.model expression uses native COPILOT_MODEL with fallback", + name: "Copilot agent keeps composite expression model and adds JS fallback env", engine: "copilot", - model: "${{ vars.MY_MODEL }}", - expectedEnvVar: constants.CopilotCLIModelEnvVar, - expectedEnvVal: "${{ vars.MY_MODEL || vars." + constants.EnvVarModelAgentCopilot + " || vars." + compilerenv.DefaultModelCopilot + " || '" + constants.CopilotBYOKDefaultModel + "' }}", + model: "${{ inputs.provider }}/${{ inputs.model }}", + expectedModelEnvVar: constants.CopilotCLIModelEnvVar, + expectedModelEnvVal: "${{ inputs.provider }}/${{ inputs.model }}", + expectedFallbackVal: "${{ vars." + constants.EnvVarModelAgentCopilot + " || vars." + compilerenv.DefaultModelCopilot + " || '" + constants.CopilotBYOKDefaultModel + "' }}", expectShellExpansion: false, }, { - name: "Claude agent with inputs.model expression uses native ANTHROPIC_MODEL with fallback", + name: "Claude agent keeps expression model and adds JS fallback env", engine: "claude", model: "${{ inputs.model }}", - expectedEnvVar: constants.ClaudeCLIModelEnvVar, - expectedEnvVal: "${{ inputs.model || vars." + constants.EnvVarModelAgentClaude + " || vars." + compilerenv.DefaultModelClaude + " || '' }}", + expectedModelEnvVar: constants.ClaudeCLIModelEnvVar, + expectedModelEnvVal: "${{ inputs.model }}", + expectedFallbackVal: "${{ vars." + constants.EnvVarModelAgentClaude + " || vars." + compilerenv.DefaultModelClaude + " || '' }}", expectShellExpansion: false, // Claude reads ANTHROPIC_MODEL natively, no shell expansion needed }, { - name: "Codex agent with inputs.model expression uses fallback chain", + name: "Codex agent keeps composite expression model and adds JS fallback env", engine: "codex", - model: "${{ inputs.model }}", - expectedEnvVar: constants.EnvVarModelAgentCodex, - expectedEnvVal: "${{ inputs.model || vars." + constants.EnvVarModelAgentCodex + " || vars." + compilerenv.DefaultModelCodex + " || '" + constants.CodexDefaultModel + "' }}", + model: "${{ inputs.provider }}/${{ inputs.model }}", + expectedModelEnvVar: constants.EnvVarModelAgentCodex, + expectedModelEnvVal: "${{ inputs.provider }}/${{ inputs.model }}", + expectedFallbackVal: "${{ vars." + constants.EnvVarModelAgentCodex + " || vars." + compilerenv.DefaultModelCodex + " || '" + constants.CodexDefaultModel + "' }}", expectShellExpansion: true, // Codex has no native model env var, uses shell expansion }, } @@ -381,18 +389,22 @@ func TestExpressionModelUsesEnvVar(t *testing.T) { } // The env var must be set to the expression value - expectedEnvLine := tt.expectedEnvVar + ": " + tt.expectedEnvVal - if !strings.Contains(stepsContent, expectedEnvLine) { - t.Errorf("Expected env line '%s' not found in steps:\n%s", expectedEnvLine, stepsContent) + expectedModelEnvLine := tt.expectedModelEnvVar + ": " + tt.expectedModelEnvVal + if !strings.Contains(stepsContent, expectedModelEnvLine) { + t.Errorf("Expected env line '%s' not found in steps:\n%s", expectedModelEnvLine, stepsContent) + } + expectedFallbackEnvLine := constants.EnvVarModelFallback + ": " + tt.expectedFallbackVal + if !strings.Contains(stepsContent, expectedFallbackEnvLine) { + t.Errorf("Expected fallback env line '%s' not found in steps:\n%s", expectedFallbackEnvLine, stepsContent) } // Check shell expansion expectation - shellExpansionPattern := "${" + tt.expectedEnvVar + ":+" + shellExpansionPattern := "${" + tt.expectedModelEnvVar + ":+" hasShellExpansion := strings.Contains(stepsContent, shellExpansionPattern) if tt.expectShellExpansion && !hasShellExpansion { - t.Errorf("Expected conditional env var usage '${%s:+' not found in steps:\n%s", tt.expectedEnvVar, stepsContent) + t.Errorf("Expected conditional env var usage '${%s:+' not found in steps:\n%s", tt.expectedModelEnvVar, stepsContent) } else if !tt.expectShellExpansion && hasShellExpansion { - t.Errorf("Unexpected conditional env var usage '${%s:+' found in steps (should use native env var):\n%s", tt.expectedEnvVar, stepsContent) + t.Errorf("Unexpected conditional env var usage '${%s:+' found in steps (should use native env var):\n%s", tt.expectedModelEnvVar, stepsContent) } }) } @@ -430,10 +442,14 @@ func TestExpressionModelDetectionJobUsesEnvVar(t *testing.T) { } stepsContent := stepsStr.String() - // Detection job for Copilot should use COPILOT_MODEL (native CLI env var) with fallback - expectedEnvLine := constants.CopilotCLIModelEnvVar + ": ${{ inputs.model || vars." + constants.EnvVarModelDetectionCopilot + " || vars." + compilerenv.DefaultModelCopilot + " || '" + constants.CopilotBYOKDefaultModel + "' }}" - if !strings.Contains(stepsContent, expectedEnvLine) { - t.Errorf("Expected env line '%s' not found in steps:\n%s", expectedEnvLine, stepsContent) + // Detection job for Copilot should keep the configured expression and expose the runtime fallback. + expectedModelEnvLine := constants.CopilotCLIModelEnvVar + ": ${{ inputs.model }}" + if !strings.Contains(stepsContent, expectedModelEnvLine) { + t.Errorf("Expected env line '%s' not found in steps:\n%s", expectedModelEnvLine, stepsContent) + } + expectedFallbackEnvLine := constants.EnvVarModelFallback + ": ${{ vars." + constants.EnvVarModelDetectionCopilot + " || vars." + compilerenv.DefaultModelCopilot + " || '" + constants.CopilotBYOKDefaultModel + "' }}" + if !strings.Contains(stepsContent, expectedFallbackEnvLine) { + t.Errorf("Expected fallback env line '%s' not found in steps:\n%s", expectedFallbackEnvLine, stepsContent) } // Must not embed expression directly in shell command From f2740c3730ed648a70869eaaa171a390a7e13220 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:31:56 +0000 Subject: [PATCH 4/7] Fix runtime fallback test module format Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/codex_harness.cjs | 4 ++-- .../js/{model_fallback.test.cjs => model_fallback.test.js} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename actions/setup/js/{model_fallback.test.cjs => model_fallback.test.js} (100%) diff --git a/actions/setup/js/codex_harness.cjs b/actions/setup/js/codex_harness.cjs index 5abe1345eb3..81fba7e97f9 100644 --- a/actions/setup/js/codex_harness.cjs +++ b/actions/setup/js/codex_harness.cjs @@ -223,10 +223,10 @@ function injectJsonFlag(args) { } function getCodexModelEnvVar(env = process.env) { - if (Object.prototype.hasOwnProperty.call(env, "GH_AW_MODEL_DETECTION_CODEX")) { + if ("GH_AW_MODEL_DETECTION_CODEX" in env) { return "GH_AW_MODEL_DETECTION_CODEX"; } - if (Object.prototype.hasOwnProperty.call(env, "GH_AW_MODEL_AGENT_CODEX")) { + if ("GH_AW_MODEL_AGENT_CODEX" in env) { return "GH_AW_MODEL_AGENT_CODEX"; } return ""; diff --git a/actions/setup/js/model_fallback.test.cjs b/actions/setup/js/model_fallback.test.js similarity index 100% rename from actions/setup/js/model_fallback.test.cjs rename to actions/setup/js/model_fallback.test.js From 00216ae5a741670f52c85d53afbab92158d25533 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 20 Jul 2026 12:51:31 +0000 Subject: [PATCH 5/7] fix: annotate model fallback logger type Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- actions/setup/js/model_fallback.cjs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/actions/setup/js/model_fallback.cjs b/actions/setup/js/model_fallback.cjs index 8991a8087e2..70c1f205ff2 100644 --- a/actions/setup/js/model_fallback.cjs +++ b/actions/setup/js/model_fallback.cjs @@ -10,6 +10,12 @@ function resolveModelWithFallback(env, primaryEnvVar) { return readTrimmedEnv(env, primaryEnvVar) || readTrimmedEnv(env, MODEL_FALLBACK_ENV_VAR); } +/** + * @param {NodeJS.ProcessEnv} env + * @param {string} primaryEnvVar + * @param {(message: string) => void} [logger] + * @returns {string} + */ function applyModelFallback(env, primaryEnvVar, logger = () => {}) { const primary = readTrimmedEnv(env, primaryEnvVar); if (primary) { From a915121a11ca2eb3f3e9f81066a35bcf1573ad85 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:53:07 +0000 Subject: [PATCH 6/7] test: cover composite claude model expressions Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- pkg/workflow/model_env_vars_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/workflow/model_env_vars_test.go b/pkg/workflow/model_env_vars_test.go index 8c27caf3128..21309bd260e 100644 --- a/pkg/workflow/model_env_vars_test.go +++ b/pkg/workflow/model_env_vars_test.go @@ -340,6 +340,15 @@ func TestExpressionModelUsesEnvVar(t *testing.T) { expectedFallbackVal: "${{ vars." + constants.EnvVarModelAgentClaude + " || vars." + compilerenv.DefaultModelClaude + " || '' }}", expectShellExpansion: false, // Claude reads ANTHROPIC_MODEL natively, no shell expansion needed }, + { + name: "Claude agent keeps composite expression model and adds JS fallback env", + engine: "claude", + model: "${{ inputs.provider }}/${{ inputs.model }}", + expectedModelEnvVar: constants.ClaudeCLIModelEnvVar, + expectedModelEnvVal: "${{ inputs.provider }}/${{ inputs.model }}", + expectedFallbackVal: "${{ vars." + constants.EnvVarModelAgentClaude + " || vars." + compilerenv.DefaultModelClaude + " || '' }}", + expectShellExpansion: false, + }, { name: "Codex agent keeps composite expression model and adds JS fallback env", engine: "codex", From 1f3575b876cf139cbe509e89a6ee2bf38ff9ee97 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:58:38 +0000 Subject: [PATCH 7/7] Fix evals model fallback handling Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../daily-cache-strategy-analyzer.lock.yml | 6 ++- .../daily-caveman-optimizer.lock.yml | 6 ++- .github/workflows/daily-doc-healer.lock.yml | 6 ++- actions/setup/js/run_evals.cjs | 3 +- actions/setup/js/run_evals.test.cjs | 14 +++++++ pkg/workflow/evals_steps.go | 41 +++++++++++++++++-- pkg/workflow/evals_steps_test.go | 34 +++++++++++++++ 7 files changed, 99 insertions(+), 11 deletions(-) diff --git a/.github/workflows/daily-cache-strategy-analyzer.lock.yml b/.github/workflows/daily-cache-strategy-analyzer.lock.yml index e52d777b303..625f9496b41 100644 --- a/.github/workflows/daily-cache-strategy-analyzer.lock.yml +++ b/.github/workflows/daily-cache-strategy-analyzer.lock.yml @@ -1876,7 +1876,8 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_EVALS_QUESTIONS: '[{"id":"cache_logs_analyzed","question":"Did the agent analyze agentic workflow logs for cache misses and misconfigured caches?"},{"id":"issue_created_or_noop","question":"Was an issue created for cache problems or improvements, or was noop used when no problems were found?"}]' - GH_AW_EVALS_MODEL: "${{ needs.activation.outputs.model_size }}" + GH_AW_EVALS_MODEL: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_DETECTION_CODEX || vars.GH_AW_DEFAULT_MODEL_CODEX || 'gpt-5.4' }} GH_AW_EVALS_PHASE: setup with: script: | @@ -2031,7 +2032,8 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_EVALS_QUESTIONS: '[{"id":"cache_logs_analyzed","question":"Did the agent analyze agentic workflow logs for cache misses and misconfigured caches?"},{"id":"issue_created_or_noop","question":"Was an issue created for cache problems or improvements, or was noop used when no problems were found?"}]' - GH_AW_EVALS_MODEL: "${{ needs.activation.outputs.model_size }}" + GH_AW_EVALS_MODEL: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_DETECTION_CODEX || vars.GH_AW_DEFAULT_MODEL_CODEX || 'gpt-5.4' }} GH_AW_EVALS_PHASE: parse GITHUB_RUN_ID: ${{ github.run_id }} with: diff --git a/.github/workflows/daily-caveman-optimizer.lock.yml b/.github/workflows/daily-caveman-optimizer.lock.yml index 8cf5fc60f5b..dbe4ac6a902 100644 --- a/.github/workflows/daily-caveman-optimizer.lock.yml +++ b/.github/workflows/daily-caveman-optimizer.lock.yml @@ -1763,7 +1763,8 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_EVALS_QUESTIONS: '[{"id":"instruction_file_optimized","question":"Did the agent apply caveman optimization to an instruction file in .github/aw or .github/agents?"},{"id":"pr_created_or_noop","question":"Was a PR created with concise improvements, or was noop used when no improvements were needed?"}]' - GH_AW_EVALS_MODEL: "${{ needs.activation.outputs.model_size }}" + GH_AW_EVALS_MODEL: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_DETECTION_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} GH_AW_EVALS_PHASE: setup with: script: | @@ -1865,7 +1866,8 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_EVALS_QUESTIONS: '[{"id":"instruction_file_optimized","question":"Did the agent apply caveman optimization to an instruction file in .github/aw or .github/agents?"},{"id":"pr_created_or_noop","question":"Was a PR created with concise improvements, or was noop used when no improvements were needed?"}]' - GH_AW_EVALS_MODEL: "${{ needs.activation.outputs.model_size }}" + GH_AW_EVALS_MODEL: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_DETECTION_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} GH_AW_EVALS_PHASE: parse GITHUB_RUN_ID: ${{ github.run_id }} with: diff --git a/.github/workflows/daily-doc-healer.lock.yml b/.github/workflows/daily-doc-healer.lock.yml index 76b6d3bbd6f..4543496fcc4 100644 --- a/.github/workflows/daily-doc-healer.lock.yml +++ b/.github/workflows/daily-doc-healer.lock.yml @@ -1870,7 +1870,8 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_EVALS_QUESTIONS: '[{"id":"gaps-confirmed","question":"Did the workflow identify at least one confirmed documentation gap to fix, or correctly conclude that no actionable gap remained?"},{"id":"pr-issue-or-noop","question":"Was a documentation pull request or issue created for confirmed gaps, or was noop used appropriately when nothing required action?"}]' - GH_AW_EVALS_MODEL: "${{ needs.activation.outputs.model_size }}" + GH_AW_EVALS_MODEL: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_DETECTION_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} GH_AW_EVALS_PHASE: setup with: script: | @@ -1972,7 +1973,8 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_EVALS_QUESTIONS: '[{"id":"gaps-confirmed","question":"Did the workflow identify at least one confirmed documentation gap to fix, or correctly conclude that no actionable gap remained?"},{"id":"pr-issue-or-noop","question":"Was a documentation pull request or issue created for confirmed gaps, or was noop used appropriately when nothing required action?"}]' - GH_AW_EVALS_MODEL: "${{ needs.activation.outputs.model_size }}" + GH_AW_EVALS_MODEL: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_DETECTION_CLAUDE || vars.GH_AW_DEFAULT_MODEL_CLAUDE || '' }} GH_AW_EVALS_PHASE: parse GITHUB_RUN_ID: ${{ github.run_id }} with: diff --git a/actions/setup/js/run_evals.cjs b/actions/setup/js/run_evals.cjs index f41af3a012e..75f2627a205 100644 --- a/actions/setup/js/run_evals.cjs +++ b/actions/setup/js/run_evals.cjs @@ -35,6 +35,7 @@ const path = require("path"); const { ERR_VALIDATION } = require("./error_codes.cjs"); const { EVALS_OUTPUT_PATH } = require("./evals_constants.cjs"); +const { resolveModelWithFallback } = require("./model_fallback.cjs"); const EVALS_DIR = "/tmp/gh-aw/evals"; const EVALS_LOG_PATH = "/tmp/gh-aw/evals/evals.log"; @@ -118,7 +119,7 @@ async function setupMain() { */ async function parseMain() { const questionsRaw = process.env.GH_AW_EVALS_QUESTIONS; - const model = process.env.GH_AW_EVALS_MODEL || ""; + const model = resolveModelWithFallback(process.env, "GH_AW_EVALS_MODEL") || ""; const runID = process.env.GITHUB_RUN_ID || "unknown"; /** @type {Array<{id: string, question: string}>} */ diff --git a/actions/setup/js/run_evals.test.cjs b/actions/setup/js/run_evals.test.cjs index 7bbf2c4ea9b..52825d72464 100644 --- a/actions/setup/js/run_evals.test.cjs +++ b/actions/setup/js/run_evals.test.cjs @@ -6,6 +6,7 @@ const EVALS_DIR = "/tmp/gh-aw/evals"; const EVALS_LOG_PATH = `${EVALS_DIR}/evals.log`; const EVALS_OUTPUT_PATH = "/tmp/gh-aw/evals.jsonl"; const require = createRequire(import.meta.url); +const { MODEL_FALLBACK_ENV_VAR } = require("./model_fallback.cjs"); const { parseMain, extractAssistantTextFromJsonlLog } = require("./run_evals.cjs"); const mockCore = { @@ -75,6 +76,19 @@ describe("run_evals.cjs", () => { expect(JSON.parse(line).runid).toBe("unknown"); }); + it("uses GH_AW_MODEL_FALLBACK when GH_AW_EVALS_MODEL resolves empty", async () => { + vi.stubEnv("GH_AW_EVALS_QUESTIONS", JSON.stringify([{ id: "labels-applied", question: "Did labels get applied?" }])); + vi.stubEnv("GH_AW_EVALS_MODEL", ""); + vi.stubEnv(MODEL_FALLBACK_ENV_VAR, "claude-sonnet-4.6"); + vi.stubEnv("GITHUB_RUN_ID", "123456789"); + fs.writeFileSync(EVALS_LOG_PATH, "labels-applied: YES\n", "utf8"); + + await parseMain(); + + const [line] = fs.readFileSync(EVALS_OUTPUT_PATH, "utf8").trim().split("\n"); + expect(JSON.parse(line).model).toBe("claude-sonnet-4.6"); + }); + it("parses answers from Pi v3 JSONL turn_end events (positional format)", async () => { vi.stubEnv( "GH_AW_EVALS_QUESTIONS", diff --git a/pkg/workflow/evals_steps.go b/pkg/workflow/evals_steps.go index 82cacd4af60..63a8f98c505 100644 --- a/pkg/workflow/evals_steps.go +++ b/pkg/workflow/evals_steps.go @@ -121,11 +121,13 @@ await main();` fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data)), " env:\n", fmt.Sprintf(" GH_AW_EVALS_QUESTIONS: '%s'\n", escapeYAMLSingleQuoted(questionsJSON)), - fmt.Sprintf(" GH_AW_EVALS_MODEL: %q\n", model), + } + steps = appendEvalsModelEnvLines(steps, c.getEvalsEngineID(data), model) + steps = append(steps, " GH_AW_EVALS_PHASE: setup\n", " with:\n", " script: |\n", - } + ) steps = append(steps, FormatJavaScriptForYAML(script)...) return steps } @@ -277,12 +279,14 @@ await main();` fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data)), " env:\n", fmt.Sprintf(" GH_AW_EVALS_QUESTIONS: '%s'\n", escapeYAMLSingleQuoted(questionsJSON)), - fmt.Sprintf(" GH_AW_EVALS_MODEL: %q\n", model), + } + steps = appendEvalsModelEnvLines(steps, c.getEvalsEngineID(data), model) + steps = append(steps, " GH_AW_EVALS_PHASE: parse\n", " GITHUB_RUN_ID: ${{ github.run_id }}\n", " with:\n", " script: |\n", - } + ) steps = append(steps, FormatJavaScriptForYAML(script)...) return steps } @@ -408,6 +412,35 @@ func (c *Compiler) resolveEvalsExecutionModel(data *WorkflowData) string { return model } +func appendEvalsModelEnvLines(steps []string, engineID, model string) []string { + hasExpression := containsExpression(model) + modelValue := fmt.Sprintf("%q", model) + if hasExpression { + modelValue = model + } + steps = append(steps, fmt.Sprintf(" GH_AW_EVALS_MODEL: %s\n", modelValue)) + if !hasExpression { + return steps + } + if fallback := buildEvalsModelFallbackExpression(engineID); fallback != "" { + steps = append(steps, fmt.Sprintf(" %s: %s\n", constants.EnvVarModelFallback, fallback)) + } + return steps +} + +func buildEvalsModelFallbackExpression(engineID string) string { + switch engineID { + case string(constants.CopilotEngine): + return compilerenv.BuildModelOverrideExpression(constants.EnvVarModelDetectionCopilot, compilerenv.DefaultModelCopilot, constants.CopilotBYOKDefaultModel) + case string(constants.ClaudeEngine): + return compilerenv.BuildModelOverrideExpressionEmptyFallback(constants.EnvVarModelDetectionClaude, compilerenv.DefaultModelClaude) + case string(constants.CodexEngine): + return compilerenv.BuildModelOverrideExpression(constants.EnvVarModelDetectionCodex, compilerenv.DefaultModelCodex, constants.CodexDefaultModel) + default: + return "" + } +} + func (c *Compiler) collectEvalsSecretReferences(data *WorkflowData) []string { if data.Evals == nil { return nil diff --git a/pkg/workflow/evals_steps_test.go b/pkg/workflow/evals_steps_test.go index a834aaaab2b..63711ca310a 100644 --- a/pkg/workflow/evals_steps_test.go +++ b/pkg/workflow/evals_steps_test.go @@ -3,10 +3,12 @@ package workflow import ( + "fmt" "strings" "testing" "github.com/github/gh-aw/pkg/constants" + "github.com/github/gh-aw/pkg/workflow/compilerenv" ) // TestBuildEvalsEngineStepsArcDindTopology verifies that the evals job @@ -198,6 +200,38 @@ func TestBuildParseEvalsResultsStepUsesResolvedExecutionModel(t *testing.T) { } } +func TestBuildParseEvalsResultsStepUsesExpressionModelAndFallbackEnv(t *testing.T) { + compiler := NewCompiler() + + data := &WorkflowData{ + AI: "copilot", + Model: "${{ inputs.model }}", + EngineConfig: &EngineConfig{ + ID: "copilot", + }, + Evals: &EvalsConfig{ + Questions: []EvalDefinition{ + {ID: "builds", Question: "Does it build?"}, + }, + }, + } + + steps := strings.Join(compiler.buildParseEvalsResultsStep(data), "") + if !strings.Contains(steps, `GH_AW_EVALS_MODEL: ${{ inputs.model }}`) { + t.Errorf("expected parse step to preserve expression-backed evals model; got:\n%s", steps) + } + expectedFallbackEnvLine := fmt.Sprintf( + "%s: ${{ vars.%s || vars.%s || '%s' }}", + constants.EnvVarModelFallback, + constants.EnvVarModelDetectionCopilot, + compilerenv.DefaultModelCopilot, + constants.CopilotBYOKDefaultModel, + ) + if !strings.Contains(steps, expectedFallbackEnvLine) { + t.Errorf("expected parse step to expose evals model fallback env %q; got:\n%s", expectedFallbackEnvLine, steps) + } +} + // TestBuildEvalsJobStepsCodexNoDuplicateContainerDownload verifies that a codex // workflow with evals does not generate a duplicate "Download container images" step. // Regression: buildEvalsJobSteps previously called buildPullAWFContainersStep