From 9db5acd9ce23ba1d296e6e103aef573b6ca1f763 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=A7=90=E6=9E=9D?= Date: Tue, 13 Oct 2026 18:03:47 +0800 Subject: [PATCH 1/3] test(runtime): verify shared OpenSandbox judges --- docs/guide/writing-evals.md | 25 +++++++++++++++++++++++++ docs/zh/guide/writing-evals.md | 20 ++++++++++++++++++++ e2e/opensandbox_shared_judge_test.go | 20 ++++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 e2e/opensandbox_shared_judge_test.go diff --git a/docs/guide/writing-evals.md b/docs/guide/writing-evals.md index b67f4e6c..03adf767 100644 --- a/docs/guide/writing-evals.md +++ b/docs/guide/writing-evals.md @@ -903,6 +903,31 @@ Judges run sequentially in the same case runtime (`none`, Docker, or OpenSandbox The runtime filesystem is shared: a judge's changes are visible to later judges. Use read-only checks when every judge must observe unchanged output. Verdicts, timeouts, context materials and session artifacts are recorded separately under each judge's ID; this does not provide filesystem or external-service isolation. The case runtime is cleaned up after all judges finish, using the configured lifecycle policy. +#### OpenSandbox shared-runtime verification + +OpenSandbox judges use the running case sandbox directly; they do not call native +snapshot, restore, pause or resume APIs. Each Agent judge uses a fresh review +session, while installed tools, filesystem changes and the case's network policy +remain in the same sandbox. Cleanup follows the normal case lifecycle. + +To run the gated live-service test, configure `OPENSANDBOX_API_KEY`, +`OPENSANDBOX_BASE_URL` and `OPENSANDBOX_IMAGE` in your environment: + +```bash +SKILL_UP_OPENSANDBOX_JUDGE_E2E=1 go test -tags e2e -run TestOpenSandboxSharedJudgeEnvironment -v ./e2e +``` + +The test creates temporary sandboxes with a five-minute TTL and test metadata. +It runs one Agent fixture, a script judge and two fresh Agent-judge fixture +sessions sequentially. It checks shared file changes, setup and Agent-installed +dependencies outside the workspace, environment configuration and per-judge +material readback. It covers PASS/PASS/PASS and script FAIL / both Agent judges +PASS / aggregate FAIL. The fixture exercises the actual judging pipeline without +calling a model provider. Actual service egress enforcement requires a separate +network test; this test does not claim external-service isolation. + + + Reports show gates, member results, and the overall decision separately. `evaluation.json` stores the complete grouped result; `grading.json` contains one top-level expectation per judge for compatibility. Adding Agent criteria does not change a script judge's weight. Multiple cases can reference the same `script_path` today; packaging helper files with a script is a separate feature. ### judge: agent_judge — LLM rubric diff --git a/docs/zh/guide/writing-evals.md b/docs/zh/guide/writing-evals.md index e1750ed7..2c2ec074 100644 --- a/docs/zh/guide/writing-evals.md +++ b/docs/zh/guide/writing-evals.md @@ -780,6 +780,26 @@ eval 和 case 都可配置 `judges`。case 级列表整体替换 eval 级列表 文件系统由所有 judge 共享：前一个 judge 的修改会被后面的 judge 看到。需要评审原始输出时，请使用只读检查。各 judge 的判定、超时、上下文材料和会话产物仍按 ID 单独记录；这不代表文件系统或外部服务隔离。所有 judge 结束后再按配置清理 case runtime。 +#### OpenSandbox 共享环境验证 + +OpenSandbox judge 直接复用 case 已启动的 sandbox，不调用原生快照、恢复、pause 或 +resume API。各 Agent judge 新开评审会话，工具、文件修改及 case 网络策略留在 +同一个 sandbox；结束后按普通 case 生命周期清理。 + +通过环境提供 `OPENSANDBOX_API_KEY`、`OPENSANDBOX_BASE_URL` 和 `OPENSANDBOX_IMAGE`： + +```bash +SKILL_UP_OPENSANDBOX_JUDGE_E2E=1 go test -tags e2e -run TestOpenSandboxSharedJudgeEnvironment -v ./e2e +``` + +测试创建带测试 metadata 和五分钟 TTL 的临时 sandbox，一次 Agent fixture 执行后， +顺序执行一个 script judge 和两个新开的 Agent judge fixture 会话。覆盖共享文件修改、 +setup/Agent 安装到工作区外的依赖、环境配置和各 judge 材料读回，以及全通过和 +script FAIL / 两个 Agent judge PASS / 总体 FAIL。测试走真实评分链路但不调用模型服务。 +实际服务的 egress 执行效果需单独网络测试，不声称外部服务隔离。 + + + 报告分别展示门控、各 judge 和总体结果。`evaluation.json` 保存完整分组结果；`grading.json` 为兼容格式，每个 judge 对应一个顶层断言。增加 Agent 评审标准不会改变脚本 judge 的权重。多个 case 现在即可引用同一个 `script_path`；连同辅助文件一起打包脚本属于后续功能。 ### judge: agent_judge — LLM 评审 diff --git a/e2e/opensandbox_shared_judge_test.go b/e2e/opensandbox_shared_judge_test.go new file mode 100644 index 00000000..4b60bf86 --- /dev/null +++ b/e2e/opensandbox_shared_judge_test.go @@ -0,0 +1,20 @@ +//go:build e2e + +package e2e + +import ( + "os" + "testing" + + "github.com/alibaba/skill-up/internal/config" +) + +func TestOpenSandboxSharedJudgeEnvironment(t *testing.T) { + if os.Getenv("SKILL_UP_OPENSANDBOX_JUDGE_E2E") != "1" { + t.Skip("set SKILL_UP_OPENSANDBOX_JUDGE_E2E=1 with an OpenSandbox service") + } + if os.Getenv("OPENSANDBOX_API_KEY") == "" || os.Getenv("OPENSANDBOX_BASE_URL") == "" || os.Getenv("OPENSANDBOX_IMAGE") == "" { + t.Fatal("OPENSANDBOX_API_KEY, OPENSANDBOX_BASE_URL and OPENSANDBOX_IMAGE are required") + } + runSharedJudgeEnvironment(t, config.Environment{Type: "opensandbox", Image: os.Getenv("OPENSANDBOX_IMAGE"), NetworkPolicy: "deny_all", ReadyTimeoutSeconds: 60}) +} From b2703b2c2c07c14f68d1fb01fc46137f8265ed99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=A7=90=E6=9E=9D?= Date: Wed, 14 Oct 2026 10:31:55 +0800 Subject: [PATCH 2/3] test(runtime): cover live OpenCode judge provider configuration --- docs/guide/writing-evals.md | 14 ++++ docs/zh/guide/writing-evals.md | 13 ++++ e2e/opensandbox_opencode_judge_test.go | 89 ++++++++++++++++++++++++++ 3 files changed, 116 insertions(+) create mode 100644 e2e/opensandbox_opencode_judge_test.go diff --git a/docs/guide/writing-evals.md b/docs/guide/writing-evals.md index 03adf767..e9d52125 100644 --- a/docs/guide/writing-evals.md +++ b/docs/guide/writing-evals.md @@ -926,6 +926,20 @@ PASS / aggregate FAIL. The fixture exercises the actual judging pipeline without calling a model provider. Actual service egress enforcement requires a separate network test; this test does not claim external-service isolation. +To verify the real OpenCode provider path, also inject `JUDGE_LIVE_API_KEY`, +set `SKILL_UP_OPENCODE_JUDGE_BASE_URL` and `SKILL_UP_OPENCODE_JUDGE_MODEL` for an +OpenAI-compatible provider, then run: + +```bash +SKILL_UP_OPENCODE_JUDGE_E2E=1 go test -count=1 -tags e2e -run '^TestOpenSandboxOpenCodeJudgeProvider$' -v ./e2e +``` + +This separate test starts the real OpenCode CLI and makes live model requests. +The runner fixture seeds a conflicting provider endpoint and key; two independent +judges must override them, pass their rubric and return different session IDs. +It uses a fifteen-minute sandbox TTL and normal case deletion. Optionally pin +OpenCode with `SKILL_UP_OPENCODE_JUDGE_VERSION` (validated with `1.14.24`). + Reports show gates, member results, and the overall decision separately. `evaluation.json` stores the complete grouped result; `grading.json` contains one top-level expectation per judge for compatibility. Adding Agent criteria does not change a script judge's weight. Multiple cases can reference the same `script_path` today; packaging helper files with a script is a separate feature. diff --git a/docs/zh/guide/writing-evals.md b/docs/zh/guide/writing-evals.md index 2c2ec074..faa9ed76 100644 --- a/docs/zh/guide/writing-evals.md +++ b/docs/zh/guide/writing-evals.md @@ -798,6 +798,19 @@ setup/Agent 安装到工作区外的依赖、环境配置和各 judge 材料读 script FAIL / 两个 Agent judge PASS / 总体 FAIL。测试走真实评分链路但不调用模型服务。 实际服务的 egress 执行效果需单独网络测试，不声称外部服务隔离。 +若要验证真实 OpenCode provider 链路，还需注入 `JUDGE_LIVE_API_KEY`，并设置 +OpenAI 兼容服务的 `SKILL_UP_OPENCODE_JUDGE_BASE_URL` 和 +`SKILL_UP_OPENCODE_JUDGE_MODEL`，然后运行： + +```bash +SKILL_UP_OPENCODE_JUDGE_E2E=1 go test -count=1 -tags e2e -run '^TestOpenSandboxOpenCodeJudgeProvider$' -v ./e2e +``` + +该测试启动真实 OpenCode CLI 并请求模型服务。runner fixture 预置冲突的 provider +端点和 key，两个独立 judge 必须覆盖它们、评分通过并返回不同 session ID。 +临时 sandbox 使用十五分钟 TTL 并正常删除。可用 `SKILL_UP_OPENCODE_JUDGE_VERSION` +固定 OpenCode 版本（已验证 `1.14.24`）。 + 报告分别展示门控、各 judge 和总体结果。`evaluation.json` 保存完整分组结果；`grading.json` 为兼容格式，每个 judge 对应一个顶层断言。增加 Agent 评审标准不会改变脚本 judge 的权重。多个 case 现在即可引用同一个 `script_path`；连同辅助文件一起打包脚本属于后续功能。 diff --git a/e2e/opensandbox_opencode_judge_test.go b/e2e/opensandbox_opencode_judge_test.go new file mode 100644 index 00000000..49b21974 --- /dev/null +++ b/e2e/opensandbox_opencode_judge_test.go @@ -0,0 +1,89 @@ +//go:build e2e + +package e2e + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/alibaba/skill-up/internal/agent" + "github.com/alibaba/skill-up/internal/config" + "github.com/alibaba/skill-up/internal/evaluator" + "github.com/alibaba/skill-up/internal/judge" + rt "github.com/alibaba/skill-up/internal/runtime" + "github.com/alibaba/skill-up/pkg/transcript" +) + +// TestOpenSandboxOpenCodeJudgeProvider uses the real OpenCode CLI and a live +// provider. The runner seeds a conflicting OpenCode endpoint to catch accidental +// inheritance; two judges must complete in separate sessions in the same sandbox. +func TestOpenSandboxOpenCodeJudgeProvider(t *testing.T) { + if os.Getenv("SKILL_UP_OPENCODE_JUDGE_E2E") != "1" { + t.Skip("set SKILL_UP_OPENCODE_JUDGE_E2E=1 with OpenSandbox and model-provider credentials") + } + for _, name := range []string{"OPENSANDBOX_API_KEY", "OPENSANDBOX_BASE_URL", "OPENSANDBOX_IMAGE", "JUDGE_LIVE_API_KEY", "SKILL_UP_OPENCODE_JUDGE_BASE_URL", "SKILL_UP_OPENCODE_JUDGE_MODEL"} { + if os.Getenv(name) == "" { + t.Fatalf("%s is required", name) + } + } + model := config.ModelConfig{Provider: "judge_live", Name: os.Getenv("SKILL_UP_OPENCODE_JUDGE_MODEL"), BaseURL: os.Getenv("SKILL_UP_OPENCODE_JUDGE_BASE_URL")} + timeout := 180 + members := []config.JudgeConfig{} + for _, id := range []string{"semantic", "review"} { + members = append(members, config.JudgeConfig{ID: id, Type: "agent_judge", Engine: &config.EngineConfig{Name: "opencode", Version: os.Getenv("SKILL_UP_OPENCODE_JUDGE_VERSION"), Model: model}, Criteria: []string{"The agent final message is exactly completed."}, TimeoutSeconds: &timeout}) + } + out := t.TempDir() + e := evaluator.NewEvaluator(evaluator.EvalOptions{Agent: &openCodeProviderRunner{}, OutputDir: out, DeleteWorkspace: true, EvalCfg: &config.EvalConfig{Environment: config.Environment{Type: "opensandbox", Image: os.Getenv("OPENSANDBOX_IMAGE"), ReadyTimeoutSeconds: 90, SandboxTimeoutSeconds: 900, Metadata: map[string]string{"skill_up_test": "opencode-judge-provider"}}, Judges: &members}}) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + results, err := e.EvaluateAll(ctx, []*config.CaseConfig{{ID: "provider", Input: config.Input{Prompt: "Return completed"}}}) + if err != nil { + t.Fatal(err) + } + if len(results) != 1 { + t.Fatalf("result count=%d", len(results)) + } + result := results[0] + if result.Status != judge.StatusPass { + t.Fatalf("status=%s error=%v outcomes=%+v", result.Status, result.Error, result.JudgeResults) + } + if len(result.JudgeResults) != 2 { + t.Fatalf("judge count=%d", len(result.JudgeResults)) + } + sessions := map[string]bool{} + for _, outcome := range result.JudgeResults { + if outcome.Status != judge.StatusPass || outcome.Session == nil || outcome.Session.SessionID == "" || sessions[outcome.Session.SessionID] { + t.Fatalf("invalid judge outcome: %+v", outcome) + } + sessions[outcome.Session.SessionID] = true + t.Logf("judge=%s engine=%s session=%s status=%s", outcome.ID, outcome.Session.Engine, outcome.Session.SessionID, outcome.Status) + } +} + +type openCodeProviderRunner struct{ sharedJudgeFixtureAgent } + +func (*openCodeProviderRunner) Install(ctx context.Context, current rt.Runtime) error { + path, err := current.Exec(ctx, `printf '%s' "${NVM_DIR:-$HOME/.nvm}/current/bin:$PATH"`, rt.ExecOptions{}) + if err != nil { + return err + } + if path.ExitCode != 0 || strings.TrimSpace(path.Stdout) == "" { + return errors.New("cannot determine sandbox Node path") + } + // Ground images may keep Node outside HOME; preserve their NVM location. + current.MergeEnv(map[string]string{"PATH": strings.TrimSpace(path.Stdout)}) + return nil +} + +func (*openCodeProviderRunner) InstallMCP(_ context.Context, current rt.Runtime, _ rt.MCPConfig) error { + current.MergeEnv(map[string]string{"OPENCODE_CONFIG_CONTENT": `{"provider":{"judge_live":{"npm":"@ai-sdk/openai-compatible","options":{"baseURL":"http://127.0.0.1:9/v1","apiKey":"{env:SKILL_UP_OPENCODE_API_KEY}"}}}}`, "SKILL_UP_OPENCODE_API_KEY": "wrong-runner-key"}) + return nil +} + +func (*openCodeProviderRunner) Run(_ context.Context, _ rt.Runtime, _ agent.ExecOptions, _ []transcript.Message) (*agent.SessionResult, error) { + return &agent.SessionResult{SessionID: "runner-fixture", FinalMessage: "completed"}, nil +} From 0bb437c43cd434b084a1e60774ed893cee0314fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=A7=90=E6=9E=9D?= Date: Wed, 14 Oct 2026 17:04:30 +0800 Subject: [PATCH 3/3] test(runtime): verify OpenCode default provider configuration --- docs/guide/writing-evals.md | 17 ++++++ docs/zh/guide/writing-evals.md | 15 +++++ e2e/opensandbox_opencode_config_test.go | 80 +++++++++++++++++++++++++ e2e/opensandbox_opencode_judge_test.go | 36 +++++++++-- 4 files changed, 142 insertions(+), 6 deletions(-) create mode 100644 e2e/opensandbox_opencode_config_test.go diff --git a/docs/guide/writing-evals.md b/docs/guide/writing-evals.md index e9d52125..a3ebfe17 100644 --- a/docs/guide/writing-evals.md +++ b/docs/guide/writing-evals.md @@ -940,6 +940,23 @@ judges must override them, pass their rubric and return different session IDs. It uses a fifteen-minute sandbox TTL and normal case deletion. Optionally pin OpenCode with `SKILL_UP_OPENCODE_JUDGE_VERSION` (validated with `1.14.24`). +To exercise an omitted `judge.engine.model.base_url`, set +`SKILL_UP_OPENCODE_JUDGE_PROVIDER=openai` (or `anthropic`), leave +`SKILL_UP_OPENCODE_JUDGE_BASE_URL` unset, and supply the native provider's model +and key through the same variables. The test still seeds a conflicting runner +provider; the independent judge must use its native default endpoint. Provider +initialization removes the selected provider's inherited inline settings while +preserving case MCP configuration, unrelated providers and other OpenCode settings. + +A separate configuration-only smoke test installs and preflights the real +OpenCode CLI in an OpenSandbox, resets a conflicting OpenAI provider, and reads +back the preserved MCP and unrelated provider settings. It needs the sandbox +variables above but no model-provider key or native-provider connectivity: + +```bash +SKILL_UP_OPENCODE_CONFIG_E2E=1 go test -count=1 -tags e2e -run '^TestOpenSandboxOpenCodeDefaultRouting$' -v ./e2e +``` + Reports show gates, member results, and the overall decision separately. `evaluation.json` stores the complete grouped result; `grading.json` contains one top-level expectation per judge for compatibility. Adding Agent criteria does not change a script judge's weight. Multiple cases can reference the same `script_path` today; packaging helper files with a script is a separate feature. diff --git a/docs/zh/guide/writing-evals.md b/docs/zh/guide/writing-evals.md index faa9ed76..f7ac3a66 100644 --- a/docs/zh/guide/writing-evals.md +++ b/docs/zh/guide/writing-evals.md @@ -811,6 +811,21 @@ SKILL_UP_OPENCODE_JUDGE_E2E=1 go test -count=1 -tags e2e -run '^TestOpenSandboxO 临时 sandbox 使用十五分钟 TTL 并正常删除。可用 `SKILL_UP_OPENCODE_JUDGE_VERSION` 固定 OpenCode 版本（已验证 `1.14.24`）。 +若要覆盖省略 `judge.engine.model.base_url` 的场景，设置 +`SKILL_UP_OPENCODE_JUDGE_PROVIDER=openai`（或 `anthropic`），不设置 +`SKILL_UP_OPENCODE_JUDGE_BASE_URL`，并通过相同变量提供原生服务的模型和 key。 +测试仍预置冲突的 runner provider，独立 judge 必须使用原生默认端点。 +初始化只移除所选 provider 的遗留 inline 配置，保留 case MCP、其他 provider +及 OpenCode 的其他设置。 + +另有配置读回 smoke test，在真实 OpenSandbox 安装并预检 OpenCode，重置冲突的 +OpenAI provider，再读回 MCP 和其他 provider 的保留状态。只需上述 sandbox 变量， +不需要模型 key 或原生模型服务的可达性： + +```bash +SKILL_UP_OPENCODE_CONFIG_E2E=1 go test -count=1 -tags e2e -run '^TestOpenSandboxOpenCodeDefaultRouting$' -v ./e2e +``` + 报告分别展示门控、各 judge 和总体结果。`evaluation.json` 保存完整分组结果；`grading.json` 为兼容格式，每个 judge 对应一个顶层断言。增加 Agent 评审标准不会改变脚本 judge 的权重。多个 case 现在即可引用同一个 `script_path`；连同辅助文件一起打包脚本属于后续功能。 diff --git a/e2e/opensandbox_opencode_config_test.go b/e2e/opensandbox_opencode_config_test.go new file mode 100644 index 00000000..65b6f3b0 --- /dev/null +++ b/e2e/opensandbox_opencode_config_test.go @@ -0,0 +1,80 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/json" + "os" + "testing" + "time" + + "github.com/alibaba/skill-up/internal/agent" + "github.com/alibaba/skill-up/internal/config" + rt "github.com/alibaba/skill-up/internal/runtime" +) + +// TestOpenSandboxOpenCodeDefaultRouting checks the real CLI's installation and +// default-provider initialization without requiring native-provider connectivity. +func TestOpenSandboxOpenCodeDefaultRouting(t *testing.T) { + if os.Getenv("SKILL_UP_OPENCODE_CONFIG_E2E") != "1" { + t.Skip("set SKILL_UP_OPENCODE_CONFIG_E2E=1 with OpenSandbox credentials") + } + for _, name := range []string{"OPENSANDBOX_API_KEY", "OPENSANDBOX_BASE_URL", "OPENSANDBOX_IMAGE"} { + if os.Getenv(name) == "" { + t.Fatalf("%s is required", name) + } + } + environment := config.Environment{Type: "opensandbox", Image: os.Getenv("OPENSANDBOX_IMAGE"), ReadyTimeoutSeconds: 90, SandboxTimeoutSeconds: 300, Metadata: map[string]string{"skill_up_test": "opencode-default-config"}} + cfg := environment.ToRuntimeConfig() + cfg.Delete = true + current, err := rt.NewRuntime(cfg) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute) + defer cancel() + if err := current.Create(ctx); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := current.Close(); err != nil { + t.Errorf("cleanup: %v", err) + } + }) + if err := (&openCodeProviderRunner{}).Install(ctx, current); err != nil { + t.Fatal(err) + } + current.MergeEnv(map[string]string{"OPENCODE_CONFIG_CONTENT": `{"mcp":{"keep":{"type":"remote","url":"https://mcp.example"}},"provider":{"openai":{"options":{"baseURL":"http://127.0.0.1:9/v1","apiKey":"{env:SKILL_UP_OPENCODE_API_KEY}"}},"other":{"options":{"baseURL":"https://other.example/v1"}}}}`}) + ag := agent.NewOpenCodeAgent(agent.Config{ModelProvider: "openai", ModelName: "gpt-5-mini", Version: os.Getenv("SKILL_UP_OPENCODE_JUDGE_VERSION")}) + if err := ag.Install(ctx, current); err != nil { + t.Fatal(err) + } + observation, err := agent.Preflight(ctx, current, ag) + if err != nil { + t.Fatal(err) + } + if err := ag.InstallJudgeConfig(ctx, current); err != nil { + t.Fatal(err) + } + read, err := current.Exec(ctx, `printf '%s' "$OPENCODE_CONFIG_CONTENT"`, rt.ExecOptions{}) + if err != nil || read.ExitCode != 0 { + t.Fatalf("read config: %v", err) + } + assertOpenCodeDefaultRoute(t, read.Stdout) + t.Logf("OpenCode version=%s; default openai routing reset; MCP and other provider preserved", observation.Version) +} + +func assertOpenCodeDefaultRoute(t *testing.T, content string) { + t.Helper() + var value struct { + MCP map[string]json.RawMessage `json:"mcp"` + Provider map[string]json.RawMessage `json:"provider"` + } + if err := json.Unmarshal([]byte(content), &value); err != nil { + t.Fatal(err) + } + if value.Provider["openai"] != nil || value.Provider["other"] == nil || value.MCP["keep"] == nil { + t.Fatal("default routing reset lost unrelated configuration or retained runner provider") + } +} diff --git a/e2e/opensandbox_opencode_judge_test.go b/e2e/opensandbox_opencode_judge_test.go index 49b21974..b92b75cb 100644 --- a/e2e/opensandbox_opencode_judge_test.go +++ b/e2e/opensandbox_opencode_judge_test.go @@ -4,6 +4,7 @@ package e2e import ( "context" + "encoding/json" "errors" "os" "strings" @@ -25,19 +26,30 @@ func TestOpenSandboxOpenCodeJudgeProvider(t *testing.T) { if os.Getenv("SKILL_UP_OPENCODE_JUDGE_E2E") != "1" { t.Skip("set SKILL_UP_OPENCODE_JUDGE_E2E=1 with OpenSandbox and model-provider credentials") } - for _, name := range []string{"OPENSANDBOX_API_KEY", "OPENSANDBOX_BASE_URL", "OPENSANDBOX_IMAGE", "JUDGE_LIVE_API_KEY", "SKILL_UP_OPENCODE_JUDGE_BASE_URL", "SKILL_UP_OPENCODE_JUDGE_MODEL"} { + for _, name := range []string{"OPENSANDBOX_API_KEY", "OPENSANDBOX_BASE_URL", "OPENSANDBOX_IMAGE", "JUDGE_LIVE_API_KEY", "SKILL_UP_OPENCODE_JUDGE_MODEL"} { if os.Getenv(name) == "" { t.Fatalf("%s is required", name) } } - model := config.ModelConfig{Provider: "judge_live", Name: os.Getenv("SKILL_UP_OPENCODE_JUDGE_MODEL"), BaseURL: os.Getenv("SKILL_UP_OPENCODE_JUDGE_BASE_URL")} + provider := os.Getenv("SKILL_UP_OPENCODE_JUDGE_PROVIDER") + if provider == "" { + provider = "judge_live" + } + endpoint := os.Getenv("SKILL_UP_OPENCODE_JUDGE_BASE_URL") + if endpoint == "" && provider != "openai" && provider != "anthropic" { + t.Fatal("SKILL_UP_OPENCODE_JUDGE_BASE_URL is required for a custom provider") + } + t.Setenv(strings.ToUpper(provider)+"_API_KEY", os.Getenv("JUDGE_LIVE_API_KEY")) + t.Setenv(strings.ToUpper(provider)+"_BASE_URL", endpoint) + t.Setenv(strings.ToUpper(provider)+"_MODEL", "") + model := config.ModelConfig{Provider: provider, Name: os.Getenv("SKILL_UP_OPENCODE_JUDGE_MODEL"), BaseURL: endpoint} timeout := 180 members := []config.JudgeConfig{} for _, id := range []string{"semantic", "review"} { members = append(members, config.JudgeConfig{ID: id, Type: "agent_judge", Engine: &config.EngineConfig{Name: "opencode", Version: os.Getenv("SKILL_UP_OPENCODE_JUDGE_VERSION"), Model: model}, Criteria: []string{"The agent final message is exactly completed."}, TimeoutSeconds: &timeout}) } out := t.TempDir() - e := evaluator.NewEvaluator(evaluator.EvalOptions{Agent: &openCodeProviderRunner{}, OutputDir: out, DeleteWorkspace: true, EvalCfg: &config.EvalConfig{Environment: config.Environment{Type: "opensandbox", Image: os.Getenv("OPENSANDBOX_IMAGE"), ReadyTimeoutSeconds: 90, SandboxTimeoutSeconds: 900, Metadata: map[string]string{"skill_up_test": "opencode-judge-provider"}}, Judges: &members}}) + e := evaluator.NewEvaluator(evaluator.EvalOptions{Agent: &openCodeProviderRunner{provider: provider}, OutputDir: out, DeleteWorkspace: true, EvalCfg: &config.EvalConfig{Environment: config.Environment{Type: "opensandbox", Image: os.Getenv("OPENSANDBOX_IMAGE"), ReadyTimeoutSeconds: 90, SandboxTimeoutSeconds: 900, Metadata: map[string]string{"skill_up_test": "opencode-judge-provider"}}, Judges: &members}}) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() results, err := e.EvaluateAll(ctx, []*config.CaseConfig{{ID: "provider", Input: config.Input{Prompt: "Return completed"}}}) @@ -64,7 +76,11 @@ func TestOpenSandboxOpenCodeJudgeProvider(t *testing.T) { } } -type openCodeProviderRunner struct{ sharedJudgeFixtureAgent } +type openCodeProviderRunner struct { + sharedJudgeFixtureAgent + + provider string +} func (*openCodeProviderRunner) Install(ctx context.Context, current rt.Runtime) error { path, err := current.Exec(ctx, `printf '%s' "${NVM_DIR:-$HOME/.nvm}/current/bin:$PATH"`, rt.ExecOptions{}) @@ -79,8 +95,16 @@ func (*openCodeProviderRunner) Install(ctx context.Context, current rt.Runtime) return nil } -func (*openCodeProviderRunner) InstallMCP(_ context.Context, current rt.Runtime, _ rt.MCPConfig) error { - current.MergeEnv(map[string]string{"OPENCODE_CONFIG_CONTENT": `{"provider":{"judge_live":{"npm":"@ai-sdk/openai-compatible","options":{"baseURL":"http://127.0.0.1:9/v1","apiKey":"{env:SKILL_UP_OPENCODE_API_KEY}"}}}}`, "SKILL_UP_OPENCODE_API_KEY": "wrong-runner-key"}) +func (a *openCodeProviderRunner) InstallMCP(_ context.Context, current rt.Runtime, _ rt.MCPConfig) error { + options := map[string]string{"baseURL": "http://127.0.0.1:9/v1"} + options["apiKey"] = "{env:SKILL_UP_OPENCODE_API_KEY}" + content, err := json.Marshal(map[string]any{"provider": map[string]any{a.provider: map[string]any{ + "npm": "@ai-sdk/openai-compatible", "options": options, + }}}) + if err != nil { + return err + } + current.MergeEnv(map[string]string{"OPENCODE_CONFIG_CONTENT": string(content), "SKILL_UP_OPENCODE_API_KEY": "wrong-runner-key"}) return nil }