diff --git a/BENCHMARKING.md b/BENCHMARKING.md index bb993967..3b0e0670 100644 --- a/BENCHMARKING.md +++ b/BENCHMARKING.md @@ -327,7 +327,7 @@ recipes directly into `pipeline.SelectionOnly`: | `stages.selection.model` | `SelectionModelOverride` | | `stages.selection.effort` | `SelectionEffortOverride` | | `stages.selection.prompt` | file contents loaded into `SelectionPromptInstructions` | -| `stages.reviewers.agent_dirs[]` | `AgentDirs` | +| `stages.reviewers.agent_dirs[]` | `AgentDirs`, each resolved against the suite file directory | | `review_base_sha` / `review_head_sha` | pinned review SHAs | Selector-only benchmarks do not run reviewer agents, do not run synthesis, do diff --git a/internal/cmd/benchmarkcmd/select.go b/internal/cmd/benchmarkcmd/select.go index d85a8a20..5c8937e5 100644 --- a/internal/cmd/benchmarkcmd/select.go +++ b/internal/cmd/benchmarkcmd/select.go @@ -288,7 +288,7 @@ func executeBenchmarkSelectRun(ctx context.Context, logger *progress.Logger, sui ProfileName: state.profileName, Profile: state.profile, PostingIdentity: postingIdentity, - AgentDirs: append([]string(nil), candidate.Stages.Reviewers.AgentDirs...), + AgentDirs: resolveSuiteAgentDirs(suiteDir, candidate.Stages.Reviewers.AgentDirs), ArtifactDir: runDir, ReviewBaseSHA: benchCase.ReviewBaseSHA, ReviewHeadSHA: benchCase.ReviewHeadSHA, @@ -389,6 +389,20 @@ func selectionRunArtifacts(runDir string) (runArtifacts, error) { }, nil } +// resolveSuiteAgentDirs resolves candidate agent directories against the suite +// file's directory, as `benchmark run` does for its child reviews, so a suite +// selects from the same catalog whichever directory it is invoked from. +func resolveSuiteAgentDirs(suiteDir string, dirs []string) []string { + if dirs == nil { + return nil + } + out := make([]string, 0, len(dirs)) + for _, dir := range dirs { + out = append(out, benchmark.ResolveSuitePath(suiteDir, dir)) + } + return out +} + func loadSelectionPromptInstructions(suiteDir, configured string) (string, error) { configured = strings.TrimSpace(configured) if configured == "" { diff --git a/internal/cmd/benchmarkcmd/select_test.go b/internal/cmd/benchmarkcmd/select_test.go index 4a5bda97..9da9b8ea 100644 --- a/internal/cmd/benchmarkcmd/select_test.go +++ b/internal/cmd/benchmarkcmd/select_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "reflect" "strings" "testing" "time" @@ -583,3 +584,60 @@ func TestSelectionReportMarkdownListsSelectedReviewers(t *testing.T) { t.Fatalf("report = %s, want selected reviewer table", report) } } + +func TestSelectResolvesRelativeAgentDirsAgainstSuiteDirectory(t *testing.T) { + cmd, _ := newTestCommand(t) + suitePath := writeBenchmarkSuite(t, ` +suite: + id: suite1 + name: Suite One + version: 1 +candidates: + - id: first + profile: home + stages: + selection: + model: claude-sonnet-4-6 + effort: high + reviewers: + model: claude-sonnet-4-6 + agent_dirs: + - ../agents +cases: + - id: case_one + pr: https://github.com/open-cli-collective/codereview-cli/pull/1 +`) + // Invoke from an unrelated directory: a cwd-relative "../agents" would + // point somewhere else entirely. + t.Chdir(t.TempDir()) + var requests []pipeline.SelectionRequest + withBenchmarkSelectSeams(t, + func(context.Context, string, bool, config.File, config.Profile, gitprovider.PRRef) (app.SelectionRuntime, error) { + return app.SelectionRuntime{Cleanup: func() {}}, nil + }, + func(_ context.Context, _ pipeline.Options, req pipeline.SelectionRequest) (pipeline.SelectionResult, error) { + requests = append(requests, req) + return pipeline.SelectionResult{ + Artifacts: pipeline.ArtifactPathsFromDir(req.ArtifactDir), + SelectionSession: pipeline.SelectionSession{Response: llm.Response{ + StructuredOutput: []byte(`{"schema_version":1,"selected_agents":[],"thread_actions":[],"reasoning":"ok"}`), + }}, + }, nil + }, + ) + + if err := root.Execute(cmd, []string{ + "benchmark", "select", suitePath, + "--results-dir", filepath.Join(t.TempDir(), "results"), + "--json", + }); err != nil { + t.Fatalf("Execute: %v", err) + } + want := []string{filepath.Join(filepath.Dir(filepath.Dir(suitePath)), "agents")} + if len(requests) != 1 { + t.Fatalf("selection requests = %d, want 1", len(requests)) + } + if got := requests[0].AgentDirs; !reflect.DeepEqual(got, want) { + t.Fatalf("selection agent dirs = %#v, want suite-relative %#v", got, want) + } +}