From d54d7751865496d176771ce8be828ccb98679949 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Mon, 17 Aug 2026 14:39:20 -0700 Subject: [PATCH 1/4] docs: add agentic coding replication pseudocode Signed-off-by: Elyas Mehtabuddin --- paper-2604.16529-replication-pseudocode.md | 2502 ++++++++++++++++++++ 1 file changed, 2502 insertions(+) create mode 100644 paper-2604.16529-replication-pseudocode.md diff --git a/paper-2604.16529-replication-pseudocode.md b/paper-2604.16529-replication-pseudocode.md new file mode 100644 index 000000000..96d916e57 --- /dev/null +++ b/paper-2604.16529-replication-pseudocode.md @@ -0,0 +1,2502 @@ +# Replication pseudocode for *Scaling Test-Time Compute for Agentic Coding* + +Source: [arXiv:2604.16529v1](https://arxiv.org/abs/2604.16529), submitted April 16, +2026. The manuscript date printed in the PDF is April 21, 2026. + +This document is a standalone implementation specification for the paper's Recursive +Tournament Voting (RTV), agentic Parallel-Distill-Refine (PDR), combined PDR+RTV method, +ablations, and analyses. It includes the appendices' operational details and example-derived +summary shapes. It does not require the paper to understand or implement the method. + +## 1. Replication contract and disclosure labels + +Use these labels throughout the implementation: + +- `PAPER`: stated in the paper or recoverable exactly from its equations, tables, figures, or + supplied appendix examples. +- `EXAMPLE-DERIVED`: observed in an appendix output, but not published as a required prompt, + schema, parser, or runtime policy. +- `RECONSTRUCTED`: an implementation choice needed to make the method executable, but not + specified by the paper. Keep it configurable and record its value in every run manifest. +- `REPO`: a behavior confirmed later against public author code. A repository audit should + replace a `RECONSTRUCTED` choice with `REPO` only when the code unambiguously establishes it. + +Do not silently invent a value for a replication-critical field. The controller must either load +it from an experiment config or stop with an error naming the missing field. + +The paper does not train a new model. It runs inference-time control flow around existing +coding agents. For each evaluated model, the same language model acts as: + +1. the coding agent that creates each rollout; +2. the summarizer of its own rollout; +3. the judge that compares summaries of its own rollouts. + +The method makes selection decisions without official ground-truth outcomes, hidden tests, test +samples, reward values, or grader traces. Those values must never enter summaries, judge prompts, +refinement prompts, tournament pairing, or tie-breaking. Agents may inspect and run tests that the +benchmark deliberately exposes inside their task environment; the ban applies to official hidden +evaluation data. + +Public-code audit, August 17, 2026: no official or author-linked implementation was found. The two +located repositories explicitly identify themselves as independent reproductions, so they do not +resolve any `RECONSTRUCTED` setting and nothing in this document is labeled `REPO`. + +## 2. Exact experiment constants stated by the paper + +```text +MAIN_PAPER_CONSTANTS = { + # Parallel rollout count per iteration. + N: 16, # PAPER + + # Total rollout iterations: iteration 0 and iteration 1. + T: 2, # PAPER + + # Number of iteration-0 survivors placed in the refinement context. + K: 4, # PAPER + + # RTV comparison group size. + G: 2, # PAPER + + # Independent judge votes per comparison group. + V: 8, # PAPER + + # Each iteration-1 rollout starts from the task's original clean image/snapshot. + fresh_environment_per_rollout: true, # PAPER + + # Reuse summaries only; do not copy prior patches, files, shells, or containers. + persistent_workspace_between_iterations: false, # PAPER +} +``` + +Benchmarks and task sets: + +```text +BENCHMARKS = [ + { + name: "SWE-Bench Verified", + task_count: 500, # full test set; PAPER + harness: "mini-SWE-agent", + harness_mode: "bash-only", # PAPER + outcome: binary pass/fail, + }, + { + name: "Terminal-Bench v2.0", + task_count: 88, + available_task_count: 89, + harness: "Terminus 1", # PAPER + outcome: binary pass/fail, + # The paper does not identify the excluded task. + excluded_task_id: REQUIRED_CONFIG, + }, +] +``` + +Models: + +```text +MODELS = [ + "Claude-4.5-Opus", + "Gemini-3.1-Pro", + "Claude-4.5-Sonnet", + "Gemini-3-Flash", + "GPT-5-0825", +] +``` + +The paper does not state exact provider model IDs, API revisions, sampling temperatures, top-p, +token limits, context limits, agent step/time limits, retry policies, seeds, container digests, +or harness commit hashes. Treat each as a required run-manifest field until public code or author +configuration supplies it. + +## 3. State and record types + +Use immutable persisted records for scientific bookkeeping. A local builder may change while a +rollout or tournament is running, but `PERSIST` freezes it; later facts use append-only link or +evaluation records. Store large text blobs by content hash, but preserve a lossless copy. + +```text +record TaskSpec: + benchmark_name: string + task_id: string + problem_text: string + clean_environment_image: string # immutable image/digest + clean_repository_revision: string | null + official_grader_config: map + public_task_assets: list[Asset] + +record InferenceRoleConfig: + provider: string + exact_model_id: string + api_revision: string | null + temperature: float + top_p: float | null + max_output_tokens: int + context_window: int + stop_sequences: list[string] + seed_policy: string + prompt_text: string + prompt_sha256: string + tool_protocol: string | null + retry_policy: RetryPolicy + +record RetryPolicy: + max_attempts: int + retryable_error_kinds: list[string] + deterministic_backoff_policy: string + +record SummaryPolicy: + serializer_revision: string + overflow_policy: string + max_content_attempts: int + malformed_output_policy: string + exhausted_call_policy: enum {ABORT_RUN, RETRY_CONTENT_ATTEMPT} + +record InvalidVotePolicy: + mode: enum {ABORT_RUN, REPLACE_UNTIL_V} + max_replacement_calls_per_group: int + +record PaperConstants: + N: int + T: int + K: int + G: int + V: int + fresh_environment_per_rollout: bool + persistent_workspace_between_iterations: bool + +record ExecutionPolicies: + summary_policy: SummaryPolicy + summary_schema_or_null: map | null + pairing_policy: string + display_order_policy: string + tie_policy: string + invalid_vote_policy: InvalidVotePolicy + +record ExperimentConfig: + paper_constants: PaperConstants + execution: ExecutionPolicies + benchmark_manifest: map + agent_limits_by_benchmark_and_model: map + experiment_seed: int + replication_label: enum {EXACT_REPLICATION, CONCEPTUAL_REPLICATION} + unresolved_field_names: list[string] + manifest_schema_revision: string + +record ModelBundle: + logical_name: string + action_role: InferenceRoleConfig + summary_role: InferenceRoleConfig + comparison_role: InferenceRoleConfig + refinement_prompt_sha256: string + +record Asset: + path: string + sha256: string + visibility: enum {PUBLIC_TO_AGENT, GRADER_ONLY} + +record Artifact: + path: string + sha256: string + mode: int + size_bytes: int + +record Usage: + input_tokens: int | null + output_tokens: int | null + cached_input_tokens: int | null + provider_cost: decimal | null + +record AgentLimits: + max_steps: int + wall_time_seconds: int + per_command_time_seconds: int + max_observation_bytes_per_step: int + max_total_trajectory_bytes: int + +record Action: + thought: string # T_i + bash_commands: list[string] # B_i = {b_1, ..., b_m} + scaffold_action: map # lossless fields such as keystrokes/is_blocking/timeout + raw_model_response: string + model_usage: Usage + +record Observation: + command: string + exit_code: int | null + stdout: bytes + stderr: bytes + timed_out: bool + started_at: timestamp + finished_at: timestamp + +record TrajectoryStep: + step_index: int + context_hash_before: string + action: Action # A_i = (T_i, B_i) + observations: list[Observation] # O_i + context_hash_after: string + +record Rollout: + run_id: string + task_id: string + model_id: string + iteration: int + rollout_index: int + seed: int + environment_id: string + refinement_summary_ids: list[string] + steps: list[TrajectoryStep] + termination_reason: enum { + AGENT_FINISHED, + MAX_STEPS, + WALL_TIME, + COMMAND_TIMEOUT, + MODEL_ERROR, + PARSE_ERROR, + INFRASTRUCTURE_ERROR + } + final_agent_message: string | null + final_patch: bytes | null # SWE-Bench-like output + final_artifact_manifest: list[Artifact] # Terminal-Bench-like output + final_environment_snapshot: string + +record StructuredSummary: + summary_id: string + rollout_id: string + benchmark_name: string + schema_version: string | null + json_value: map + raw_model_response: string + parse_attempts: int + validation_warnings: list[string] + model_usage: Usage + +record RolloutSummaryLink: + rollout_id: string + summary_id: string + +record RolloutEvaluation: + rollout_id: string + grader_revision: string + binary_outcome: bool + score_details: map + +record Candidate: + candidate_id: string + rollout_id: string + summary_id: string | null + display_payload: string # summary JSON, or raw trace in one ablation + +record Vote: + tournament_id: string + iteration: int + round_index: int + group_index: int + vote_index: int + ordered_candidate_ids: list[string] + prompt_sha256: string + raw_model_response: string | null + selected_display_position: int | null + selected_candidate_id: string | null + parse_error: string | null + model_usage: Usage + +record GroupDecision: + round_index: int + group_index: int + effective_group_size: int + input_candidate_ids: list[string] + votes: list[Vote] + vote_counts: map[candidate_id, int] + selected_candidate_id: string + tie_break_record: map | null + +record Tournament: + tournament_id: string + task_id: string + model_id: string + rollout_iteration: int + target_survivor_count: int + configured_G: int + V: int + root_seed: int + pairing_policy_hash: string + display_order_policy_hash: string + tie_policy_hash: string + rounds: list[list[GroupDecision]] + population_ids_by_round: list[list[string]] + survivor_candidate_ids: list[string] + +record TournamentCheckpoint: + checkpoint_id: string + tournament_id: string + survivor_count: int + completed_round_count: int + survivor_candidate_ids: list[string] + population_sha256: string + +record EvaluationRow: + task_id: string + model_id: string + stage: enum {ITER_0, SELECT_K, ITER_1, FINAL} + candidate_ids: list[string] + binary_outcomes: list[bool] + +record TaskRun: + task_id: string + model_logical_name: string + iter0_rollout_ids: list[string] + iter0_summary_ids: list[string] + iter0_tournament_id: string + select_k_checkpoint_id: string + refinement_summary_ids: list[string] + iter1_rollout_ids: list[string] + iter1_summary_ids: list[string] + final_tournament_id: string + final_rollout_id: string + output_patch: bytes | null + output_artifacts: list[Artifact] + output_environment_snapshot: string +``` + +Every record must also carry the experiment manifest hash, code revision, prompt revision, model +configuration, harness revision, container digest, and creation timestamp, either directly or by +foreign key. + +## 4. Agent rollout dynamics + +The paper defines a rollout as interleaved agent actions and environment observations. At step +`i`, the accumulated context is `C_(i-1)`: + +```text +A_i = (T_i, B_i) = LM(P_action(P_in; C_(i-1))) +O_i = Environment(C_(i-1); B_i) +C_i = append(C_(i-1), (A_i, O_i)) +``` + +Executable pseudocode: + +```text +interface AgentScaffoldAdapter: + # Returns user/tool context only; CALL_ROLE adds the versioned role prompt once. + build_initial_context(task, refinement_summaries) + parse_action(raw_response) -> ScaffoldAction + execute_action(environment, ScaffoldAction, limits) -> list[Observation] + is_complete(ScaffoldAction) -> bool + final_message(ScaffoldAction) -> string | null + serialize_step_for_next_context(context, action, observations) + serialize_lossless_trajectory(rollout) + extract_patch(environment) -> bytes | null + list_output_artifacts(environment) -> list[Artifact] + +SWE_ADAPTER = MiniSWEAgentBashAdapter( + exact_revision = REQUIRED_CONFIG, + prompt_and_protocol_revision = REQUIRED_CONFIG, + observation_truncation_policy = REQUIRED_CONFIG, +) +TERMINAL_ADAPTER = Terminus1Adapter( + exact_revision = REQUIRED_CONFIG, + prompt_and_protocol_revision = REQUIRED_CONFIG, + observation_truncation_policy = REQUIRED_CONFIG, +) + +function RUN_ROLLOUT(task, model_bundle, scaffold, limits, iteration, rollout_index, + refinement_summaries, seed) -> Rollout: + assert iteration == 0 implies refinement_summaries is empty + assert iteration > 0 implies len(refinement_summaries) > 0 + + env = CREATE_FRESH_ENVIRONMENT( + image = task.clean_environment_image, + repository_revision = task.clean_repository_revision, + public_assets = task.public_task_assets, + ) + + # Critical paper invariant: a new container/snapshot is created for every rollout, + # including every refined rollout. No filesystem state crosses rollout boundaries. + assert env.has_no_parent_rollout_state() + + context = scaffold.build_initial_context( + task = task, + refinement_summaries = refinement_summaries, + ) + + rollout = new Rollout( + run_id = UNIQUE_ID(), + task_id = task.task_id, + model_id = model_bundle.action_role.exact_model_id, + iteration = iteration, + rollout_index = rollout_index, + seed = seed, + environment_id = env.id, + refinement_summary_ids = [s.summary_id for s in refinement_summaries], + steps = [], + ) + + deadline = NOW() + limits.wall_time_seconds + + for step_index in 0 .. limits.max_steps - 1: + if NOW() >= deadline: + rollout.termination_reason = WALL_TIME + break + + response = CALL_ROLE_WITH_RECORDED_RETRIES( + role = model_bundle.action_role, + messages = context, + tools = model_bundle.action_role.tool_protocol, + seed = DERIVE_SEED(seed, "action", step_index), + ) + + if response.failed: + rollout.termination_reason = MODEL_ERROR + RECORD_FAILURE(response) + break + + parse = scaffold.parse_action(response.raw) + if parse.failed: + rollout.termination_reason = PARSE_ERROR + RECORD_FAILURE(parse) + break + + action = Action( + thought = parse.action.thought, + bash_commands = parse.action.commands, + scaffold_action = parse.action.lossless_value, + raw_model_response = response.raw, + model_usage = response.usage, + ) + + observations = scaffold.execute_action(env, parse.action, limits) + new_context = scaffold.serialize_step_for_next_context( + context, parse.action, observations + ) + rollout.steps.append(TrajectoryStep( + step_index = step_index, + context_hash_before = SHA256(context), + action = action, + observations = observations, + context_hash_after = SHA256(new_context), + )) + context = new_context + + if scaffold.is_complete(parse.action): + rollout.termination_reason = AGENT_FINISHED + rollout.final_agent_message = scaffold.final_message(parse.action) + break + else: + rollout.termination_reason = MAX_STEPS + + rollout.final_patch = scaffold.extract_patch(env) + rollout.final_artifact_manifest = scaffold.list_output_artifacts(env) + rollout.final_environment_snapshot = env.SNAPSHOT_IMMUTABLY() + scaffold.serialize_lossless_trajectory(rollout) + PERSIST_LOSSLESS(rollout) + return rollout +``` + +The appendix shows two different scaffold protocols, so one global `BASH_TOOL_SCHEMA` or parser is +not sufficient. The mini-SWE-agent trace uses `THOUGHT`, ``, ``, and ``. +The Terminus 1 trace uses JSON fields including `state_analysis`, `explanation`, +`bash_commands[{keystrokes,is_blocking,timeout_sec}]`, and `is_task_complete`. Exact scaffold +prompts, revisions, and serialization remain required configuration. + +The scaffold's `build_initial_context` is the only conceptual difference between initial and +refined rollouts: + +```text +function BUILD_INITIAL_AGENT_CONTEXT(task, refinement_summaries): + # User-context messages only. CALL_ROLE_WITH_RECORDED_RETRIES adds the action-role prompt. + messages = [] + + if refinement_summaries is not empty: + messages.append({role: "user", content: REFINEMENT_PREAMBLE}) + for position, summary in enumerate(refinement_summaries, start = 1): + messages.append({ + role: "user", + content: + "PRIOR ATTEMPT SUMMARY " + position + "\n" + + CANONICAL_JSON(summary.json_value), + }) + messages.append({role: "user", content: REFINEMENT_POSTAMBLE}) + + messages.append({role: "user", content: FORMAT_ORIGINAL_TASK(task.problem_text)}) + return messages +``` + +The paper states that the first refined action is conditioned on the original task and the +distilled prior summaries; later actions retain that refinement context while following normal +agent dynamics. Do not present refinement summaries as verified facts. They can describe failed +or mistaken attempts. + +## 5. Structured rollout summarization + +For rollout `R_i`, the paper computes: + +```text +S_i = LM(P_sum(R_i)) +``` + +The paper does not disclose how `R_i` is serialized, reduced when it exceeds context, or combined +with the task, patch, and artifact list. It also does not disclose a formal output schema, parsing, +repair, retry, exclusion, or replacement policy. A faithful implementation therefore makes those +choices part of a versioned adapter instead of presenting them as paper behavior. + +```text +interface SummaryInputAdapter: + serialize(task, rollout, policy) -> string + parse(raw_response, configured_schema_or_null) -> { + json_value: map | null, + validation_warnings: list[string], + } + +function SUMMARIZE_ROLLOUT(task, rollout, model_bundle, input_adapter, + summary_policy, configured_schema_or_null) -> StructuredSummary: + summary_input = input_adapter.serialize( + task = task, + rollout = rollout, + policy = summary_policy, + ) + + for attempt in 1 .. summary_policy.max_content_attempts: + response = CALL_ROLE_WITH_RECORDED_RETRIES( + role = model_bundle.summary_role, + messages = [{role: "user", content: summary_input}], + seed = DERIVE_SEED(rollout.seed, "summary", attempt), + ) + PERSIST_RAW_RESPONSE_BEFORE_PARSING(response) + if response.failed: + disposition = HANDLE_EXHAUSTED_MODEL_CALL( + call_kind = "summary", + policy = summary_policy.exhausted_call_policy, + attempt = attempt, + max_content_attempts = summary_policy.max_content_attempts, + failed_response = response, + ) + if disposition == RETRY_CONTENT_ATTEMPT: + continue + raise SummaryModelCallFailed(rollout.run_id, response.attempt_records) + + parsed = input_adapter.parse(response.raw, configured_schema_or_null) + + if parsed.json_value is not null: + summary = StructuredSummary( + summary_id = UNIQUE_ID(), + rollout_id = rollout.run_id, + benchmark_name = task.benchmark_name, + schema_version = VERSION(configured_schema_or_null), + json_value = parsed.json_value, + raw_model_response = response.raw, + parse_attempts = attempt, + validation_warnings = parsed.validation_warnings, + model_usage = response.usage, + ) + PERSIST(summary) + PERSIST(RolloutSummaryLink(rollout.run_id, summary.summary_id)) + return summary + + summary_input = APPLY_CONFIGURED_MALFORMED_SUMMARY_POLICY( + policy = summary_policy.malformed_output_policy, + original_input = summary_input, + invalid_output = response.raw, + warnings = parsed.validation_warnings, + ) + + raise SummaryGenerationFailed(rollout.run_id) +``` + +### 5.1 SWE-Bench summary shape observed in the appendix + +`EXAMPLE-DERIVED`: the paper's one SWE-Bench example is a JSON object with the following keys and +observed value shapes. It is not a published JSON Schema. A formal schema, if used, must tolerate +the example's string/list and evidence-value variation or convert it under a recorded policy. + +```text +OBSERVED_SWE_SUMMARY_EXAMPLE_SHAPE = { + issue_requirements: { + primary_objective: string, + expected_behavior: string, + current_behavior: string, + reproduction_steps: string, + explicit_constraints: list[string], + success_criteria: list[string] + }, + agent_actions: { + exploration: { + files_examined: list[string], + codebase_navigation: list[string], + commands_used: list[string], + key_discoveries: list[string] + }, + solution_approach: { + strategy: string, + stated_reasoning: string, + approach_changes: list[string] + }, + implementation: { + modification_scope: string, + files_modified: list[string], + edit_methods: list[string], + command_dependencies: list[string] + } + }, + code_changes: list[{ + description: string, + file_path: string, + function_method: string, + condition_added: string, + construct_added: string, + before_code: string, + after_code: string + }], + command_execution: { + commands_with_response: list[{ + step: string, + command: string, + return_code: int, + stdout_summary: string, + stderr_summary: string + }], + commands_without_response: list[{ + step: string, + command: string, + intended_purpose: string, + depended_on_by: list[string] + }], + error_responses: list[{ + step: string, + command: string, + return_code: int, + error_message: string, + addressed_in_subsequent_steps: bool + }] + }, + error_analysis: { + intermediate_errors: list[{ + error_type: string, + classification: string, + evidence: string, + resolved_before_submission: bool, + resolution_evidence: string + }], + final_state: { + status: string, + errors: list[{ + error_type: string, + classification: string, + evidence: string + }] + } + }, + evidence_inventory: { + confirmed_outcomes: list[{ + step: string, + command: string, + outcome: string, + key_output: string + }], + error_outcomes: list[{ + step: string, + command: string, + return_code: int, + outcome: string, + error_message: string + }], + unconfirmed_actions: list[{ + step: string, + command: string, + intended_purpose: string, + dependent_commands: string | list[string] + }] + }, + final_state: { + agent_claims: { + completion_claim: string, + reasoning_provided: string, + uncertainty_expressed: string | list[string] + }, + expected_artifacts: list[{ + artifact_or_change: string, + existence_checked: bool | string, + content_examined: bool | string, + observation: string + }], + unresolved_issues: list[string] + }, + patch_status: { + status: string, + success_patterns_found: list[string], + failure_patterns_found: list[string] + }, + requirement_coverage: list[{ + requirement: string, + evidence_locations: list[string], + related_actions: list[string], + related_code_changes: list[string] + }], + verification_record: { + verification_commands: list[{ + step: string, + command: string, + return_code: int, + what_it_checked: string, + has_response_block: bool + }], + verification_coverage: { + verified_with_response: list[string], + attempted_but_unconfirmed: list[string], + not_verified: list[string], + test_commands_executed: list[string], + test_results: string | list[string] + } + } +} +``` + +### 5.2 Terminal-Bench summary shape observed in the appendix + +`EXAMPLE-DERIVED`: the paper's one Terminal-Bench summary has the following keys and observed +value shapes. It likewise does not establish a formal required schema. + +```text +OBSERVED_TERMINAL_SUMMARY_EXAMPLE_SHAPE = { + task_requirements: { + primary_objective: string, + output_specifications: list[{ + artifact: string, + format: string, + verbatim_evidence: string + }], + explicit_constraints: list[string], + success_criteria: list[{ + criterion: string, + verbatim_evidence: string + }], + task_assertions: {broken_or_needs_fixing: string} + }, + agent_actions: { + exploration: { + files_directories_examined: list[string], + diagnostic_commands: list[string], + key_information_discovered: list[string] + }, + solution_approach: { + strategy: string, + stated_reasoning: string, + approach_changes: list[string] + }, + implementation_structure: { + structure_type: string, + intermediate_files: list[string], + command_dependencies: list[{command: string, depends_on: string}] + } + }, + command_execution_record: { + commands_with_response: list[{ + step: string, + command: string, + return_code: int, + stdout_summary: string, + stderr_summary: string + }], + commands_without_response: list[{ + step: string, + command: string, + intended_purpose: string, + depended_on_by: list[string] + }], + error_responses: list[map] + }, + evidence_inventory: { + confirmed_outcomes: list[{ + step: string, + command: string, + description: string, + key_output: string + }], + error_outcomes: list[map], + unconfirmed_actions: list[{ + step: string, + command: string, + intended_purpose: string, + depends_on_this: list[string] + }], + unverified_aspects: list[{aspect: string, factual_relevance: string}] + }, + final_state: { + agent_assessment: { + completion_claim: string, + reasoning_provided: string, + uncertainty_expressed: string | list[string], + verbatim_quote: string + }, + expected_artifacts: list[{ + artifact: string, + existence_checked: bool, + existence_check_command: string, + existence_check_step: string, + content_examined: bool, + content_examine_command: string, + content_examine_step: string, + observation: string + }], + unresolved_issues: list[string] + }, + resource_access: { + resources_accessed: { + input_files_read: list[string], + intermediate_files: list[string], + output_files_created: list[{file_path: string, command_used: string}], + external_resources: list[string] + }, + access_timing: list[{resource: string, timing: string, description: string}], + runtime_dependencies: { + files_required_at_runtime: list[string], + tools_invoked_at_runtime: list[string], + environment_conditions: list[string] + } + }, + solution_characteristics: { + code_script_generation: { + applicable: bool, + execution_behavior: string, + external_files_accessed_at_runtime: list[string], + access_method: string, + contains_embedded_data: bool + }, + debugging_fix: { + applicable: bool, + problem_identification: string, + fix_implemented: string, + no_fix_conclusion: string, + test_inputs_used: list[string] + }, + data_processing: { + applicable: bool, + processing_method: string, + input_files_used: list[string], + intermediate_files_created: list[string], + final_output_produced: string + } + }, + verification_record: { + verification_commands: list[{ + step: string, + command: string, + return_code: int, + what_it_checked: string, + has_response_block: bool + }], + verification_coverage: { + verified_with_response: list[string], + attempted_but_unconfirmed: list[string], + not_verified: list[string], + inputs_scenarios_tested: list[string], + inputs_scenarios_not_tested: list[string] + } + } +} +``` + +The examples show that summaries distinguish actions with captured responses from actions merely +claimed by the agent, record intermediate errors and whether they were resolved, identify final +artifacts and runtime dependencies, and map verification evidence to requirements. Preserve these +distinctions; they are important inputs to the judge. + +## 6. Recursive Tournament Voting + +RTV receives `N` candidates, partitions them into groups of `G`, collects `V` independent model +votes per group, retains one candidate per group, and repeats until a requested survivor count is +reached. Main experiments use pairwise groups and eight votes. + +For group `j` in round `r`, the paper defines the selected display position as: + +```text +g_j^(r) = argmax over g in {1, ..., G} of + sum over v in {1, ..., V} of + 1[ LM(P_comp(P_in; S_(j,1)^(r), ..., S_(j,G)^(r))) == g ] +``` + +The selected rollouts become the next population. Summaries remain attached to their original +rollouts; survivors are not re-summarized between tournament rounds. + +```text +function COLLECT_RTV_VOTE(tournament, rollout_iteration, round_index, group_index, + vote_record_index, displayed_group, prompt, model_bundle, + effective_G, call_seed) -> Vote: + response = CALL_ROLE_WITH_RECORDED_RETRIES( + role = model_bundle.comparison_role, + messages = [{role: "user", content: prompt}], + seed = call_seed, + ) + PERSIST_RAW_RESPONSE_BEFORE_PARSING(response) + PERSIST_PROVIDER_ATTEMPTS(response.attempt_records) + + common = { + tournament_id: tournament.tournament_id, + iteration: rollout_iteration, + round_index: round_index, + group_index: group_index, + vote_index: vote_record_index, + ordered_candidate_ids: IDS(displayed_group), + prompt_sha256: SHA256(prompt), + model_usage: response.usage, + } + + if response.failed: + return Vote( + **common, + raw_model_response = null, + selected_display_position = null, + selected_candidate_id = null, + parse_error = "MODEL_CALL_FAILED:" + response.error_kind, + ) + + position = PARSE_FINAL_VERDICT(response.raw, allowed = 1 .. effective_G) + if position is invalid: + return Vote( + **common, + raw_model_response = response.raw, + selected_display_position = null, + selected_candidate_id = null, + parse_error = position.error, + ) + + return Vote( + **common, + raw_model_response = response.raw, + selected_display_position = position, + selected_candidate_id = displayed_group[position - 1].candidate_id, + parse_error = null, + ) + +function RUN_RTV(task, model_bundle, candidates, configured_G, V, target_survivors, + policies, seed, freeze_checkpoint_at = null) + -> (Tournament, TournamentCheckpoint | null): + assert 1 <= target_survivors <= len(candidates) + assert configured_G >= 2 + + # The paper's main N=16, G=2, K=4 cases divide exactly in every used round. + # Fail instead of inventing bye behavior for a non-divisible population unless an explicit + # policy is configured. + population = COPY(candidates) + rollout_iteration = UNIQUE_VALUE([ + RESOLVE_ROLLOUT(candidate.rollout_id).iteration for candidate in candidates + ]) + tournament = new Tournament( + tournament_id = UNIQUE_ID(), + task_id = task.task_id, + model_id = model_bundle.logical_name, + rollout_iteration = rollout_iteration, + target_survivor_count = target_survivors, + configured_G = configured_G, + V = V, + root_seed = seed, + pairing_policy_hash = HASH(policies.pairing_policy), + display_order_policy_hash = HASH(policies.display_order_policy), + tie_policy_hash = HASH(policies.tie_policy), + rounds = [], + population_ids_by_round = [], + survivor_candidate_ids = [], + ) + frozen_checkpoint = null + round_index = 0 + tournament.population_ids_by_round.append(IDS(population)) + + while len(population) > target_survivors: + # The G=8 ablation is 16 -> 2 -> 1, so the last round must shrink the configured group + # size from eight to two. G=4 similarly uses [4, 4]. + effective_G = MIN(configured_G, len(population)) + if len(population) mod effective_G != 0: + raise UndefinedByePolicy(len(population), effective_G) + if len(population) / effective_G < target_survivors: + raise TournamentWouldOvershootTarget( + population_size = len(population), + effective_group_size = effective_G, + requested_survivors = target_survivors, + ) + + ordered_population = APPLY_PAIRING_POLICY( + population, + policies.pairing_policy, + DERIVE_SEED(seed, "pairing", round_index), + ) + groups = CHUNK(ordered_population, size = effective_G) + results = ARRAY(size = len(groups)) + + PARALLEL_FOR group_index, group in enumerate(groups): + votes = [] + vote_contexts = ARRAY(size = V) + + for vote_index in 0 .. V - 1: + displayed_group = APPLY_DISPLAY_ORDER_POLICY( + group, + policies.display_order_policy, + DERIVE_SEED(seed, "display", round_index, group_index, vote_index), + ) + prompt = RENDER_COMPARISON_PROMPT( + task_text = task.problem_text, + candidates = displayed_group, + benchmark_name = task.benchmark_name, + ) + vote_contexts[vote_index] = { + original_vote_index: vote_index, + displayed_group: displayed_group, + prompt: prompt, + } + votes.append(COLLECT_RTV_VOTE( + tournament = tournament, + rollout_iteration = rollout_iteration, + round_index = round_index, + group_index = group_index, + vote_record_index = vote_index, + displayed_group = displayed_group, + prompt = prompt, + model_bundle = model_bundle, + effective_G = effective_G, + call_seed = DERIVE_SEED( + seed, "vote", round_index, group_index, vote_index + ), + )) + + valid_votes = [v for v in votes if v.selected_candidate_id is not null] + if len(valid_votes) != V: + invalid_contexts = [ + vote_contexts[index] + for index in 0 .. V - 1 + if votes[index].selected_candidate_id is null + ] + policy = policies.invalid_vote_policy + if policy.mode == ABORT_RUN: + raise IncompleteVoteSet( + expected = V, + actual = len(valid_votes), + vote_records = votes, + ) + + assert policy.mode == REPLACE_UNTIL_V + replacement_index = 0 + while ( + len(valid_votes) < V + and replacement_index < policy.max_replacement_calls_per_group + ): + source = invalid_contexts[replacement_index mod len(invalid_contexts)] + replacement = COLLECT_RTV_VOTE( + tournament = tournament, + rollout_iteration = rollout_iteration, + round_index = round_index, + group_index = group_index, + vote_record_index = V + replacement_index, + displayed_group = source.displayed_group, + prompt = source.prompt, + model_bundle = model_bundle, + effective_G = effective_G, + call_seed = DERIVE_SEED( + seed, + "replacement-vote", + round_index, + group_index, + source.original_vote_index, + replacement_index, + ), + ) + votes.append(replacement) + if replacement.selected_candidate_id is not null: + valid_votes.append(replacement) + replacement_index += 1 + + if len(valid_votes) != V: + # The paper's equation sums exactly V valid votes. A recorded replacement may + # supply an invalid slot, but the run aborts when the configured bound is exhausted. + raise IncompleteVoteSet( + expected = V, + actual = len(valid_votes), + vote_records = votes, + ) + + counts = COUNT_BY(valid_votes, key = selected_candidate_id) + assert counts is not empty + winners = ARGMAX_KEYS(counts) + if len(winners) == 1: + selected_id = winners[0] + tie_record = null + else: + selected_id, tie_record = BREAK_TIE_WITH_RECORDED_POLICY( + winners = winners, + group = group, + policy = policies.tie_policy, + seed = DERIVE_SEED(seed, "tie", round_index, group_index), + ) + + decision = GroupDecision( + round_index = round_index, + group_index = group_index, + effective_group_size = effective_G, + input_candidate_ids = IDS(group), + votes = votes, + vote_counts = counts, + selected_candidate_id = selected_id, + tie_break_record = tie_record, + ) + results[group_index] = { + decision: decision, + survivor: FIND_BY_ID(group, selected_id), + } + + round_decisions = [results[j].decision for j in 0 .. len(groups) - 1] + next_population = [results[j].survivor for j in 0 .. len(groups) - 1] + tournament.rounds.append(round_decisions) + population = next_population + tournament.population_ids_by_round.append(IDS(population)) + round_index += 1 + + if freeze_checkpoint_at is not null and len(population) == freeze_checkpoint_at: + assert frozen_checkpoint is null + frozen_checkpoint = TournamentCheckpoint( + checkpoint_id = UNIQUE_ID(), + tournament_id = tournament.tournament_id, + survivor_count = len(population), + completed_round_count = round_index, + survivor_candidate_ids = IDS(population), + population_sha256 = HASH(IDS(population)), + ) + PERSIST_IMMUTABLE(frozen_checkpoint) + + tournament.survivor_candidate_ids = IDS(population) + if freeze_checkpoint_at is not null: + assert frozen_checkpoint is not null + PERSIST(tournament) + return tournament, frozen_checkpoint +``` + +For `N=16, G=2`: + +```text +16 -> 8 -> 4 # Select-K; stop with K=4 +16 -> 8 -> 4 -> 2 -> 1 # Final RTV; stop with one +``` + +The group-size ablation uses these per-round effective group sizes: + +```text +configured G=16: [16] # 16 -> 1 +configured G=8: [8, 2] # 16 -> 2 -> 1 +configured G=4: [4, 4] # 16 -> 4 -> 1 +configured G=2: [2, 2, 2, 2] # 16 -> 8 -> 4 -> 2 -> 1 +``` + +The paper's analysis continues the iteration-0 tournament from four to one even though the method +uses the four-candidate population as the refinement context. Preserve the `K=4` checkpoint before +continuing the diagnostic tournament. + +### 6.1 Comparison output shape and reconstructed prompt contract + +`EXAMPLE-DERIVED`: appendix outputs show long, evidence-based comparisons followed by an exact +line such as: + +```text +Final verdict: Solution 1 +``` + +The paper defines `P_comp` but does not publish its text. The following rubric is reconstructed +from two example outputs. The comparison input must include the original task and each candidate's +structured summary, and must not include official pass/fail outcomes: + +1. restate the task requirements and explicit constraints; +2. detect disqualifying evidence, including a missing/unapplied patch, missing artifacts, dirty or + inconsistent final state, and unresolved fatal errors; +3. assess code/output completeness and correctness; +4. assess scope and whether the solution treats a root cause or only masks a symptom; +5. map verification commands to requirements and ensure verification happened after final edits; +6. distinguish confirmed command output from unconfirmed agent claims; +7. compare alternative interpretations of an ambiguous task; +8. rank candidates with the priority order shown by the SWE appendix example: + `disqualification > code correctness > code completeness > verification validity > test + results > fix scope > execution evidence > interpretation`; +9. end with exactly one parseable verdict naming a displayed candidate number. + +For Terminal-Bench, additionally assess constraint compliance, self-contained reproducibility, +required output files, runtime dependencies, input coverage, and whether available tests were run. + +```text +function PARSE_FINAL_VERDICT(text, allowed): + matches = REGEX_FIND_ALL( + pattern = case_insensitive("^\\s*Final verdict:\\s*Solution\\s+(\\d+)\\s*$"), + text = text, + multiline = true, + ) + if len(matches) != 1: + return INVALID + position = INT(matches[0].group(1)) + return position if position in allowed else INVALID +``` + +## 7. PDR refinement variants + +Let iteration-0 rollouts be `R_1 ... R_N` and summaries be `S_1^(0) ... S_N^(0)`. + +### 7.1 Single-rollout refinement ablation + +```text +function SINGLE_ROLLOUT_REFINEMENT(task, model_bundle, scaffold, limits, iter0_rollouts): + assert len(iter0_rollouts) == N + iter1 = ARRAY(size = N) + PARALLEL_FOR i in 0 .. N - 1: + iter1[i] = RUN_ROLLOUT( + task = task, + model_bundle = model_bundle, + scaffold = scaffold, + limits = limits, + iteration = 1, + rollout_index = i, + refinement_summaries = [SUMMARY_FOR_ROLLOUT(iter0_rollouts[i].run_id)], + seed = DERIVE_SEED( + EXPERIMENT_SEED, task.task_id, model_bundle.logical_name, 1, i + ), + ) + return iter1 +``` + +### 7.2 Random-K PDR ablation + +The paper describes sampling a separate size-`K` subset for each next-iteration rollout: + +```text +function RANDOM_K_REFINEMENT(task, model_bundle, scaffold, limits, iter0_rollouts, K): + summaries = [SUMMARY_FOR_ROLLOUT(r.run_id) for r in iter0_rollouts] + iter1 = ARRAY(size = N) + PARALLEL_FOR i in 0 .. N - 1: + J_i = SAMPLE_WITHOUT_REPLACEMENT( + population = 0 .. N - 1, + sample_size = K, + seed = DERIVE_SEED( + EXPERIMENT_SEED, task.task_id, model_bundle.logical_name, "random-k", i + ), + ) + iter1[i] = RUN_ROLLOUT( + task = task, + model_bundle = model_bundle, + scaffold = scaffold, + limits = limits, + iteration = 1, + rollout_index = i, + refinement_summaries = [summaries[j] for j in J_i], + seed = DERIVE_SEED( + EXPERIMENT_SEED, task.task_id, model_bundle.logical_name, 1, i + ), + ) + return iter1 +``` + +The original PDR procedure selects one rollout as its final answer. For this paper's pure-PDR +ablation, do not select a single final rollout: report iteration-1 performance as average pass@1 +over all N refined rollouts for every task. + +```text +function PURE_PDR_ABLATION_SCORE(task_runs): + return AVERAGE_PASS_AT_1([ + RESOLVE_ROLLOUTS(run.iter1_rollout_ids) for run in task_runs + ]) +``` + +### 7.3 Select-K refinement used by the full method + +Run RTV on the 16 iteration-0 candidates and stop after two pairwise rounds, leaving the same four +selected summaries as the refinement context for every fresh iteration-1 rollout. + +```text +function SELECT_K_REFINEMENT(task, model_bundle, scaffold, limits, iter0_candidates, K, + execution_policies, experiment_seed): + selection, unused_checkpoint = RUN_RTV( + task = task, + model_bundle = model_bundle, + candidates = iter0_candidates, + configured_G = 2, + V = 8, + target_survivors = K, + policies = execution_policies, + seed = DERIVE_SEED( + experiment_seed, task.task_id, model_bundle.logical_name, "select-k-helper" + ), + ) + assert unused_checkpoint is null + selected = RESOLVE_CANDIDATES(selection.survivor_candidate_ids) + selected_summaries = [SUMMARY_FOR_ROLLOUT(c.rollout_id) for c in selected] + + iter1 = ARRAY(size = N) + PARALLEL_FOR i in 0 .. N - 1: + iter1[i] = RUN_ROLLOUT( + task = task, + model_bundle = model_bundle, + scaffold = scaffold, + limits = limits, + iteration = 1, + rollout_index = i, + refinement_summaries = selected_summaries, + seed = DERIVE_SEED( + experiment_seed, task.task_id, model_bundle.logical_name, 1, i + ), + ) + return selection, iter1 +``` + +## 8. Complete PDR+RTV controller + +```text +function VALIDATE_EXPERIMENT_CONFIG(config, model_bundle, task, scaffold, summary_adapter): + c = config.paper_constants + assert c.N > 0 and c.T == 2 and 0 < c.K <= c.N + assert c.G >= 2 and c.V >= 1 + assert c.fresh_environment_per_rollout + assert not c.persistent_workspace_between_iterations + assert config.execution.summary_policy.max_content_attempts >= 1 + vote_policy = config.execution.invalid_vote_policy + if vote_policy.mode == ABORT_RUN: + assert vote_policy.max_replacement_calls_per_group == 0 + else: + assert vote_policy.mode == REPLACE_UNTIL_V + assert vote_policy.max_replacement_calls_per_group >= 1 + for role in [ + model_bundle.action_role, + model_bundle.summary_role, + model_bundle.comparison_role, + ]: + assert role.retry_policy.max_attempts >= 1 + assert role.prompt_text is not empty + assert SHA256(role.prompt_text) == role.prompt_sha256 + assert LOOKUP_REQUIRED( + config.agent_limits_by_benchmark_and_model, + task.benchmark_name, + model_bundle.logical_name, + ) is AgentLimits + + complete_manifest = { + experiment_config: config, + model_bundle: model_bundle, + benchmark_manifest: config.benchmark_manifest, + task_spec: task, + scaffold_adapter: scaffold, + summary_input_adapter: summary_adapter, + } + # The schema marks which nullable values are legal. It reports missing keys, null required + # values, and any REQUIRED_CONFIG sentinel recursively; it does not trust a hand-written list. + derived_unresolved = VALIDATE_AGAINST_REQUIRED_MANIFEST_SCHEMA( + complete_manifest, + schema_revision = config.manifest_schema_revision, + ).unresolved_field_paths + assert SORT(config.unresolved_field_names) == SORT(derived_unresolved) + if config.replication_label == EXACT_REPLICATION: + assert derived_unresolved is empty + +function RUN_PDR_RTV_FOR_TASK(task, model_bundle, config) -> TaskRun: + scaffold = ADAPTER_FOR(task.benchmark_name) + summary_adapter = SUMMARY_INPUT_ADAPTER_FOR(task.benchmark_name) + VALIDATE_EXPERIMENT_CONFIG(config, model_bundle, task, scaffold, summary_adapter) + constants = config.paper_constants + policies = config.execution + if constants != MAIN_PAPER_CONSTANTS: + RECORD_DEVIATIONS(constants) + ASSERT_SAME_PROVIDER_MODEL_REVISION( + model_bundle.action_role, + model_bundle.summary_role, + model_bundle.comparison_role, + ) + assert constants.T == 2 + assert constants.N == 16 + assert constants.K == 4 + assert constants.G == 2 + assert constants.V == 8 + limits = LOOKUP_REQUIRED( + config.agent_limits_by_benchmark_and_model, + task.benchmark_name, + model_bundle.logical_name, + ) + + # Stage 1: iteration 0, 16 independent clean-environment rollouts. + iter0_rollouts = ARRAY(size = constants.N) + PARALLEL_FOR i in 0 .. constants.N - 1: + iter0_rollouts[i] = RUN_ROLLOUT( + task = task, + model_bundle = model_bundle, + scaffold = scaffold, + limits = limits, + iteration = 0, + rollout_index = i, + refinement_summaries = [], + seed = DERIVE_SEED( + config.experiment_seed, task.task_id, model_bundle.logical_name, 0, i + ), + ) + + # Summaries are generated after rollout completion and before official grading results are + # exposed to the method. + iter0_summaries = ARRAY(size = constants.N) + PARALLEL_FOR i in 0 .. constants.N - 1: + iter0_summaries[i] = SUMMARIZE_ROLLOUT( + task, + iter0_rollouts[i], + model_bundle, + summary_adapter, + policies.summary_policy, + policies.summary_schema_or_null, + ) + iter0_candidates = MAKE_CANDIDATES(iter0_rollouts, iter0_summaries) + + # Stage 2: run one iteration-0 RTV instance 16 -> 8 -> 4 -> 2 -> 1. Freeze the exact + # top-four population after round two for refinement; the last two rounds exist only for + # the paper's iteration-0 RTV diagnostics and cannot change that immutable checkpoint. + iter0_tournament, select_k_checkpoint = RUN_RTV( + task, model_bundle, iter0_candidates, + configured_G = constants.G, + V = constants.V, + target_survivors = 1, + policies = policies, + seed = DERIVE_SEED( + config.experiment_seed, task.task_id, model_bundle.logical_name, "iteration-0-rtv" + ), + freeze_checkpoint_at = constants.K, + ) + assert select_k_checkpoint.survivor_count == constants.K + selected_iter0 = RESOLVE_CANDIDATES(select_k_checkpoint.survivor_candidate_ids) + refinement_summaries = [SUMMARY_FOR_ROLLOUT(c.rollout_id) for c in selected_iter0] + + # Stage 3: iteration 1, 16 new clean-environment rollouts. All receive the same selected four + # summaries, but have separate model sampling seeds and separate environments. + iter1_rollouts = ARRAY(size = constants.N) + PARALLEL_FOR i in 0 .. constants.N - 1: + iter1_rollouts[i] = RUN_ROLLOUT( + task = task, + model_bundle = model_bundle, + scaffold = scaffold, + limits = limits, + iteration = 1, + rollout_index = i, + refinement_summaries = refinement_summaries, + seed = DERIVE_SEED( + config.experiment_seed, task.task_id, model_bundle.logical_name, 1, i + ), + ) + + iter1_summaries = ARRAY(size = constants.N) + PARALLEL_FOR i in 0 .. constants.N - 1: + iter1_summaries[i] = SUMMARIZE_ROLLOUT( + task, + iter1_rollouts[i], + model_bundle, + summary_adapter, + policies.summary_policy, + policies.summary_schema_or_null, + ) + iter1_candidates = MAKE_CANDIDATES(iter1_rollouts, iter1_summaries) + + # Stage 4: final RTV 16 -> 8 -> 4 -> 2 -> 1. + final_tournament, unused_checkpoint = RUN_RTV( + task, model_bundle, iter1_candidates, + configured_G = constants.G, + V = constants.V, + target_survivors = 1, + policies = policies, + seed = DERIVE_SEED( + config.experiment_seed, task.task_id, model_bundle.logical_name, "final" + ), + ) + assert unused_checkpoint is null + final_candidate = RESOLVE_CANDIDATE(final_tournament.survivor_candidate_ids[0]) + final_rollout = RESOLVE_ROLLOUT(final_candidate.rollout_id) + + # The output is the surviving rollout's actual patch/artifacts/environment, not the judge's + # prose and not a synthesized merge of several workspaces. + result = TaskRun( + task_id = task.task_id, + model_logical_name = model_bundle.logical_name, + iter0_rollout_ids = IDS(iter0_rollouts), + iter0_summary_ids = IDS(iter0_summaries), + iter0_tournament_id = iter0_tournament.tournament_id, + select_k_checkpoint_id = select_k_checkpoint.checkpoint_id, + refinement_summary_ids = IDS(refinement_summaries), + iter1_rollout_ids = IDS(iter1_rollouts), + iter1_summary_ids = IDS(iter1_summaries), + final_tournament_id = final_tournament.tournament_id, + final_rollout_id = final_rollout.run_id, + output_patch = final_rollout.final_patch, + output_artifacts = final_rollout.final_artifact_manifest, + output_environment_snapshot = final_rollout.final_environment_snapshot, + ) + PERSIST(result) + return result +``` + +Run this independently for every `(benchmark, model, task)` combination. Never mix summaries or +votes across tasks or models. + +## 9. Prompt templates required for a faithful implementation + +The paper publishes example outputs but not the exact prompts. Keep all reconstructed templates +versioned and save the exact rendered prompt for every call. + +At manifest construction, copy the scaffold's exact action prompt into `action_role.prompt_text`, +set `summary_role.prompt_text` to the chosen version of Section 9.1, and set +`comparison_role.prompt_text` to the chosen version of Section 9.3. The scaffold adapter inserts +the Section 9.2 refinement text into each refined rollout's user context. Store every rendered +prompt hash; `CALL_ROLE_WITH_RECORDED_RETRIES` sends the role prompt as the system/instruction +message exactly once. + +### 9.1 Reconstructed summary prompt + +```text +You are given an agentic coding task and one attempt serialized under the recorded summary-input +policy. Produce one structured summary. If the configured protocol supplies a JSON Schema, +produce one valid JSON object that conforms to it; otherwise follow the versioned JSON-object +contract. Schema enforcement is optional, but the summary record always stores one JSON object. + +Report evidence, not just the agent's claims. Distinguish: +- commands with captured responses from commands merely proposed or issued without a response; +- successful outputs from errors; +- errors fixed later from errors still present at submission; +- files that were proved to exist from files the agent only claimed to create; +- tests run after the final edit from tests run before it; +- task requirements verified from requirements not checked; +- runtime dependencies and external files from development-only dependencies. + +Preserve exact paths, functions, command text, return codes, decisive output, before/after code, +unresolved issues, and uncertainty. Do not infer official hidden-test success. Do not include any +official grader result. Output JSON only. +``` + +### 9.2 Reconstructed refinement preamble/postamble + +```text +PREAMBLE: +You are starting a new independent attempt in a freshly initialized environment. Below are +structured summaries of K prior attempts on the same task. The summaries may describe successes, +failures, partial progress, conflicting diagnoses, or unverified claims. Use them as evidence, +not as ground truth. + +POSTAMBLE: +Synthesize common findings, retain useful diversity, reconcile conflicts using concrete evidence, +avoid repeated dead ends, and verify the resulting solution in this fresh environment. You do not +have prior files or patches unless you recreate them yourself. +``` + +### 9.3 Reconstructed comparison prompt + +```text +Compare the numbered candidate summaries for the original task. You cannot run hidden tests and +must not assume that a candidate passed. Rank candidates from their recorded code/artifacts, +commands, outputs, error state, and verification coverage. + +Follow the benchmark-specific rubric in Section 6.1. Explain the decisive evidence. End with one +line in exactly this format, where N is one displayed candidate number: + +Final verdict: Solution N +``` + +## 10. Benchmark adapters and outcome isolation + +### 10.1 SWE-Bench Verified + +```text +function EVALUATE_SWE_ROLLOUT(task, rollout): + # Evaluation occurs in a grader clone/snapshot, never in a future judge/refinement context. + grader_env = RESTORE_CLEAN_TASK_ENV(task) + APPLY_PATCH(grader_env, rollout.final_patch) + result = RUN_OFFICIAL_SWE_BENCH_GRADER(grader_env, task.official_grader_config) + evaluation = RolloutEvaluation( + rollout_id = rollout.run_id, + grader_revision = OFFICIAL_GRADER_REVISION, + binary_outcome = result.resolved, + score_details = result, + ) + PERSIST(evaluation) + return evaluation +``` + +Use the full 500-task SWE-Bench Verified test set and the bash-only mini-SWE-agent scaffold. Record +the exact dataset revision, repository base commit per task, harness commit, image digest, and +grader version. + +### 10.2 Terminal-Bench v2.0 + +```text +function EVALUATE_TERMINAL_ROLLOUT(task, rollout): + grader_env = RESTORE_SNAPSHOT(rollout.final_environment_snapshot) + result = RUN_OFFICIAL_TERMINAL_BENCH_GRADER(grader_env, task.official_grader_config) + evaluation = RolloutEvaluation( + rollout_id = rollout.run_id, + grader_revision = OFFICIAL_GRADER_REVISION, + binary_outcome = result.passed, + score_details = result, + ) + PERSIST(evaluation) + return evaluation +``` + +Use the Terminus 1 scaffold on the exact 88-task subset. Record the omitted task ID; the paper does +not state it. + +### 10.3 Leakage-safe sequencing + +`RECONSTRUCTED leakage-safe implementation`: the paper requires outcome-free selection but does +not state when grading ran. For maximum assurance, run the experiment in two phases: + +```text +PHASE A: + produce all rollouts, summaries, select-K decisions, refined rollouts, and final decisions + freeze and hash all records + +PHASE B: + run official graders for every iteration-0 and iteration-1 rollout + append one immutable RolloutEvaluation per rollout ID + compute metrics and plots +``` + +If grading must happen earlier for operational reasons, keep results in an access-controlled store +that the generation, summary, refinement, and comparison workers cannot read. + +## 11. Metrics + +Let `y[t, i, q]` be the binary outcome for task `q`, iteration `t`, rollout `i`. + +```text +function AVERAGE_PASS_AT_1(candidate_sets_by_task): + # Called "average pass@1" in the paper: mean binary reward over all candidates and tasks. + values = [] + for task_candidates in candidate_sets_by_task: + values.extend([OUTCOME(c) for c in task_candidates]) + return 100 * MEAN(values) + +function PASS_AT_N(candidate_sets_by_task): + # Fraction of tasks with at least one passing remaining candidate. + return 100 * MEAN([ + ANY(OUTCOME(c) for c in task_candidates) + for task_candidates in candidate_sets_by_task + ]) + +function MIXED_TASK_COUNT(candidate_sets_by_task): + return COUNT(task_candidates where + ANY(OUTCOME(c) == true) and ANY(OUTCOME(c) == false)) + +function STAGE_METRICS(all_task_runs): + return { + ITER_0: AVERAGE_PASS_AT_1([ + RESOLVE_ROLLOUTS(run.iter0_rollout_ids) for run in all_task_runs + ]), + SELECT_K: AVERAGE_PASS_AT_1([ + RESOLVE_ROLLOUTS_FOR_CANDIDATES( + RESOLVE_TOURNAMENT_CHECKPOINT( + run.select_k_checkpoint_id + ).survivor_candidate_ids + ) + for run in all_task_runs + ]), + ITER_1: AVERAGE_PASS_AT_1([ + RESOLVE_ROLLOUTS(run.iter1_rollout_ids) for run in all_task_runs + ]), + FINAL: AVERAGE_PASS_AT_1([ + [RESOLVE_ROLLOUT(run.final_rollout_id)] for run in all_task_runs + ]), + } +``` + +For these binary benchmarks, a population's pass@N is the upper bound on post-selection pass@1: +an oracle selector reaches it by retaining a successful rollout whenever one exists. + +### 11.1 RTV round dynamics + +At tournament round `r`, compute average pass@1 over every remaining candidate and pass@N over the +set of remaining candidates for each task. Average pass@1 should usually rise; pass@N can only stay +the same or fall because eliminating a successful candidate can remove the last success for a task. + +```text +function RTV_ROUND_CURVES(tournaments): + population_counts = [len(t.population_ids_by_round) for t in tournaments] + assert ALL(count == population_counts[0] for count in population_counts) + population_count = population_counts[0] + for round_index in 0 .. population_count - 1: + populations = [ + RESOLVE_ROLLOUTS_FOR_CANDIDATES(t.population_ids_by_round[round_index]) + for t in tournaments + ] + emit(round_index, + average_pass_at_1 = AVERAGE_PASS_AT_1(populations), + pass_at_n = PASS_AT_N(populations)) +``` + +### 11.2 Groupwise comparison accuracy + +Only score groups that contain at least one passing and one failing input rollout. A decision is +correct when its selected rollout passes. + +```text +function GROUPWISE_JUDGE_ACCURACY(tournaments): + rows = [] + for tournament in tournaments: + for round in tournament.rounds: + for decision in round: + outcomes = [OUTCOME(id) for id in decision.input_candidate_ids] + if ANY(outcomes) and not ALL(outcomes): + rows.append({ + benchmark: BENCHMARK_FOR(tournament.task_id), + model: tournament.model_id, + iteration: tournament.rollout_iteration, + round_index: decision.round_index, + correct: OUTCOME(decision.selected_candidate_id), + }) + return { + per_round: GROUP_MEAN( + rows, by = [benchmark, model, iteration, round_index] + ), + pooled_average: GROUP_MEAN( + rows, by = [benchmark, model, iteration] + ), + } +``` + +Do not compare these accuracies as controlled judge benchmarks across models: each model judges a +different pool of its own trajectories and summaries. + +### 11.3 Sequential pass-count transition matrix + +```text +function PASS_COUNT_TRANSITION_MATRIX(task_runs, N = 16): + matrix = ZEROS(rows = N + 1, columns = N + 1) + for run in task_runs: + p0 = SUM(OUTCOME(r) for r in RESOLVE_ROLLOUTS(run.iter0_rollout_ids)) + p1 = SUM(OUTCOME(r) for r in RESOLVE_ROLLOUTS(run.iter1_rollout_ids)) + matrix[p0, p1] += 1 + return matrix +``` + +Rows are iteration-0 pass counts and columns are iteration-1 pass counts. Cells above the diagonal +are improvements; cells below are regressions. + +### 11.4 Context-quality analysis + +```text +function CONTEXT_QUALITY_BUCKETS(task_runs, N = 16, K = 4): + buckets = {p: [] for p in 0 .. K} + for run in task_runs: + for refined_rollout in RESOLVE_ROLLOUTS(run.iter1_rollout_ids): + source_ids = refined_rollout.refinement_summary_ids + assert len(source_ids) == K + context_pass_count = SUM( + OUTCOME(ROLLOUT_FOR_SUMMARY(summary_id)) for summary_id in source_ids + ) + buckets[context_pass_count].append(OUTCOME(refined_rollout)) + + return { + p: { + context_count: len(buckets[p]), + average_task_equivalents: len(buckets[p]) / N, + iteration1_pass_at_1: + null if len(buckets[p]) == 0 else 100 * MEAN(buckets[p]), + binary_outcomes: buckets[p], + } + for p in 0 .. K + } +``` + +For select-K, each task's 16 refined rollouts share one context, so dividing context count by 16 +produces an integer task count. For random-K, each refined rollout can have a different context; +the same division produces the paper's fractional “average tasks” values. + +### 11.5 Step efficiency + +```text +function STEP_STATISTICS(task_runs): + rows = FLATTEN([ + { + benchmark: BENCHMARK_FOR(run.task_id), + model: run.model_logical_name, + iteration: r.iteration, + passed: OUTCOME(r), + steps: len(r.steps), + } + for run in task_runs + for r in RESOLVE_ROLLOUTS(run.iter0_rollout_ids + run.iter1_rollout_ids) + ]) + results = {} + for benchmark, model, iteration in DISTINCT_KEYS(rows): + group = FILTER(rows, matching = [benchmark, model, iteration]) + results[benchmark, model, iteration] = { + all: MEAN([row.steps for row in group]), + pass: MEAN([row.steps for row in group if row.passed]), + fail: MEAN([row.steps for row in group if not row.passed]), + } + return results +``` + +The paper counts agent steps, not shell commands. One action containing several commands is one +step. + +### 11.6 New-solution discovery + +```text +function NEW_SOLUTION_TASKS(task_runs): + return [ + run.task_id for run in task_runs + if not ANY(OUTCOME(r) for r in RESOLVE_ROLLOUTS(run.iter0_rollout_ids)) + and ANY(OUTCOME(r) for r in RESOLVE_ROLLOUTS(run.iter1_rollout_ids)) + ] +``` + +### 11.7 Pairwise model capability matrix + +For models `M_i, M_j`, count tasks for which `M_i` has at least one iteration-0 success and `M_j` +has zero iteration-0 successes. + +```text +matrix[i, j] = COUNT(task q where + ANY(y[M_i, iteration=0, :, q]) and + not ANY(y[M_j, iteration=0, :, q])) +``` + +For the cross-model new-solution analysis: + +```text +discovery_tasks = UNION(NEW_SOLUTION_TASKS(runs_for_model) for model in MODELS) +for task_id in discovery_tasks: + for model in MODELS: + already_solved_iter0[task_id, model] = ANY( + OUTCOME(r) for r in ITER0_ROLLOUTS(task_id, model) + ) +``` + +## 12. Complete experiment matrix + +### 12.1 Parallel aggregation: summaries versus raw trajectories + +Run on both benchmarks with Claude-4.5-Sonnet and Gemini-3-Flash. + +The paper does not disclose the held-fixed group size, vote count, or role call settings for this +figure. Require them in the ablation manifest: + +```text +SUMMARY_VS_RAW_HELD_FIXED = { + G: REQUIRED_CONFIG, + V: REQUIRED_CONFIG, + summary_and_comparison_call_settings: REQUIRED_CONFIG, +} +``` + +```text +for representation in [STRUCTURED_SUMMARY, FULL_ROLLOUT_TRACE]: + run identical N=16 tournaments + if STRUCTURED_SUMMARY: + Candidate.display_payload = canonical summary JSON + else: + Candidate.display_payload = full raw rollout trajectory + compare average pass@1 after every round and final pass@1 +``` + +Keep every other parameter, task set, initial rollout pool, pairing, vote seed, and judge model +fixed. The paper reports that structured summaries win consistently, especially in later rounds. + +### 12.2 RTV group-size ablation + +Use Gemini-3-Flash on both benchmarks, `N=16`, and: + +```text +GROUP_SIZE_ABLATION_HELD_FIXED = { + V: REQUIRED_CONFIG, # not disclosed in the group-size passage/caption + summary_and_comparison_call_settings: REQUIRED_CONFIG, +} +``` + +```text +G in [16, 8, 4, 2] + +G=16: 16 -> 1 +G=8: 16 -> 2 -> 1 +G=4: 16 -> 4 -> 1 +G=2: 16 -> 8 -> 4 -> 2 -> 1 +``` + +Use the same rollout pool. The paper reports `G=2` as best. + +### 12.3 RTV vote-count ablation + +Use Gemini-3-Flash on both benchmarks, `N=16`, `G=2`, and: + +```text +V in [1, 2, 4, 8, 16] +``` + +Use identical candidates and pairings. The paper reports gains with more votes and diminishing +returns beginning around `V=8`. +Keep summary/comparison role settings fixed at explicitly recorded `REQUIRED_CONFIG` values; the +paper does not publish those API settings. + +### 12.4 Standalone RTV + +For all five models and both benchmarks, run `N=16, G=2, V=8` on iteration-0 summaries until one +candidate remains. Compare initial average pass@1 to final RTV pass@1. + +Textual check reported by the paper for Claude-4.5-Sonnet: + +```text +SWE-Bench Verified: 67.4 -> 73.6 +Terminal-Bench v2.0: 40.6 -> 54.6 +``` + +### 12.5 Sequential refinement ablation + +Use 100 randomly sampled SWE-Bench Verified tasks, Claude-4.5-Sonnet and Gemini-3.1-Pro, +`N=16`, `K=4`. Record the exact sampled task IDs and random seed. Run: + +1. single-rollout refinement; +2. random-K refinement; +3. select-K refinement using RTV. + +Use the same iteration-0 rollouts for all three variants. Compare average iteration-0 and +iteration-1 pass@1, pass-count distributions, and iteration-1 success stratified by the number of +passing summaries in each refinement context. +Although original PDR chooses one final rollout, this paper estimates each pure-PDR variant by +averaging the binary scores of all 16 iteration-1 rollouts; do not evaluate only one chosen answer. + +The paper's check values are: + +| Model | Single iter 0 -> 1 | Random-K iter 0 -> 1 | Select-K iter 0 -> 1 | +|---|---:|---:|---:| +| Claude-4.5-Sonnet | 69.87 -> 70.87 | 69.87 -> 75.06 | 69.87 -> 78.06 | +| Gemini-3.1-Pro | 72.69 -> 73.75 | 72.69 -> 76.94 | 72.69 -> 79.25 | + +Context-quality ablation checks are `iteration-1 pass@1 (average task equivalents)`. A dash means +the bucket had no contexts: + +| Model/method | 0/4 | 1/4 | 2/4 | 3/4 | 4/4 | +|---|---:|---:|---:|---:|---:| +| Sonnet random-K | 2.5 (17.8) | 47.1 (6.5) | 80.4 (6.7) | 88.8 (16.2) | 98.1 (52.8) | +| Sonnet select-K | 1.2 (16) | 12.5 (3) | - (0) | 90.5 (19) | 97.3 (62) | +| Gemini Pro random-K | 1.9 (16.2) | 50.0 (6.4) | 66.4 (7.2) | 90.6 (10.6) | 99.1 (58.5) | +| Gemini Pro select-K | 2.2 (17) | 40.6 (2) | 52.1 (3) | 81.2 (7) | 99.7 (71) | + +For Sonnet, the number of 100 sampled tasks with 16/16 passing iteration-1 rollouts is 40 under +single-rollout refinement and 51 under random-K refinement. + +### 12.6 Main experiment + +For each model and benchmark task, run the controller in Section 8. Report stage average pass@1: + +| Model | SWE iter 0 | SWE select-K | SWE iter 1 | SWE final | Terminal iter 0 | Terminal select-K | Terminal iter 1 | Terminal final | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| Claude-4.5-Opus | 70.94 | 75.00 | 76.04 | 77.60 | 46.95 | 54.26 | 52.49 | 59.09 | +| Gemini-3.1-Pro | 72.25 | 75.30 | 76.16 | 76.60 | 52.49 | 59.66 | 56.89 | 64.77 | +| Claude-4.5-Sonnet | 67.41 | 72.60 | 74.01 | 75.60 | 40.62 | 50.85 | 50.00 | 56.82 | +| Gemini-3-Flash | 70.79 | 73.55 | 74.28 | 76.00 | 37.93 | 45.45 | 43.68 | 48.86 | +| GPT-5-0825 | 61.41 | 65.25 | 67.73 | 69.80 | 31.32 | 35.23 | 35.30 | 38.64 | + +Treat these values as result checks, not algorithm inputs. + +### 12.7 Required analysis outputs + +Generate all of the following from stored per-rollout outcomes and tournament logs: + +1. stage metrics: iteration 0, select-K, iteration 1, final; +2. per-iteration average pass@1, pass@16, and mixed-task count; +3. average steps for all, passing, and failing rollouts by iteration/model/benchmark; +4. iteration-0 to iteration-1 pass-count transition matrices; +5. pass-count distributions for both iterations; +6. context-quality buckets from zero to four passing selected summaries; +7. average pass@1 and pass@N after every RTV round for both iterations; +8. groupwise comparison accuracy for mixed groups by model, benchmark, iteration, and round; +9. pairwise model capability matrices; +10. tasks with zero iteration-0 successes and at least one iteration-1 success; +11. qualitative traces showing how refined rollouts reuse consensus, avoid repeated failures, and + resolve disagreements among prior summaries. + +### 12.8 Additional numeric checks from the paper and appendices + +Main rollout statistics use tuples `(average pass@1, pass@16, mixed-task count)`: + +```text +SWE-Bench Verified iteration 0 iteration 1 +Claude-4.5-Opus (70.94, 85.40, 218) (76.04, 81.20, 56) +Gemini-3.1-Pro (72.25, 86.00, 200) (76.16, 82.00, 59) +Claude-4.5-Sonnet (67.41, 83.40, 259) (74.01, 79.20, 71) +Gemini-3-Flash (70.79, 84.00, 251) (74.28, 79.80, 179) +GPT-5-0825 (61.41, 79.00, 257) (67.73, 73.40, 96) + +Terminal-Bench v2.0 iteration 0 iteration 1 +Claude-4.5-Opus (46.95, 70.45, 36) (52.49, 65.91, 22) +Gemini-3.1-Pro (52.49, 76.14, 35) (56.89, 72.73, 23) +Claude-4.5-Sonnet (40.62, 67.05, 38) (50.00, 62.50, 19) +Gemini-3-Flash (37.93, 60.23, 37) (43.68, 56.82, 18) +GPT-5-0825 (31.32, 51.14, 30) (35.30, 43.18, 10) +``` + +Average step counts use tuples `(all, passing, failing)`: + +```text +Model SWE iteration 0 -> 1 Terminal iteration 0 -> 1 +Opus (41.23,33.83,59.30) -> (14.31,13.16,17.97) (24.43,24.66,24.23) -> (12.14,10.96,13.45) +Pro (35.56,33.88,39.92) -> (17.95,17.05,20.82) (21.57,17.47,26.09) -> (10.95, 9.20,13.25) +Sonnet (49.24,46.13,55.67) -> (25.02,24.39,26.84) (21.74,19.47,23.30) -> ( 7.78, 6.29, 9.26) +Flash (51.10,48.39,57.65) -> (28.80,27.37,32.94) (16.01,15.68,16.22) -> ( 7.80, 6.16, 9.07) +``` + +Main select-K context checks are `iteration-1 pass@1 (task count)` for context pass counts +`[0/4, 1/4, 2/4, 3/4, 4/4]`: + +```text +Opus SWE: 0.1(81), 33.4(29), 55.5(25), 85.4(39), 99.2(326) +Opus Terminal: 1.8(31), 31.2(4), 43.0(8), 78.5(9), 94.1(36) + +Pro SWE: 0.6(87), 36.9(22), 38.4(22), 87.0(36), 99.8(333) +Pro Terminal: 3.8(23), 37.5(7), 45.8(12), 57.5(5), 93.1(41) + +Sonnet SWE: 1.7(94), 18.8(16), 65.4(28), 88.1(68), 99.7(294) +Sonnet Terminal: 3.0(33), 30.2(6), 58.0(7), 76.4(9), 91.7(33) + +Flash SWE: 0.0(93), 34.7(18), 73.1(20), 88.1(63), 96.4(306) +Flash Terminal: 1.0(37), 23.2(7), 68.1(9), 60.0(5), 91.0(30) +``` + +Groupwise judge accuracy checks use `R0, R1, R2, R3, pooled Avg`. The average is pooled over all +qualifying mixed groups; it is not the arithmetic mean of the four round percentages. + +```text +Iter 0 SWE Terminal +Opus 69.1,68.3,60.6,55.6,67.0 81.4,78.9,67.9,64.3,77.9 +Pro 66.9,64.5,51.1,65.1,64.5 84.2,81.4,72.2,62.5,80.7 +Sonnet 70.6,66.2,53.3,55.6,66.6 80.5,87.0,78.6,62.5,81.7 +Flash 66.3,55.6,53.3,54.5,61.3 85.1,79.4,80.0,73.7,82.3 + +Iter 1 SWE Terminal +Opus 54.4,62.9,60.0,71.4,58.2 75.4,74.3,84.2,83.3,77.2 +Pro 43.4,47.4,65.8,60.0,48.3 77.4,68.6,85.7,88.9,76.7 +Sonnet 63.2,57.5,59.0,60.0,60.9 78.9,77.4,76.2,77.8,78.0 +Flash 67.3,51.5,55.8,50.0,62.1 77.9,71.1,72.4,90.9,75.8 +``` + +### 12.9 Analysis coverage matrix + +```text +All five models: + main stage results; main rollout statistics; standalone RTV; pairwise model comparisons + +Opus, Pro, Sonnet, and Flash only: + step statistics; pass-count transitions; main context-quality analysis; + pass-count distributions; RTV round dynamics; groupwise judge accuracy + +Sonnet and Pro only, on the sampled 100 SWE tasks: + single-rollout/random-K/select-K sequential ablation + +Sonnet and Flash only: + structured-summary versus raw-trajectory ablation + +Flash only: + G and V RTV parameter ablations +``` + +Several plotted series are present only as curves or heatmaps in the supplied figure PDFs, not as +machine-readable values or TeX tables. Their exact numeric points cannot be reconstructed from the +source text alone. Treat these files as visual checks; digitize the source assets under a recorded +method if numeric comparison is required: + +```text +rtv_summary_vs_rollouts_colm_v1.pdf +rtv_parameter_search_colm_v1.pdf +rtv_main_results_colm_v1.pdf +pdr_pass_count_random_k.pdf +pdr_rtv_confusion_matrices.pdf +pdr_rtv_pass_count_distributions_swe_bench.pdf +pdr_rtv_pass_count_distributions_terminal_bench.pdf +pdr_rtv_parallel_analysis_swe_bench.pdf +pdr_rtv_parallel_analysis_terminal_bench.pdf +pdr_top_k_pass_rate_analysis.pdf +rollout_matchups_iter0_colm_v1.pdf +``` + +## 13. Expected appendix checks + +The paper reports these new-solution tasks. Use them as end-to-end data-integrity checks, not as +inputs to generation or selection. + +```text +SWE-Bench Verified: + Claude-4.5-Opus: django_django-11951 + Gemini-3.1-Pro: sphinx-doc_sphinx-9602 + Claude-4.5-Sonnet: django_django-13964, scikit-learn_scikit-learn-25102 + Gemini-3-Flash: none + GPT-5-0825: pydata_xarray-4687 + +Terminal-Bench v2.0: + Claude-4.5-Opus: + caffe-cifar-10, chess-best-move, gpt2-codegolf, nginx-request-logging, + vulnerable-secret + Gemini-3.1-Pro: + configure-git-webserver, gcode-to-text, git-leak-recovery, + large-scale-text-editing, openssl-selfsigned-cert + Claude-4.5-Sonnet: + mcmc-sampling-stan, sparql-university + Gemini-3-Flash: + mcmc-sampling-stan, regex-chess + GPT-5-0825: + mcmc-sampling-stan, schemelike-metacircular-eval, vulnerable-secret +``` + +The paper then checks whether each model had already solved every discovery task in iteration 0. +The table captions incorrectly describe the checkmark as self-improvement; the table values and +surrounding prose show that it means “this model had at least one iteration-0 success.” The +corrected matrices below use `Y` for that meaning: + +```text +SWE discovery task Opus Pro Sonnet Flash GPT-5 +django_django-11951 N N Y Y Y +sphinx-doc_sphinx-9602 N N Y Y N +django_django-13964 Y Y N Y Y +scikit-learn_scikit-learn-25102 Y N Y N Y +pydata_xarray-4687 Y Y Y Y N + +Terminal discovery task Opus Pro Sonnet Flash GPT-5 +caffe-cifar-10 N Y N Y Y +chess-best-move N Y Y Y Y +configure-git-webserver N N Y N Y +gcode-to-text Y N N N N +git-leak-recovery N N Y N N +gpt2-codegolf N Y N N N +large-scale-text-editing N N N N N +mcmc-sampling-stan N Y N N N +nginx-request-logging N Y N N N +openssl-selfsigned-cert Y N N N N +regex-chess N N Y N N +schemelike-metacircular-eval Y Y Y Y N +sparql-university Y Y N Y N +vulnerable-secret N Y N N N +``` + +The qualitative examples emphasize four refinement behaviors that the trace viewer should make +auditable: + +1. consensus synthesis across multiple summaries; +2. direct reuse of a precise diagnosis, file, line, command, or test; +3. explicit reconciliation of conflicting approaches using recorded evidence; +4. avoidance of prior environment/setup failures in the fresh rollout. + +Exact appendix example inventory: + +```text +PDR+RTV qualitative excerpts: + Opus / SWE / django_django-13033 + Gemini Pro / SWE / sympy_sympy-17318 + Opus / Terminal / sparql-university + Sonnet / Terminal / sqlite-db-truncate + +Initial trajectories: + Gemini Pro / SWE / sympy__sympy-17630 + Opus / Terminal / openssl-selfsigned-cert + +Refined trajectories: + Opus / Terminal / gpt2-codegolf + Gemini Pro / Terminal / large-scale-text-editing + +Structured summaries: + Gemini Pro / SWE / sympy__sympy-17630 + Opus / Terminal / openssl-selfsigned-cert + +Group comparisons: + Gemini Pro / SWE / django__django-15973 + Opus / Terminal / video-processing +``` + +Task-specific trace checks include direct reuse of Django line 730 and the `pieces[-1]` fix, +preinstallation of `asgiref`, `pytz`, and `sqlparse`, reuse of the SymPy call-chain diagnosis and a +prior failing test, selection of a two-file SymPy fix, separation of the SPARQL EU and student-count +conditions, and selection of the successful SQLite serial-type interpretation before building the +B-tree leaf-page parser. + +## 14. Resume, concurrency, and failure handling + +The paper reports infrastructure-related Gemini-3.1-Pro API failures during final RTV, but does not +specify recovery semantics. Failure policy can change results, so implement it explicitly: + +```text +function CALL_ROLE_WITH_RECORDED_RETRIES(role, messages, tools = null, seed): + request_messages = [ + {role: "system", content: role.prompt_text}, + *messages, + ] + assert SHA256(role.prompt_text) == role.prompt_sha256 + + for attempt in 1 .. role.retry_policy.max_attempts: + response = PROVIDER_CALL( + role = role, + messages = request_messages, + tools = tools, + seed = DERIVE_SEED(seed, "provider-attempt", attempt), + ) + record request ID, attempt, status, latency, usage, and provider error + if response is valid: + return response + if response.error_kind not in role.retry_policy.retryable_error_kinds: + break + wait DETERMINISTIC_BACKOFF( + attempt, seed, role.retry_policy.deterministic_backoff_policy + ) + return FAILED_RESPONSE(all_attempts) + +function HANDLE_EXHAUSTED_MODEL_CALL(call_kind, policy, attempt, max_content_attempts, + failed_response): + if policy == RETRY_CONTENT_ATTEMPT and attempt < max_content_attempts: + RECORD_REPLACEMENT_LINK(failed_response, next_content_attempt = attempt + 1) + return RETRY_CONTENT_ATTEMPT + raise RequiredModelCallFailed(call_kind, failed_response.attempt_records) +``` + +Required operational rules: + +- Use an idempotency key derived from experiment/task/model/iteration/rollout/step/call kind. +- Resume from immutable records; never regenerate a successful call during resume. +- Never replace a failed rollout or vote with an unrecorded extra sample. +- If replacement is permitted by the configured protocol, label the replacement and include it in + denominators exactly once. +- Keep rollout environments isolated even when executing them concurrently. +- Rate-limit at provider/model level without changing task membership. +- Save raw provider responses before parsing. +- Freeze pairing and display order before launching parallel judge calls. +- Refuse to compute final metrics while required records are missing or duplicated. +- If calls are cached, key the cache by task ID, lossless trajectory/input hash, prompt and schema + version, exact model identity, and the complete role generation configuration. + +## 15. Compute accounting + +Per task and model in the main `T=2` experiment: + +```text +agent trajectories: 16 iteration 0 + 16 iteration 1 = 32 +summary calls: 16 iteration 0 + 16 iteration 1 = 32 + +select-K RTV group decisions: 8 + 4 = 12 +select-K judge calls: 12 * V=8 = 96 + +final RTV group decisions: 8 + 4 + 2 + 1 = 15 +final judge calls: 15 * V=8 = 120 + +comparison calls that affect method output: 216 +iteration-0 diagnostic continuation: 24 +total comparison calls executed: 240 +total summary + comparison calls: 272 +``` + +Each trajectory contains many action-generation model calls, one per agent step, so do not report +the 32 trajectories as 32 ordinary language-model requests. Compute total provider calls, input +tokens, output tokens, wall time, and cost by call kind: action, summary, comparison, and retry. + +The 24 diagnostic calls are the last two rounds of the same frozen-checkpoint iteration-0 +tournament. They are required for the paper's roundwise iteration-0 analysis but cannot change +the four summaries already recorded for refinement. + +## 16. Deterministic validation and unit tests + +Before spending model compute, test the controller with scripted fake models and fake graders. + +```text +test_main_population_sizes: + assert the frozen select-K checkpoint populations are [16, 8, 4] + assert its continued iteration-0 diagnostic tournament populations are [16, 8, 4, 2, 1] + assert final populations are [16, 8, 4, 2, 1] + +test_refinement_checkpoint_is_immutable: + freeze the iteration-0 top-four candidate IDs and hash + continue the same tournament to two and one survivors + assert the checkpoint IDs/hash and every refined rollout's four summary IDs are unchanged + +test_group_size_ablation_schedules: + assert configured G=16 uses effective sizes [16] + assert configured G=8 uses effective sizes [8, 2] + assert configured G=4 uses effective sizes [4, 4] + assert configured G=2 uses effective sizes [2, 2, 2, 2] + +test_call_counts: + assert decisions through the select-K checkpoint == 12 + assert votes through the select-K checkpoint == 96 + assert diagnostic continuation decisions == 3 + assert diagnostic continuation votes == 24 + assert final group decisions == 15 + assert final votes == 120 + +test_fresh_environments: + create a unique file in every iteration-0 environment + assert no iteration-1 environment contains any such file + +test_same_selected_context_for_select_k: + assert every iteration-1 rollout has exactly the same four summary IDs + +test_random_k_is_per_rollout: + assert every random-K context has K distinct IDs + assert sampled ID sets are recorded for each rollout + +test_no_grader_leakage: + tag every object by provenance and authorize only task-visible inputs plus method records + assert no grader-owned reward, score, verifier result, hidden test, grader trace, or + evaluation-store object/reference reaches summary inputs, summaries, judge cards, + comparison prompts, refinement contexts, pairing, ties, or fallbacks + allow ordinary task-visible text to mention words such as "hidden tests" + assert action, summary, comparison, pairing, tie-breaking, and invalid-vote fallback + workers cannot read grader records or the evaluation store before method outputs freeze + +test_same_model_for_all_roles: + assert every rollout uses the configured logical model identity + assert every summary and every RTV vote uses that same identity + reject aliases that resolve to different provider model revisions + +test_surviving_artifact_identity: + assert final output patch/artifacts hash equals the selected rollout's stored output hash + +test_verdict_parser: + accept exactly one "Final verdict: Solution N" line in range + reject missing, duplicate, conflicting, or out-of-range verdicts + assert malformed, empty, ambiguous, and out-of-range verdicts never default or clamp + to a displayed candidate + +test_vote_mapping_after_display_permutation: + assert displayed position maps back to the correct stable candidate ID + +test_tie_is_explicit: + force a 4-4 vote split and assert the configured tie policy is recorded + +test_exact_vote_count: + assert every non-bye group records exactly V valid or explicitly replaced votes + assert faithful mode never stops voting after an early unflippable majority + +test_metric_denominators: + synthetic tasks with known outcomes reproduce average pass@1, pass@N, mixed counts, + transition matrices, group accuracy, and context buckets + +test_summary_example_shapes: + parse both appendix-derived example JSON files without type loss + if a configured schema is used, validate that both examples satisfy its union/normalization + policy; do not claim the examples publish a normative schema + +test_scaffold_adapters: + parse and round-trip the appendix mini-SWE-agent and Terminus 1 trajectory examples + +test_configuration_effects: + every declared behavior-controlling field alters execution exactly as documented + or is rejected as unsupported; provenance-only fields may be observational +``` + +## 17. Public code audit and paper/code differences + +No official or author-linked implementation was public when this audit ran on August 17, 2026. +The paper source has no repository or code-availability statement, and searches of the authors', +Meta's, and GitHub's public pages found no matching official repository. Two later repositories +explicitly describe themselves as independent reproductions: + +- [genji970/facebook-paper_harness-inference-scale-agent at + `e4d81a67198cd55798330ac3e6ce261a131e1852`](https://github.com/genji970/facebook-paper_harness-inference-scale-agent/tree/e4d81a67198cd55798330ac3e6ce261a131e1852), + dated May 3, 2026. Its README says it is unofficial, covers only Gemini plus SWE-Bench, and was + written because no public paper implementation existed. +- [zy95-12/Agentic-Coding at + `30442021b3f9d2ae065cdcd1b9bbdbf1e5c38740`](https://github.com/zy95-12/Agentic-Coding/tree/30442021b3f9d2ae065cdcd1b9bbdbf1e5c38740), + dated July 10, 2026. Its README calls it an unaffiliated reproduction and says its runs are + engineering checks rather than a reproduction of the headline results. + +Neither repository can turn an unresolved field into `REPO` evidence. Do not copy their prompts, +fallbacks, runtime constants, or result claims into a paper-faithful configuration. + +The `zy95-12` reproduction follows the broad `iteration 0 -> summarize -> select K -> iteration 1 +-> summarize -> final RTV` sequence, but differs materially: + +- it uses GLM and Terminus 2 instead of the paper's five models and Terminus 1; +- its controller's `--model` setting controls summarization/judging but is not passed to Harbor + rollout generation, so generator and judge identities can diverge; +- it reads official verifier rewards, places them in summaries and judge cards, tells the judge to + use them, ranks reward first in a fallback, and writes prior reward into refinement context; +- it summarizes rule-extracted evidence truncated to 45,000 characters rather than a disclosed + paper serializer; +- it can stop voting when a majority cannot flip instead of collecting the equation's exact `V` + votes; +- malformed verdicts can be clamped/defaulted to a candidate; +- it uses adjacent groups, singleton byes, and a lexicographic trial-ID tie-break, none of which is + paper evidence; +- its GLM route, rollout temperature `0.2`, 128,000/8,192 token limits, Terminus 2 settings, + concurrency, timeouts, and evidence cap are third-party choices, not missing paper constants. + +The `genji970` reproduction also differs: + +- generator and summary/judge model IDs are configured independently; +- summary and judge calls force temperature zero, so repeated identical prompts do not provide a + meaningful independent-vote mechanism; +- the comparison parser takes the first integer and maps malformed/out-of-range output to the + first candidate; tied counts also choose the first displayed candidate; +- it collapses mini-SWE-agent execution into one synthetic step plus an optional patch step, + keeps only the last 12,000 stdout/stderr characters, and truncates that patch step to 4,000 + characters by default; +- it treats zero process exit plus any nonempty patch as success before official grading; +- its final record does not always save and hash the selected patch; +- its `num_iterations` option does not control execution, which is hard-coded to two iterations. + +The `zy95-12` repository also contains paths that must not be mistaken for its main experiment: +`src/json_command_agent.py` is abandoned; `rtv_select.py::_select_group` is unused and calls +`_vote` without its required `debug_log`; and its reward-first semantic top-K context is +generated but replaced by `rtv_refinement_context.md` before iteration 1. + +`genji970` grades only the selected final rollout. `zy95-12` verifies every Harbor trial but +exposes those rewards before summarization and selection. Neither implements a leakage-safe +freeze-then-grade phase or the paper's complete analyses. Across the two reproductions, omitted +study components include one benchmark or the other, four of five models, the complete ablation +matrix, transition matrices, roundwise pass@1/pass@N, mixed-group judge accuracy, and full +benchmark scale. The pseudocode in this document retains those paper requirements and explicitly +rejects reward leakage, early majority stopping, silent verdict defaults, and model-identity +drift. + +## 18. Replication-critical details not disclosed by the paper + +Numerically exact reproduction is impossible from the PDF alone until these values are recovered +from public code/configuration or the authors: + +1. exact agent, summarization, comparison, and refinement prompts; +2. exact summary output schema, whether the API enforced it, and whether every task in a benchmark + used the same schema; +3. malformed-summary parse, repair, retry, exclusion, and replacement policy; +4. exact model provider IDs and API revisions for the action, summary, and comparison roles; +5. temperature, top-p, token budgets, context management, stop settings, and seed behavior for + each role; +6. exact mini-SWE-agent and Terminus 1 action protocols, prompts, and serializer revisions; +7. agent step, command, and wall-time limits; +8. summary-input serialization and over-context truncation/reduction policy; +9. pairing order between rounds and whether populations are shuffled; +10. candidate display-order randomization; +11. tie-breaking for even `V=8` vote splits; +12. invalid/malformed judge-output parsing and retry behavior; +13. model/API error retry and replacement policy; +14. experiment and ablation seeds, including the 100-task SWE-Bench sample; +15. exact benchmark, harness, grader, repository, and container revisions; +16. the one omitted Terminal-Bench task; +17. whether summaries include the original task, final patch, and artifact manifest separately + from the trajectory; +18. whether selected summaries use a fixed concatenation order in every refined rollout; +19. exact command-output truncation presented to the agent and summarizer; +20. how unfinished/invalid rollouts enter summarization, tournaments, and metric denominators; +21. concurrency and provider rate-limit settings; +22. held-fixed group size, vote count, and role call settings for the summary-versus-raw and + group-size parameter ablations. + +The implementation must print this unresolved-field list at startup and refuse a run labeled +`exact_replication=true` while any field remains unresolved. A run may proceed as a +`conceptual_replication` when every reconstructed choice is explicit in the manifest. + +## 19. End-to-end execution checklist + +```text +1. Pin dataset, harness, grader, image, repository, model, and prompt revisions. +2. Freeze task lists, including the exact 88 Terminal-Bench tasks. +3. Freeze all seeds and controller policies. +4. For each benchmark/model/task, create 16 clean iteration-0 environments and run rollouts. +5. Generate 16 structured summaries with the same model; parse/validate them under the recorded + summary protocol. +6. Run iteration-0 RTV 16 -> 8 -> 4 with 8 votes per pair; freeze the four survivors, then + continue that same tournament 4 -> 2 -> 1 only for the paper's diagnostic analysis. +7. Create 16 new clean environments; give all 16 the same four selected summaries. +8. Run the 16 iteration-1 rollouts and generate 16 new summaries. +9. Run final RTV 16 -> 8 -> 4 -> 2 -> 1 with 8 votes per pair. +10. Return the selected rollout's actual patch/artifacts/environment. +11. Freeze all method outputs before exposing official grader results. +12. Grade all 32 rollouts per task so every stage and analysis can be computed. +13. Compute stage scores, pass@16, mixed counts, step statistics, transitions, context buckets, + RTV round curves, mixed-group judge accuracy, model matchups, and new-solution tasks. +14. Run the summary/raw, G, V, standalone RTV, and three refinement ablations with fixed pools. +15. Compare to the paper's result checks without feeding those checks back into the method. +16. Publish the full manifest, task list, prompts, raw call logs, summaries, tournament records, + patches/artifact manifests, grader versions, and metric-generation code. +``` From a70a4a2c930eeaae3bf451d217414cf1afb1d39f Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 18 Aug 2026 23:06:21 +0000 Subject: [PATCH 2/4] feat(benchmark): add test-time scaling workflow Signed-off-by: Elyas Mehtabuddin --- Cargo.lock | 24 + Cargo.toml | 3 + benchmark/README.md | 38 ++ benchmark/agent-versions.env | 1 + benchmark/prepare_harbor_dataset.py | 41 +- benchmark/run-baseline.sh | 56 +- benchmark/test_time_scaling_config.py | 256 +++++++++ benchmark/test_time_scaling_grades.py | 147 ++++++ benchmark/test_time_scaling_harbor_agent.py | 33 ++ benchmark/test_time_scaling_patch_agent.py | 71 +++ benchmark/test_time_scaling_rollouts.py | 275 ++++++++++ .../Cargo.toml | 20 + .../src/backend.rs | 191 +++++++ .../src/command.rs | 64 +++ .../src/config.rs | 169 ++++++ .../src/main.rs | 97 ++++ .../src/model_client.rs | 214 ++++++++ .../switchyard-test-time-scaling/Cargo.toml | 22 + .../src/config.rs | 132 +++++ .../src/controller.rs | 286 +++++++++++ .../switchyard-test-time-scaling/src/error.rs | 68 +++ .../src/evaluation.rs | 204 ++++++++ .../switchyard-test-time-scaling/src/lib.rs | 44 ++ .../src/manifest.rs | 135 +++++ .../switchyard-test-time-scaling/src/model.rs | 238 +++++++++ .../switchyard-test-time-scaling/src/ports.rs | 36 ++ .../src/prompts.rs | 39 ++ .../src/record.rs | 25 + .../switchyard-test-time-scaling/src/seed.rs | 24 + .../src/tournament.rs | 416 +++++++++++++++ .../src/verdict.rs | 61 +++ .../tests/evaluation.rs | 49 ++ .../tests/manifest.rs | 70 +++ .../tests/workflow.rs | 484 ++++++++++++++++++ tests/test_prepare_harbor_dataset.py | 15 +- tests/test_run_baseline_script.py | 406 ++++++++------- tests/test_test_time_scaling_adapters.py | 157 ++++++ 37 files changed, 4398 insertions(+), 213 deletions(-) create mode 100644 benchmark/test_time_scaling_config.py create mode 100644 benchmark/test_time_scaling_grades.py create mode 100644 benchmark/test_time_scaling_harbor_agent.py create mode 100644 benchmark/test_time_scaling_patch_agent.py create mode 100644 benchmark/test_time_scaling_rollouts.py create mode 100644 crates/switchyard-test-time-scaling-runner/Cargo.toml create mode 100644 crates/switchyard-test-time-scaling-runner/src/backend.rs create mode 100644 crates/switchyard-test-time-scaling-runner/src/command.rs create mode 100644 crates/switchyard-test-time-scaling-runner/src/config.rs create mode 100644 crates/switchyard-test-time-scaling-runner/src/main.rs create mode 100644 crates/switchyard-test-time-scaling-runner/src/model_client.rs create mode 100644 crates/switchyard-test-time-scaling/Cargo.toml create mode 100644 crates/switchyard-test-time-scaling/src/config.rs create mode 100644 crates/switchyard-test-time-scaling/src/controller.rs create mode 100644 crates/switchyard-test-time-scaling/src/error.rs create mode 100644 crates/switchyard-test-time-scaling/src/evaluation.rs create mode 100644 crates/switchyard-test-time-scaling/src/lib.rs create mode 100644 crates/switchyard-test-time-scaling/src/manifest.rs create mode 100644 crates/switchyard-test-time-scaling/src/model.rs create mode 100644 crates/switchyard-test-time-scaling/src/ports.rs create mode 100644 crates/switchyard-test-time-scaling/src/prompts.rs create mode 100644 crates/switchyard-test-time-scaling/src/record.rs create mode 100644 crates/switchyard-test-time-scaling/src/seed.rs create mode 100644 crates/switchyard-test-time-scaling/src/tournament.rs create mode 100644 crates/switchyard-test-time-scaling/src/verdict.rs create mode 100644 crates/switchyard-test-time-scaling/tests/evaluation.rs create mode 100644 crates/switchyard-test-time-scaling/tests/manifest.rs create mode 100644 crates/switchyard-test-time-scaling/tests/workflow.rs create mode 100644 tests/test_test_time_scaling_adapters.py diff --git a/Cargo.lock b/Cargo.lock index b8410e755..0f2e04b47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2387,6 +2387,30 @@ dependencies = [ "tokio", ] +[[package]] +name = "switchyard-test-time-scaling" +version = "0.2.0" +dependencies = [ + "async-trait", + "futures", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", +] + +[[package]] +name = "switchyard-test-time-scaling-runner" +version = "0.2.0" +dependencies = [ + "async-trait", + "reqwest", + "serde", + "serde_json", + "switchyard-test-time-scaling", + "tokio", +] + [[package]] name = "switchyard-translation" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 07d133bb3..d7fdaf6ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,8 @@ members = [ "crates/protocol", "crates/switchyard-server", "crates/switchyard-skill-distillation", + "crates/switchyard-test-time-scaling", + "crates/switchyard-test-time-scaling-runner", "crates/switchyard-translation", ] @@ -39,6 +41,7 @@ switchyard-libsy = { path = "crates/libsy", version = "0.2.0" } switchyard-llm-client = { path = "crates/libsy-llm-client", version = "0.2.0" } switchyard-protocol = { path = "crates/protocol", version = "0.2.0" } switchyard-server = { path = "crates/switchyard-server", version = "0.2.0" } +switchyard-test-time-scaling = { path = "crates/switchyard-test-time-scaling", version = "0.2.0" } switchyard-translation = { path = "crates/switchyard-translation", version = "0.2.0" } thiserror = "2" tokio = { version = "1", features = ["full"] } diff --git a/benchmark/README.md b/benchmark/README.md index 86feef084..e654f7541 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -129,6 +129,15 @@ uv run --no-sync python benchmark/prepare_harbor_dataset.py \ --overwrite ``` +SWE-bench Verified is the benchmark used by the test-time scaling runner: + +```bash +uv run --no-sync python benchmark/prepare_harbor_dataset.py \ + --source-dataset swebench-verified@1.0 \ + --output-dir benchmark/datasets/swebench-verified-closed-book \ + --overwrite +``` + SWE-Bench Pro is supported with the Harbor dataset `cais/swebenchpro`. The generated dataset uses the same pinned-agent and closed-book proxy path without opening dataset-specific agent egress. @@ -139,6 +148,35 @@ uv run --no-sync python benchmark/prepare_harbor_dataset.py \ --overwrite ``` +## Run Paper Test-Time Scaling + +The test-time scaling runner follows the paper's two-iteration schedule. One full task uses 32 agent +attempts, 32 summaries, and 240 comparison calls. Start with smaller settings before running the +paper defaults. + +The config writer records every choice that the paper does not state. It labels the result as a +conceptual replication because those missing choices prevent an exact numerical replication. + +```bash +export NVIDIA_API_KEY="..." + +uv run --no-sync python benchmark/test_time_scaling_config.py \ + --dataset benchmark/datasets/swebench-verified-closed-book \ + --task astropy__astropy-7606 \ + --model azure/anthropic/claude-sonnet-4-5 \ + --output benchmark/tb_runs/test-time-scaling-canary + +cargo run -p switchyard-test-time-scaling-runner -- \ + benchmark/tb_runs/test-time-scaling-canary/config/runner.json +``` + +The default settings are N=16, K=4, G=2, and V=8. For a small end-to-end check, add +`--rollouts 2 --refinement-count 1 --votes 1` to the config command. + +The runner saves `run.json` before it starts official verification. Selection cannot read verifier +results because Harbor does not create them during the agent attempts. After selection, the grader +applies every saved patch to a fresh task container and writes `evaluation.json`. + ## Run Without Switchyard Omit `--server-config` to fully disable Switchyard. The runner still creates the benchmark diff --git a/benchmark/agent-versions.env b/benchmark/agent-versions.env index a064b2b37..0842ae193 100644 --- a/benchmark/agent-versions.env +++ b/benchmark/agent-versions.env @@ -4,6 +4,7 @@ CLAUDE_CODE_VERSION=2.1.211 CODEX_VERSION=0.144.5 OPENCODE_VERSION=1.18.3 +MINI_SWE_AGENT_VERSION=2.4.6 NODE_VERSION=20.11.1 # Hermes (NousResearch hermes-agent), installed from GitHub at dataset-bake time. # Must be a full 40-character commit SHA; anything else is rejected at build time. diff --git a/benchmark/prepare_harbor_dataset.py b/benchmark/prepare_harbor_dataset.py index 79d5c1993..299b34f04 100644 --- a/benchmark/prepare_harbor_dataset.py +++ b/benchmark/prepare_harbor_dataset.py @@ -3,8 +3,6 @@ """Prepare a local closed-book Harbor dataset with prebaked coding agents.""" -from __future__ import annotations - import argparse import hashlib import json @@ -155,7 +153,11 @@ def _find_exported_dataset_root(download_root: Path, source_dataset: str) -> Pat return candidate dirs_with_tasks = sorted( - {task.parent.parent for task in download_root.rglob("task.toml") if task.parent != download_root} + { + task.parent.parent + for task in download_root.rglob("task.toml") + if task.parent != download_root + } ) if len(dirs_with_tasks) == 1: return dirs_with_tasks[0] @@ -164,7 +166,9 @@ def _find_exported_dataset_root(download_root: Path, source_dataset: str) -> Pat raise FileNotFoundError(f"could not find exported tasks under {download_root}") -def _run_download(source_dataset: str, download_root: Path, harbor_command: str, overwrite: bool) -> Path: +def _run_download( + source_dataset: str, download_root: Path, harbor_command: str, overwrite: bool +) -> Path: download_root.mkdir(parents=True, exist_ok=True) command = [ *shlex.split(harbor_command), @@ -199,6 +203,7 @@ def _install_layer(pins: dict[str, str]) -> str: node_version = pins["NODE_VERSION"] claude_version = pins["CLAUDE_CODE_VERSION"] codex_version = pins["CODEX_VERSION"] + mini_swe_agent_version = pins["MINI_SWE_AGENT_VERSION"] opencode_version = pins["OPENCODE_VERSION"] # Hermes (NousResearch hermes-agent) is a per-user uv app installed from # GitHub, not an npm package. Baking it here (build-time, with host network) @@ -228,7 +233,7 @@ def _install_layer(pins: dict[str, str]) -> str: return f""" # Switchyard benchmark prebaked coding agents. -ENV SWITCHYARD_PREBAKED_AGENT_VERSIONS="claude-code={claude_version},codex={codex_version},opencode={opencode_version},node={node_version},hermes={hermes_version}" +ENV SWITCHYARD_PREBAKED_AGENT_VERSIONS="claude-code={claude_version},codex={codex_version},mini-swe-agent={mini_swe_agent_version},opencode={opencode_version},node={node_version},hermes={hermes_version}" RUN set -eux; \\ if command -v apt-get >/dev/null 2>&1; then \\ apt-get update; \\ @@ -282,6 +287,12 @@ def _install_layer(pins: dict[str, str]) -> str: curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/{hermes_version}/scripts/install.sh \\ | bash -s -- --skip-setup --commit {hermes_version} --force-commit; \\ hermes version +RUN set -eux; \\ + export HOME=/root; \\ + export PATH="/root/.local/bin:$PATH"; \\ + curl -LsSf https://astral.sh/uv/0.7.13/install.sh | sh; \\ + uv tool install mini-swe-agent=={mini_swe_agent_version}; \\ + uv tool list | grep -F "mini-swe-agent v{mini_swe_agent_version}" """ @@ -329,9 +340,7 @@ def _rewrite_task_image(task_dir: Path, pins: dict[str, str]) -> dict[str, Any]: entrypoint_layer = _entrypoint_layer() if docker_image: - dockerfile.write_text( - f"FROM {docker_image}\nUSER root\n{layer.lstrip()}{entrypoint_layer}" - ) + dockerfile.write_text(f"FROM {docker_image}\nUSER root\n{layer.lstrip()}{entrypoint_layer}") task_toml.write_text( _remove_toml_key_from_table(task_toml.read_text(), "environment", "docker_image") ) @@ -389,7 +398,9 @@ def _append_proxy_allowlist(proxy_assets: Path, hosts: tuple[str, ...]) -> None: allowlist_path = proxy_assets / "allowlist-base.txt" base = allowlist_path.read_text().rstrip() additions = "\n".join(dict.fromkeys(hosts)) - allowlist_path.write_text(f"{base}\n\n# Dataset-required package and data sources.\n{additions}\n") + allowlist_path.write_text( + f"{base}\n\n# Dataset-required package and data sources.\n{additions}\n" + ) def _merge_compose(task_dir: Path, proxy_allowlist_hosts: tuple[str, ...]) -> dict[str, Any]: @@ -449,7 +460,11 @@ def _merge_compose(task_dir: Path, proxy_allowlist_hosts: tuple[str, ...]) -> di "proxy": {"condition": "service_healthy"}, } main["volumes"] = [ - *([str(item) for item in main.get("volumes", [])] if isinstance(main.get("volumes"), list) else []), + *( + [str(item) for item in main.get("volumes", [])] + if isinstance(main.get("volumes"), list) + else [] + ), "proxy-ca-public:/etc/proxy-ca:ro", ] @@ -506,7 +521,6 @@ def _merge_compose(task_dir: Path, proxy_allowlist_hosts: tuple[str, ...]) -> di def prepare_dataset( - *, source_dataset: str, source_dir: Path | None, output_dir: Path, @@ -518,6 +532,7 @@ def prepare_dataset( "CLAUDE_CODE_VERSION", "CODEX_VERSION", "HERMES_VERSION", + "MINI_SWE_AGENT_VERSION", "NODE_VERSION", "OPENCODE_VERSION", } @@ -582,7 +597,9 @@ def _cli_main(argv: list[str] | None = None) -> int: parser.add_argument("--source-dataset", default=DEFAULT_SOURCE_DATASET) parser.add_argument("--source-dir", type=Path, default=None) parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) - parser.add_argument("--harbor-command", default=os.environ.get("HARBOR_COMMAND", "uv run --no-sync harbor")) + parser.add_argument( + "--harbor-command", default=os.environ.get("HARBOR_COMMAND", "uv run --no-sync harbor") + ) parser.add_argument("--overwrite", action="store_true") ns = parser.parse_args(argv) diff --git a/benchmark/run-baseline.sh b/benchmark/run-baseline.sh index bcadfc043..6b7e31d44 100755 --- a/benchmark/run-baseline.sh +++ b/benchmark/run-baseline.sh @@ -24,6 +24,7 @@ fi CLAUDE_CODE_VERSION="${CLAUDE_CODE_VERSION:-2.1.211}" CODEX_VERSION="${CODEX_VERSION:-0.144.5}" OPENCODE_VERSION="${OPENCODE_VERSION:-1.18.3}" +MINI_SWE_AGENT_VERSION="${MINI_SWE_AGENT_VERSION:-2.4.6}" NODE_VERSION="${NODE_VERSION:-20.11.1}" DEFAULT_HARBOR_MODEL="openai/gpt-5.2" @@ -38,6 +39,7 @@ SERVER_CONFIG="" MODEL="" ROUTE_MODEL="" AGENT="terminus-2" +AGENT_IMPORT_PATH="" HARBOR_MODEL="${DEFAULT_HARBOR_MODEL}" HARBOR_PATH="" HARBOR_BIN="${HARBOR_BIN:-}" @@ -101,6 +103,7 @@ Main options: agents, and MODEL for claude-code/codex. --route-model MODEL Deprecated alias for --model in Switchyard mode. --agent NAME Harbor agent (default: terminus-2) + --agent-import-path PATH Custom Harbor agent in module.path:ClassName form. --harbor-model MODEL Explicit Harbor model label override. --upstream-base-url URL Direct-upstream OpenAI-compatible base URL (default: https://openrouter.ai/api/v1). @@ -305,6 +308,7 @@ while [[ $# -gt 0 ]]; do --model) MODEL="$2"; shift 2 ;; --route-model) ROUTE_MODEL="$2"; shift 2 ;; --agent) AGENT="$2"; shift 2 ;; + --agent-import-path) AGENT_IMPORT_PATH="$2"; shift 2 ;; --harbor-model) HARBOR_MODEL="$2"; HARBOR_MODEL_SET=1; shift 2 ;; --upstream-base-url) UPSTREAM_BASE_URL="$2"; shift 2 ;; --upstream-api-key-env) UPSTREAM_API_KEY_ENV="$2"; shift 2 ;; @@ -538,15 +542,23 @@ fi WRAPPER="${RUN_DIR}/run-background.sh" HARBOR_JOB_DIR="${RUN_DIR}/jobs/${JOB_NAME}" -HARBOR_CMD=("${HARBOR_CMD_PREFIX[@]}" run - --agent "${AGENT}" - --model "${HARBOR_MODEL}" +HARBOR_CMD=("${HARBOR_CMD_PREFIX[@]}" run) +if [[ -n "${AGENT_IMPORT_PATH}" ]]; then + HARBOR_CMD+=(--agent-import-path "${AGENT_IMPORT_PATH}") +else + HARBOR_CMD+=(--agent "${AGENT}") +fi +HARBOR_CMD+=(--model "${HARBOR_MODEL}" --jobs-dir "${RUN_DIR}/jobs" --job-name "${JOB_NAME}" -n "${N_CONCURRENT}" --max-retries "${MAX_RETRIES}" --agent-timeout-multiplier "${AGENT_TIMEOUT_MULTIPLIER}") +if [[ "${AGENT}" == "mini-swe-agent" ]]; then + HARBOR_CMD+=(--ak "version=${MINI_SWE_AGENT_VERSION}") +fi + HARBOR_CMD+=(--path "${HARBOR_PATH}") if [[ -n "${CODEX_MODEL_CATALOG_HOST}" ]]; then @@ -710,7 +722,7 @@ print_resolved() { shell_join "${HARBOR_PYTHON_CMD[@]}" echo echo " harbor_patch: verified" - echo " agent/model: ${AGENT} / ${HARBOR_MODEL}" + echo " agent/model: ${AGENT_IMPORT_PATH:-${AGENT}} / ${HARBOR_MODEL}" echo " reasoning: ${REASONING_EFFORT:-unset}" if [[ -n "${CODEX_MODEL_CATALOG_HOST}" ]]; then echo " codex_catalog: ${CODEX_MODEL_CATALOG_HOST}" @@ -747,7 +759,8 @@ fi AGENT_VERSIONS_JSON="$(json_object_from_pairs \ claude_code "${CLAUDE_CODE_VERSION}" codex "${CODEX_VERSION}" \ - opencode "${OPENCODE_VERSION}" node "${NODE_VERSION}")" + opencode "${OPENCODE_VERSION}" mini_swe_agent "${MINI_SWE_AGENT_VERSION}" \ + node "${NODE_VERSION}")" HARBOR_EXTRA_JSON="$(json_array)" if [[ "${#HARBOR_EXTRA[@]}" -gt 0 ]]; then HARBOR_EXTRA_JSON="$(json_array "${HARBOR_EXTRA[@]}")" @@ -855,11 +868,31 @@ cleanup() { docker rm -f "\${SWITCHYARD_DOCKER_CONTAINER}" >/dev/null 2>&1 || true fi if [[ "\${DOCKER_NETWORK_CREATED}" == "1" ]]; then - docker network rm "\${SWITCHYARD_DOCKER_NETWORK}" >/dev/null 2>&1 || true + docker_with_timeout network rm "\${SWITCHYARD_DOCKER_NETWORK}" >/dev/null 2>&1 || true fi } trap cleanup EXIT +docker_with_timeout() { + if command -v timeout >/dev/null 2>&1; then + timeout 30 docker "\$@" + else + docker "\$@" + fi +} + +ensure_docker_network() { + if docker_with_timeout network inspect "\${SWITCHYARD_DOCKER_NETWORK}" >/dev/null 2>&1; then + return + fi + if ! docker_with_timeout network create "\${SWITCHYARD_DOCKER_NETWORK}" >/dev/null; then + echo "ERROR: Docker could not inspect or create network \${SWITCHYARD_DOCKER_NETWORK}." >&2 + echo " Check the Docker network API before retrying this benchmark." >&2 + exit 127 + fi + DOCKER_NETWORK_CREATED=1 +} + export PYTHONHASHSEED=0 export LC_ALL=C.UTF-8 @@ -872,10 +905,7 @@ if [[ "\${SERVER_ENABLED}" == "1" ]]; then if [[ "\${SWITCHYARD_DOCKER_BUILD}" != "0" ]]; then docker build -f "\${SWITCHYARD_DOCKERFILE}" -t "\${SWITCHYARD_DOCKER_IMAGE}" . fi - if ! docker network inspect "\${SWITCHYARD_DOCKER_NETWORK}" >/dev/null 2>&1; then - docker network create "\${SWITCHYARD_DOCKER_NETWORK}" - DOCKER_NETWORK_CREATED=1 - fi + ensure_docker_network docker rm -f "\${SWITCHYARD_DOCKER_CONTAINER}" >/dev/null 2>&1 || true DOCKER_RUN_ARGS=( -d --rm @@ -920,10 +950,7 @@ else echo "Switchyard disabled; using direct upstream" echo " network: \${SWITCHYARD_DOCKER_NETWORK}" echo " upstream: \${HARBOR_BASE_URL}" - if ! docker network inspect "\${SWITCHYARD_DOCKER_NETWORK}" >/dev/null 2>&1; then - docker network create "\${SWITCHYARD_DOCKER_NETWORK}" - DOCKER_NETWORK_CREATED=1 - fi + ensure_docker_network } > "\${SERVER_LOG}" 2>&1 echo "\${SWITCHYARD_DOCKER_NETWORK}" > "\${RUN_DIR}/docker_network" fi @@ -979,6 +1006,7 @@ else export OPENAI_API_KEY="\${!UPSTREAM_API_KEY_ENV}" fi export OPENAI_BASE_URL="\${HARBOR_BASE_URL}" +export OPENAI_API_BASE="\${HARBOR_BASE_URL}" if [[ "\${BOOK_MODE}" == "closed" ]]; then export CLOSED_BOOK_MODE=1 else diff --git a/benchmark/test_time_scaling_config.py b/benchmark/test_time_scaling_config.py new file mode 100644 index 000000000..7f15c2ecb --- /dev/null +++ b/benchmark/test_time_scaling_config.py @@ -0,0 +1,256 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Write configs for one SWE-bench test-time scaling run.""" + +import argparse +import hashlib +import json +import subprocess +from pathlib import Path +from typing import Any + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("--dataset", type=Path, required=True) + parser.add_argument("--task", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--base-url", default="https://inference-api.nvidia.com/v1") + parser.add_argument("--api-key-env", default="NVIDIA_API_KEY") + parser.add_argument("--rollouts", type=int, default=16) + parser.add_argument("--refinement-count", type=int, default=4) + parser.add_argument("--group-size", type=int, default=2) + parser.add_argument("--votes", type=int, default=8) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--concurrency", type=int, default=16) + return parser + + +def _choice(value: str, source: str = "reconstructed") -> dict[str, str]: + return {"source": source, "value": value} + + +def _revision(repo: Path) -> str: + head = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repo, + check=True, + text=True, + capture_output=True, + ).stdout.strip() + dirty = subprocess.run( + ["git", "status", "--porcelain"], + cwd=repo, + check=True, + text=True, + capture_output=True, + ).stdout + return f"{head}-dirty" if dirty else head + + +def _manifest( + revision: str, + model: str, + dataset_label: str, + base_url: str, + rollouts: int, + refinement_count: int, + group_size: int, + votes: int, + seed: int, + concurrency: int, +) -> dict[str, Any]: + fields = { + "exact_prompts": _choice(f"prompt constants in source revision {revision}"), + "summary_schema": _choice("one JSON object; no fixed JSON Schema"), + "malformed_summary_policy": _choice( + "accept a bare JSON object or one JSON code fence; retry once, then stop" + ), + "model_ids_and_api_revisions": _choice(f"{model} through {base_url}"), + "role_inference_settings": _choice( + "same model for every role; mini-swe-agent action defaults; summary temperature 0.0; vote temperature 0.4; provider seed omitted" + ), + "scaffold_revisions_and_protocols": _choice( + "Harbor from uv.lock; mini-swe-agent 2.4.6; native tool-calling" + ), + "agent_limits": _choice("task timeout with multiplier 1.0"), + "summary_serialization": _choice("original task plus patch and native trajectory JSON"), + "pairing_order": _choice("in order"), + "display_order": _choice("in order"), + "tie_break": _choice("first candidate in the group"), + "invalid_vote_policy": _choice("stop the run"), + "model_retry_policy": _choice( + "Harbor retries a failed rollout twice; summary and vote calls use up to three HTTP attempts; summary JSON uses two content attempts" + ), + "experiment_seeds": _choice(f"root seed {seed}; provider seed omitted"), + "benchmark_revisions": _choice(dataset_label, "repository"), + "terminal_bench_task_list": _choice("not applicable to SWE-bench Verified"), + "summary_input_contents": _choice( + "task, patch, and trajectory only; official verification runs after selection" + ), + "refinement_summary_order": _choice("tournament survivor order"), + "observation_truncation": _choice("keep the first and last 100000 characters"), + "unfinished_rollout_policy": _choice("keep the returned patch and trajectory"), + "concurrency_and_rate_limits": _choice( + f"Harbor and model concurrency {concurrency}; gateway retries bounded" + ), + "ablation_fixed_settings": _choice( + f"N={rollouts}, K={refinement_count}, G={group_size}, V={votes}", + "paper" + if (rollouts, refinement_count, group_size, votes) == (16, 4, 2, 8) + else "reconstructed", + ), + } + return { + "schema_version": 1, + "replication_mode": "conceptual", + "code_revision": revision, + "model_id": model, + "fields": fields, + } + + +def write_configs(args: argparse.Namespace, repo: Path) -> tuple[Path, Path]: + """Write the Harbor and Rust runner configs and return their paths.""" + dataset = args.dataset.resolve() + task_dir = dataset / args.task + instruction = task_dir / "instruction.md" + dataset_manifest = dataset / "switchyard_dataset_manifest.json" + if not instruction.is_file() or not dataset_manifest.is_file(): + raise FileNotFoundError( + "dataset must contain the task and switchyard_dataset_manifest.json" + ) + if ( + min( + args.rollouts, + args.refinement_count, + args.group_size, + args.votes, + args.concurrency, + ) + <= 0 + ): + raise ValueError("counts and concurrency must be positive") + if args.refinement_count > args.rollouts: + raise ValueError("refinement count must not exceed rollout count") + if args.group_size < 2: + raise ValueError("group size must be at least two") + output = args.output.resolve() + config_dir = output / "config" + config_dir.mkdir(parents=True, exist_ok=False) + harbor_path = config_dir / "harbor.json" + runner_path = config_dir / "runner.json" + grade_root = output / "private-grades" + dataset_data = json.loads(dataset_manifest.read_text()) + dataset_digest = hashlib.sha256(dataset_manifest.read_bytes()).hexdigest() + dataset_label = ( + f"{dataset_data.get('source_dataset', dataset.name)}; manifest_sha256={dataset_digest}" + ) + revision = _revision(repo) + + harbor = { + "repo_root": str(repo), + "dataset_root": str(dataset), + "run_baseline": str(repo / "benchmark" / "run-baseline.sh"), + "method_root": str(output / "harbor"), + "private_grade_root": str(grade_root), + "run_record": str(output / "method" / "run.json"), + "task_id": args.task, + "model_id": args.model, + "harbor_model": f"openai/{args.model}", + "agent_import_path": ( + "benchmark.test_time_scaling_harbor_agent:TestTimeScalingMiniSweAgent" + ), + "n_concurrent": args.concurrency, + "max_retries": 2, + "agent_timeout_multiplier": 1.0, + "upstream_base_url": args.base_url, + "upstream_api_key_env": args.api_key_env, + "harbor_command": ["uv", "run", "--no-sync", "harbor"], + } + runner = { + "task": { + "id": args.task, + "benchmark": "swebench-verified", + "prompt": instruction.read_text(), + }, + "scaling": { + "rollout_count": args.rollouts, + "refinement_count": args.refinement_count, + "group_size": args.group_size, + "votes_per_group": args.votes, + "seed": args.seed, + "pairing_order": "in_order", + "display_order": "in_order", + "tie_policy": "first_in_group", + "invalid_vote_policy": {"mode": "abort"}, + }, + "manifest": _manifest( + revision, + args.model, + dataset_label, + args.base_url, + args.rollouts, + args.refinement_count, + args.group_size, + args.votes, + args.seed, + args.concurrency, + ), + "output_dir": str(output / "method"), + "rollout_command": { + "argv": [ + "uv", + "run", + "--no-sync", + "python", + "benchmark/test_time_scaling_rollouts.py", + "--config", + str(harbor_path), + ] + }, + "evaluation_command": { + "argv": [ + "uv", + "run", + "--no-sync", + "python", + "benchmark/test_time_scaling_grades.py", + "--config", + str(harbor_path), + ] + }, + "model": { + "base_url": args.base_url, + "api_key_env": args.api_key_env, + "max_concurrency": args.concurrency, + "summary_max_tokens": 4096, + "comparison_max_tokens": 4096, + "max_summary_input_chars": 200000, + "summary_content_attempts": 2, + "http_attempts": 3, + "request_timeout_seconds": 180, + "summary_temperature": 0.0, + "comparison_temperature": 0.4, + "send_seed": False, + }, + } + harbor_path.write_text(json.dumps(harbor, indent=2)) + runner_path.write_text(json.dumps(runner, indent=2)) + return runner_path, harbor_path + + +def main() -> int: + args = _parser().parse_args() + repo = Path(__file__).resolve().parents[1] + runner_path, harbor_path = write_configs(args, repo) + print(f"wrote runner config: {runner_path}") + print(f"wrote Harbor config: {harbor_path}") + print(f"run: cargo run -p switchyard-test-time-scaling-runner -- {runner_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/test_time_scaling_grades.py b/benchmark/test_time_scaling_grades.py new file mode 100644 index 000000000..d5fb9d102 --- /dev/null +++ b/benchmark/test_time_scaling_grades.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run official Harbor verification after test-time selection finishes.""" + +import argparse +import json +import os +import secrets +import subprocess +import sys +from pathlib import Path +from typing import Any + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("--config", type=Path, required=True) + return parser + + +def _object(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text()) + if not isinstance(value, dict): + raise ValueError(f"{path} must contain one JSON object") + return value + + +def _records(path: Path) -> list[dict[str, Any]]: + value = json.loads(path.read_text()) + if not isinstance(value, list) or any(not isinstance(item, dict) for item in value): + raise ValueError(f"{path} must contain a list of objects") + return value + + +def _string_list(value: Any, name: str) -> list[str]: + if not isinstance(value, list) or not value or any(not isinstance(item, str) for item in value): + raise ValueError(f"{name} must contain one or more strings") + return value + + +def _grade_records(grade_root: Path) -> list[dict[str, Any]]: + mappings = sorted(grade_root.glob("iteration-*.json")) + if len(mappings) != 2: + raise RuntimeError(f"expected two patch maps; found {len(mappings)}") + return [record for mapping in mappings for record in _records(mapping)] + + +def _passed(trial_dir: Path) -> bool: + reward_path = trial_dir / "verifier" / "reward.txt" + if not reward_path.is_file(): + raise FileNotFoundError(f"missing official reward: {reward_path}") + return float(reward_path.read_text().strip()) > 0.0 + + +def _run_trial( + harbor_command: list[str], + task_dir: Path, + trials_dir: Path, + rollout_id: str, + patch_file: Path, + verifier_proxy: str, + repo_root: Path, + env: dict[str, str], +) -> bool: + command = [ + *harbor_command, + "trial", + "start", + "--path", + str(task_dir), + "--trial-name", + rollout_id, + "--trials-dir", + str(trials_dir), + "--agent-import-path", + "benchmark.test_time_scaling_patch_agent:TestTimeScalingPatchAgent", + "--agent-kwarg", + f"patch_file={patch_file}", + "--ve", + f"HTTP_PROXY={verifier_proxy}", + "--ve", + f"HTTPS_PROXY={verifier_proxy}", + "--ve", + f"http_proxy={verifier_proxy}", + "--ve", + f"https_proxy={verifier_proxy}", + "--ve", + "NO_PROXY=localhost,127.0.0.1,proxy", + "--ve", + "no_proxy=localhost,127.0.0.1,proxy", + ] + subprocess.run( + command, cwd=repo_root, env=env, check=True, stdout=sys.stderr, stderr=sys.stderr + ) + return _passed(trials_dir / rollout_id) + + +def main() -> int: + args = _parser().parse_args() + config = _object(args.config) + run_record = Path(config["run_record"]).resolve() + if not run_record.is_file(): + raise FileNotFoundError(f"method record must be saved before grading: {run_record}") + grade_root = Path(config["private_grade_root"]).resolve() + repo_root = Path(config["repo_root"]).resolve() + task_dir = Path(config["dataset_root"]).resolve() / str(config["task_id"]) + trials_dir = grade_root / "trials" + trials_dir.mkdir(parents=True, exist_ok=True) + network = f"switchyard-grade-{secrets.token_hex(6)}" + token = secrets.token_urlsafe(32) + verifier_proxy = f"http://verifier:{token}@proxy:3129" + env = dict(os.environ) + env.update( + { + "ALLOWED_HOSTS": "", + "CLOSED_BOOK_MODE": "1", + "SWITCHYARD_DOCKER_NETWORK": network, + "SWITCHYARD_VERIFIER_PROXY_TOKEN": token, + } + ) + subprocess.run(["docker", "network", "create", network], check=True, timeout=30) + try: + outcomes = [ + { + "rollout_id": str(record["rollout_id"]), + "passed": _run_trial( + _string_list(config.get("harbor_command"), "harbor_command"), + task_dir, + trials_dir, + str(record["rollout_id"]), + Path(record["patch_file"]), + verifier_proxy, + repo_root, + env, + ), + } + for record in _grade_records(grade_root) + ] + finally: + subprocess.run(["docker", "network", "rm", network], check=False, timeout=30) + json.dump(outcomes, sys.stdout) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/test_time_scaling_harbor_agent.py b/benchmark/test_time_scaling_harbor_agent.py new file mode 100644 index 000000000..d5ad7a8d3 --- /dev/null +++ b/benchmark/test_time_scaling_harbor_agent.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Harbor mini-SWE-agent adapter that saves the final repository patch.""" + +from harbor.agents.installed.mini_swe_agent import MiniSweAgent +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext + +PATCH_COMMAND = ( + "mkdir -p /logs/artifacts && " + "git -C /testbed add -A && " + "git -C /testbed diff --cached --binary --no-ext-diff " + "> /logs/artifacts/patch.diff" +) + + +class TestTimeScalingMiniSweAgent(MiniSweAgent): # type: ignore[misc] + """Run mini-SWE-agent and save the final repository patch.""" + + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + try: + await super().run(instruction, environment, context) + finally: + await self.exec_as_agent( + environment, + command=PATCH_COMMAND, + ) diff --git a/benchmark/test_time_scaling_patch_agent.py b/benchmark/test_time_scaling_patch_agent.py new file mode 100644 index 000000000..b98595fec --- /dev/null +++ b/benchmark/test_time_scaling_patch_agent.py @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Harbor agent that applies one saved patch without calling a model.""" + +import base64 +import hashlib +import logging +import shlex +from pathlib import Path + +from harbor.agents.base import BaseAgent +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext +from harbor.models.task.config import MCPServerConfig + + +class TestTimeScalingPatchAgent(BaseAgent): # type: ignore[misc] + """Apply a saved patch so Harbor can run the official verifier later.""" + + def __init__( + self, + logs_dir: Path, + patch_file: str, + model_name: str | None = None, + logger: logging.Logger | None = None, + mcp_servers: list[MCPServerConfig] | None = None, + skills_dir: str | None = None, + extra_env: dict[str, str] | None = None, + ) -> None: + super().__init__( + logs_dir=logs_dir, + model_name=model_name, + logger=logger, + mcp_servers=mcp_servers, + skills_dir=skills_dir, + ) + self.patch_file = Path(patch_file) + del extra_env + + @staticmethod + def name() -> str: + """Return the name stored in the Harbor result.""" + return "test-time-scaling-patch" + + def version(self) -> str: + """Return this small agent's record format version.""" + return "1" + + async def setup(self, environment: BaseEnvironment) -> None: + """Use the task image without installing an agent.""" + + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + """Apply the patch in the clean task checkout.""" + del instruction + patch = self.patch_file.read_bytes() + context.metadata = {"patch_sha256": hashlib.sha256(patch).hexdigest()} + if not patch: + return + encoded = base64.b64encode(patch).decode() + result = await environment.exec( + command=(f"printf %s {shlex.quote(encoded)} | base64 --decode | git apply --binary -"), + cwd="/testbed", + ) + if result.return_code != 0: + raise RuntimeError(f"could not apply saved patch: {result.stderr}") diff --git a/benchmark/test_time_scaling_rollouts.py b/benchmark/test_time_scaling_rollouts.py new file mode 100644 index 000000000..55e6e11b7 --- /dev/null +++ b/benchmark/test_time_scaling_rollouts.py @@ -0,0 +1,275 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run one iteration of fresh Harbor rollouts and return method-only records.""" + +import argparse +import hashlib +import json +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("--config", type=Path, required=True) + return parser + + +def _read_object(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text()) + if not isinstance(value, dict): + raise ValueError(f"{path} must contain one JSON object") + return value + + +def _read_batch() -> dict[str, Any]: + value = json.load(sys.stdin) + if not isinstance(value, dict): + raise ValueError("stdin must contain one JSON object") + return value + + +def _single_iteration(requests: list[dict[str, Any]]) -> int: + iterations: set[int] = set() + indexes: list[int] = [] + for request in requests: + iteration = request.get("iteration") + rollout_index = request.get("rollout_index") + if not isinstance(iteration, int) or not isinstance(rollout_index, int): + raise ValueError("iteration and rollout_index must be integers") + iterations.add(iteration) + indexes.append(rollout_index) + if len(iterations) != 1: + raise ValueError("one Harbor batch must contain one iteration") + iteration = iterations.pop() + if iteration not in (0, 1): + raise ValueError("iteration must be 0 or 1") + expected = list(range(len(requests))) + actual = sorted(indexes) + if actual != expected: + raise ValueError("rollout indexes must be unique and contiguous") + return iteration + + +def _safe_task_id(task_id: str) -> str: + if not task_id or task_id in (".", "..") or Path(task_id).name != task_id: + raise ValueError("task id must be one path component") + return task_id + + +def _prepare_dataset( + source_root: Path, + method_root: Path, + task: dict[str, Any], + requests: list[dict[str, Any]], + iteration: int, +) -> Path: + task_id = _safe_task_id(str(task.get("id", ""))) + source_task = source_root / task_id + if not source_task.is_dir(): + raise FileNotFoundError(f"task not found: {source_task}") + dataset_root = method_root / f"dataset-iteration-{iteration}" + if dataset_root.exists(): + raise FileExistsError(f"iteration dataset already exists: {dataset_root}") + dataset_root.mkdir(parents=True) + shutil.copy2(source_root / "switchyard_dataset_manifest.json", dataset_root) + copied_task = dataset_root / task_id + shutil.copytree(source_task, copied_task) + + instruction_path = copied_task / "instruction.md" + original = instruction_path.read_text() + expected_prompt = str(task.get("prompt", "")) + if original.strip() != expected_prompt.strip(): + raise ValueError("configured task prompt does not match instruction.md") + refinement_prompts = {request.get("refinement_prompt") for request in requests} + if iteration == 0 and refinement_prompts != {None}: + raise ValueError("iteration zero must not contain refinement context") + if iteration == 1: + if len(refinement_prompts) != 1 or None in refinement_prompts: + raise ValueError("every refined rollout must use the same context") + refinement = refinement_prompts.pop() + instruction_path.write_text(f"{refinement}\n\nORIGINAL TASK\n{original}") + return dataset_root + + +def _harbor_command( + config: dict[str, Any], + dataset_root: Path, + task_id: str, + attempt_count: int, + iteration: int, +) -> list[str]: + run_root = Path(config["method_root"]).resolve() / f"harbor-iteration-{iteration}" + return [ + "bash", + str(Path(config["run_baseline"]).resolve()), + "--output-dir", + str(run_root), + "--harbor-path", + str(dataset_root), + "--model", + str(config["model_id"]), + "--agent", + "mini-swe-agent", + "--agent-import-path", + str(config["agent_import_path"]), + "--harbor-model", + str(config["harbor_model"]), + "--n-concurrent", + str(config["n_concurrent"]), + "--max-retries", + str(config["max_retries"]), + "--agent-timeout-multiplier", + str(config["agent_timeout_multiplier"]), + "--task-id", + task_id, + "--upstream-base-url", + str(config["upstream_base_url"]), + "--upstream-api-key-env", + str(config["upstream_api_key_env"]), + "--harbor-extra", + "-k", + "--harbor-extra", + str(attempt_count), + "--harbor-extra", + "--disable-verification", + "--foreground", + ] + + +def _run_harbor( + config: dict[str, Any], + dataset_root: Path, + task_id: str, + attempt_count: int, + iteration: int, +) -> Path: + repo_root = Path(config["repo_root"]).resolve() + run_root = Path(config["method_root"]).resolve() / f"harbor-iteration-{iteration}" + if run_root.exists(): + raise FileExistsError(f"Harbor output already exists: {run_root}") + command = _harbor_command(config, dataset_root, task_id, attempt_count, iteration) + subprocess.run(command, cwd=repo_root, check=True, stdout=sys.stderr, stderr=sys.stderr) + runs = [path for path in run_root.iterdir() if path.is_dir()] + if len(runs) != 1: + raise RuntimeError(f"expected one Harbor run under {run_root}; found {len(runs)}") + return runs[0] + + +def _trial_directories(run_dir: Path) -> list[Path]: + job_roots = [path for path in (run_dir / "jobs").iterdir() if path.is_dir()] + if len(job_roots) != 1: + raise RuntimeError(f"expected one Harbor job under {run_dir}") + return sorted( + path for path in job_roots[0].iterdir() if path.is_dir() and (path / "agent").is_dir() + ) + + +def _read_trajectory(trial_dir: Path) -> Any: + native = trial_dir / "agent" / "mini-swe-agent.trajectory.json" + if native.is_file(): + return json.loads(native.read_text()) + converted = trial_dir / "agent" / "trajectory.json" + if converted.is_file(): + return json.loads(converted.read_text()) + log = trial_dir / "agent" / "mini-swe-agent.txt" + return {"agent_log": log.read_text() if log.is_file() else ""} + + +def _public_rollout( + task_id: str, + model_id: str, + request: dict[str, Any], + trial_dir: Path, +) -> dict[str, Any]: + patch_path = trial_dir / "artifacts" / "patch.diff" + patch = patch_path.read_text() if patch_path.is_file() else "" + digest = hashlib.sha256(patch.encode()).hexdigest() + iteration = int(request["iteration"]) + rollout_index = int(request["rollout_index"]) + rollout_id = f"{task_id}-iteration-{iteration}-rollout-{rollout_index}" + return { + "id": rollout_id, + "iteration": iteration, + "rollout_index": rollout_index, + "model_id": model_id, + "environment_id": trial_dir.name, + "output_digest": f"sha256:{digest}", + "output": { + "patch": patch, + "trajectory": _read_trajectory(trial_dir), + }, + } + + +def _write_private_grade_map( + grade_root: Path, + iteration: int, + pairs: list[tuple[dict[str, Any], Path]], +) -> None: + grade_root.mkdir(parents=True, exist_ok=True) + path = grade_root / f"iteration-{iteration}.json" + if path.exists(): + raise FileExistsError(f"grade map already exists: {path}") + patch_root = grade_root / "patches" + patch_root.mkdir(exist_ok=True) + records = [] + for rollout, _trial_dir in pairs: + patch_path = patch_root / f"{rollout['id']}.diff" + patch_path.write_text(str(rollout["output"]["patch"])) + records.append({"rollout_id": rollout["id"], "patch_file": str(patch_path.resolve())}) + path.write_text(json.dumps(records, indent=2)) + + +def main() -> int: + args = _parser().parse_args() + config = _read_object(args.config) + expected_harbor_model = f"openai/{config['model_id']}" + if config.get("harbor_model") != expected_harbor_model: + raise ValueError(f"harbor_model must be {expected_harbor_model}") + batch = _read_batch() + task = batch.get("task") + requests = batch.get("requests") + if not isinstance(task, dict) or not isinstance(requests, list) or not requests: + raise ValueError("batch must contain a task and at least one request") + if any(not isinstance(request, dict) for request in requests): + raise ValueError("every rollout request must be an object") + iteration = _single_iteration(requests) + method_root = Path(config["method_root"]).resolve() + method_root.mkdir(parents=True, exist_ok=True) + source_root = Path(config["dataset_root"]).resolve() + dataset_root = _prepare_dataset(source_root, method_root, task, requests, iteration) + task_id = _safe_task_id(str(task["id"])) + run_dir = _run_harbor( + config, + dataset_root, + task_id, + len(requests), + iteration, + ) + trials = _trial_directories(run_dir) + if len(trials) != len(requests): + raise RuntimeError(f"Harbor returned {len(trials)} trials; expected {len(requests)}") + ordered_requests = sorted(requests, key=lambda request: request["rollout_index"]) + pairs = [ + ( + _public_rollout(task_id, str(config["model_id"]), request, trial), + trial, + ) + for request, trial in zip(ordered_requests, trials, strict=True) + ] + _write_private_grade_map( + Path(config["private_grade_root"]).resolve(), + iteration, + pairs, + ) + json.dump([rollout for rollout, _trial in pairs], sys.stdout) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/crates/switchyard-test-time-scaling-runner/Cargo.toml b/crates/switchyard-test-time-scaling-runner/Cargo.toml new file mode 100644 index 000000000..316d002ee --- /dev/null +++ b/crates/switchyard-test-time-scaling-runner/Cargo.toml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "switchyard-test-time-scaling-runner" +version.workspace = true +description = "Harbor and model runner for agentic test-time scaling" +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +async-trait.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +switchyard-test-time-scaling.workspace = true +tokio.workspace = true diff --git a/crates/switchyard-test-time-scaling-runner/src/backend.rs b/crates/switchyard-test-time-scaling-runner/src/backend.rs new file mode 100644 index 000000000..1a739f5d5 --- /dev/null +++ b/crates/switchyard-test-time-scaling-runner/src/backend.rs @@ -0,0 +1,191 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Live Harbor and model backend. + +use async_trait::async_trait; +use serde::Serialize; +use serde_json::{Map, Value}; +use std::path::PathBuf; +use switchyard_test_time_scaling::{ + ComparisonRequest, ComparisonResponse, Result, Rollout, RolloutRequest, ScalingBackend, + ScalingError, Summary, Task, +}; + +use crate::command::call_json; +use crate::config::{CommandConfig, ModelConfig}; +use crate::model_client::ModelClient; + +const SUMMARY_PROMPT: &str = "You are given an agentic coding task and one recorded attempt. Produce one structured summary as a JSON object. Report evidence, not just the agent's claims. Separate commands with captured responses from proposed commands, successful outputs from errors, fixed errors from unresolved errors, files proved to exist from files only claimed, tests run after the final edit from earlier tests, and verified requirements from unchecked requirements. Preserve exact paths, functions, commands, return codes, decisive output, code changes, unresolved issues, and uncertainty. Do not infer hidden-test success. Do not include any official grader result. Output JSON only."; + +#[derive(Serialize)] +struct RolloutBatch<'a> { + task: &'a Task, + requests: Vec, +} + +/// Adapter used by the core scaling controller. +pub struct LiveBackend { + model_id: String, + rollout_command: CommandConfig, + model_config: ModelConfig, + model: ModelClient, +} + +impl LiveBackend { + /// Creates a backend after loading the configured API key. + pub fn new( + model_id: String, + rollout_command: CommandConfig, + model_config: ModelConfig, + call_log_path: PathBuf, + ) -> std::result::Result { + let model = ModelClient::new(model_id.clone(), &model_config, call_log_path)?; + Ok(Self { + model_id, + rollout_command, + model_config, + model, + }) + } +} + +#[async_trait] +impl ScalingBackend for LiveBackend { + type Output = Value; + + fn model_id(&self) -> &str { + &self.model_id + } + + async fn run_rollouts( + &self, + task: &Task, + requests: Vec, + ) -> Result>> { + call_json(&self.rollout_command, &RolloutBatch { task, requests }) + .await + .map_err(ScalingError::Backend) + } + + async fn summarize(&self, task: &Task, rollout: &Rollout) -> Result { + let serialized = serde_json::to_string(&rollout.output) + .map_err(|error| ScalingError::Backend(error.to_string()))?; + let input = truncate_middle(&serialized, self.model_config.max_summary_input_chars); + let mut prompt = format!( + "{SUMMARY_PROMPT}\n\nOriginal task:\n{}\n\nRecorded attempt:\n{}", + task.prompt, input + ); + + for attempt in 1..=self.model_config.summary_content_attempts { + let reply = self + .model + .complete( + "summary", + &prompt, + self.model_config.summary_max_tokens, + self.model_config.summary_temperature, + rollout.rollout_index as u64 + + u64::from(rollout.iteration) * 1_000 + + attempt as u64, + ) + .await + .map_err(ScalingError::Backend)?; + if let Some(value) = parse_json_object(&reply.content) { + return Ok(Summary { + id: format!("summary-{}", rollout.id), + rollout_id: rollout.id.clone(), + model_id: self.model_id.clone(), + value, + raw_response: reply.raw_response, + generation_attempts: attempt, + }); + } + prompt.push_str(&format!( + "\n\nThe previous response was not one JSON object. Return a corrected JSON object only. Previous response:\n{}", + truncate_middle(&reply.content, 4_000) + )); + } + Err(ScalingError::Backend(format!( + "summary for {} was not a JSON object after {} attempts", + rollout.id, self.model_config.summary_content_attempts + ))) + } + + async fn compare( + &self, + _task: &Task, + request: ComparisonRequest, + ) -> Result { + let reply = self + .model + .complete( + "comparison", + &request.prompt, + self.model_config.comparison_max_tokens, + self.model_config.comparison_temperature, + request.seed, + ) + .await + .map_err(ScalingError::Backend)?; + Ok(ComparisonResponse { + model_id: self.model_id.clone(), + content: reply.content, + }) + } +} + +fn parse_json_object(content: &str) -> Option> { + let trimmed = content.trim(); + if let Ok(value) = serde_json::from_str::>(trimmed) { + return Some(value); + } + let fenced = trimmed + .strip_prefix("```json") + .or_else(|| trimmed.strip_prefix("```JSON")) + .or_else(|| trimmed.strip_prefix("```"))? + .strip_suffix("```")? + .trim(); + serde_json::from_str(fenced).ok() +} + +fn truncate_middle(text: &str, limit: usize) -> String { + if text.chars().count() <= limit { + return text.to_string(); + } + let left_count = limit / 2; + let right_count = limit - left_count; + let left: String = text.chars().take(left_count).collect(); + let mut right: Vec = text.chars().rev().take(right_count).collect(); + right.reverse(); + format!( + "{left}\n[... middle removed by recorded summary input limit ...]\n{}", + right.into_iter().collect::() + ) +} + +#[cfg(test)] +mod tests { + use super::{parse_json_object, truncate_middle}; + + #[test] + fn truncation_keeps_both_ends() { + assert_eq!(truncate_middle("short", 5), "short"); + let value = truncate_middle("abcdefghij", 6); + assert!(value.starts_with("abc")); + assert!(value.ends_with("hij")); + } + + #[test] + fn summary_parser_accepts_an_object_with_one_optional_fence() { + assert_eq!( + parse_json_object(r#"{"result":"ok"}"#).unwrap()["result"], + "ok" + ); + assert_eq!( + parse_json_object("```json\n{\"result\":\"ok\"}\n```").unwrap()["result"], + "ok" + ); + assert!(parse_json_object("before {\"result\":\"ok\"}").is_none()); + } +} diff --git a/crates/switchyard-test-time-scaling-runner/src/command.rs b/crates/switchyard-test-time-scaling-runner/src/command.rs new file mode 100644 index 000000000..dccdbcfca --- /dev/null +++ b/crates/switchyard-test-time-scaling-runner/src/command.rs @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shell-free JSON command calls. + +use std::process::Stdio; + +use serde::Serialize; +use serde::de::DeserializeOwned; +use tokio::io::AsyncWriteExt; +use tokio::process::Command; + +use crate::config::CommandConfig; + +/// Sends one JSON value to a command and parses its JSON output. +pub async fn call_json(command: &CommandConfig, input: &I) -> Result +where + I: Serialize, + O: DeserializeOwned, +{ + let Some((program, arguments)) = command.argv.split_first() else { + return Err("command is empty".to_string()); + }; + let mut child = Command::new(program) + .args(arguments) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .map_err(|error| format!("could not start {program}: {error}"))?; + let bytes = serde_json::to_vec(input).map_err(|error| error.to_string())?; + let mut stdin = child + .stdin + .take() + .ok_or_else(|| "command stdin is unavailable".to_string())?; + stdin + .write_all(&bytes) + .await + .map_err(|error| format!("could not write command input: {error}"))?; + drop(stdin); + + let output = child + .wait_with_output() + .await + .map_err(|error| format!("command did not finish: {error}"))?; + if !output.status.success() { + return Err(format!( + "command exited with {}; stderr: {}", + output.status, + bounded_text(&output.stderr, 8_000) + )); + } + serde_json::from_slice(&output.stdout).map_err(|error| { + format!( + "command returned invalid JSON: {error}; stdout: {}", + bounded_text(&output.stdout, 8_000) + ) + }) +} + +fn bounded_text(bytes: &[u8], limit: usize) -> String { + String::from_utf8_lossy(&bytes[..bytes.len().min(limit)]).into_owned() +} diff --git a/crates/switchyard-test-time-scaling-runner/src/config.rs b/crates/switchyard-test-time-scaling-runner/src/config.rs new file mode 100644 index 000000000..1036ad7f6 --- /dev/null +++ b/crates/switchyard-test-time-scaling-runner/src/config.rs @@ -0,0 +1,169 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Runner configuration. + +use std::path::PathBuf; + +use serde::Deserialize; +use switchyard_test_time_scaling::{ExperimentManifest, ScalingConfig, Task}; + +/// One command invoked without a shell. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CommandConfig { + /// Executable followed by its arguments. + pub argv: Vec, +} + +/// NVIDIA-compatible chat completion settings. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ModelConfig { + /// OpenAI-compatible API base URL. + pub base_url: String, + /// Environment variable that contains the API key. + pub api_key_env: String, + /// Simultaneous summary and comparison calls. + pub max_concurrency: usize, + /// Maximum output tokens for a summary. + pub summary_max_tokens: usize, + /// Maximum output tokens for a comparison. + pub comparison_max_tokens: usize, + /// Maximum serialized rollout characters shown to the summarizer. + pub max_summary_input_chars: usize, + /// Content attempts made when a summary is not a JSON object. + pub summary_content_attempts: usize, + /// HTTP attempts for retryable model errors. + pub http_attempts: usize, + /// Request timeout in seconds. + pub request_timeout_seconds: u64, + /// Sampling temperature used for summary calls. + pub summary_temperature: f64, + /// Sampling temperature used for comparison calls. + pub comparison_temperature: f64, + /// Whether to send the recorded logical seed to the provider. + pub send_seed: bool, +} + +/// Complete input for one task run. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RunConfig { + /// Task to run. + pub task: Task, + /// PDR and RTV settings. + pub scaling: ScalingConfig, + /// Replication choices and source labels. + pub manifest: ExperimentManifest, + /// Directory for the method record and post-run evaluation. + pub output_dir: PathBuf, + /// Harbor batch adapter command. It reads a request from stdin and writes rollouts to stdout. + pub rollout_command: CommandConfig, + /// Optional post-selection grader command. It writes rollout outcomes to stdout. + pub evaluation_command: Option, + /// Model API settings. + pub model: ModelConfig, +} + +impl RunConfig { + /// Rejects missing or unsafe runner settings. + pub fn validate(&self) -> Result<(), String> { + validate_command(&self.rollout_command, "rollout_command")?; + if let Some(command) = &self.evaluation_command { + validate_command(command, "evaluation_command")?; + } + if self.output_dir.as_os_str().is_empty() { + return Err("output_dir must not be empty".to_string()); + } + if self.model.base_url.trim().is_empty() || self.model.api_key_env.trim().is_empty() { + return Err("model base_url and api_key_env must not be empty".to_string()); + } + if self.model.max_concurrency == 0 + || self.model.summary_max_tokens == 0 + || self.model.comparison_max_tokens == 0 + || self.model.max_summary_input_chars < 2 + || self.model.summary_content_attempts == 0 + || self.model.http_attempts == 0 + || self.model.request_timeout_seconds == 0 + { + return Err( + "model counts, token limits, input limit, attempts, and timeout must be positive" + .to_string(), + ); + } + if !self.model.summary_temperature.is_finite() + || self.model.summary_temperature < 0.0 + || !self.model.comparison_temperature.is_finite() + || self.model.comparison_temperature < 0.0 + { + return Err("model temperatures must be finite and non-negative".to_string()); + } + Ok(()) + } +} + +fn validate_command(command: &CommandConfig, name: &str) -> Result<(), String> { + if command.argv.is_empty() || command.argv.iter().any(|part| part.trim().is_empty()) { + return Err(format!("{name} must contain non-empty arguments")); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{CommandConfig, ModelConfig, RunConfig}; + use std::collections::BTreeMap; + use std::path::PathBuf; + use switchyard_test_time_scaling::{ + ExperimentManifest, MANIFEST_SCHEMA_VERSION, ReplicationMode, ScalingConfig, Task, + }; + + fn config() -> RunConfig { + RunConfig { + task: Task { + id: "task".to_string(), + benchmark: "benchmark".to_string(), + prompt: "prompt".to_string(), + }, + scaling: ScalingConfig::default(), + manifest: ExperimentManifest { + schema_version: MANIFEST_SCHEMA_VERSION, + replication_mode: ReplicationMode::Conceptual, + code_revision: "revision".to_string(), + model_id: "model".to_string(), + fields: BTreeMap::new(), + }, + output_dir: PathBuf::from("output"), + rollout_command: CommandConfig { + argv: vec!["runner".to_string()], + }, + evaluation_command: None, + model: ModelConfig { + base_url: "https://example.test/v1".to_string(), + api_key_env: "TEST_KEY".to_string(), + max_concurrency: 1, + summary_max_tokens: 1, + comparison_max_tokens: 1, + max_summary_input_chars: 2, + summary_content_attempts: 1, + http_attempts: 1, + request_timeout_seconds: 1, + summary_temperature: 0.0, + comparison_temperature: 0.0, + send_seed: false, + }, + } + } + + #[test] + fn rejects_empty_command_and_zero_limits() { + let mut value = config(); + assert!(value.validate().is_ok()); + value.rollout_command.argv.clear(); + assert!(value.validate().is_err()); + value = config(); + value.model.max_concurrency = 0; + assert!(value.validate().is_err()); + } +} diff --git a/crates/switchyard-test-time-scaling-runner/src/main.rs b/crates/switchyard-test-time-scaling-runner/src/main.rs new file mode 100644 index 000000000..33eb15c80 --- /dev/null +++ b/crates/switchyard-test-time-scaling-runner/src/main.rs @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +mod backend; +mod command; +mod config; +mod model_client; + +use std::path::Path; + +use backend::LiveBackend; +use command::call_json; +use config::RunConfig; +use serde::Serialize; +use switchyard_test_time_scaling::{ + ExperimentMetrics, RolloutEvaluation, ScalingController, TaskEvaluation, encode_run, + evaluate_run, experiment_metrics, +}; + +#[derive(Serialize)] +struct EvaluationRecord { + task: TaskEvaluation, + metrics: ExperimentMetrics, +} + +#[tokio::main] +async fn main() -> Result<(), String> { + let mut arguments = std::env::args_os(); + let program = arguments.next().unwrap_or_default(); + let Some(config_path) = arguments.next() else { + return Err(format!( + "usage: {} CONFIG.json", + Path::new(&program).display() + )); + }; + if arguments.next().is_some() { + return Err("expected exactly one config path".to_string()); + } + + let bytes = tokio::fs::read(&config_path) + .await + .map_err(|error| error.to_string())?; + let config: RunConfig = serde_json::from_slice(&bytes).map_err(|error| error.to_string())?; + config.validate()?; + config + .manifest + .validate() + .map_err(|error| error.to_string())?; + if tokio::fs::try_exists(&config.output_dir) + .await + .map_err(|error| error.to_string())? + { + return Err(format!( + "output directory already exists: {}", + config.output_dir.display() + )); + } + let backend = LiveBackend::new( + config.manifest.model_id.clone(), + config.rollout_command.clone(), + config.model.clone(), + config.output_dir.join("model_calls.jsonl"), + )?; + tokio::fs::create_dir_all(&config.output_dir) + .await + .map_err(|error| error.to_string())?; + let controller = ScalingController::new(backend, config.scaling, config.manifest) + .map_err(|error| error.to_string())?; + let run = controller + .run(config.task) + .await + .map_err(|error| error.to_string())?; + let run_path = config.output_dir.join("run.json"); + let run_bytes = encode_run(&run).map_err(|error| error.to_string())?; + tokio::fs::write(&run_path, run_bytes) + .await + .map_err(|error| error.to_string())?; + println!("saved method record: {}", run_path.display()); + + if let Some(command) = &config.evaluation_command { + let outcomes: Vec = call_json(command, &()).await?; + let task = evaluate_run(&run, outcomes).map_err(|error| error.to_string())?; + let metrics = + experiment_metrics(std::slice::from_ref(&task)).map_err(|error| error.to_string())?; + let record = EvaluationRecord { task, metrics }; + let evaluation_path = config.output_dir.join("evaluation.json"); + let bytes = serde_json::to_vec_pretty(&record).map_err(|error| error.to_string())?; + tokio::fs::write(&evaluation_path, bytes) + .await + .map_err(|error| error.to_string())?; + println!( + "saved post-selection evaluation: {}", + evaluation_path.display() + ); + } + Ok(()) +} diff --git a/crates/switchyard-test-time-scaling-runner/src/model_client.rs b/crates/switchyard-test-time-scaling-runner/src/model_client.rs new file mode 100644 index 000000000..741d7586f --- /dev/null +++ b/crates/switchyard-test-time-scaling-runner/src/model_client.rs @@ -0,0 +1,214 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Small OpenAI-compatible chat client. + +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use reqwest::StatusCode; +use serde_json::{Value, json}; +use tokio::io::AsyncWriteExt; +use tokio::sync::Mutex; +use tokio::sync::Semaphore; +use tokio::time::sleep; + +use crate::config::ModelConfig; + +/// One lossless chat response and its text content. +pub struct ModelReply { + /// Message content used by the method. + pub content: String, + /// Complete provider response. + pub raw_response: String, +} + +/// Bounded client shared by summary and comparison roles. +pub struct ModelClient { + client: reqwest::Client, + endpoint: String, + api_key: String, + model_id: String, + attempts: usize, + send_seed: bool, + limit: Arc, + call_log_path: PathBuf, + call_log_lock: Arc>, +} + +impl ModelClient { + /// Builds a client without logging the API key. + pub fn new( + model_id: String, + config: &ModelConfig, + call_log_path: PathBuf, + ) -> Result { + let api_key = std::env::var(&config.api_key_env) + .map_err(|_| format!("{} is not set", config.api_key_env))?; + if api_key.trim().is_empty() { + return Err(format!("{} is empty", config.api_key_env)); + } + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(config.request_timeout_seconds)) + .build() + .map_err(|error| format!("could not build model client: {error}"))?; + Ok(Self { + client, + endpoint: format!("{}/chat/completions", config.base_url.trim_end_matches('/')), + api_key, + model_id, + attempts: config.http_attempts, + send_seed: config.send_seed, + limit: Arc::new(Semaphore::new(config.max_concurrency)), + call_log_path, + call_log_lock: Arc::new(Mutex::new(())), + }) + } + + /// Sends one user message. + pub async fn complete( + &self, + role: &str, + prompt: &str, + max_tokens: usize, + temperature: f64, + seed: u64, + ) -> Result { + let _permit = self + .limit + .acquire() + .await + .map_err(|_| "model concurrency limit closed".to_string())?; + let mut body = json!({ + "model": self.model_id, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": temperature, + }); + if self.send_seed { + body["seed"] = json!(seed); + } + + for attempt in 1..=self.attempts { + let response = self + .client + .post(&self.endpoint) + .bearer_auth(&self.api_key) + .json(&body) + .send() + .await; + match response { + Ok(response) => { + let status = response.status(); + let raw = response + .text() + .await + .map_err(|error| format!("could not read model response: {error}"))?; + self.record_attempt(role, attempt, &body, Some(status), Some(&raw), None) + .await?; + if status.is_success() { + return parse_reply(raw, &self.model_id); + } + if !retryable_status(status) || attempt == self.attempts { + return Err(format!("model returned {status}: {}", bounded(&raw, 2_000))); + } + } + Err(error) => { + self.record_attempt(role, attempt, &body, None, None, Some(&error.to_string())) + .await?; + if attempt == self.attempts { + return Err(format!("model request failed: {error}")); + } + } + } + sleep(Duration::from_secs(attempt.min(5) as u64)).await; + } + Err("model request exhausted attempts".to_string()) + } + + async fn record_attempt( + &self, + role: &str, + attempt: usize, + request: &Value, + status: Option, + response: Option<&str>, + error: Option<&str>, + ) -> Result<(), String> { + let record = json!({ + "role": role, + "attempt": attempt, + "request": request, + "status": status.map(|value| value.as_u16()), + "response": response, + "error": error, + }); + let mut bytes = serde_json::to_vec(&record).map_err(|error| error.to_string())?; + bytes.push(b'\n'); + let _guard = self.call_log_lock.lock().await; + let mut file = tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&self.call_log_path) + .await + .map_err(|error| format!("could not open model call log: {error}"))?; + file.write_all(&bytes) + .await + .map_err(|error| format!("could not write model call log: {error}")) + } +} + +fn retryable_status(status: StatusCode) -> bool { + status == StatusCode::TOO_MANY_REQUESTS || status.is_server_error() +} + +fn parse_reply(raw_response: String, expected_model_id: &str) -> Result { + let value: Value = serde_json::from_str(&raw_response) + .map_err(|error| format!("model returned invalid JSON: {error}"))?; + let response_model_id = value + .get("model") + .and_then(Value::as_str) + .ok_or_else(|| "model response has no model ID".to_string())?; + if response_model_id != expected_model_id { + return Err(format!( + "model response used {response_model_id}; expected {expected_model_id}" + )); + } + let content = value + .pointer("/choices/0/message/content") + .and_then(Value::as_str) + .ok_or_else(|| "model response has no text content".to_string())? + .to_string(); + Ok(ModelReply { + content, + raw_response, + }) +} + +fn bounded(text: &str, limit: usize) -> &str { + &text[..text.floor_char_boundary(limit.min(text.len()))] +} + +#[cfg(test)] +mod tests { + use super::{bounded, parse_reply}; + + #[test] + fn parses_chat_content_and_bounds_utf8() { + let reply = parse_reply( + r#"{"model":"model-1","choices":[{"message":{"content":"done"}}]}"#.to_string(), + "model-1", + ) + .expect("chat response"); + assert_eq!(reply.content, "done"); + assert_eq!(bounded("aéz", 2), "a"); + assert!( + parse_reply( + r#"{"model":"model-2","choices":[{"message":{"content":"done"}}]}"#.to_string(), + "model-1", + ) + .is_err() + ); + } +} diff --git a/crates/switchyard-test-time-scaling/Cargo.toml b/crates/switchyard-test-time-scaling/Cargo.toml new file mode 100644 index 000000000..535b525e2 --- /dev/null +++ b/crates/switchyard-test-time-scaling/Cargo.toml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "switchyard-test-time-scaling" +version.workspace = true +description = "Harness-neutral test-time scaling for agentic coding" +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +async-trait.workspace = true +futures.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true + +[dev-dependencies] +tokio.workspace = true diff --git a/crates/switchyard-test-time-scaling/src/config.rs b/crates/switchyard-test-time-scaling/src/config.rs new file mode 100644 index 000000000..35ba39de0 --- /dev/null +++ b/crates/switchyard-test-time-scaling/src/config.rs @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Settings that change the scaling workflow. + +use serde::{Deserialize, Serialize}; + +use crate::{Result, ScalingError}; + +/// How candidates are paired before each tournament round. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PairingOrder { + /// Keep the current candidate order. + #[default] + InOrder, + /// Shuffle candidates from the configured seed. + Shuffle, +} + +/// How candidates are shown to each judge vote. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DisplayOrder { + /// Keep the group order. + #[default] + InOrder, + /// Shuffle the displayed positions for every vote. + Shuffle, +} + +/// How a tied vote is resolved. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TiePolicy { + /// Select the earliest tied candidate in the original group. + #[default] + FirstInGroup, + /// Select a tied candidate from the configured seed. + SeededRandom, +} + +/// How malformed or failed judge votes are handled. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", tag = "mode")] +pub enum InvalidVotePolicy { + /// Stop the tournament when any required vote is invalid. + #[default] + Abort, + /// Replace invalid votes up to the stated call limit per group. + Replace { + /// Maximum replacement calls made for one group. + max_calls_per_group: usize, + }, +} + +/// Settings for the two-iteration PDR and RTV workflow. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ScalingConfig { + /// Independent attempts run in each iteration. + pub rollout_count: usize, + /// Iteration-zero summaries given to every refined attempt. + pub refinement_count: usize, + /// Maximum comparison group size. + pub group_size: usize, + /// Valid votes required for every comparison group. + pub votes_per_group: usize, + /// Root seed used for repeatable ordering and adapter calls. + pub seed: u64, + /// Pairing order used by tournaments. + pub pairing_order: PairingOrder, + /// Display order used by judge calls. + pub display_order: DisplayOrder, + /// Tie policy used by group decisions. + pub tie_policy: TiePolicy, + /// Invalid-vote policy used by group decisions. + pub invalid_vote_policy: InvalidVotePolicy, +} + +impl Default for ScalingConfig { + fn default() -> Self { + Self { + rollout_count: 16, + refinement_count: 4, + group_size: 2, + votes_per_group: 8, + seed: 0, + pairing_order: PairingOrder::InOrder, + display_order: DisplayOrder::InOrder, + tie_policy: TiePolicy::FirstInGroup, + invalid_vote_policy: InvalidVotePolicy::Abort, + } + } +} + +impl ScalingConfig { + /// Rejects settings that cannot run the two-iteration workflow. + pub fn validate(&self) -> Result<()> { + if self.rollout_count == 0 { + return Err(ScalingError::InvalidConfig( + "rollout_count must be at least 1".to_string(), + )); + } + if self.refinement_count == 0 || self.refinement_count > self.rollout_count { + return Err(ScalingError::InvalidConfig( + "refinement_count must be between 1 and rollout_count".to_string(), + )); + } + if self.group_size < 2 { + return Err(ScalingError::InvalidConfig( + "group_size must be at least 2".to_string(), + )); + } + if self.votes_per_group == 0 { + return Err(ScalingError::InvalidConfig( + "votes_per_group must be at least 1".to_string(), + )); + } + if matches!( + self.invalid_vote_policy, + InvalidVotePolicy::Replace { + max_calls_per_group: 0 + } + ) { + return Err(ScalingError::InvalidConfig( + "replacement vote limit must be at least 1".to_string(), + )); + } + Ok(()) + } +} diff --git a/crates/switchyard-test-time-scaling/src/controller.rs b/crates/switchyard-test-time-scaling/src/controller.rs new file mode 100644 index 000000000..cfec9b56c --- /dev/null +++ b/crates/switchyard-test-time-scaling/src/controller.rs @@ -0,0 +1,286 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Two-iteration scaling controller. + +use std::collections::HashSet; + +use futures::future::try_join_all; + +use crate::prompts::refinement_prompt; +use crate::seed; +use crate::{ + Attempt, Candidate, ExperimentManifest, Result, Rollout, RolloutRequest, ScalingBackend, + ScalingConfig, ScalingError, ScalingRun, Summary, Task, run_tournament, +}; + +/// Runs parallel-distill-refine followed by recursive tournament voting. +pub struct ScalingController { + backend: B, + config: ScalingConfig, + manifest: ExperimentManifest, +} + +impl ScalingController +where + B: ScalingBackend, +{ + /// Creates a controller after validating its configuration. + pub fn new(backend: B, config: ScalingConfig, manifest: ExperimentManifest) -> Result { + config.validate()?; + validate_controller_schedule(&config)?; + manifest.validate()?; + if backend.model_id().trim().is_empty() { + return Err(ScalingError::InvalidConfig( + "backend model_id must not be empty".to_string(), + )); + } + if backend.model_id() != manifest.model_id { + return Err(ScalingError::InvalidConfig( + "backend model_id must match the experiment manifest".to_string(), + )); + } + Ok(Self { + backend, + config, + manifest, + }) + } + + /// Returns the active workflow settings. + pub fn config(&self) -> &ScalingConfig { + &self.config + } + + /// Runs the complete workflow for one task. + pub async fn run(&self, task: Task) -> Result> { + validate_task(&task)?; + let iteration_zero = self.run_iteration(&task, 0, Vec::new()).await?; + let zero_candidates: Vec = + iteration_zero.iter().map(Candidate::from_attempt).collect(); + let iteration_zero_tournament = run_tournament( + &self.backend, + &task, + zero_candidates.clone(), + &self.config, + 1, + Some(self.config.refinement_count), + seed::derive(self.config.seed, &[10]), + ) + .await?; + let checkpoint = iteration_zero_tournament + .checkpoint + .as_ref() + .ok_or_else(|| { + ScalingError::InvalidRecord( + "iteration-zero tournament did not save refinement candidates".to_string(), + ) + })?; + let refinement_summaries = summaries_for_ids(&zero_candidates, &checkpoint.candidate_ids)?; + let refinement_summary_ids = refinement_summaries + .iter() + .map(|summary| summary.id.clone()) + .collect(); + + let iteration_one = self.run_iteration(&task, 1, refinement_summaries).await?; + validate_attempt_records(&iteration_zero, &iteration_one)?; + let one_candidates: Vec = + iteration_one.iter().map(Candidate::from_attempt).collect(); + let final_tournament = run_tournament( + &self.backend, + &task, + one_candidates, + &self.config, + 1, + None, + seed::derive(self.config.seed, &[20]), + ) + .await?; + let Some(final_candidate_id) = final_tournament.survivor_candidate_ids.first() else { + return Err(ScalingError::InvalidRecord( + "final tournament has no survivor".to_string(), + )); + }; + let final_rollout = iteration_one + .iter() + .find(|attempt| &attempt.rollout.id == final_candidate_id) + .map(|attempt| attempt.rollout.clone()) + .ok_or_else(|| { + ScalingError::InvalidRecord( + "final tournament survivor has no matching rollout".to_string(), + ) + })?; + + Ok(ScalingRun { + manifest: self.manifest.clone(), + task, + model_id: self.backend.model_id().to_string(), + config: self.config.clone(), + iteration_zero, + iteration_zero_tournament, + refinement_summary_ids, + iteration_one, + final_tournament, + final_rollout, + }) + } + + async fn run_iteration( + &self, + task: &Task, + iteration: u8, + refinement_summaries: Vec, + ) -> Result>> { + let rendered_refinement = + (!refinement_summaries.is_empty()).then(|| refinement_prompt(&refinement_summaries)); + let requests: Vec = (0..self.config.rollout_count) + .map(|rollout_index| RolloutRequest { + iteration, + rollout_index, + seed: seed::derive( + self.config.seed, + &[30, iteration as u64, rollout_index as u64], + ), + refinement_summaries: refinement_summaries.clone(), + refinement_prompt: rendered_refinement.clone(), + }) + .collect(); + let rollouts = self.backend.run_rollouts(task, requests.clone()).await?; + if rollouts.len() != requests.len() { + return Err(ScalingError::InvalidRecord(format!( + "backend returned {} rollouts; expected {}", + rollouts.len(), + requests.len() + ))); + } + let summaries: Vec = try_join_all( + rollouts + .iter() + .map(|rollout| self.backend.summarize(task, rollout)), + ) + .await?; + + requests + .into_iter() + .zip(rollouts) + .zip(summaries) + .map(|((request, rollout), summary)| { + validate_attempt(&request, &rollout, &summary, self.backend.model_id())?; + Ok(Attempt { + request, + rollout, + summary, + }) + }) + .collect() + } +} + +fn summaries_for_ids(candidates: &[Candidate], ids: &[String]) -> Result> { + ids.iter() + .map(|id| { + candidates + .iter() + .find(|candidate| &candidate.id == id) + .map(|candidate| candidate.summary.clone()) + .ok_or_else(|| { + ScalingError::InvalidRecord( + "checkpoint candidate has no matching summary".to_string(), + ) + }) + }) + .collect() +} + +fn validate_task(task: &Task) -> Result<()> { + if task.id.trim().is_empty() + || task.benchmark.trim().is_empty() + || task.prompt.trim().is_empty() + { + return Err(ScalingError::InvalidConfig( + "task id, benchmark, and prompt must not be empty".to_string(), + )); + } + Ok(()) +} + +fn validate_attempt( + request: &RolloutRequest, + rollout: &Rollout, + summary: &Summary, + expected_model_id: &str, +) -> Result<()> { + if rollout.id.trim().is_empty() + || rollout.environment_id.trim().is_empty() + || rollout.output_digest.trim().is_empty() + { + return Err(ScalingError::InvalidRecord( + "rollout, environment, and output identifiers must not be empty".to_string(), + )); + } + if rollout.model_id != expected_model_id || summary.model_id != expected_model_id { + return Err(ScalingError::InvalidRecord( + "rollout and summary model IDs must match the experiment model".to_string(), + )); + } + if rollout.iteration != request.iteration || rollout.rollout_index != request.rollout_index { + return Err(ScalingError::InvalidRecord( + "rollout iteration and index must match its request".to_string(), + )); + } + if summary.id.trim().is_empty() + || summary.rollout_id != rollout.id + || summary.raw_response.trim().is_empty() + || summary.generation_attempts == 0 + { + return Err(ScalingError::InvalidRecord( + "summary must identify its rollout and record generation".to_string(), + )); + } + Ok(()) +} + +fn validate_controller_schedule(config: &ScalingConfig) -> Result<()> { + let mut population = config.rollout_count; + let mut reaches_refinement_count = population == config.refinement_count; + while population > 1 { + let group_size = config.group_size.min(population); + if !population.is_multiple_of(group_size) { + return Err(ScalingError::InvalidConfig(format!( + "rollout_count {population} is not divisible by group size {group_size}" + ))); + } + population /= group_size; + reaches_refinement_count |= population == config.refinement_count; + } + if !reaches_refinement_count { + return Err(ScalingError::InvalidConfig( + "refinement_count is not reached by the tournament schedule".to_string(), + )); + } + Ok(()) +} + +fn validate_attempt_records(zero: &[Attempt], one: &[Attempt]) -> Result<()> { + let mut rollout_ids = HashSet::with_capacity(zero.len() + one.len()); + let mut summary_ids = HashSet::with_capacity(zero.len() + one.len()); + let mut environment_ids = HashSet::with_capacity(zero.len() + one.len()); + for attempt in zero.iter().chain(one) { + if !rollout_ids.insert(&attempt.rollout.id) { + return Err(ScalingError::InvalidRecord( + "rollout identifiers must be unique".to_string(), + )); + } + if !summary_ids.insert(&attempt.summary.id) { + return Err(ScalingError::InvalidRecord( + "summary identifiers must be unique".to_string(), + )); + } + if !environment_ids.insert(&attempt.rollout.environment_id) { + return Err(ScalingError::InvalidRecord( + "every rollout must use a fresh environment identifier".to_string(), + )); + } + } + Ok(()) +} diff --git a/crates/switchyard-test-time-scaling/src/error.rs b/crates/switchyard-test-time-scaling/src/error.rs new file mode 100644 index 000000000..17142ecbc --- /dev/null +++ b/crates/switchyard-test-time-scaling/src/error.rs @@ -0,0 +1,68 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Errors returned by the scaling workflow. + +use thiserror::Error; + +/// Result type used by this crate. +pub type Result = std::result::Result; + +/// A scaling run could not continue safely. +#[derive(Clone, Debug, Error, PartialEq, Eq)] +pub enum ScalingError { + /// The experiment manifest is incomplete or makes an unsupported claim. + #[error("invalid experiment manifest: {0}")] + InvalidManifest(String), + + /// Configuration values do not describe a valid workflow. + #[error("invalid scaling config: {0}")] + InvalidConfig(String), + + /// The adapter failed to run an attempt or summary call. + #[error("backend failed: {0}")] + Backend(String), + + /// An adapter returned records that do not match the request. + #[error("invalid backend record: {0}")] + InvalidRecord(String), + + /// Official outcomes do not cover one completed run exactly once. + #[error("invalid evaluation: {0}")] + InvalidEvaluation(String), + + /// A completed run could not be encoded or decoded. + #[error("run record failed: {0}")] + Record(String), + + /// The population cannot be divided without an unspecified bye policy. + #[error("population {population} is not divisible by group size {group_size}")] + UnevenGroups { + /// Number of current candidates. + population: usize, + /// Effective group size for this round. + group_size: usize, + }, + + /// A round would reduce the population below the requested survivor count. + #[error( + "group size {group_size} would reduce population {population} below {target} survivors" + )] + OvershootsTarget { + /// Number of current candidates. + population: usize, + /// Effective group size for this round. + group_size: usize, + /// Requested survivor count. + target: usize, + }, + + /// A group did not produce the required number of valid votes. + #[error("group produced {actual} valid votes; expected {expected}")] + IncompleteVotes { + /// Required valid-vote count. + expected: usize, + /// Actual valid-vote count. + actual: usize, + }, +} diff --git a/crates/switchyard-test-time-scaling/src/evaluation.rs b/crates/switchyard-test-time-scaling/src/evaluation.rs new file mode 100644 index 000000000..8f695d12b --- /dev/null +++ b/crates/switchyard-test-time-scaling/src/evaluation.rs @@ -0,0 +1,204 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Post-run grading records and paper stage metrics. + +use std::collections::{HashMap, HashSet}; + +use serde::{Deserialize, Serialize}; + +use crate::{Result, ScalingError, ScalingRun}; + +/// Official result for one rollout, produced only after selection finishes. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RolloutEvaluation { + /// Rollout that was graded. + pub rollout_id: String, + /// Whether the official grader accepted the rollout. + pub passed: bool, +} + +/// Stage scores for one completed task. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TaskEvaluation { + /// Task that was graded. + pub task_id: String, + /// Iteration-zero outcomes in rollout order. + pub iteration_zero: Vec, + /// Outcomes for the four summaries selected for refinement. + pub selected_for_refinement: Vec, + /// Iteration-one outcomes in rollout order. + pub iteration_one: Vec, + /// Outcome of the rollout selected by the final tournament. + pub final_passed: bool, +} + +/// Aggregate values reported for one stage over several tasks. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StageMetrics { + /// Average outcome across all rollouts in this stage. + pub average_pass_at_one: f64, + /// Fraction of tasks with at least one passing rollout. + pub pass_at_n: f64, + /// Tasks containing both passing and failing rollouts. + pub mixed_task_count: usize, +} + +/// Paper stage metrics over a set of completed tasks. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ExperimentMetrics { + /// Number of evaluated tasks. + pub task_count: usize, + /// Metrics for the first 16 rollouts. + pub iteration_zero: StageMetrics, + /// Metrics for the summaries selected for refinement. + pub selected_for_refinement: StageMetrics, + /// Metrics for the second 16 rollouts. + pub iteration_one: StageMetrics, + /// Fraction of tasks whose final selected rollout passed. + pub final_pass_at_one: f64, +} + +/// Joins official outcomes to a completed run without exposing them to the controller. +pub fn evaluate_run( + run: &ScalingRun, + evaluations: Vec, +) -> Result { + let expected_ids: HashSet<&str> = run + .iteration_zero + .iter() + .chain(&run.iteration_one) + .map(|attempt| attempt.rollout.id.as_str()) + .collect(); + let mut outcomes = HashMap::with_capacity(evaluations.len()); + for evaluation in evaluations { + if !expected_ids.contains(evaluation.rollout_id.as_str()) { + return Err(ScalingError::InvalidEvaluation(format!( + "unknown rollout {}", + evaluation.rollout_id + ))); + } + if outcomes + .insert(evaluation.rollout_id.clone(), evaluation.passed) + .is_some() + { + return Err(ScalingError::InvalidEvaluation(format!( + "duplicate rollout {}", + evaluation.rollout_id + ))); + } + } + if outcomes.len() != expected_ids.len() { + return Err(ScalingError::InvalidEvaluation(format!( + "received {} outcomes; expected {}", + outcomes.len(), + expected_ids.len() + ))); + } + + let lookup = |rollout_id: &str| { + outcomes + .get(rollout_id) + .copied() + .ok_or_else(|| ScalingError::InvalidEvaluation(format!("missing rollout {rollout_id}"))) + }; + let iteration_zero = run + .iteration_zero + .iter() + .map(|attempt| lookup(&attempt.rollout.id)) + .collect::>>()?; + let checkpoint = run + .iteration_zero_tournament + .checkpoint + .as_ref() + .ok_or_else(|| { + ScalingError::InvalidEvaluation( + "iteration-zero tournament has no refinement checkpoint".to_string(), + ) + })?; + let selected_for_refinement = checkpoint + .candidate_ids + .iter() + .map(|id| lookup(id)) + .collect::>>()?; + let iteration_one = run + .iteration_one + .iter() + .map(|attempt| lookup(&attempt.rollout.id)) + .collect::>>()?; + let final_passed = lookup(&run.final_rollout.id)?; + + Ok(TaskEvaluation { + task_id: run.task.id.clone(), + iteration_zero, + selected_for_refinement, + iteration_one, + final_passed, + }) +} + +/// Computes the paper's main stage metrics from completed task evaluations. +pub fn experiment_metrics(tasks: &[TaskEvaluation]) -> Result { + if tasks.is_empty() { + return Err(ScalingError::InvalidEvaluation( + "at least one task evaluation is required".to_string(), + )); + } + let mut task_ids = HashSet::with_capacity(tasks.len()); + for task in tasks { + if task.task_id.trim().is_empty() || !task_ids.insert(&task.task_id) { + return Err(ScalingError::InvalidEvaluation( + "task evaluation identifiers must be non-empty and unique".to_string(), + )); + } + if task.iteration_zero.is_empty() + || task.selected_for_refinement.is_empty() + || task.iteration_one.is_empty() + { + return Err(ScalingError::InvalidEvaluation( + "every evaluated stage must contain at least one rollout".to_string(), + )); + } + } + + Ok(ExperimentMetrics { + task_count: tasks.len(), + iteration_zero: stage_metrics(tasks, |task| &task.iteration_zero), + selected_for_refinement: stage_metrics(tasks, |task| &task.selected_for_refinement), + iteration_one: stage_metrics(tasks, |task| &task.iteration_one), + final_pass_at_one: tasks.iter().filter(|task| task.final_passed).count() as f64 + / tasks.len() as f64, + }) +} + +fn stage_metrics<'a>( + tasks: &'a [TaskEvaluation], + outcomes: impl Fn(&'a TaskEvaluation) -> &'a [bool], +) -> StageMetrics { + let total = tasks.iter().map(|task| outcomes(task).len()).sum::(); + let passed = tasks + .iter() + .map(|task| outcomes(task).iter().filter(|value| **value).count()) + .sum::(); + let tasks_with_pass = tasks + .iter() + .filter(|task| outcomes(task).iter().any(|value| *value)) + .count(); + let mixed_task_count = tasks + .iter() + .filter(|task| { + let values = outcomes(task); + values.iter().any(|value| *value) && values.iter().any(|value| !*value) + }) + .count(); + + StageMetrics { + average_pass_at_one: passed as f64 / total as f64, + pass_at_n: tasks_with_pass as f64 / tasks.len() as f64, + mixed_task_count, + } +} diff --git a/crates/switchyard-test-time-scaling/src/lib.rs b/crates/switchyard-test-time-scaling/src/lib.rs new file mode 100644 index 000000000..9e3639905 --- /dev/null +++ b/crates/switchyard-test-time-scaling/src/lib.rs @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Test-time scaling for agentic coding tasks. +//! +//! The controller runs independent attempts, summarizes them, selects useful summaries for a +//! second set of fresh attempts, and selects one final result. Callers provide the agent harness +//! and model calls through [`ScalingBackend`]. + +#![deny(missing_docs)] + +mod config; +mod controller; +mod error; +mod evaluation; +mod manifest; +mod model; +mod ports; +mod prompts; +mod record; +mod seed; +mod tournament; +mod verdict; + +pub use config::{DisplayOrder, InvalidVotePolicy, PairingOrder, ScalingConfig, TiePolicy}; +pub use controller::ScalingController; +pub use error::{Result, ScalingError}; +pub use evaluation::{ + ExperimentMetrics, RolloutEvaluation, StageMetrics, TaskEvaluation, evaluate_run, + experiment_metrics, +}; +pub use manifest::{ + ExperimentManifest, MANIFEST_SCHEMA_VERSION, ManifestSource, ManifestValue, + REQUIRED_MANIFEST_FIELDS, ReplicationMode, +}; +pub use model::{ + Attempt, Candidate, ComparisonRequest, ComparisonResponse, GroupDecision, Rollout, + RolloutRequest, ScalingRun, Summary, Task, Tournament, TournamentCheckpoint, Vote, +}; +pub use ports::ScalingBackend; +pub use prompts::refinement_prompt; +pub use record::{decode_run, encode_run}; +pub use tournament::run_tournament; +pub use verdict::parse_verdict; diff --git a/crates/switchyard-test-time-scaling/src/manifest.rs b/crates/switchyard-test-time-scaling/src/manifest.rs new file mode 100644 index 000000000..25dbc8245 --- /dev/null +++ b/crates/switchyard-test-time-scaling/src/manifest.rs @@ -0,0 +1,135 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Settings needed to interpret and repeat one experiment. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use crate::{Result, ScalingError}; + +/// Current serialized manifest version. +pub const MANIFEST_SCHEMA_VERSION: u16 = 1; + +/// Paper-critical settings that the paper does not fully disclose. +pub const REQUIRED_MANIFEST_FIELDS: [&str; 22] = [ + "exact_prompts", + "summary_schema", + "malformed_summary_policy", + "model_ids_and_api_revisions", + "role_inference_settings", + "scaffold_revisions_and_protocols", + "agent_limits", + "summary_serialization", + "pairing_order", + "display_order", + "tie_break", + "invalid_vote_policy", + "model_retry_policy", + "experiment_seeds", + "benchmark_revisions", + "terminal_bench_task_list", + "summary_input_contents", + "refinement_summary_order", + "observation_truncation", + "unfinished_rollout_policy", + "concurrency_and_rate_limits", + "ablation_fixed_settings", +]; + +/// Whether a run claims exact or conceptual replication. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReplicationMode { + /// Every paper-critical setting comes from the paper or author code. + Exact, + /// Missing paper details use explicit, recorded choices. + Conceptual, +} + +/// Where one manifest value came from. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ManifestSource { + /// The paper states the value. + Paper, + /// Author code or configuration states the value. + Repository, + /// The replication chooses a value the paper does not state. + Reconstructed, +} + +/// One recorded paper-critical setting. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManifestValue { + /// Evidence used for this value. + pub source: ManifestSource, + /// Exact value or revision used by the run. + pub value: String, +} + +/// Complete description of one experiment setup. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ExperimentManifest { + /// Serialized manifest version. + pub schema_version: u16, + /// Replication claim made by this run. + pub replication_mode: ReplicationMode, + /// Source revision that ran the experiment. + pub code_revision: String, + /// Exact model shared by rollout, summary, and comparison calls. + pub model_id: String, + /// Required paper-critical settings. + pub fields: BTreeMap, +} + +impl ExperimentManifest { + /// Rejects missing, unknown, empty, or unsupported manifest fields. + pub fn validate(&self) -> Result<()> { + if self.schema_version != MANIFEST_SCHEMA_VERSION { + return Err(ScalingError::InvalidManifest(format!( + "unsupported schema version {}; expected {MANIFEST_SCHEMA_VERSION}", + self.schema_version + ))); + } + if self.code_revision.trim().is_empty() || self.model_id.trim().is_empty() { + return Err(ScalingError::InvalidManifest( + "code_revision and model_id must not be empty".to_string(), + )); + } + + for name in REQUIRED_MANIFEST_FIELDS { + let Some(field) = self.fields.get(name) else { + return Err(ScalingError::InvalidManifest(format!( + "missing required field {name}" + ))); + }; + if field.value.trim().is_empty() { + return Err(ScalingError::InvalidManifest(format!( + "field {name} must not be empty" + ))); + } + if self.replication_mode == ReplicationMode::Exact + && field.source == ManifestSource::Reconstructed + { + return Err(ScalingError::InvalidManifest(format!( + "exact replication cannot use reconstructed field {name}" + ))); + } + } + + if let Some(name) = self + .fields + .keys() + .find(|name| !REQUIRED_MANIFEST_FIELDS.contains(&name.as_str())) + { + return Err(ScalingError::InvalidManifest(format!( + "unknown field {name}" + ))); + } + Ok(()) + } +} diff --git a/crates/switchyard-test-time-scaling/src/model.rs b/crates/switchyard-test-time-scaling/src/model.rs new file mode 100644 index 000000000..7a9e403fd --- /dev/null +++ b/crates/switchyard-test-time-scaling/src/model.rs @@ -0,0 +1,238 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Records produced by one scaling run. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +/// One agentic coding task. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Task { + /// Stable task identifier. + pub id: String, + /// Benchmark or task family name. + pub benchmark: String, + /// Original task text shown to every attempt. + pub prompt: String, +} + +/// Request for one independent agent attempt. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RolloutRequest { + /// Zero for initial attempts and one for refined attempts. + pub iteration: u8, + /// Zero-based attempt position within the iteration. + pub rollout_index: usize, + /// Seed for this attempt. + pub seed: u64, + /// Prior summaries supplied to a refined attempt. + pub refinement_summaries: Vec, + /// Rendered prior-attempt context, absent for initial attempts. + pub refinement_prompt: Option, +} + +/// Result returned by an agent harness. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Rollout { + /// Stable attempt identifier. + pub id: String, + /// Zero for an initial attempt and one for a refined attempt. + pub iteration: u8, + /// Zero-based attempt position within the iteration. + pub rollout_index: usize, + /// Exact model used for this attempt. + pub model_id: String, + /// Identifier of the fresh environment used for this attempt. + pub environment_id: String, + /// Content digest of the returned patch, artifacts, or environment snapshot. + pub output_digest: String, + /// Patch, artifact set, snapshot, or other harness-owned result. + pub output: O, +} + +/// Structured summary of one attempt. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Summary { + /// Stable summary identifier. + pub id: String, + /// Attempt summarized by this record. + pub rollout_id: String, + /// Exact model used to generate this summary. + pub model_id: String, + /// JSON object presented to refinement and comparison calls. + pub value: Map, + /// Lossless model response saved before parsing. + pub raw_response: String, + /// Number of content-generation attempts used to produce the object. + pub generation_attempts: usize, +} + +/// One completed attempt together with its request and summary. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Attempt { + /// Request used to start the attempt. + pub request: RolloutRequest, + /// Agent harness result. + pub rollout: Rollout, + /// Structured summary of the result. + pub summary: Summary, +} + +/// Summary-backed candidate used by a tournament. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Candidate { + /// Stable candidate identifier. + pub id: String, + /// Attempt represented by the candidate. + pub rollout_id: String, + /// Structured summary shown to judges. + pub summary: Summary, +} + +impl Candidate { + /// Builds a candidate from a completed attempt. + pub fn from_attempt(attempt: &Attempt) -> Self { + Self { + id: attempt.rollout.id.clone(), + rollout_id: attempt.rollout.id.clone(), + summary: attempt.summary.clone(), + } + } +} + +/// One comparison call sent to the backend. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ComparisonRequest { + /// Fully rendered comparison prompt. + pub prompt: String, + /// Candidates in the exact order used by the prompt. + pub candidates: Vec, + /// Zero-based tournament round. + pub round_index: usize, + /// Zero-based group within the round. + pub group_index: usize, + /// Stable vote record position, including replacements. + pub vote_index: usize, + /// Seed for this judge call. + pub seed: u64, +} + +/// Successful response from one comparison model call. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ComparisonResponse { + /// Exact model used for this comparison. + pub model_id: String, + /// Lossless response returned by the model. + pub content: String, +} + +/// One recorded judge call and its parsed choice. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Vote { + /// Stable vote record position, including replacements. + pub vote_index: usize, + /// Candidate identifiers in displayed order. + pub ordered_candidate_ids: Vec, + /// Fully rendered prompt sent to the judge. + pub prompt: String, + /// Seed used for the judge call. + pub seed: u64, + /// Exact model used when the call succeeded. + pub model_id: Option, + /// Raw judge response when the model call succeeded. + pub raw_response: Option, + /// Selected one-based displayed position when parsing succeeded. + pub selected_position: Option, + /// Stable candidate identifier selected by the vote. + pub selected_candidate_id: Option, + /// Short failure reason for an invalid vote. + pub error: Option, +} + +/// Result of one tournament comparison group. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct GroupDecision { + /// Zero-based round position. + pub round_index: usize, + /// Zero-based group position. + pub group_index: usize, + /// Candidate identifiers before display reordering. + pub input_candidate_ids: Vec, + /// Initial and replacement vote records. + pub votes: Vec, + /// Valid vote totals by candidate identifier. + pub vote_counts: BTreeMap, + /// Candidate that advances to the next round. + pub selected_candidate_id: String, + /// Recorded tie policy when the highest count was shared. + pub tie_break: Option, +} + +/// Immutable survivor list captured during a tournament. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TournamentCheckpoint { + /// Number of completed rounds. + pub completed_rounds: usize, + /// Candidate identifiers frozen at this point. + pub candidate_ids: Vec, +} + +/// Complete record of one recursive tournament. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Tournament { + /// Settings used for this tournament. + pub config: crate::ScalingConfig, + /// Root seed used for pairing, display order, votes, and ties. + pub root_seed: u64, + /// Candidate identifiers at the start and after every round. + pub populations: Vec>, + /// Group decisions in round order. + pub rounds: Vec>, + /// Requested final survivor count. + pub target_survivors: usize, + /// Optional survivor list frozen before the tournament finished. + pub checkpoint: Option, + /// Final surviving candidate identifiers. + pub survivor_candidate_ids: Vec, +} + +/// Result of the complete two-iteration scaling workflow. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ScalingRun { + /// Experiment settings needed to interpret and repeat this run. + pub manifest: crate::ExperimentManifest, + /// Task processed by the controller. + pub task: Task, + /// Model identity reported by the shared backend. + pub model_id: String, + /// Settings used for the complete run. + pub config: crate::ScalingConfig, + /// Initial independent attempts. + pub iteration_zero: Vec>, + /// Tournament that selected refinement context and continued to one diagnostic survivor. + pub iteration_zero_tournament: Tournament, + /// Summary identifiers given to every refined attempt. + pub refinement_summary_ids: Vec, + /// Fresh attempts conditioned on the selected summaries. + pub iteration_one: Vec>, + /// Tournament that selected the final result. + pub final_tournament: Tournament, + /// Exact rollout selected by the final tournament. + pub final_rollout: Rollout, +} diff --git a/crates/switchyard-test-time-scaling/src/ports.rs b/crates/switchyard-test-time-scaling/src/ports.rs new file mode 100644 index 000000000..deed6e79d --- /dev/null +++ b/crates/switchyard-test-time-scaling/src/ports.rs @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Adapter boundary for agent harnesses and model calls. + +use async_trait::async_trait; + +use crate::{ + ComparisonRequest, ComparisonResponse, Result, Rollout, RolloutRequest, Summary, Task, +}; + +/// Runs all three model roles for one logical model identity. +/// +/// Every rollout returned by [`run_rollouts`](Self::run_rollouts) must use a fresh environment. +/// Official grader results must not be included in summaries or comparison responses. +#[async_trait] +pub trait ScalingBackend: Send + Sync { + /// Harness-owned rollout output, such as a patch or artifact manifest. + type Output: Clone + Send + Sync + 'static; + + /// Exact model identity used for attempts, summaries, and comparisons. + fn model_id(&self) -> &str; + + /// Runs one iteration of independent agent attempts in fresh environments. + async fn run_rollouts( + &self, + task: &Task, + requests: Vec, + ) -> Result>>; + + /// Produces one structured JSON summary for a completed attempt. + async fn summarize(&self, task: &Task, rollout: &Rollout) -> Result; + + /// Compares candidates and returns a response ending in one strict verdict line. + async fn compare(&self, task: &Task, request: ComparisonRequest) -> Result; +} diff --git a/crates/switchyard-test-time-scaling/src/prompts.rs b/crates/switchyard-test-time-scaling/src/prompts.rs new file mode 100644 index 000000000..68ce2f09d --- /dev/null +++ b/crates/switchyard-test-time-scaling/src/prompts.rs @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Small prompt builders used by the controller. + +use crate::{Candidate, Summary, Task}; + +const COMPARISON_INSTRUCTIONS: &str = "Compare the numbered candidate summaries for the original task. Use only recorded evidence and do not assume hidden tests passed. Reject a candidate with a missing patch or artifact, an inconsistent final state, or an unresolved fatal error. Then compare code correctness, completeness, verification after the final edit, test results, root-cause coverage, confirmed command output, and reasonable task interpretation, in that order. Explain the decisive evidence. End with exactly: Final verdict: Solution N"; +const REFINEMENT_PREAMBLE: &str = "You are starting a new independent attempt in a fresh environment. The prior summaries may contain successes, failures, conflicting diagnoses, or unverified claims. Use them as evidence, not as ground truth."; +const REFINEMENT_POSTAMBLE: &str = "Combine useful findings, avoid repeated dead ends, and verify the solution in this fresh environment. Prior files and patches are not available unless you recreate them."; + +/// Renders the shared prior-attempt context for a refined rollout. +pub fn refinement_prompt(summaries: &[Summary]) -> String { + let mut prompt = format!("{REFINEMENT_PREAMBLE}\n"); + for (index, summary) in summaries.iter().enumerate() { + prompt.push_str(&format!( + "\nPRIOR ATTEMPT SUMMARY {}\n{}\n", + index + 1, + serde_json::Value::Object(summary.value.clone()) + )); + } + prompt.push_str(&format!("\n{REFINEMENT_POSTAMBLE}")); + prompt +} + +pub(crate) fn comparison_prompt(task: &Task, candidates: &[Candidate]) -> String { + let mut prompt = format!( + "{COMPARISON_INSTRUCTIONS}\n\nOriginal task:\n{}\n\nCandidates:\n", + task.prompt + ); + for (index, candidate) in candidates.iter().enumerate() { + prompt.push_str(&format!( + "\nSolution {}:\n{}\n", + index + 1, + serde_json::Value::Object(candidate.summary.value.clone()) + )); + } + prompt +} diff --git a/crates/switchyard-test-time-scaling/src/record.rs b/crates/switchyard-test-time-scaling/src/record.rs new file mode 100644 index 000000000..95e3560e0 --- /dev/null +++ b/crates/switchyard-test-time-scaling/src/record.rs @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! JSON encoding for saved and replayed scaling runs. + +use serde::Serialize; +use serde::de::DeserializeOwned; + +use crate::{Result, ScalingError, ScalingRun}; + +/// Encodes a completed run as readable JSON. +pub fn encode_run(run: &ScalingRun) -> Result> +where + O: Serialize, +{ + serde_json::to_vec_pretty(run).map_err(|error| ScalingError::Record(error.to_string())) +} + +/// Decodes a previously saved run. +pub fn decode_run(bytes: &[u8]) -> Result> +where + O: DeserializeOwned, +{ + serde_json::from_slice(bytes).map_err(|error| ScalingError::Record(error.to_string())) +} diff --git a/crates/switchyard-test-time-scaling/src/seed.rs b/crates/switchyard-test-time-scaling/src/seed.rs new file mode 100644 index 000000000..0ef128a50 --- /dev/null +++ b/crates/switchyard-test-time-scaling/src/seed.rs @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Repeatable seed helpers with no runtime state. + +pub(crate) fn derive(root: u64, parts: &[u64]) -> u64 { + parts.iter().fold(mix(root), |seed, part| mix(seed ^ part)) +} + +pub(crate) fn shuffle(values: &mut [T], seed: u64) { + let mut state = seed; + for end in (1..values.len()).rev() { + state = mix(state); + let index = (state as usize) % (end + 1); + values.swap(index, end); + } +} + +fn mix(mut value: u64) -> u64 { + value = value.wrapping_add(0x9e37_79b9_7f4a_7c15); + value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + value ^ (value >> 31) +} diff --git a/crates/switchyard-test-time-scaling/src/tournament.rs b/crates/switchyard-test-time-scaling/src/tournament.rs new file mode 100644 index 000000000..583475a1e --- /dev/null +++ b/crates/switchyard-test-time-scaling/src/tournament.rs @@ -0,0 +1,416 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Recursive tournament selection. + +use std::collections::{BTreeMap, HashSet}; + +use futures::future::try_join_all; + +use crate::config::{DisplayOrder, InvalidVotePolicy, PairingOrder, TiePolicy}; +use crate::prompts::comparison_prompt; +use crate::seed; +use crate::{ + Candidate, ComparisonRequest, GroupDecision, Result, ScalingBackend, ScalingConfig, + ScalingError, Task, Tournament, TournamentCheckpoint, Vote, parse_verdict, +}; + +#[derive(Clone, Copy)] +struct VotePosition { + round: usize, + group: usize, + display: usize, + vote: usize, +} + +/// Runs recursive tournament voting until `target_survivors` candidates remain. +/// +/// When `checkpoint_at` is set, the exact survivor population is copied when it first reaches +/// that size. The tournament then continues to its final target. +pub async fn run_tournament( + backend: &B, + task: &Task, + candidates: Vec, + config: &ScalingConfig, + target_survivors: usize, + checkpoint_at: Option, + root_seed: u64, +) -> Result +where + B: ScalingBackend, +{ + config.validate()?; + validate_candidates(&candidates)?; + if target_survivors == 0 || target_survivors > candidates.len() { + return Err(ScalingError::InvalidConfig( + "target_survivors must be between 1 and the candidate count".to_string(), + )); + } + if checkpoint_at.is_some_and(|count| count < target_survivors || count > candidates.len()) { + return Err(ScalingError::InvalidConfig( + "checkpoint size must be between target_survivors and the candidate count".to_string(), + )); + } + + let mut population = candidates; + let mut populations = vec![candidate_ids(&population)]; + let mut rounds = Vec::new(); + let mut checkpoint = checkpoint_at + .filter(|count| *count == population.len()) + .map(|_| TournamentCheckpoint { + completed_rounds: 0, + candidate_ids: candidate_ids(&population), + }); + + while population.len() > target_survivors { + let round_index = rounds.len(); + let group_size = config.group_size.min(population.len()); + if !population.len().is_multiple_of(group_size) { + return Err(ScalingError::UnevenGroups { + population: population.len(), + group_size, + }); + } + if population.len() / group_size < target_survivors { + return Err(ScalingError::OvershootsTarget { + population: population.len(), + group_size, + target: target_survivors, + }); + } + + let mut ordered = population; + if config.pairing_order == PairingOrder::Shuffle { + seed::shuffle( + &mut ordered, + seed::derive(root_seed, &[1, round_index as u64]), + ); + } + let groups: Vec> = ordered + .chunks(group_size) + .map(<[Candidate]>::to_vec) + .collect(); + let results = try_join_all(groups.into_iter().enumerate().map(|(group_index, group)| { + decide_group( + backend, + task, + group, + config, + round_index, + group_index, + root_seed, + ) + })) + .await?; + + let mut decisions = Vec::with_capacity(results.len()); + let mut next_population = Vec::with_capacity(results.len()); + for (decision, survivor) in results { + decisions.push(decision); + next_population.push(survivor); + } + rounds.push(decisions); + population = next_population; + populations.push(candidate_ids(&population)); + + if checkpoint.is_none() && checkpoint_at == Some(population.len()) { + checkpoint = Some(TournamentCheckpoint { + completed_rounds: rounds.len(), + candidate_ids: candidate_ids(&population), + }); + } + } + + if checkpoint_at.is_some() && checkpoint.is_none() { + return Err(ScalingError::InvalidConfig( + "checkpoint size is not reached by this tournament schedule".to_string(), + )); + } + + Ok(Tournament { + config: config.clone(), + root_seed, + populations, + rounds, + target_survivors, + checkpoint, + survivor_candidate_ids: candidate_ids(&population), + }) +} + +async fn decide_group( + backend: &B, + task: &Task, + group: Vec, + config: &ScalingConfig, + round_index: usize, + group_index: usize, + root_seed: u64, +) -> Result<(GroupDecision, Candidate)> +where + B: ScalingBackend, +{ + let initial = (0..config.votes_per_group).map(|vote_index| { + collect_vote( + backend, + task, + &group, + config, + VotePosition { + round: round_index, + group: group_index, + display: vote_index, + vote: vote_index, + }, + root_seed, + ) + }); + let mut votes = try_join_all(initial).await?; + let invalid_slots: Vec = votes + .iter() + .enumerate() + .filter_map(|(index, vote)| vote.selected_candidate_id.is_none().then_some(index)) + .collect(); + let mut valid_count = votes.len() - invalid_slots.len(); + + if valid_count != config.votes_per_group { + match config.invalid_vote_policy { + InvalidVotePolicy::Abort => { + return Err(ScalingError::IncompleteVotes { + expected: config.votes_per_group, + actual: valid_count, + }); + } + InvalidVotePolicy::Replace { + max_calls_per_group, + } => { + for replacement_index in 0..max_calls_per_group { + if valid_count == config.votes_per_group { + break; + } + let Some(source_vote_index) = invalid_slots + .get(replacement_index % invalid_slots.len()) + .copied() + else { + break; + }; + let vote_index = config.votes_per_group + replacement_index; + let vote = collect_vote( + backend, + task, + &group, + config, + VotePosition { + round: round_index, + group: group_index, + display: source_vote_index, + vote: vote_index, + }, + root_seed, + ) + .await?; + if vote.selected_candidate_id.is_some() { + valid_count += 1; + } + votes.push(vote); + } + } + } + } + if valid_count != config.votes_per_group { + return Err(ScalingError::IncompleteVotes { + expected: config.votes_per_group, + actual: valid_count, + }); + } + + let mut counts = BTreeMap::new(); + for candidate_id in votes + .iter() + .filter_map(|vote| vote.selected_candidate_id.as_ref()) + { + *counts.entry(candidate_id.clone()).or_insert(0) += 1; + } + let Some(highest_count) = counts.values().copied().max() else { + return Err(ScalingError::InvalidRecord( + "a completed group has no vote counts".to_string(), + )); + }; + let winners: Vec<&Candidate> = group + .iter() + .filter(|candidate| counts.get(&candidate.id) == Some(&highest_count)) + .collect(); + let Some(first_winner) = winners.first().copied() else { + return Err(ScalingError::InvalidRecord( + "vote counts do not match any group candidate".to_string(), + )); + }; + let (selected_id, tie_break) = if winners.len() == 1 { + (first_winner.id.clone(), None) + } else { + let selected = match config.tie_policy { + TiePolicy::FirstInGroup => first_winner, + TiePolicy::SeededRandom => { + let index = seed::derive(root_seed, &[5, round_index as u64, group_index as u64]) + as usize + % winners.len(); + winners.get(index).copied().ok_or_else(|| { + ScalingError::InvalidRecord("tie selection is out of range".to_string()) + })? + } + }; + let policy = match config.tie_policy { + TiePolicy::FirstInGroup => "first_in_group", + TiePolicy::SeededRandom => "seeded_random", + }; + (selected.id.clone(), Some(policy.to_string())) + }; + let Some(survivor) = group + .iter() + .find(|candidate| candidate.id == selected_id) + .cloned() + else { + return Err(ScalingError::InvalidRecord( + "selected candidate is not in its group".to_string(), + )); + }; + + Ok(( + GroupDecision { + round_index, + group_index, + input_candidate_ids: candidate_ids(&group), + votes, + vote_counts: counts, + selected_candidate_id: selected_id, + tie_break, + }, + survivor, + )) +} + +async fn collect_vote( + backend: &B, + task: &Task, + group: &[Candidate], + config: &ScalingConfig, + position: VotePosition, + root_seed: u64, +) -> Result +where + B: ScalingBackend, +{ + let mut displayed = group.to_vec(); + if config.display_order == DisplayOrder::Shuffle { + seed::shuffle( + &mut displayed, + seed::derive( + root_seed, + &[ + 2, + position.round as u64, + position.group as u64, + position.display as u64, + ], + ), + ); + } + let ordered_candidate_ids = candidate_ids(&displayed); + let prompt = comparison_prompt(task, &displayed); + let call_seed = seed::derive( + root_seed, + &[ + 3, + position.round as u64, + position.group as u64, + position.display as u64, + position.vote as u64, + ], + ); + let request = ComparisonRequest { + prompt: prompt.clone(), + candidates: displayed.clone(), + round_index: position.round, + group_index: position.group, + vote_index: position.vote, + seed: call_seed, + }; + + match backend.compare(task, request).await { + Ok(response) => { + if response.model_id != backend.model_id() { + return Err(ScalingError::InvalidRecord( + "comparison model ID must match the experiment model".to_string(), + )); + } + let selected_position = parse_verdict(&response.content, displayed.len()); + let selected_candidate_id = selected_position + .and_then(|value| displayed.get(value - 1)) + .map(|candidate| candidate.id.clone()); + let error = selected_candidate_id + .is_none() + .then(|| "judge response has no single in-range verdict".to_string()); + Ok(Vote { + vote_index: position.vote, + ordered_candidate_ids, + prompt, + seed: call_seed, + model_id: Some(response.model_id), + raw_response: Some(response.content), + selected_position, + selected_candidate_id, + error, + }) + } + Err(error) => Ok(Vote { + vote_index: position.vote, + ordered_candidate_ids, + prompt, + seed: call_seed, + model_id: None, + raw_response: None, + selected_position: None, + selected_candidate_id: None, + error: Some(error.to_string()), + }), + } +} + +fn candidate_ids(candidates: &[Candidate]) -> Vec { + candidates + .iter() + .map(|candidate| candidate.id.clone()) + .collect() +} + +fn validate_candidates(candidates: &[Candidate]) -> Result<()> { + if candidates.is_empty() { + return Err(ScalingError::InvalidConfig( + "at least one tournament candidate is required".to_string(), + )); + } + let mut ids = HashSet::with_capacity(candidates.len()); + for candidate in candidates { + if candidate.id.trim().is_empty() || candidate.rollout_id.trim().is_empty() { + return Err(ScalingError::InvalidRecord( + "candidate identifiers must not be empty".to_string(), + )); + } + if !ids.insert(&candidate.id) { + return Err(ScalingError::InvalidRecord( + "candidate identifiers must be unique".to_string(), + )); + } + if candidate.rollout_id != candidate.summary.rollout_id { + return Err(ScalingError::InvalidRecord( + "candidate summary points to a different rollout".to_string(), + )); + } + if candidate.summary.model_id.trim().is_empty() { + return Err(ScalingError::InvalidRecord( + "candidate summary model ID must not be empty".to_string(), + )); + } + } + Ok(()) +} diff --git a/crates/switchyard-test-time-scaling/src/verdict.rs b/crates/switchyard-test-time-scaling/src/verdict.rs new file mode 100644 index 000000000..15f385008 --- /dev/null +++ b/crates/switchyard-test-time-scaling/src/verdict.rs @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Strict comparison-verdict parsing. + +/// Parses exactly one `Final verdict: Solution N` line in range. +pub fn parse_verdict(text: &str, candidate_count: usize) -> Option { + let mut matches = text.lines().filter_map(parse_line); + let position = matches.next()?; + if matches.next().is_some() || position == 0 || position > candidate_count { + return None; + } + Some(position) +} + +fn parse_line(line: &str) -> Option { + let (label, choice) = line.trim().split_once(':')?; + if !label.trim().eq_ignore_ascii_case("final verdict") { + return None; + } + let choice = choice.trim(); + let prefix = choice.get(..8)?; + if !prefix.eq_ignore_ascii_case("solution") { + return None; + } + let suffix = choice.get(8..)?; + if !suffix.starts_with(char::is_whitespace) { + return None; + } + let number = suffix.trim(); + if number.is_empty() || !number.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + number.parse().ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn accepts_one_verdict_in_range() { + assert_eq!( + parse_verdict("Evidence.\nFinal verdict: Solution 2", 2), + Some(2) + ); + assert_eq!(parse_verdict(" final VERDICT: solution 1 ", 2), Some(1)); + } + + #[test] + fn rejects_missing_duplicate_and_out_of_range_verdicts() { + assert_eq!(parse_verdict("Solution 1", 2), None); + assert_eq!( + parse_verdict("Final verdict: Solution 1\nFinal verdict: Solution 2", 2), + None + ); + assert_eq!(parse_verdict("Final verdict: Solution 3", 2), None); + assert_eq!(parse_verdict("Final verdict: Solution 1 later", 2), None); + assert_eq!(parse_verdict("Final verdict: Solution1", 2), None); + } +} diff --git a/crates/switchyard-test-time-scaling/tests/evaluation.rs b/crates/switchyard-test-time-scaling/tests/evaluation.rs new file mode 100644 index 000000000..2cc940eda --- /dev/null +++ b/crates/switchyard-test-time-scaling/tests/evaluation.rs @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use switchyard_test_time_scaling::{Result, TaskEvaluation, experiment_metrics}; + +#[test] +fn stage_metrics_use_task_and_rollout_denominators() -> Result<()> { + let metrics = experiment_metrics(&[ + TaskEvaluation { + task_id: "task-1".to_string(), + iteration_zero: vec![true, false], + selected_for_refinement: vec![true], + iteration_one: vec![true, true], + final_passed: true, + }, + TaskEvaluation { + task_id: "task-2".to_string(), + iteration_zero: vec![false, false], + selected_for_refinement: vec![false], + iteration_one: vec![false, true], + final_passed: false, + }, + ])?; + + assert_eq!(metrics.task_count, 2); + assert_eq!(metrics.iteration_zero.average_pass_at_one, 0.25); + assert_eq!(metrics.iteration_zero.pass_at_n, 0.5); + assert_eq!(metrics.iteration_zero.mixed_task_count, 1); + assert_eq!(metrics.selected_for_refinement.average_pass_at_one, 0.5); + assert_eq!(metrics.iteration_one.average_pass_at_one, 0.75); + assert_eq!(metrics.iteration_one.pass_at_n, 1.0); + assert_eq!(metrics.iteration_one.mixed_task_count, 1); + assert_eq!(metrics.final_pass_at_one, 0.5); + Ok(()) +} + +#[test] +fn metrics_reject_empty_or_duplicate_tasks() { + assert!(experiment_metrics(&[]).is_err()); + + let task = TaskEvaluation { + task_id: "same".to_string(), + iteration_zero: vec![false], + selected_for_refinement: vec![false], + iteration_one: vec![false], + final_passed: false, + }; + assert!(experiment_metrics(&[task.clone(), task]).is_err()); +} diff --git a/crates/switchyard-test-time-scaling/tests/manifest.rs b/crates/switchyard-test-time-scaling/tests/manifest.rs new file mode 100644 index 000000000..96c10d037 --- /dev/null +++ b/crates/switchyard-test-time-scaling/tests/manifest.rs @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::BTreeMap; + +use switchyard_test_time_scaling::{ + ExperimentManifest, MANIFEST_SCHEMA_VERSION, ManifestSource, ManifestValue, + REQUIRED_MANIFEST_FIELDS, ReplicationMode, +}; + +fn manifest(source: ManifestSource) -> ExperimentManifest { + ExperimentManifest { + schema_version: MANIFEST_SCHEMA_VERSION, + replication_mode: ReplicationMode::Conceptual, + code_revision: "revision".to_string(), + model_id: "model".to_string(), + fields: BTreeMap::from_iter(REQUIRED_MANIFEST_FIELDS.map(|name| { + ( + name.to_string(), + ManifestValue { + source, + value: "recorded value".to_string(), + }, + ) + })), + } +} + +#[test] +fn conceptual_manifest_requires_every_recorded_choice() { + let mut value = manifest(ManifestSource::Reconstructed); + assert!(value.validate().is_ok()); + + value.fields.remove("tie_break"); + assert!(value.validate().is_err()); +} + +#[test] +fn exact_manifest_rejects_reconstructed_choices() { + let mut value = manifest(ManifestSource::Reconstructed); + value.replication_mode = ReplicationMode::Exact; + assert!(value.validate().is_err()); + + for field in value.fields.values_mut() { + field.source = ManifestSource::Paper; + } + assert!(value.validate().is_ok()); +} + +#[test] +fn manifest_rejects_unknown_and_empty_fields() { + let mut value = manifest(ManifestSource::Paper); + value.fields.insert( + "typo".to_string(), + ManifestValue { + source: ManifestSource::Paper, + value: "value".to_string(), + }, + ); + assert!(value.validate().is_err()); + + value.fields.remove("typo"); + value + .fields + .get_mut("agent_limits") + .expect("agent_limits fixture") + .value + .clear(); + assert!(value.validate().is_err()); +} diff --git a/crates/switchyard-test-time-scaling/tests/workflow.rs b/crates/switchyard-test-time-scaling/tests/workflow.rs new file mode 100644 index 000000000..2d699c87c --- /dev/null +++ b/crates/switchyard-test-time-scaling/tests/workflow.rs @@ -0,0 +1,484 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::BTreeMap; +use std::collections::HashSet; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use serde_json::{Map, json}; +use switchyard_test_time_scaling::{ + Candidate, ComparisonRequest, ComparisonResponse, DisplayOrder, ExperimentManifest, + InvalidVotePolicy, MANIFEST_SCHEMA_VERSION, ManifestSource, ManifestValue, + REQUIRED_MANIFEST_FIELDS, Result, Rollout, RolloutEvaluation, RolloutRequest, ScalingBackend, + ScalingConfig, ScalingController, ScalingError, ScalingRun, Summary, Task, TiePolicy, + decode_run, encode_run, evaluate_run, run_tournament, +}; + +#[derive(Clone, Copy)] +enum JudgeScript { + First, + Tie, + FirstVoteInvalid, + RolloutZero, + WrongSummaryModel, + WrongComparisonModel, +} + +struct FakeBackend { + judge_script: JudgeScript, + rollout_requests: Arc>>, + comparison_calls: Arc, +} + +impl FakeBackend { + fn new(judge_script: JudgeScript) -> Self { + Self { + judge_script, + rollout_requests: Arc::new(Mutex::new(Vec::new())), + comparison_calls: Arc::new(AtomicUsize::new(0)), + } + } +} + +#[async_trait] +impl ScalingBackend for FakeBackend { + type Output = String; + + fn model_id(&self) -> &str { + "test-model-v1" + } + + async fn run_rollouts( + &self, + _task: &Task, + requests: Vec, + ) -> Result>> { + self.rollout_requests + .lock() + .expect("request log lock") + .extend(requests.iter().cloned()); + Ok(requests + .into_iter() + .map(|request| { + let suffix = format!("{}-{}", request.iteration, request.rollout_index); + Rollout { + id: format!("rollout-{suffix}"), + iteration: request.iteration, + rollout_index: request.rollout_index, + model_id: self.model_id().to_string(), + environment_id: format!("environment-{suffix}"), + output_digest: format!("digest-{suffix}"), + output: format!("output-{suffix}"), + } + }) + .collect()) + } + + async fn summarize(&self, _task: &Task, rollout: &Rollout) -> Result { + let value = json!({"evidence": rollout.id}); + Ok(Summary { + id: format!("summary-{}", rollout.id), + rollout_id: rollout.id.clone(), + model_id: if matches!(self.judge_script, JudgeScript::WrongSummaryModel) { + "different-model".to_string() + } else { + self.model_id().to_string() + }, + value: value + .as_object() + .cloned() + .expect("summary fixture is an object"), + raw_response: value.to_string(), + generation_attempts: 1, + }) + } + + async fn compare( + &self, + _task: &Task, + request: ComparisonRequest, + ) -> Result { + self.comparison_calls.fetch_add(1, Ordering::Relaxed); + let content = match self.judge_script { + JudgeScript::First => "Evidence.\nFinal verdict: Solution 1".to_string(), + JudgeScript::Tie => { + let position = if request.vote_index < 4 { 1 } else { 2 }; + format!("Final verdict: Solution {position}") + } + JudgeScript::FirstVoteInvalid if request.vote_index == 0 => "no verdict".to_string(), + JudgeScript::FirstVoteInvalid => "Final verdict: Solution 1".to_string(), + JudgeScript::RolloutZero => { + let position = request + .candidates + .iter() + .position(|candidate| candidate.id == "rollout-0") + .map(|index| index + 1) + .unwrap_or(1); + format!("Final verdict: Solution {position}") + } + JudgeScript::WrongSummaryModel | JudgeScript::WrongComparisonModel => { + "Final verdict: Solution 1".to_string() + } + }; + Ok(ComparisonResponse { + model_id: if matches!(self.judge_script, JudgeScript::WrongComparisonModel) { + "different-model".to_string() + } else { + self.model_id().to_string() + }, + content, + }) + } +} + +fn task() -> Task { + Task { + id: "task-1".to_string(), + benchmark: "test-benchmark".to_string(), + prompt: "fix the test task".to_string(), + } +} + +fn manifest() -> ExperimentManifest { + ExperimentManifest { + schema_version: MANIFEST_SCHEMA_VERSION, + replication_mode: switchyard_test_time_scaling::ReplicationMode::Conceptual, + code_revision: "test-revision".to_string(), + model_id: "test-model-v1".to_string(), + fields: BTreeMap::from_iter(REQUIRED_MANIFEST_FIELDS.map(|name| { + ( + name.to_string(), + ManifestValue { + source: ManifestSource::Reconstructed, + value: "test choice".to_string(), + }, + ) + })), + } +} + +fn candidates(count: usize) -> Vec { + (0..count) + .map(|index| { + let rollout_id = format!("rollout-{index}"); + Candidate { + id: rollout_id.clone(), + rollout_id: rollout_id.clone(), + summary: Summary { + id: format!("summary-{index}"), + rollout_id, + model_id: "test-model-v1".to_string(), + value: Map::from_iter([("index".to_string(), json!(index))]), + raw_response: format!(r#"{{"index":{index}}}"#), + generation_attempts: 1, + }, + } + }) + .collect() +} + +#[tokio::test] +async fn main_workflow_keeps_attempts_isolated_and_returns_the_selected_output() -> Result<()> { + let backend = FakeBackend::new(JudgeScript::First); + let request_log = Arc::clone(&backend.rollout_requests); + let comparison_calls = Arc::clone(&backend.comparison_calls); + let controller = ScalingController::new(backend, ScalingConfig::default(), manifest())?; + + let run = controller.run(task()).await?; + + let population_sizes = + |populations: &[Vec]| populations.iter().map(Vec::len).collect::>(); + assert_eq!( + population_sizes(&run.iteration_zero_tournament.populations), + vec![16, 8, 4, 2, 1] + ); + assert_eq!( + population_sizes(&run.final_tournament.populations), + vec![16, 8, 4, 2, 1] + ); + let checkpoint = run + .iteration_zero_tournament + .checkpoint + .as_ref() + .expect("refinement checkpoint"); + assert_eq!(checkpoint.completed_rounds, 2); + assert_eq!(checkpoint.candidate_ids.len(), 4); + assert_eq!( + checkpoint.candidate_ids, + run.iteration_zero_tournament.populations[2] + ); + assert_eq!(run.refinement_summary_ids.len(), 4); + + for attempt in &run.iteration_one { + let refinement_prompt = attempt + .request + .refinement_prompt + .as_deref() + .expect("refined attempt has rendered context"); + assert!(refinement_prompt.contains("PRIOR ATTEMPT SUMMARY 4")); + let summary_ids: Vec<&str> = attempt + .request + .refinement_summaries + .iter() + .map(|summary| summary.id.as_str()) + .collect(); + assert_eq!( + summary_ids, + run.refinement_summary_ids + .iter() + .map(String::as_str) + .collect::>() + ); + assert_eq!( + attempt.request.refinement_prompt, + run.iteration_one[0].request.refinement_prompt + ); + } + + let environments: HashSet<&str> = run + .iteration_zero + .iter() + .chain(&run.iteration_one) + .map(|attempt| attempt.rollout.environment_id.as_str()) + .collect(); + assert_eq!(environments.len(), 32); + assert_eq!(comparison_calls.load(Ordering::Relaxed), 240); + assert_eq!( + run.iteration_zero_tournament + .rounds + .iter() + .flatten() + .count(), + 15 + ); + assert_eq!( + run.iteration_zero_tournament + .rounds + .iter() + .flatten() + .flat_map(|decision| &decision.votes) + .count(), + 120 + ); + assert_eq!( + run.final_rollout.id, + run.final_tournament.survivor_candidate_ids[0] + ); + assert_eq!(run.final_rollout.output, "output-1-0"); + assert_eq!(run.final_rollout.output_digest, "digest-1-0"); + assert_eq!(run.model_id, "test-model-v1"); + let encoded = encode_run(&run)?; + let replayed: ScalingRun = decode_run(&encoded)?; + assert_eq!(replayed, run); + + let evaluations = run + .iteration_zero + .iter() + .chain(&run.iteration_one) + .map(|attempt| RolloutEvaluation { + rollout_id: attempt.rollout.id.clone(), + passed: attempt.rollout.id.ends_with("-0"), + }) + .collect(); + let evaluated = evaluate_run(&run, evaluations)?; + assert_eq!(evaluated.iteration_zero.len(), 16); + assert_eq!(evaluated.selected_for_refinement.len(), 4); + assert_eq!(evaluated.iteration_one.len(), 16); + assert!(evaluated.final_passed); + + let requests = request_log.lock().expect("request log lock"); + assert_eq!(requests.len(), 32); + assert_eq!( + requests + .iter() + .filter(|request| request.iteration == 0) + .count(), + 16 + ); + assert!( + requests + .iter() + .filter(|request| request.iteration == 0) + .all(|request| request.refinement_summaries.is_empty() + && request.refinement_prompt.is_none()) + ); + Ok(()) +} + +#[tokio::test] +async fn group_size_schedules_match_the_paper() -> Result<()> { + let cases = [ + (16, vec![16]), + (8, vec![8, 2]), + (4, vec![4, 4]), + (2, vec![2, 2, 2, 2]), + ]; + for (group_size, expected) in cases { + let backend = FakeBackend::new(JudgeScript::First); + let config = ScalingConfig { + group_size, + votes_per_group: 1, + ..ScalingConfig::default() + }; + let tournament = + run_tournament(&backend, &task(), candidates(16), &config, 1, None, 42).await?; + let actual: Vec = tournament + .rounds + .iter() + .map(|round| round[0].input_candidate_ids.len()) + .collect(); + assert_eq!(actual, expected); + } + Ok(()) +} + +#[tokio::test] +async fn ties_and_replacement_votes_are_recorded() -> Result<()> { + let tie_backend = FakeBackend::new(JudgeScript::Tie); + let tie_config = ScalingConfig { + rollout_count: 2, + refinement_count: 1, + votes_per_group: 8, + tie_policy: TiePolicy::FirstInGroup, + ..ScalingConfig::default() + }; + let tied = run_tournament( + &tie_backend, + &task(), + candidates(2), + &tie_config, + 1, + None, + 7, + ) + .await?; + let decision = &tied.rounds[0][0]; + assert_eq!( + decision.vote_counts.values().copied().collect::>(), + [4, 4] + ); + assert_eq!(decision.tie_break.as_deref(), Some("first_in_group")); + + let replacement_backend = FakeBackend::new(JudgeScript::FirstVoteInvalid); + let replacement_config = ScalingConfig { + rollout_count: 2, + refinement_count: 1, + votes_per_group: 2, + invalid_vote_policy: InvalidVotePolicy::Replace { + max_calls_per_group: 1, + }, + ..ScalingConfig::default() + }; + let replaced = run_tournament( + &replacement_backend, + &task(), + candidates(2), + &replacement_config, + 1, + None, + 7, + ) + .await?; + assert_eq!(replaced.rounds[0][0].votes.len(), 3); + assert_eq!(replaced.rounds[0][0].votes[0].selected_candidate_id, None); + assert!( + replaced.rounds[0][0].votes[2] + .selected_candidate_id + .is_some() + ); + Ok(()) +} + +#[tokio::test] +async fn abort_policy_rejects_an_invalid_vote() { + let backend = FakeBackend::new(JudgeScript::FirstVoteInvalid); + let config = ScalingConfig { + rollout_count: 2, + refinement_count: 1, + votes_per_group: 2, + invalid_vote_policy: InvalidVotePolicy::Abort, + ..ScalingConfig::default() + }; + let result = run_tournament(&backend, &task(), candidates(2), &config, 1, None, 7).await; + assert_eq!( + result, + Err(ScalingError::IncompleteVotes { + expected: 2, + actual: 1, + }) + ); +} + +#[tokio::test] +async fn display_permutations_map_back_to_the_stable_candidate() -> Result<()> { + let backend = FakeBackend::new(JudgeScript::RolloutZero); + let config = ScalingConfig { + rollout_count: 2, + refinement_count: 1, + votes_per_group: 8, + display_order: DisplayOrder::Shuffle, + ..ScalingConfig::default() + }; + let tournament = run_tournament(&backend, &task(), candidates(2), &config, 1, None, 9).await?; + let decision = &tournament.rounds[0][0]; + + assert_eq!(decision.selected_candidate_id, "rollout-0"); + assert!( + decision + .votes + .iter() + .all(|vote| vote.selected_candidate_id.as_deref() == Some("rollout-0")) + ); + assert!( + decision + .votes + .iter() + .any(|vote| vote.ordered_candidate_ids[0] != "rollout-0") + ); + Ok(()) +} + +#[tokio::test] +async fn model_drift_is_rejected() -> Result<()> { + let mut wrong_manifest = manifest(); + wrong_manifest.model_id = "different-model".to_string(); + assert!( + ScalingController::new( + FakeBackend::new(JudgeScript::First), + ScalingConfig::default(), + wrong_manifest, + ) + .is_err() + ); + + let small_config = ScalingConfig { + rollout_count: 2, + refinement_count: 1, + votes_per_group: 1, + ..ScalingConfig::default() + }; + let controller = ScalingController::new( + FakeBackend::new(JudgeScript::WrongSummaryModel), + small_config.clone(), + manifest(), + )?; + assert!(matches!( + controller.run(task()).await, + Err(ScalingError::InvalidRecord(_)) + )); + + let result = run_tournament( + &FakeBackend::new(JudgeScript::WrongComparisonModel), + &task(), + candidates(2), + &small_config, + 1, + None, + 1, + ) + .await; + assert!(matches!(result, Err(ScalingError::InvalidRecord(_)))); + Ok(()) +} diff --git a/tests/test_prepare_harbor_dataset.py b/tests/test_prepare_harbor_dataset.py index 74dd620bf..ca132c616 100644 --- a/tests/test_prepare_harbor_dataset.py +++ b/tests/test_prepare_harbor_dataset.py @@ -1,8 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from __future__ import annotations - import importlib.util import json from pathlib import Path @@ -37,7 +35,6 @@ def _write_task(root: Path, name: str, task_toml: str, dockerfile: str | None = def _prepare( tmp_path: Path, source: Path, - *, source_dataset: str = "openthoughts-tblite@2.0", ) -> Path: module = _load_generator_module() @@ -190,6 +187,8 @@ def test_prebuilt_docker_image_task_becomes_derived_dockerfile(tmp_path: Path) - assert dockerfile.startswith("FROM python:3.12-slim\nUSER root\n") assert "@anthropic-ai/claude-code@2.1.211" in dockerfile assert "@openai/codex@0.144.5" in dockerfile + assert "uv tool install mini-swe-agent==2.4.6" in dockerfile + assert 'grep -F "mini-swe-agent v2.4.6"' in dockerfile assert "opencode-ai@1.18.3" in dockerfile @@ -286,6 +285,7 @@ def test_generated_dataset_manifest_records_pins_tasks_and_digests(tmp_path: Pat "CLAUDE_CODE_VERSION": "2.1.211", "CODEX_VERSION": "0.144.5", "HERMES_VERSION": "3c27eb6234bf91b8ceee9e9071591b31e9b148cb", + "MINI_SWE_AGENT_VERSION": "2.4.6", "NODE_VERSION": "20.11.1", "OPENCODE_VERSION": "1.18.3", } @@ -309,6 +309,7 @@ def test_generated_compose_bakes_task_id_into_proxy_env(tmp_path: Path) -> None: assert "SWITCHYARD_TASK_ID=task-id-check" in proxy_env assert "SWITCHYARD_TRIAL_DIR=${HOST_AGENT_LOGS_PATH:-}" in proxy_env + def test_a_hermes_ref_that_is_not_a_commit_sha_is_rejected() -> None: """Only a full commit SHA can be recorded as a pin. @@ -321,6 +322,7 @@ def test_a_hermes_ref_that_is_not_a_commit_sha_is_rejected() -> None: base = { "CLAUDE_CODE_VERSION": "1", "CODEX_VERSION": "2", + "MINI_SWE_AGENT_VERSION": "2.4.6", "OPENCODE_VERSION": "3", "NODE_VERSION": "4", } @@ -345,6 +347,7 @@ def test_the_hermes_installer_is_fetched_at_the_pinned_commit() -> None: pins = { "CLAUDE_CODE_VERSION": "1", "CODEX_VERSION": "2", + "MINI_SWE_AGENT_VERSION": "2.4.6", "OPENCODE_VERSION": "3", "NODE_VERSION": "4", "HERMES_VERSION": sha, @@ -368,6 +371,7 @@ def test_the_hermes_pin_is_applied_by_commit_and_forced() -> None: pins = { "CLAUDE_CODE_VERSION": "1", "CODEX_VERSION": "2", + "MINI_SWE_AGENT_VERSION": "2.4.6", "OPENCODE_VERSION": "3", "NODE_VERSION": "4", "HERMES_VERSION": sha, @@ -385,6 +389,7 @@ def test_the_alpine_branch_installs_the_shell_the_installer_needs() -> None: pins = { "CLAUDE_CODE_VERSION": "1", "CODEX_VERSION": "2", + "MINI_SWE_AGENT_VERSION": "2.4.6", "OPENCODE_VERSION": "3", "NODE_VERSION": "4", "HERMES_VERSION": "3c27eb6234bf91b8ceee9e9071591b31e9b148cb", @@ -400,7 +405,8 @@ def test_a_missing_hermes_pin_is_reported_with_the_other_pins(tmp_path: Path) -> module = _load_generator_module() versions = tmp_path / "agent-versions.env" versions.write_text( - "CLAUDE_CODE_VERSION=1\nCODEX_VERSION=2\nOPENCODE_VERSION=3\nNODE_VERSION=4\n" + "CLAUDE_CODE_VERSION=1\nCODEX_VERSION=2\nMINI_SWE_AGENT_VERSION=5\n" + "OPENCODE_VERSION=3\nNODE_VERSION=4\n" ) module.AGENT_VERSIONS_FILE = versions source = tmp_path / "source" @@ -414,4 +420,3 @@ def test_a_missing_hermes_pin_is_reported_with_the_other_pins(tmp_path: Path) -> harbor_command="harbor", overwrite=False, ) - diff --git a/tests/test_run_baseline_script.py b/tests/test_run_baseline_script.py index 86e19e97e..0348d2388 100644 --- a/tests/test_run_baseline_script.py +++ b/tests/test_run_baseline_script.py @@ -1,8 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from __future__ import annotations - import json import os import shlex @@ -38,6 +36,7 @@ def _write_fake_harbor(bin_dir: Path) -> Path: { printf 'OPENAI_API_KEY=%s\\n' "${OPENAI_API_KEY:-}" printf 'OPENAI_BASE_URL=%s\\n' "${OPENAI_BASE_URL:-}" + printf 'OPENAI_API_BASE=%s\\n' "${OPENAI_API_BASE:-}" printf 'ANTHROPIC_BASE_URL=%s\\n' "${ANTHROPIC_BASE_URL:-}" printf 'ANTHROPIC_AUTH_TOKEN=%s\\n' "${ANTHROPIC_AUTH_TOKEN:-}" printf 'ANTHROPIC_API_KEY=%s\\n' "${ANTHROPIC_API_KEY:-}" @@ -66,8 +65,8 @@ def _write_fake_docker(bin_dir: Path) -> Path: docker = bin_dir / "docker" docker.write_text( "#!/usr/bin/env bash\n" - "if [[ -n \"${FAKE_DOCKER_LOG:-}\" ]]; then\n" - " printf '%s\\n' \"$*\" >> \"${FAKE_DOCKER_LOG}\"\n" + 'if [[ -n "${FAKE_DOCKER_LOG:-}" ]]; then\n' + ' printf \'%s\\n\' "$*" >> "${FAKE_DOCKER_LOG}"\n' "fi\n" "exit 0\n" ) @@ -124,7 +123,7 @@ def _write_fake_harbor_patch(tmp_path: Path) -> Path: return patch_file -def _write_fake_harbor_python(tmp_path: Path, *, patched: bool = True) -> Path: +def _write_fake_harbor_python(tmp_path: Path, patched: bool = True) -> Path: tmp_path.mkdir(parents=True, exist_ok=True) harbor_site = tmp_path / "fake-site-packages" / "harbor" base = harbor_site / "agents" / "installed" / "base.py" @@ -147,11 +146,11 @@ def _write_fake_harbor_python(tmp_path: Path, *, patched: bool = True) -> Path: def _run_baseline( tmp_path: Path, - *args: str, + args: list[str], env: dict[str, str] | None = None, include_dataset: bool = True, fake_server: bool = False, -): +) -> subprocess.CompletedProcess[str]: fake_bin = tmp_path / "default-bin" fake_bin.mkdir(exist_ok=True) _write_fake_harbor(fake_bin) @@ -249,7 +248,7 @@ def _write_closed_book_dataset(path: Path) -> Path: def test_direct_mode_requires_model(tmp_path: Path) -> None: - result = _run_baseline(tmp_path, "--dry-run") + result = _run_baseline(tmp_path, ["--dry-run"]) assert result.returncode != 0 assert "--model is required when running direct upstream" in result.stderr @@ -258,21 +257,20 @@ def test_direct_mode_requires_model(tmp_path: Path) -> None: def test_direct_mode_defaults_to_openrouter_upstream_without_switchyard(tmp_path: Path) -> None: result = _run_baseline( tmp_path, - "--model", - "openai/gpt-5.2", - "--agent", - "codex", - "--dry-run", + ["--model", "openai/gpt-5.2", "--agent", "codex", "--dry-run"], ) assert result.returncode == 0, result.stderr harbor = _line_argv(result.stdout, "HARBOR_CMD: ") assert _option_value(harbor, "--model") == "openai/gpt-5.2" agent_env = _option_values(harbor, "--ae") - ca_env_prefixes = ("SSL_CERT_FILE=", "REQUESTS_CA_BUNDLE=", "CURL_CA_BUNDLE=", "GIT_SSL_CAINFO=") - assert not any( - value.startswith(ca_env_prefixes) for value in agent_env + ca_env_prefixes = ( + "SSL_CERT_FILE=", + "REQUESTS_CA_BUNDLE=", + "CURL_CA_BUNDLE=", + "GIT_SSL_CAINFO=", ) + assert not any(value.startswith(ca_env_prefixes) for value in agent_env) assert "server_preset: direct" in result.stdout assert "server_mode: direct" in result.stdout assert "upstream_url: https://openrouter.ai/api/v1" in result.stdout @@ -285,11 +283,7 @@ def test_direct_mode_defaults_to_openrouter_upstream_without_switchyard(tmp_path def test_direct_mode_uses_generic_upstream_key_when_set(tmp_path: Path) -> None: result = _run_baseline( tmp_path, - "--model", - "provider/model", - "--agent", - "codex", - "--dry-run", + ["--model", "provider/model", "--agent", "codex", "--dry-run"], env={ "UPSTREAM_API_KEY": "upstream-test", # pragma: allowlist secret "UPSTREAM_BASE_URL": "https://provider.example/v1", @@ -301,12 +295,56 @@ def test_direct_mode_uses_generic_upstream_key_when_set(tmp_path: Path) -> None: assert "api_key_env: UPSTREAM_API_KEY" in result.stdout +def test_direct_foreground_sets_both_openai_base_variables(tmp_path: Path) -> None: + result = _run_baseline( + tmp_path, + ["--model", "provider/model", "--agent", "mini-swe-agent", "--foreground"], + fake_server=True, + env={ + "UPSTREAM_API_KEY": "upstream-test", # pragma: allowlist secret + "UPSTREAM_BASE_URL": "https://provider.example/v1", + }, + ) + + assert result.returncode == 0, result.stderr + run_dir = next((tmp_path / "out").glob("baseline-direct-direct-*")) + env_text = (next((run_dir / "jobs").iterdir()) / "env.txt").read_text() + assert "OPENAI_BASE_URL=https://provider.example/v1" in env_text + assert "OPENAI_API_BASE=https://provider.example/v1" in env_text + manifest = json.loads((run_dir / "run_manifest.json").read_text()) + assert manifest["closed_book"]["agent_versions"]["mini_swe_agent"] == "2.4.6" + + +def test_custom_agent_import_replaces_named_agent_and_pins_mini_swe(tmp_path: Path) -> None: + result = _run_baseline( + tmp_path, + [ + "--model", + "provider/model", + "--agent", + "mini-swe-agent", + "--agent-import-path", + "benchmark.module:Agent", + "--dry-run", + ], + env={ + "UPSTREAM_API_KEY": "upstream-test", # pragma: allowlist secret + "UPSTREAM_BASE_URL": "https://provider.example/v1", + }, + ) + + assert result.returncode == 0, result.stderr + harbor = _line_argv(result.stdout, "HARBOR_CMD: ") + assert _option_value(harbor, "--agent-import-path") == "benchmark.module:Agent" + assert "--agent" not in harbor + assert "version=2.4.6" in _option_values(harbor, "--ak") + assert "agent/model: benchmark.module:Agent" in result.stdout + + def test_direct_mode_requires_selected_api_key_env(tmp_path: Path) -> None: result = _run_baseline( tmp_path, - "--model", - "openai/gpt-5.2", - "--dry-run", + ["--model", "openai/gpt-5.2", "--dry-run"], env={"OPENROUTER_API_KEY": ""}, ) @@ -317,11 +355,7 @@ def test_direct_mode_requires_selected_api_key_env(tmp_path: Path) -> None: def test_direct_mode_rejects_server_only_options(tmp_path: Path) -> None: result = _run_baseline( tmp_path, - "--model", - "openai/gpt-5.2", - "--server-extra", - "--log-level=debug", - "--dry-run", + ["--model", "openai/gpt-5.2", "--server-extra", "--log-level=debug", "--dry-run"], ) assert result.returncode != 0 @@ -329,9 +363,7 @@ def test_direct_mode_rejects_server_only_options(tmp_path: Path) -> None: result = _run_baseline( tmp_path, - "--route-model", - "tb-lite-random-routing", - "--dry-run", + ["--route-model", "tb-lite-random-routing", "--dry-run"], ) assert result.returncode != 0 @@ -341,11 +373,7 @@ def test_direct_mode_rejects_server_only_options(tmp_path: Path) -> None: def test_legacy_routing_profiles_are_rejected(tmp_path: Path) -> None: result = _run_baseline( tmp_path, - "--routing-profiles", - str(tmp_path / "routes.yaml"), - "--model", - "switchyard", - "--dry-run", + ["--routing-profiles", str(tmp_path / "routes.yaml"), "--model", "switchyard", "--dry-run"], ) assert result.returncode != 0 @@ -356,13 +384,15 @@ def test_legacy_routing_profiles_are_rejected(tmp_path: Path) -> None: def test_dry_run_claude_code_opus_defaults_high_reasoning(tmp_path: Path) -> None: result = _run_baseline( tmp_path, - "--server-config", - str(REPO / "benchmark" / "server-configs" / "tb-lite-single-opus-4-7.toml"), - "--route-model", - "tb-lite-single-opus-4-7", - "--agent", - "claude-code", - "--dry-run", + [ + "--server-config", + str(REPO / "benchmark" / "server-configs" / "tb-lite-single-opus-4-7.toml"), + "--route-model", + "tb-lite-single-opus-4-7", + "--agent", + "claude-code", + "--dry-run", + ], ) assert result.returncode == 0, result.stderr @@ -378,13 +408,15 @@ def test_dry_run_claude_code_opus_defaults_high_reasoning(tmp_path: Path) -> Non def test_dry_run_codex_gpt_defaults_high_reasoning(tmp_path: Path) -> None: result = _run_baseline( tmp_path, - "--server-config", - str(REPO / "benchmark" / "server-configs" / "tb-lite-single-gpt-5-5.toml"), - "--route-model", - "tb-lite-single-gpt-5-5", - "--agent", - "codex", - "--dry-run", + [ + "--server-config", + str(REPO / "benchmark" / "server-configs" / "tb-lite-single-gpt-5-5.toml"), + "--route-model", + "tb-lite-single-gpt-5-5", + "--agent", + "codex", + "--dry-run", + ], ) assert result.returncode == 0, result.stderr @@ -406,20 +438,24 @@ def test_dry_run_explicit_empty_reasoning_omits_kwarg(tmp_path: Path) -> None: result = _run_baseline( tmp_path, - "--server-config", - str(profile), - "--route-model", - "tb-lite-random-routing", - "--agent", - "claude-code", - "--reasoning-effort", - "", - "--dry-run", + [ + "--server-config", + str(profile), + "--route-model", + "tb-lite-random-routing", + "--agent", + "claude-code", + "--reasoning-effort", + "", + "--dry-run", + ], ) assert result.returncode == 0, result.stderr harbor = _line_argv(result.stdout, "HARBOR_CMD: ") - assert not any(value.startswith("reasoning_effort=") for value in _option_values(harbor, "--ak")) + assert not any( + value.startswith("reasoning_effort=") for value in _option_values(harbor, "--ak") + ) assert not any(value.startswith("thinking=") for value in _option_values(harbor, "--ak")) assert "reasoning: unset" in result.stdout @@ -429,15 +465,17 @@ def test_dry_run_explicit_reasoning_overrides_default(tmp_path: Path) -> None: result = _run_baseline( tmp_path, - "--server-config", - str(profile), - "--route-model", - "tb-lite-random-routing", - "--agent", - "claude-code", - "--reasoning-effort", - "medium", - "--dry-run", + [ + "--server-config", + str(profile), + "--route-model", + "tb-lite-random-routing", + "--agent", + "claude-code", + "--reasoning-effort", + "medium", + "--dry-run", + ], ) assert result.returncode == 0, result.stderr @@ -451,19 +489,21 @@ def test_dry_run_explicit_claude_thinking_overrides_adaptive_default(tmp_path: P result = _run_baseline( tmp_path, - "--server-config", - str(profile), - "--route-model", - "tb-lite-random-routing", - "--agent", - "claude-code", - "--reasoning-effort", - "high", - "--harbor-extra", - "--ak", - "--harbor-extra", - "thinking=disabled", - "--dry-run", + [ + "--server-config", + str(profile), + "--route-model", + "tb-lite-random-routing", + "--agent", + "claude-code", + "--reasoning-effort", + "high", + "--harbor-extra", + "--ak", + "--harbor-extra", + "thinking=disabled", + "--dry-run", + ], ) assert result.returncode == 0, result.stderr @@ -477,11 +517,7 @@ def test_harbor_path_is_required(tmp_path: Path) -> None: result = _run_baseline( tmp_path, - "--server-config", - str(profile), - "--route-model", - "tb-lite-random-routing", - "--dry-run", + ["--server-config", str(profile), "--route-model", "tb-lite-random-routing", "--dry-run"], include_dataset=False, ) @@ -496,15 +532,17 @@ def test_closed_book_dataset_rejects_unpatched_harbor(tmp_path: Path) -> None: result = _run_baseline( tmp_path, - "--harbor-path", - str(dataset), - "--server-config", - str(profile), - "--route-model", - "tb-lite-random-routing", - "--harbor-python", - str(fake_python), - "--dry-run", + [ + "--harbor-path", + str(dataset), + "--server-config", + str(profile), + "--route-model", + "tb-lite-random-routing", + "--harbor-python", + str(fake_python), + "--dry-run", + ], include_dataset=False, ) @@ -520,11 +558,7 @@ def test_dry_run_server_config_uses_rust_switchyard_server(tmp_path: Path) -> No result = _run_baseline( tmp_path, - "--server-config", - str(profile), - "--model", - "tb-lite-random-routing", - "--dry-run", + ["--server-config", str(profile), "--model", "tb-lite-random-routing", "--dry-run"], ) assert result.returncode == 0, result.stderr @@ -545,11 +579,13 @@ def test_foreground_server_run_collects_rust_stats_endpoint(tmp_path: Path) -> N result = _run_baseline( tmp_path, - "--server-config", - str(profile), - "--route-model", - "tb-lite-random-routing", - "--foreground", + [ + "--server-config", + str(profile), + "--route-model", + "tb-lite-random-routing", + "--foreground", + ], fake_server=True, env={"FAKE_DOCKER_LOG": str(docker_log)}, ) @@ -570,11 +606,7 @@ def test_dry_run_route_model_alias_uses_rust_switchyard_server(tmp_path: Path) - result = _run_baseline( tmp_path, - "--server-config", - str(profile), - "--route-model", - "tb-lite-random-routing", - "--dry-run", + ["--server-config", str(profile), "--route-model", "tb-lite-random-routing", "--dry-run"], ) assert result.returncode == 0, result.stderr @@ -591,13 +623,15 @@ def test_dry_run_server_config_honors_explicit_harbor_model(tmp_path: Path) -> N result = _run_baseline( tmp_path, - "--server-config", - str(profile), - "--route-model", - "tb-lite-random-routing", - "--harbor-model", - "openai/custom-route-label", - "--dry-run", + [ + "--server-config", + str(profile), + "--route-model", + "tb-lite-random-routing", + "--harbor-model", + "openai/custom-route-label", + "--dry-run", + ], ) assert result.returncode == 0, result.stderr @@ -610,15 +644,17 @@ def test_dry_run_harbor_extra(tmp_path: Path) -> None: result = _run_baseline( tmp_path, - "--server-config", - str(profile), - "--route-model", - "tb-lite-random-routing", - "--harbor-extra", - "--include-task-name=hello", - "--harbor-extra", - "--no-upload", - "--dry-run", + [ + "--server-config", + str(profile), + "--route-model", + "tb-lite-random-routing", + "--harbor-extra", + "--include-task-name=hello", + "--harbor-extra", + "--no-upload", + "--dry-run", + ], ) assert result.returncode == 0, result.stderr @@ -636,15 +672,17 @@ def test_dry_run_harbor_path_uses_local_dataset_and_closed_book_artifact( result = _run_baseline( tmp_path, - "--harbor-path", - str(dataset), - "--server-config", - str(profile), - "--route-model", - "tb-lite-random-routing", - "--agent", - "codex", - "--dry-run", + [ + "--harbor-path", + str(dataset), + "--server-config", + str(profile), + "--route-model", + "tb-lite-random-routing", + "--agent", + "codex", + "--dry-run", + ], include_dataset=False, ) @@ -679,17 +717,19 @@ def test_dry_run_open_book_uses_proxy_topology_without_tool_disables(tmp_path: P result = _run_baseline( tmp_path, - "--harbor-path", - str(dataset), - "--server-config", - str(profile), - "--route-model", - "tb-lite-random-routing", - "--agent", - "codex", - "--book-mode", - "open", - "--dry-run", + [ + "--harbor-path", + str(dataset), + "--server-config", + str(profile), + "--route-model", + "tb-lite-random-routing", + "--agent", + "codex", + "--book-mode", + "open", + "--dry-run", + ], include_dataset=False, ) @@ -711,19 +751,21 @@ def test_dry_run_claude_closed_book_merges_disallowed_tools(tmp_path: Path) -> N result = _run_baseline( tmp_path, - "--harbor-path", - str(dataset), - "--server-config", - str(profile), - "--route-model", - "tb-lite-random-routing", - "--agent", - "claude-code", - "--harbor-extra", - "--ak", - "--harbor-extra", - "disallowed_tools=Bash", - "--dry-run", + [ + "--harbor-path", + str(dataset), + "--server-config", + str(profile), + "--route-model", + "tb-lite-random-routing", + "--agent", + "claude-code", + "--harbor-extra", + "--ak", + "--harbor-extra", + "disallowed_tools=Bash", + "--dry-run", + ], include_dataset=False, ) @@ -739,15 +781,17 @@ def test_dry_run_opencode_closed_book_disables_webfetch(tmp_path: Path) -> None: result = _run_baseline( tmp_path, - "--harbor-path", - str(dataset), - "--server-config", - str(profile), - "--route-model", - "tb-lite-random-routing", - "--agent", - "opencode", - "--dry-run", + [ + "--harbor-path", + str(dataset), + "--server-config", + str(profile), + "--route-model", + "tb-lite-random-routing", + "--agent", + "opencode", + "--dry-run", + ], include_dataset=False, ) @@ -765,13 +809,15 @@ def test_task_list_file_expands_to_include_task_name(tmp_path: Path) -> None: result = _run_baseline( tmp_path, - "--server-config", - str(profile), - "--route-model", - "tb-lite-random-routing", - "--task-list-file", - str(task_list), - "--dry-run", + [ + "--server-config", + str(profile), + "--route-model", + "tb-lite-random-routing", + "--task-list-file", + str(task_list), + "--dry-run", + ], ) assert result.returncode == 0, result.stderr @@ -787,11 +833,7 @@ def test_dry_run_uses_harbor_bin_override(tmp_path: Path) -> None: result = _run_baseline( tmp_path, - "--server-config", - str(profile), - "--route-model", - "tb-lite-random-routing", - "--dry-run", + ["--server-config", str(profile), "--route-model", "tb-lite-random-routing", "--dry-run"], env={"HARBOR_BIN": str(expected_harbor)}, ) diff --git a/tests/test_test_time_scaling_adapters.py b/tests/test_test_time_scaling_adapters.py new file mode 100644 index 000000000..8056402d3 --- /dev/null +++ b/tests/test_test_time_scaling_adapters.py @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import importlib.util +import json +from pathlib import Path +from types import ModuleType + +REPO = Path(__file__).resolve().parents[1] +ROLLOUTS = REPO / "benchmark" / "test_time_scaling_rollouts.py" +GRADES = REPO / "benchmark" / "test_time_scaling_grades.py" +CONFIG = REPO / "benchmark" / "test_time_scaling_config.py" +HARBOR_AGENT = REPO / "benchmark" / "test_time_scaling_harbor_agent.py" + + +def _load(path: Path, name: str) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _source_dataset(tmp_path: Path) -> Path: + root = tmp_path / "source" + task = root / "task-1" + task.mkdir(parents=True) + (root / "switchyard_dataset_manifest.json").write_text("{}") + (task / "instruction.md").write_text("Fix the task.\n") + (task / "task.toml").write_text("[agent]\ntimeout_sec = 10\n") + return root + + +def test_refined_dataset_prepends_one_shared_context(tmp_path: Path) -> None: + module = _load(ROLLOUTS, "switchyard_test_time_scaling_rollouts") + source = _source_dataset(tmp_path) + requests = [ + {"iteration": 1, "rollout_index": index, "refinement_prompt": "Prior evidence."} + for index in range(2) + ] + + prepared = module._prepare_dataset( + source, + tmp_path / "method", + {"id": "task-1", "prompt": "Fix the task."}, + requests, + 1, + ) + + instruction = (prepared / "task-1" / "instruction.md").read_text() + assert instruction == "Prior evidence.\n\nORIGINAL TASK\nFix the task.\n" + assert (source / "task-1" / "instruction.md").read_text() == "Fix the task.\n" + + +def test_public_rollout_excludes_private_grade_data(tmp_path: Path) -> None: + rollouts = _load(ROLLOUTS, "switchyard_test_time_scaling_rollouts_public") + grades = _load(GRADES, "switchyard_test_time_scaling_grades") + trial = tmp_path / "trial-a" + (trial / "agent").mkdir(parents=True) + (trial / "artifacts").mkdir() + (trial / "verifier").mkdir() + (trial / "agent" / "mini-swe-agent.trajectory.json").write_text( + json.dumps({"messages": [{"role": "assistant", "content": "changed code"}]}) + ) + (trial / "artifacts" / "patch.diff").write_text("diff --git a/a b/a\n") + (trial / "verifier" / "reward.txt").write_text("1\n") + + public = rollouts._public_rollout( + "task-1", + "model-1", + {"iteration": 0, "rollout_index": 0}, + trial, + ) + + encoded = json.dumps(public) + assert "reward" not in encoded + assert "verifier" not in encoded + assert str(trial) not in encoded + assert public["output_digest"].startswith("sha256:") + assert grades._passed(trial) is True + + grade_root = tmp_path / "private-grades" + rollouts._write_private_grade_map(grade_root, 0, [(public, trial)]) + mapping_text = (grade_root / "iteration-0.json").read_text() + assert "trial_dir" not in mapping_text + assert "reward" not in mapping_text + mapping = json.loads(mapping_text) + assert Path(mapping[0]["patch_file"]).read_text() == "diff --git a/a b/a\n" + + +def test_rollout_disables_verification(tmp_path: Path) -> None: + module = _load(ROLLOUTS, "switchyard_test_time_scaling_rollouts_command") + method_root = tmp_path / "method" + config = { + "repo_root": str(REPO), + "method_root": str(method_root), + "run_baseline": str(REPO / "benchmark" / "run-baseline.sh"), + "model_id": "model-1", + "agent_import_path": "module:Agent", + "harbor_model": "openai/model-1", + "n_concurrent": 1, + "max_retries": 1, + "agent_timeout_multiplier": 1.0, + "upstream_base_url": "https://example.test/v1", + "upstream_api_key_env": "TEST_KEY", + } + + command = module._harbor_command(config, tmp_path / "dataset", "task-1", 2, 0) + + assert "--disable-verification" in command + assert command[command.index("--harbor-extra") + 1] == "-k" + + +def test_config_writer_uses_paper_defaults(tmp_path: Path) -> None: + module = _load(CONFIG, "switchyard_test_time_scaling_config") + dataset = _source_dataset(tmp_path) + output = tmp_path / "output" + args = module._parser().parse_args( + [ + "--dataset", + str(dataset), + "--task", + "task-1", + "--model", + "azure/anthropic/claude-sonnet-4-5", + "--output", + str(output), + ] + ) + runner_path, harbor_path = module.write_configs(args, REPO) + + runner = json.loads(runner_path.read_text()) + harbor = json.loads(harbor_path.read_text()) + assert runner["scaling"] == { + "rollout_count": 16, + "refinement_count": 4, + "group_size": 2, + "votes_per_group": 8, + "seed": 0, + "pairing_order": "in_order", + "display_order": "in_order", + "tie_policy": "first_in_group", + "invalid_vote_policy": {"mode": "abort"}, + } + assert len(runner["manifest"]["fields"]) == 22 + assert runner["manifest"]["fields"]["ablation_fixed_settings"]["source"] == "paper" + assert runner["evaluation_command"]["argv"][-2:] == ["--config", str(harbor_path)] + assert harbor["harbor_model"] == "openai/azure/anthropic/claude-sonnet-4-5" + assert harbor["run_record"] == str(output / "method" / "run.json") + + +def test_saved_patch_includes_new_files() -> None: + module = _load(HARBOR_AGENT, "switchyard_test_time_scaling_harbor_agent") + + assert "git -C /testbed add -A" in module.PATCH_COMMAND + assert "diff --cached --binary" in module.PATCH_COMMAND From 7ab4fccb50f2bb817a218bf09210da6b5cbf64be Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Wed, 19 Aug 2026 10:31:02 -0700 Subject: [PATCH 3/4] fix(test-time-scaling): accept markdown-wrapped comparison verdicts Signed-off-by: Elyas Mehtabuddin --- .../src/verdict.rs | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/crates/switchyard-test-time-scaling/src/verdict.rs b/crates/switchyard-test-time-scaling/src/verdict.rs index 15f385008..650419a0b 100644 --- a/crates/switchyard-test-time-scaling/src/verdict.rs +++ b/crates/switchyard-test-time-scaling/src/verdict.rs @@ -14,11 +14,20 @@ pub fn parse_verdict(text: &str, candidate_count: usize) -> Option { } fn parse_line(line: &str) -> Option { + // Tolerate Markdown emphasis models add around the verdict (**bold**, *italic*, `code`, + // _underline_). Trailing prose ("Solution 1 later") and a missing space ("Solution1") stay + // rejected: only emphasis punctuation is stripped, never letters, digits, or interior words. + let emphasis = |c: char| matches!(c, '*' | '_' | '`'); let (label, choice) = line.trim().split_once(':')?; - if !label.trim().eq_ignore_ascii_case("final verdict") { + if !label + .trim() + .trim_matches(emphasis) + .trim() + .eq_ignore_ascii_case("final verdict") + { return None; } - let choice = choice.trim(); + let choice = choice.trim().trim_matches(emphasis).trim(); let prefix = choice.get(..8)?; if !prefix.eq_ignore_ascii_case("solution") { return None; @@ -27,7 +36,7 @@ fn parse_line(line: &str) -> Option { if !suffix.starts_with(char::is_whitespace) { return None; } - let number = suffix.trim(); + let number = suffix.trim().trim_matches(emphasis).trim(); if number.is_empty() || !number.bytes().all(|byte| byte.is_ascii_digit()) { return None; } @@ -47,6 +56,23 @@ mod tests { assert_eq!(parse_verdict(" final VERDICT: solution 1 ", 2), Some(1)); } + #[test] + fn accepts_markdown_wrapped_verdict() { + // Real judges (e.g. Claude Haiku) end with a bold verdict; strip the emphasis, keep the vote. + assert_eq!( + parse_verdict("Analysis.\n**Final verdict: Solution 1**", 2), + Some(1) + ); + assert_eq!(parse_verdict("Final verdict: **Solution 2**", 2), Some(2)); + assert_eq!(parse_verdict("`Final verdict: Solution 1`", 2), Some(1)); + // Emphasis stripping must not start accepting trailing prose or a missing space. + assert_eq!( + parse_verdict("**Final verdict: Solution 1 later**", 2), + None + ); + assert_eq!(parse_verdict("**Final verdict: Solution1**", 2), None); + } + #[test] fn rejects_missing_duplicate_and_out_of_range_verdicts() { assert_eq!(parse_verdict("Solution 1", 2), None); From a9c9e09109d8410a7ee31651216e8a143fe3efe3 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Wed, 19 Aug 2026 12:13:39 -0700 Subject: [PATCH 4/4] fix(test-time-scaling): skip harbor adapter test when harbor is unavailable Signed-off-by: Elyas Mehtabuddin --- tests/test_test_time_scaling_adapters.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_test_time_scaling_adapters.py b/tests/test_test_time_scaling_adapters.py index 8056402d3..c3f370bfa 100644 --- a/tests/test_test_time_scaling_adapters.py +++ b/tests/test_test_time_scaling_adapters.py @@ -6,6 +6,8 @@ from pathlib import Path from types import ModuleType +import pytest + REPO = Path(__file__).resolve().parents[1] ROLLOUTS = REPO / "benchmark" / "test_time_scaling_rollouts.py" GRADES = REPO / "benchmark" / "test_time_scaling_grades.py" @@ -151,6 +153,8 @@ def test_config_writer_uses_paper_defaults(tmp_path: Path) -> None: def test_saved_patch_includes_new_files() -> None: + # The Harbor agent adapter imports `harbor`, which requires Python >= 3.12; skip where absent. + pytest.importorskip("harbor") module = _load(HARBOR_AGENT, "switchyard_test_time_scaling_harbor_agent") assert "git -C /testbed add -A" in module.PATCH_COMMAND