diff --git a/.gitignore b/.gitignore index c2d3459..a5b3d38 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,9 @@ build/ dist/ *.egg-info/ .yada/ +# Single-task and suite evaluation outputs, including workspaces and traces. eval-results/ +# Local background-run supervisor files from the documented suite command. +eval_run.log +eval_run.pid logs/ diff --git a/README-cn.md b/README-cn.md index 09f6273..939b3b6 100644 --- a/README-cn.md +++ b/README-cn.md @@ -32,12 +32,19 @@ git clone https://github.com/GenTang/Yada.git cd Yada uv sync --locked --dev -export DEEPSEEK_API_KEY="sk-..." +install -d -m 700 ~/.config/yada +(umask 077; touch ~/.config/yada/deepseek_api_key) +chmod 600 ~/.config/yada/deepseek_api_key +${EDITOR:-vi} ~/.config/yada/deepseek_api_key uv run yada "修复 parser 的边界问题，并运行相关测试" \ --workspace /path/to/repository ``` +文件中只写 API Key。对于 Docker、云主机或 Secret Manager 挂载，也可以使用 +`--api-key-file /run/secrets/deepseek_api_key`；详见 +[配置文档](docs/configuration.md#deepseek-credentials)。 + Yada 默认会在运行仓库命令前请求确认。只有在可信、一次性的隔离环境中才应使用 `--yes`： diff --git a/README.md b/README.md index b2a71e9..1e66a5c 100644 --- a/README.md +++ b/README.md @@ -32,12 +32,19 @@ git clone https://github.com/GenTang/Yada.git cd Yada uv sync --locked --dev -export DEEPSEEK_API_KEY="sk-..." +install -d -m 700 ~/.config/yada +(umask 077; touch ~/.config/yada/deepseek_api_key) +chmod 600 ~/.config/yada/deepseek_api_key +${EDITOR:-vi} ~/.config/yada/deepseek_api_key uv run yada "Fix the failing parser edge case and run the relevant tests" \ --workspace /path/to/repository ``` +Enter only the API key in that file. Yada also accepts +`--api-key-file /run/secrets/deepseek_api_key` for mounted secret stores; see +[Configuration](docs/configuration.md#deepseek-credentials). + Yada asks before running repository commands. Use `--yes` only inside a trusted, disposable environment: diff --git a/benchmarks/suites/swebench-verified-canary-v1.json b/benchmarks/suites/swebench-verified-canary-v1.json new file mode 100644 index 0000000..3face25 --- /dev/null +++ b/benchmarks/suites/swebench-verified-canary-v1.json @@ -0,0 +1,16 @@ +{ + "schema_version": 1, + "suite_id": "swebench-verified-canary-v1", + "benchmark": "swebench-verified", + "description": "A small, versioned SWE-bench Verified canary for development signal before larger runs.", + "instances": [ + "pytest-dev__pytest-10051", + "pytest-dev__pytest-10081", + "pytest-dev__pytest-10356", + "django__django-15987", + "sympy__sympy-19637", + "sphinx-doc__sphinx-9367", + "scikit-learn__scikit-learn-13439", + "pydata__xarray-6461" + ] +} diff --git a/docs/cli-reference.md b/docs/cli-reference.md index b06edeb..af4b0ce 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -32,6 +32,7 @@ uv run yada --task-file issue.md --workspace /path/to/repository | `TASK` | Natural-language coding task. | — | | `--task-file PATH` | Read the task from a UTF-8 file. | — | | `--workspace PATH` | Target Git workspace. | Current directory | +| `--api-key-file PATH` | Read the DeepSeek key from a private file. | Per-user config file, then compatibility environment fallback | | `--model NAME` | DeepSeek model name. | `DEEPSEEK_MODEL` or `deepseek-v4-pro` | | `--base-url URL` | DeepSeek-compatible API base URL. | `DEEPSEEK_BASE_URL` or `https://api.deepseek.com` | | `--reasoning-effort high\|max` | Thinking effort. | `max` | @@ -290,6 +291,7 @@ container; Yada's automatic Agent command container applies only to the native | `--max-steps N` | Model-turn budget. | `30` | | `--wall-time SECONDS` | Comparable wall-time budget. | `1800` | | `--max-output-tokens N` | Per-completion token limit. | `16384` | +| `--api-key-file PATH` | Private DeepSeek credential file for the native Yada agent. | Per-user config file, then compatibility environment fallback | | `--editing-strategy patch-only\|replace-first` | Native Yada editing policy. | `replace-first` | The native agent also accepts the model, thinking, timeout, command-policy, and diff --git a/docs/configuration.md b/docs/configuration.md index ee3d51d..da282d3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -28,21 +28,35 @@ After editable installation, use `.venv/bin/yada`; after `uv sync`, use ## DeepSeek credentials -`DEEPSEEK_API_KEY` is required when the native Yada agent calls DeepSeek: +The recommended setup is a private file at +`~/.config/yada/deepseek_api_key`: ```bash -export DEEPSEEK_API_KEY="sk-..." +install -d -m 700 ~/.config/yada +(umask 077; touch ~/.config/yada/deepseek_api_key) +chmod 600 ~/.config/yada/deepseek_api_key +${EDITOR:-vi} ~/.config/yada/deepseek_api_key ``` -PowerShell: +Put only the API key in the file. On Linux and macOS, it must not grant any +permissions to group or others (`0600` is recommended; `0400` is also valid). +To use another location, pass it explicitly: -```powershell -$env:DEEPSEEK_API_KEY = "sk-..." +```bash +uv run yada "Fix the failing test" \ + --api-key-file /run/secrets/deepseek_api_key \ + --workspace /path/to/repository ``` -Do not put the key in a task file, trace, issue, or commit. Yada sends it only in -the DeepSeek authorization header and removes secret-looking environment -variables from repository subprocesses. +Credential resolution order is: + +1. `--api-key-file PATH`; +2. `DEEPSEEK_API_KEY_FILE`, containing a file path; +3. `~/.config/yada/deepseek_api_key`; and +4. `DEEPSEEK_API_KEY`, retained for compatibility. + +On Windows, the default file is `%APPDATA%\Yada\deepseek_api_key`. Never put +the key in a command argument, task, trace, image, issue, or commit. ## Model and endpoint @@ -240,9 +254,12 @@ network access: ```bash docker build -t yada . docker run --rm -it \ - -e DEEPSEEK_API_KEY \ + -v "$HOME/.config/yada/deepseek_api_key:/run/secrets/deepseek_api_key:ro" \ -v "/path/to/repository:/workspace" \ - yada "Fix the failing test" --workspace /workspace --yes + yada "Fix the failing test" \ + --workspace /workspace \ + --api-key-file /run/secrets/deepseek_api_key \ + --yes ``` Use a stronger sandbox when the repository or its dependencies are untrusted. diff --git a/docs/dev/debugging.md b/docs/dev/debugging.md index fe085d5..5e0ad24 100644 --- a/docs/dev/debugging.md +++ b/docs/dev/debugging.md @@ -243,7 +243,9 @@ SWE-bench score. Use the official Docker grader for published results; see the ## Common failure signals -- **`DEEPSEEK_API_KEY is not set`**: export the key in the shell launching Yada. +- **`DeepSeek API key not found`**: create the private default credential file, + pass `--api-key-file`, or configure `DEEPSEEK_API_KEY_FILE`; see + [DeepSeek credentials](../configuration.md#deepseek-credentials). - **No request payload in a trace**: rerun with `--trace-level debug`. - **`finish_task` rejected**: run a successful `test` or `build` after the latest patch. - **Tool reports `ok` but tests failed**: inspect the command `exit_code`. diff --git a/docs/evaluation.md b/docs/evaluation.md index aef9377..f2ccacc 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -43,7 +43,8 @@ Before this pipeline starts, the CLI: 2. resolves the result and artifact paths; 3. creates a run ID containing UTC time and a random suffix; 4. configures the native Yada or external command agent; and -5. requires `DEEPSEEK_API_KEY` when the native Yada agent is selected. +5. resolves a private API key file (or the compatibility environment fallback) + when the native Yada agent is selected. The default result name uses system-local time at minute precision. A collision adds `(1)`, `(2)`, and so on. The result and artifact directory always receive @@ -84,6 +85,136 @@ Without Docker, use direct `yada ...` for normal repository work or may choose to invoke Docker itself, but Docker is not a Yada requirement for this path. +## Choose the smallest useful evaluation scale + +Evaluation should expand in explicit stages. A larger stage costs more time, +model tokens, network traffic, and Docker storage; it does not replace the +faster signal from the stages before it. + +| Scale | Use it for | Typical task set | Meaning | +| --- | --- | --- | --- | +| Local case | Developing or debugging a prompt, tool, adapter, or grader change | One checked-in `--case` recipe | Fast project-specific regression signal; not an official SWE-bench result | +| Canary suite | Checking that an evaluation-affecting change behaves plausibly across projects before spending on a pilot | The versioned Canary-8 manifest | Repeatable development signal with aggregate metrics and resumable execution | +| Pilot sample | Estimating variance, failures, and operational cost before a full run | A fixed, reviewed sample larger than the canary | Preflight evidence; record and version the sample rather than selecting tasks during the run | +| Full benchmark | Producing the final result after the implementation and run configuration are frozen | All 500 SWE-bench Verified tasks | The only full-dataset result, still requiring complete methodology and provenance for any comparison | + +Run the checked-in Canary-8 suite with the pinned Harness environment: + +```bash +uv run --with 'swebench==4.1.0' \ + python scripts/eval_suite.py run \ + benchmarks/suites/swebench-verified-canary-v1.json \ + --api-key-file ~/.config/yada/deepseek_api_key +``` + +The development-only runner calls the existing single-task command once per +instance, sequentially: + +```text +python -m yada eval --swebench INSTANCE_ID +``` + +It does not add a public `yada eval-suite` command or code under `src/yada`. +Every instance runs once unless the whole suite receives `--repeat N`; there are +no per-instance repeat overrides. Model, budget, and secret-file options also +apply uniformly to every attempt. Only the secret path may enter suite run +metadata; the key value is never copied. + +The runner prints the new suite directory under `eval-results/suites/`. That +directory contains a manifest snapshot, run metadata, isolated attempt +directories, and deterministic `summary.json` and `summary.md` files. It is +already covered by the repository's `eval-results/` ignore rule, so raw +workspaces, traces, Harness logs, and Docker data are not committed. + +After Ctrl-C or another interruption, resume the printed directory: + +```bash +uv run --with 'swebench==4.1.0' \ + python scripts/eval_suite.py run \ + benchmarks/suites/swebench-verified-canary-v1.json \ + --resume eval-results/suites/SUITE_DIRECTORY +``` + +Resume requires the same manifest hash, Yada commit, model, budgets, and global +repeat count. Completed attempts are skipped. An attempt whose atomic Yada +result was written immediately before interruption is recovered; an incomplete +workspace is retained for diagnosis and the retry uses a new execution +directory. Ordinary resolved, unresolved, and error outcomes never stop later +tasks. + +The summaries report outcome counts and resolution rate; per-attempt and +per-instance steps, token usage, and Agent duration; repeated-run min, max, and +median; semantic patch IDs and patch convergence; and result, artifact, and +trace paths. They record only safe run configuration and never copy API keys +from any credential source. Canary and pilot summaries are development +evidence, not official leaderboard scores. + +## Long-running suites on a remote host + +Yada calls a remote model API and runs SWE-bench in Docker, so this workload +does not need a GPU. Prefer an `x86_64` Linux host: upstream SWE-bench recommends +at least 8 CPU cores, 16 GB RAM, and 120 GB of free storage, and describes ARM +support as experimental. The suite runner is intentionally sequential, so a +larger machine improves an individual build or test but does not make multiple +instances run concurrently. + +For AWS, a current-generation general-purpose Intel instance such as +`m8i.2xlarge` (8 vCPU, 32 GiB), with `m7i.2xlarge` as a broadly available +fallback, is a practical starting point. Use an encrypted persistent `gp3` EBS +volume rather than instance storage. The upstream 120 GB figure covers the +Harness baseline; Yada also retains result workspaces and traces. Start around +300 GB for canary and pilot work, and size a full 500-task volume from measured +pilot growth—500 GB to 1 TB is a safer initial range when retaining every raw +workspace. Monitor both `df -h` and `docker system df`. + +Use an EC2 instance role, not long-lived AWS access keys. Give it only +`secretsmanager:GetSecretValue` for the DeepSeek secret, the permissions needed +for Systems Manager, and optional access to a dedicated result bucket. Systems +Manager Session Manager allows administration without an inbound SSH rule. +Materialize the DeepSeek value into a private memory-backed file before the run: + +```bash +install -d -m 700 /dev/shm/yada +umask 077 +aws secretsmanager get-secret-value \ + --secret-id yada/deepseek-api-key \ + --query SecretString \ + --output text \ + --no-cli-pager \ + > /dev/shm/yada/deepseek_api_key +chmod 600 /dev/shm/yada/deepseek_api_key +``` + +Then run with unbuffered logs and place both the checkout and output directory +on EBS: + +```bash +nohup env PYTHONUNBUFFERED=1 \ + uv run --with 'swebench==4.1.0' \ + python -u scripts/eval_suite.py run \ + benchmarks/suites/swebench-verified-canary-v1.json \ + --api-key-file /dev/shm/yada/deepseek_api_key \ + --repeat 3 \ + --max-steps 60 \ + --output-dir /data/yada-results/canary-v1-r3 \ + > /data/yada-results/canary-v1-r3.log 2>&1 < /dev/null & +``` + +Use On-Demand for the first long run. Spot can reduce compute cost after resume +has been tested, but interruption notice is short and a terminated instance can +lose its root volume by default. Keep the suite directory on persistent EBS, +preserve that volume on termination, and expect the current attempt to resume +in a new execution directory. Copy `summary.json`, `summary.md`, and any needed +diagnostics to encrypted object storage; raw traces and workspaces may contain +sensitive source or model reasoning and should not be uploaded indiscriminately. + +References: [SWE-bench Docker setup](https://www.swebench.com/SWE-bench/guides/docker_setup/), +[EC2 general-purpose instances](https://aws.amazon.com/ec2/instance-types/general-purpose/), +[EBS gp3](https://docs.aws.amazon.com/ebs/latest/userguide/general-purpose.html), +[EC2 IAM roles](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html), +[Secrets Manager GetSecretValue](https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html), +and [Session Manager](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager.html). + ## Local case: `--case PATH` Example: diff --git a/scripts/eval_suite.py b/scripts/eval_suite.py new file mode 100644 index 0000000..da1cd5c --- /dev/null +++ b/scripts/eval_suite.py @@ -0,0 +1,1069 @@ +#!/usr/bin/env python3 +"""Run a versioned SWE-bench suite through Yada's single-task eval command. + +This development script deliberately uses only the Python standard library. It +keeps suite orchestration outside ``src/yada`` and treats every underlying +``python -m yada eval --swebench`` invocation as an isolated, durable attempt. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import statistics +import subprocess +import sys +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Sequence + +SUITE_RUN_SCHEMA_VERSION = 1 +SUMMARY_SCHEMA_VERSION = 1 +ATTEMPT_SCHEMA_VERSION = 1 +DEFAULT_MODEL = "deepseek-v4-pro" +DEFAULT_BUDGETS = { + "max_steps": 30, + "wall_time_seconds": 1_800, + "max_output_tokens": 16_384, +} +_SUITE_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_INSTANCE_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_HUNK_PATTERN = re.compile(r"^@@ -\d+(?:,\d+)? \+\d+(?:,\d+)? @@(?P.*)$") + + +class SuiteError(RuntimeError): + """A user-facing suite configuration or state error.""" + + +@dataclass(frozen=True) +class SuiteManifest: + """Validated fields from one versioned suite manifest.""" + + path: Path + suite_id: str + benchmark: str + instances: tuple[str, ...] + sha256: str + raw: bytes + + +def build_parser() -> argparse.ArgumentParser: + """Build the development-only suite runner parser.""" + + parser = argparse.ArgumentParser( + description="Run and resume a versioned SWE-bench evaluation suite." + ) + commands = parser.add_subparsers(dest="command", required=True) + run = commands.add_parser( + "run", + help="Run missing attempts sequentially and refresh suite summaries.", + ) + run.add_argument("manifest", type=Path, help="Versioned suite JSON manifest.") + destination = run.add_mutually_exclusive_group() + destination.add_argument( + "--output-dir", + "--output", + type=Path, + help=("Suite directory. An existing directory with suite-run.json is resumed."), + ) + destination.add_argument( + "--resume", + type=Path, + help="Existing suite directory to resume.", + ) + run.add_argument( + "--repeat", + type=_positive_int, + default=None, + help="Run every instance N times (default: 1; no per-instance overrides).", + ) + run.add_argument("--model", default=None) + run.add_argument( + "--api-key-file", + type=Path, + default=None, + help="Private DeepSeek API key file forwarded to every Yada attempt.", + ) + run.add_argument("--max-steps", type=_positive_int, default=None) + run.add_argument("--wall-time", type=_positive_int, default=None) + run.add_argument("--max-output-tokens", type=_positive_int, default=None) + run.add_argument("--reasoning-effort", choices=("high", "max"), default=None) + run.add_argument( + "--thinking", + action=argparse.BooleanOptionalAction, + default=None, + ) + run.add_argument( + "--editing-strategy", + choices=("patch-only", "replace-first"), + default=None, + ) + run.add_argument("--api-timeout", type=_positive_int, default=None) + run.add_argument("--command-timeout", type=_positive_int, default=None) + run.add_argument("--trace-level", choices=("summary", "debug"), default=None) + run.add_argument( + "--python", + dest="python_executable", + default=None, + help=("Python used for 'python -m yada eval' (default: this interpreter)."), + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the command and return a process-style exit code.""" + + args = build_parser().parse_args(argv) + try: + if args.command == "run": + return run_suite(args) + except SuiteError as exc: + print(f"eval-suite: {exc}", file=sys.stderr) + return 2 + raise AssertionError(f"unsupported command: {args.command}") + + +def run_suite(args: argparse.Namespace) -> int: + """Create or resume one suite directory, running all missing attempts.""" + + manifest = load_manifest(args.manifest) + repository = Path(__file__).resolve().parents[1] + yada_commit = git_head(repository) + suite_dir, metadata = _open_suite_run( + args, + manifest=manifest, + repository=repository, + yada_commit=yada_commit, + ) + repeat = _require_int(metadata["configuration"], "repeat") + total = len(manifest.instances) * repeat + print(f"Suite: {manifest.suite_id}") + print(f"Directory: {suite_dir}") + print(f"Attempts: {total} ({len(manifest.instances)} instances x {repeat})") + + try: + for instance_index, instance_id in enumerate(manifest.instances, 1): + for attempt_number in range(1, repeat + 1): + attempt_dir = _attempt_dir( + suite_dir, + instance_index, + instance_id, + attempt_number, + ) + marker = attempt_dir / "attempt.json" + if marker.is_file(): + _load_attempt(marker, instance_id, attempt_number) + print( + f"Skip completed: {instance_id} " + f"(attempt {attempt_number}/{repeat})" + ) + continue + + recovered = _recover_attempt( + suite_dir, + attempt_dir, + instance_id, + attempt_number, + ) + if recovered is not None: + _write_json(marker, recovered) + print( + f"Recovered completed: {instance_id} " + f"(attempt {attempt_number}/{repeat})" + ) + write_summaries(suite_dir, manifest, metadata) + continue + + execution_dir = _next_execution_dir(attempt_dir) + print( + f"Run: {instance_id} (attempt {attempt_number}/{repeat}, " + f"{execution_dir.name})" + ) + record = _run_attempt( + suite_dir=suite_dir, + execution_dir=execution_dir, + instance_id=instance_id, + instance_index=instance_index, + attempt_number=attempt_number, + metadata=metadata, + repository=repository, + ) + _write_json(marker, record) + write_summaries(suite_dir, manifest, metadata) + print(f"Outcome: {record['status']}") + except KeyboardInterrupt: + write_summaries(suite_dir, manifest, metadata) + print( + f"\nInterrupted. Resume with --resume {suite_dir}", + file=sys.stderr, + ) + return 130 + + summary = write_summaries(suite_dir, manifest, metadata) + counts = summary["counts"] + print( + "Complete: " + f"{counts['resolved']} resolved, {counts['unresolved']} unresolved, " + f"{counts['error']} error" + ) + print(f"Summary: {suite_dir / 'summary.json'}") + return 0 + + +def load_manifest(path: Path) -> SuiteManifest: + """Load and strictly validate a suite manifest.""" + + resolved = path.expanduser().resolve() + try: + raw = resolved.read_bytes() + except OSError as exc: + raise SuiteError(f"cannot read suite manifest {resolved}: {exc}") from exc + try: + data = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise SuiteError(f"invalid UTF-8 JSON suite manifest: {exc}") from exc + if not isinstance(data, dict): + raise SuiteError("suite manifest must be a JSON object") + if data.get("schema_version") != 1: + raise SuiteError("suite manifest schema_version must be 1") + suite_id = data.get("suite_id") + if not isinstance(suite_id, str) or not _SUITE_ID_PATTERN.fullmatch(suite_id): + raise SuiteError("suite_id must contain only letters, digits, '.', '_' or '-'") + benchmark = data.get("benchmark") + if benchmark != "swebench-verified": + raise SuiteError("benchmark must be 'swebench-verified'") + instances = data.get("instances") + if not isinstance(instances, list) or not instances: + raise SuiteError("instances must be a non-empty array") + if any( + not isinstance(item, str) or not _INSTANCE_ID_PATTERN.fullmatch(item) + for item in instances + ): + raise SuiteError("each instance ID must be a path-safe non-empty string") + if len(set(instances)) != len(instances): + raise SuiteError("suite instance IDs must be unique") + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + return SuiteManifest( + path=resolved, + suite_id=suite_id, + benchmark=benchmark, + instances=tuple(instances), + sha256=digest, + raw=raw, + ) + + +def git_head(repository: Path) -> str | None: + """Return the repository commit without making Git a run dependency.""" + + try: + result = subprocess.run( + ["git", "-C", str(repository), "rev-parse", "HEAD"], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=5, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + value = result.stdout.strip() + return value if result.returncode == 0 and value else None + + +def semantic_patch_id(patch: str) -> str: + """Hash a canonical unified diff while ignoring non-semantic Git metadata.""" + + canonical: list[str] = [] + for line in patch.replace("\r\n", "\n").replace("\r", "\n").splitlines(): + if line.startswith(("index ", "--- ", "+++ ")): + continue + hunk = _HUNK_PATTERN.match(line) + if hunk: + canonical.append("@@" + hunk.group("context")) + else: + canonical.append(line) + payload = "\n".join(canonical).encode("utf-8") + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def write_summaries( + suite_dir: Path, + manifest: SuiteManifest, + metadata: dict[str, Any], +) -> dict[str, Any]: + """Regenerate deterministic JSON and Markdown from durable attempt markers.""" + + summary = build_summary(suite_dir, manifest, metadata) + _write_json(suite_dir / "summary.json", summary) + _write_text(suite_dir / "summary.md", render_markdown(summary)) + return summary + + +def build_summary( + suite_dir: Path, + manifest: SuiteManifest, + metadata: dict[str, Any], +) -> dict[str, Any]: + """Build a deterministic aggregate in manifest and attempt order.""" + + configuration = metadata["configuration"] + repeat = _require_int(configuration, "repeat") + all_attempts: list[dict[str, Any]] = [] + instance_summaries: list[dict[str, Any]] = [] + for instance_index, instance_id in enumerate(manifest.instances, 1): + attempts: list[dict[str, Any]] = [] + for attempt_number in range(1, repeat + 1): + marker = ( + _attempt_dir( + suite_dir, + instance_index, + instance_id, + attempt_number, + ) + / "attempt.json" + ) + if marker.is_file(): + attempts.append(_load_attempt(marker, instance_id, attempt_number)) + all_attempts.extend(attempts) + counts = _outcome_counts(attempts) + patch_ids = sorted( + { + item["patch_id"] + for item in attempts + if isinstance(item.get("patch_id"), str) + } + ) + converged: bool | None + if repeat == 1 or len(attempts) < repeat: + converged = None + elif any(not isinstance(item.get("patch_id"), str) for item in attempts): + converged = False + else: + converged = len(patch_ids) == 1 + instance_summaries.append( + { + "instance_id": instance_id, + "expected_attempts": repeat, + "completed_attempts": len(attempts), + "pending_attempts": repeat - len(attempts), + "counts": counts, + "resolution_rate": _resolution_rate(counts), + "metrics": _metrics(attempts), + "patch_ids": patch_ids, + "patches_converged": converged, + "attempts": [_attempt_summary(item) for item in attempts], + } + ) + counts = _outcome_counts(all_attempts) + expected = len(manifest.instances) * repeat + return { + "schema_version": SUMMARY_SCHEMA_VERSION, + "suite_id": manifest.suite_id, + "benchmark": manifest.benchmark, + "suite_sha256": manifest.sha256, + "yada_commit": metadata["yada_commit"], + "model": configuration["model"], + "budgets": configuration["budgets"], + "repeat": repeat, + "expected_attempts": expected, + "completed_attempts": len(all_attempts), + "pending_attempts": expected - len(all_attempts), + "counts": counts, + "resolution_rate": _resolution_rate(counts), + "metrics": _metrics(all_attempts), + "instances": instance_summaries, + } + + +def render_markdown(summary: dict[str, Any]) -> str: + """Render a stable human-readable suite summary.""" + + counts = summary["counts"] + budgets = summary["budgets"] + lines = [ + f"# SWE-bench suite summary: {summary['suite_id']}", + "", + ( + "Development canary only; these results are not an official " + "leaderboard score." + ), + "", + f"- Suite hash: `{summary['suite_sha256']}`", + f"- Yada commit: `{summary['yada_commit'] or 'unknown'}`", + f"- Model: `{summary['model']}`", + ( + "- Budgets: " + f"steps={budgets['max_steps']}, " + f"wall={budgets['wall_time_seconds']}s, " + f"output tokens={budgets['max_output_tokens']}" + ), + f"- Repeat: {summary['repeat']} per instance", + ( + f"- Progress: {summary['completed_attempts']}/" + f"{summary['expected_attempts']} attempts" + ), + "", + "## Overall", + "", + "| Resolved | Unresolved | Error | Resolution rate |", + "| ---: | ---: | ---: | ---: |", + ( + f"| {counts['resolved']} | {counts['unresolved']} | " + f"{counts['error']} | {_percent(summary['resolution_rate'])} |" + ), + "", + "| Agent metric | Min | Max | Median |", + "| --- | ---: | ---: | ---: |", + ] + for label, key in ( + ("Steps", "steps"), + ("Tokens", "tokens"), + ("Duration (ms)", "agent_duration_ms"), + ): + values = summary["metrics"][key] + lines.append( + f"| {label} | {_value(values['min'])} | {_value(values['max'])} | " + f"{_value(values['median'])} |" + ) + + lines.extend( + [ + "", + "## Per-instance results", + "", + ( + "| Instance | R/U/E | Rate | Steps min/max/median | " + "Tokens min/max/median | Agent ms min/max/median | " + "Patches converge |" + ), + "| --- | ---: | ---: | ---: | ---: | ---: | --- |", + ] + ) + for instance in summary["instances"]: + metrics = instance["metrics"] + count = instance["counts"] + lines.append( + f"| `{_escape(instance['instance_id'])}` | " + f"{count['resolved']}/{count['unresolved']}/{count['error']} | " + f"{_percent(instance['resolution_rate'])} | " + f"{_range_text(metrics['steps'])} | " + f"{_range_text(metrics['tokens'])} | " + f"{_range_text(metrics['agent_duration_ms'])} | " + f"{_convergence_text(instance['patches_converged'])} |" + ) + + lines.extend( + [ + "", + "## Attempts", + "", + ( + "| Instance | Attempt | Outcome | Steps | Tokens | Agent ms | " + "Patch ID | Result | Trace |" + ), + "| --- | ---: | --- | ---: | ---: | ---: | --- | --- | --- |", + ] + ) + for instance in summary["instances"]: + for attempt in instance["attempts"]: + lines.append( + f"| `{_escape(instance['instance_id'])}` | " + f"{attempt['attempt']} | {attempt['status']} | " + f"{_value(attempt['steps'])} | {_value(attempt['tokens'])} | " + f"{_value(attempt['agent_duration_ms'])} | " + f"{_code(attempt['patch_id'])} | " + f"{_path_link(attempt['result_path'], 'result')} | " + f"{_path_link(attempt['trace_path'], 'trace')} |" + ) + return "\n".join(lines) + "\n" + + +def _open_suite_run( + args: argparse.Namespace, + *, + manifest: SuiteManifest, + repository: Path, + yada_commit: str | None, +) -> tuple[Path, dict[str, Any]]: + requested = args.resume or args.output_dir + if requested is None: + suite_dir = _next_default_suite_dir(repository, manifest.suite_id) + existing = False + else: + suite_dir = requested.expanduser().resolve() + existing = (suite_dir / "suite-run.json").is_file() + if args.resume is not None and not existing: + raise SuiteError(f"resume directory has no suite-run.json: {suite_dir}") + + if existing: + metadata = _read_json(suite_dir / "suite-run.json") + _validate_resume( + args, + metadata=metadata, + manifest=manifest, + yada_commit=yada_commit, + ) + return suite_dir, metadata + + configuration = _new_configuration(args) + if suite_dir.exists() and any(suite_dir.iterdir()): + raise SuiteError(f"new suite directory is not empty: {suite_dir}") + suite_dir.mkdir(parents=True, exist_ok=True) + metadata = { + "schema_version": SUITE_RUN_SCHEMA_VERSION, + "suite_id": manifest.suite_id, + "benchmark": manifest.benchmark, + "suite_sha256": manifest.sha256, + "manifest_path": _display_path(manifest.path, repository), + "instances": list(manifest.instances), + "yada_commit": yada_commit, + "configuration": configuration, + "created_at": _utc_now(), + } + _write_bytes(suite_dir / "suite-manifest.json", manifest.raw) + _write_json(suite_dir / "suite-run.json", metadata) + return suite_dir, metadata + + +def _new_configuration(args: argparse.Namespace) -> dict[str, Any]: + repeat = args.repeat if args.repeat is not None else 1 + model = args.model or os.environ.get("DEEPSEEK_MODEL", DEFAULT_MODEL) + if not isinstance(model, str) or not model.strip(): + raise SuiteError("model must not be empty") + return { + "repeat": repeat, + "model": model, + "api_key_file": ( + str(args.api_key_file.expanduser().resolve()) + if args.api_key_file is not None + else None + ), + "budgets": { + "max_steps": args.max_steps or DEFAULT_BUDGETS["max_steps"], + "wall_time_seconds": ( + args.wall_time or DEFAULT_BUDGETS["wall_time_seconds"] + ), + "max_output_tokens": ( + args.max_output_tokens or DEFAULT_BUDGETS["max_output_tokens"] + ), + }, + "reasoning_effort": args.reasoning_effort or "max", + "thinking": True if args.thinking is None else args.thinking, + "editing_strategy": args.editing_strategy or "replace-first", + "api_timeout_seconds": args.api_timeout or 300, + "command_timeout_seconds": args.command_timeout or 120, + "trace_level": args.trace_level or "summary", + "python_executable": args.python_executable or sys.executable, + "autonomous_commands": True, + } + + +def _validate_resume( + args: argparse.Namespace, + *, + metadata: dict[str, Any], + manifest: SuiteManifest, + yada_commit: str | None, +) -> None: + if metadata.get("schema_version") != SUITE_RUN_SCHEMA_VERSION: + raise SuiteError("unsupported suite-run.json schema_version") + expected = { + "suite_id": manifest.suite_id, + "benchmark": manifest.benchmark, + "suite_sha256": manifest.sha256, + "instances": list(manifest.instances), + "yada_commit": yada_commit, + } + for key, value in expected.items(): + if metadata.get(key) != value: + raise SuiteError( + f"cannot resume: {key} changed ({metadata.get(key)!r} != {value!r})" + ) + configuration = metadata.get("configuration") + if not isinstance(configuration, dict): + raise SuiteError("suite-run.json configuration must be an object") + stored_budgets = configuration.get("budgets") + if not isinstance(stored_budgets, dict): + raise SuiteError("suite-run.json budgets must be an object") + supplied = { + "repeat": args.repeat, + "model": args.model, + "api_key_file": ( + str(args.api_key_file.expanduser().resolve()) + if args.api_key_file is not None + else None + ), + "reasoning_effort": args.reasoning_effort, + "thinking": args.thinking, + "editing_strategy": args.editing_strategy, + "api_timeout_seconds": args.api_timeout, + "command_timeout_seconds": args.command_timeout, + "trace_level": args.trace_level, + "python_executable": args.python_executable, + } + for key, value in supplied.items(): + if value is not None and configuration.get(key) != value: + raise SuiteError(f"cannot resume with a different {key}") + supplied_budgets = { + "max_steps": args.max_steps, + "wall_time_seconds": args.wall_time, + "max_output_tokens": args.max_output_tokens, + } + for key, value in supplied_budgets.items(): + if value is not None and stored_budgets.get(key) != value: + raise SuiteError(f"cannot resume with a different {key}") + + +def _run_attempt( + *, + suite_dir: Path, + execution_dir: Path, + instance_id: str, + instance_index: int, + attempt_number: int, + metadata: dict[str, Any], + repository: Path, +) -> dict[str, Any]: + execution_dir.mkdir(parents=True, exist_ok=False) + result_path = execution_dir / "result.json" + artifacts_path = execution_dir / "artifacts" + configuration = metadata["configuration"] + budgets = configuration["budgets"] + run_id = ( + f"{metadata['suite_id']}-{instance_index:03d}-" + f"r{attempt_number:03d}-{execution_dir.name}" + ) + command = [ + str(configuration["python_executable"]), + "-m", + "yada", + "eval", + "--swebench", + instance_id, + "--output", + str(result_path), + "--artifact-dir", + str(artifacts_path), + "--run-id", + run_id, + "--model", + str(configuration["model"]), + "--max-steps", + str(budgets["max_steps"]), + "--wall-time", + str(budgets["wall_time_seconds"]), + "--max-output-tokens", + str(budgets["max_output_tokens"]), + "--reasoning-effort", + str(configuration["reasoning_effort"]), + "--thinking" if configuration["thinking"] else "--no-thinking", + "--editing-strategy", + str(configuration["editing_strategy"]), + "--api-timeout", + str(configuration["api_timeout_seconds"]), + "--command-timeout", + str(configuration["command_timeout_seconds"]), + "--trace-level", + str(configuration["trace_level"]), + "--yes", + ] + if configuration.get("api_key_file"): + command.extend(["--api-key-file", str(configuration["api_key_file"])]) + started_at = _utc_now() + started = time.monotonic() + launch_error: str | None = None + return_code: int | None = None + try: + process = subprocess.run(command, cwd=repository, check=False) + return_code = process.returncode + except KeyboardInterrupt: + raise + except OSError as exc: + launch_error = f"{type(exc).__name__}: {exc}" + duration_ms = round((time.monotonic() - started) * 1000) + return _attempt_record( + suite_dir=suite_dir, + result_path=result_path, + artifacts_path=artifacts_path, + instance_id=instance_id, + attempt_number=attempt_number, + execution=execution_dir.name, + return_code=return_code, + started_at=started_at, + completed_at=_utc_now(), + suite_duration_ms=duration_ms, + recovered=False, + external_error=launch_error, + ) + + +def _recover_attempt( + suite_dir: Path, + attempt_dir: Path, + instance_id: str, + attempt_number: int, +) -> dict[str, Any] | None: + if not attempt_dir.is_dir(): + return None + executions = sorted( + ( + item + for item in attempt_dir.iterdir() + if item.is_dir() and re.fullmatch(r"execution-\d{3}", item.name) + ), + reverse=True, + ) + for execution_dir in executions: + result_path = execution_dir / "result.json" + if not result_path.is_file(): + continue + try: + result = _read_json(result_path) + except SuiteError: + continue + if result.get("instance_id") != instance_id: + continue + return _attempt_record( + suite_dir=suite_dir, + result_path=result_path, + artifacts_path=execution_dir / "artifacts", + instance_id=instance_id, + attempt_number=attempt_number, + execution=execution_dir.name, + return_code=None, + started_at=str(result.get("started_at") or "unknown"), + completed_at=_timestamp_for(result_path), + suite_duration_ms=_optional_int(result.get("duration_ms")), + recovered=True, + external_error=None, + ) + return None + + +def _attempt_record( + *, + suite_dir: Path, + result_path: Path, + artifacts_path: Path, + instance_id: str, + attempt_number: int, + execution: str, + return_code: int | None, + started_at: str, + completed_at: str, + suite_duration_ms: int | None, + recovered: bool, + external_error: str | None, +) -> dict[str, Any]: + result: dict[str, Any] | None = None + result_error = external_error + if result_path.is_file(): + try: + candidate = _read_json(result_path) + if candidate.get("instance_id") != instance_id: + result_error = ( + "result instance mismatch: " + f"{candidate.get('instance_id')!r} != {instance_id!r}" + ) + else: + result = candidate + except SuiteError as exc: + result_error = str(exc) + elif result_error is None: + result_error = f"evaluation exited with code {return_code} without result.json" + + raw_status = result.get("status") if result else None + status = raw_status if raw_status in {"resolved", "unresolved"} else "error" + agent_run = result.get("agent_run") if result else None + if not isinstance(agent_run, dict): + agent_run = {} + usage = _numeric_usage(agent_run.get("usage")) + patch = agent_run.get("patch") + patch_id = semantic_patch_id(patch) if isinstance(patch, str) else None + trace_path = agent_run.get("trace_path") + if not isinstance(trace_path, str): + fallback_trace = artifacts_path / "yada-trace.jsonl" + trace_path = str(fallback_trace) if fallback_trace.is_file() else None + error = result_error + if error is None and status == "error" and result: + value = result.get("error") + error = str(value) if value else f"evaluation status: {raw_status!r}" + return { + "schema_version": ATTEMPT_SCHEMA_VERSION, + "instance_id": instance_id, + "attempt": attempt_number, + "execution": execution, + "status": status, + "return_code": return_code, + "started_at": started_at, + "completed_at": completed_at, + "suite_duration_ms": suite_duration_ms, + "model": agent_run.get("model"), + "steps": _optional_int(agent_run.get("steps")), + "usage": usage, + "tokens": _token_count(usage), + "agent_duration_ms": _optional_int(agent_run.get("duration_ms")), + "patch_id": patch_id, + "result_path": _relative_path(result_path, suite_dir), + "artifacts_path": _relative_path(artifacts_path, suite_dir), + "trace_path": _relative_path(Path(trace_path), suite_dir) + if trace_path + else None, + "error": _redact_text(error) if error else None, + "recovered": recovered, + } + + +def _attempt_summary(attempt: dict[str, Any]) -> dict[str, Any]: + keys = ( + "attempt", + "execution", + "status", + "return_code", + "steps", + "tokens", + "usage", + "agent_duration_ms", + "suite_duration_ms", + "patch_id", + "result_path", + "artifacts_path", + "trace_path", + "error", + "recovered", + ) + return {key: attempt.get(key) for key in keys} + + +def _metrics(attempts: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + return { + key: _metric_range([item.get(key) for item in attempts]) + for key in ("steps", "tokens", "agent_duration_ms") + } + + +def _metric_range(values: list[Any]) -> dict[str, int | float | None]: + numbers = [ + value + for value in values + if isinstance(value, int) and not isinstance(value, bool) + ] + if not numbers: + return {"min": None, "max": None, "median": None} + return { + "min": min(numbers), + "max": max(numbers), + "median": statistics.median(numbers), + } + + +def _outcome_counts(attempts: list[dict[str, Any]]) -> dict[str, int]: + return { + status: sum(item.get("status") == status for item in attempts) + for status in ("resolved", "unresolved", "error") + } + + +def _resolution_rate(counts: dict[str, int]) -> float: + total = sum(counts.values()) + return round(counts["resolved"] / total, 6) if total else 0.0 + + +def _numeric_usage(value: Any) -> dict[str, int]: + if not isinstance(value, dict): + return {} + return { + str(key): item + for key, item in sorted(value.items(), key=lambda pair: str(pair[0])) + if isinstance(item, int) and not isinstance(item, bool) + } + + +def _token_count(usage: dict[str, int]) -> int | None: + if "total_tokens" in usage: + return usage["total_tokens"] + for left, right in ( + ("input_tokens", "output_tokens"), + ("prompt_tokens", "completion_tokens"), + ): + if left in usage or right in usage: + return usage.get(left, 0) + usage.get(right, 0) + return None + + +def _load_attempt( + path: Path, + instance_id: str, + attempt_number: int, +) -> dict[str, Any]: + data = _read_json(path) + if data.get("schema_version") != ATTEMPT_SCHEMA_VERSION: + raise SuiteError(f"unsupported attempt schema: {path}") + if data.get("instance_id") != instance_id or data.get("attempt") != attempt_number: + raise SuiteError(f"attempt marker identity mismatch: {path}") + if data.get("status") not in {"resolved", "unresolved", "error"}: + raise SuiteError(f"attempt marker has invalid status: {path}") + return data + + +def _attempt_dir( + suite_dir: Path, + instance_index: int, + instance_id: str, + attempt_number: int, +) -> Path: + return ( + suite_dir + / "runs" + / f"{instance_index:03d}-{instance_id}" + / f"attempt-{attempt_number:03d}" + ) + + +def _next_execution_dir(attempt_dir: Path) -> Path: + attempt_dir.mkdir(parents=True, exist_ok=True) + number = 1 + while (attempt_dir / f"execution-{number:03d}").exists(): + number += 1 + return attempt_dir / f"execution-{number:03d}" + + +def _next_default_suite_dir(repository: Path, suite_id: str) -> Path: + root = repository / "eval-results" / "suites" + timestamp = datetime.now().astimezone().strftime("%Y%m%d-%H%M%S") + candidate = root / f"{suite_id}__{timestamp}" + number = 1 + while candidate.exists(): + candidate = root / f"{suite_id}__{timestamp}({number})" + number += 1 + return candidate.resolve() + + +def _relative_path(path: Path, root: Path) -> str: + resolved = path.expanduser().resolve() + try: + return resolved.relative_to(root.resolve()).as_posix() + except ValueError: + return str(resolved) + + +def _display_path(path: Path, repository: Path) -> str: + try: + return path.relative_to(repository).as_posix() + except ValueError: + return str(path) + + +def _read_json(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise SuiteError(f"cannot read JSON {path}: {exc}") from exc + if not isinstance(value, dict): + raise SuiteError(f"expected a JSON object: {path}") + return value + + +def _write_json(path: Path, value: dict[str, Any]) -> None: + _write_text( + path, + json.dumps( + value, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + + "\n", + ) + + +def _write_text(path: Path, value: str) -> None: + _write_bytes(path, value.encode("utf-8")) + + +def _write_bytes(path: Path, value: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.tmp") + temporary.write_bytes(value) + temporary.replace(path) + + +def _require_int(mapping: dict[str, Any], key: str) -> int: + value = mapping.get(key) + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise SuiteError(f"configuration {key} must be a positive integer") + return value + + +def _optional_int(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) else None + + +def _positive_int(value: str) -> int: + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("must be a positive integer") + return parsed + + +def _timestamp_for(path: Path) -> str: + return datetime.fromtimestamp(path.stat().st_mtime, timezone.utc).isoformat() + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _redact_text(value: str) -> str: + redacted = re.sub( + r"(?i)(authorization\s*:\s*bearer\s+)[^\s,;]+", + r"\1[REDACTED]", + value, + ) + return re.sub(r"\bsk-[A-Za-z0-9_-]{8,}\b", "[REDACTED]", redacted) + + +def _percent(value: float) -> str: + return f"{value * 100:.1f}%" + + +def _value(value: Any) -> str: + if value is None: + return "—" + if isinstance(value, float) and value.is_integer(): + return str(int(value)) + return str(value) + + +def _range_text(metric: dict[str, Any]) -> str: + return "/".join(_value(metric[key]) for key in ("min", "max", "median")) + + +def _convergence_text(value: bool | None) -> str: + if value is None: + return "n/a" + return "yes" if value else "no" + + +def _escape(value: str) -> str: + return value.replace("|", "\\|").replace("`", "\\`") + + +def _code(value: str | None) -> str: + return f"`{_escape(value)}`" if value else "—" + + +def _path_link(value: str | None, label: str) -> str: + if not value: + return "—" + return f"[{label}]({_escape(value)})" + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/yada/evals/cli.py b/src/yada/evals/cli.py index 597fd9b..b282042 100644 --- a/src/yada/evals/cli.py +++ b/src/yada/evals/cli.py @@ -14,6 +14,7 @@ from yada.evals.base import RunBudget from yada.evals.benchmarks import LocalBenchmark, SWEbenchBenchmark from yada.evals.runner import EvalRunner +from yada.secrets import SecretConfigError, load_deepseek_api_key from yada.utils.naming import next_available_run_name, readable_run_name @@ -67,6 +68,14 @@ def build_parser() -> argparse.ArgumentParser: budget.add_argument("--max-output-tokens", type=int, default=16_384) model = parser.add_argument_group("Yada / DeepSeek") + model.add_argument( + "--api-key-file", + type=Path, + help=( + "Read the DeepSeek API key from a private file instead of placing " + "the secret in the environment." + ), + ) model.add_argument( "--model", default=os.environ.get("DEEPSEEK_MODEL", "deepseek-v4-pro"), @@ -141,9 +150,10 @@ def run_cli(argv: list[str] | None = None) -> int: ) if args.agent == "yada": - api_key = os.environ.get("DEEPSEEK_API_KEY", "") - if not api_key: - parser.error("DEEPSEEK_API_KEY is required for --agent yada") + try: + api_key = load_deepseek_api_key(args.api_key_file) + except SecretConfigError as exc: + parser.error(str(exc)) agent = YadaAgentAdapter( api_key=api_key, model=args.model, diff --git a/src/yada/run/cli.py b/src/yada/run/cli.py index f2978a2..3b4b6cd 100644 --- a/src/yada/run/cli.py +++ b/src/yada/run/cli.py @@ -11,6 +11,7 @@ from yada.agents import Agent from yada.editing import DEFAULT_EDITING_STRATEGY, EDITING_STRATEGY_CHOICES from yada.models import DeepSeekAPIError, DeepSeekClient +from yada.secrets import SecretConfigError, load_deepseek_api_key from yada.tools import ToolRunner from yada.traces import TraceWriter from yada.utils.naming import next_available_run_name, readable_run_name @@ -39,6 +40,14 @@ def build_parser() -> argparse.ArgumentParser: default=os.environ.get("DEEPSEEK_MODEL", "deepseek-v4-pro"), help="DeepSeek model name.", ) + parser.add_argument( + "--api-key-file", + type=Path, + help=( + "Read the DeepSeek API key from a private file instead of placing " + "the secret in the environment." + ), + ) parser.add_argument( "--base-url", default=os.environ.get("DEEPSEEK_BASE_URL", "https://api.deepseek.com"), @@ -124,9 +133,10 @@ def run_cli(argv: list[str] | None = None) -> int: workspace = args.workspace.expanduser().resolve() if not workspace.is_dir(): parser.error(f"workspace is not a directory: {workspace}") - api_key = os.environ.get("DEEPSEEK_API_KEY", "") - if not api_key: - parser.error("DEEPSEEK_API_KEY is not set") + try: + api_key = load_deepseek_api_key(args.api_key_file) + except SecretConfigError as exc: + parser.error(str(exc)) trace_path = args.trace or _default_trace_path(workspace, task) command_policy = "allow" if args.yes else args.command_policy diff --git a/src/yada/secrets.py b/src/yada/secrets.py new file mode 100644 index 0000000..d5d2f70 --- /dev/null +++ b/src/yada/secrets.py @@ -0,0 +1,116 @@ +"""Resolve provider credentials without placing secret values on argv or disk logs.""" + +from __future__ import annotations + +import os +import stat +from collections.abc import Mapping +from pathlib import Path + +_MAX_SECRET_BYTES = 4_096 +_KEY_FILE_ENVIRONMENT = "DEEPSEEK_API_KEY_FILE" +_LEGACY_KEY_ENVIRONMENT = "DEEPSEEK_API_KEY" + + +class SecretConfigError(ValueError): + """Raised when a configured secret source is missing or unsafe.""" + + +def default_deepseek_api_key_path( + environment: Mapping[str, str] | None = None, +) -> Path: + """Return the platform-appropriate default credential-file path.""" + + values = os.environ if environment is None else environment + if os.name == "nt" and values.get("APPDATA"): + return Path(values["APPDATA"]) / "Yada" / "deepseek_api_key" + config_home = values.get("XDG_CONFIG_HOME") + root = Path(config_home).expanduser() if config_home else Path.home() / ".config" + return root / "yada" / "deepseek_api_key" + + +def load_deepseek_api_key( + api_key_file: Path | None = None, + *, + environment: Mapping[str, str] | None = None, +) -> str: + """Load a DeepSeek key from a private file, with an environment fallback. + + Precedence is an explicit ``--api-key-file``, ``DEEPSEEK_API_KEY_FILE``, + the default user config file, then the legacy ``DEEPSEEK_API_KEY`` value. + The value itself is never included in an exception. + """ + + values = os.environ if environment is None else environment + configured_path: Path | None = None + required_file = False + if api_key_file is not None: + configured_path = api_key_file + required_file = True + elif values.get(_KEY_FILE_ENVIRONMENT): + configured_path = Path(values[_KEY_FILE_ENVIRONMENT]) + required_file = True + else: + default_path = default_deepseek_api_key_path(values) + if default_path.is_file(): + configured_path = default_path + + if configured_path is not None: + return _read_private_secret(configured_path, required=required_file) + + legacy_value = values.get(_LEGACY_KEY_ENVIRONMENT, "") + if legacy_value: + return _validate_secret_value(legacy_value, source=_LEGACY_KEY_ENVIRONMENT) + + default_path = default_deepseek_api_key_path(values) + raise SecretConfigError( + "DeepSeek API key not found; create the private file " + f"{default_path}, pass --api-key-file, or set {_KEY_FILE_ENVIRONMENT}" + ) + + +def _read_private_secret(path: Path, *, required: bool) -> str: + resolved = path.expanduser().resolve() + try: + metadata = resolved.stat() + except OSError as exc: + if required: + raise SecretConfigError( + f"cannot read DeepSeek API key file {resolved}: {exc}" + ) from exc + raise SecretConfigError( + f"DeepSeek API key file is unavailable: {resolved}" + ) from exc + if not stat.S_ISREG(metadata.st_mode): + raise SecretConfigError(f"DeepSeek API key path is not a file: {resolved}") + if os.name != "nt" and stat.S_IMODE(metadata.st_mode) & 0o077: + raise SecretConfigError( + f"DeepSeek API key file must not be accessible by group or others: " + f"{resolved}; run chmod 600 {resolved}" + ) + if metadata.st_size > _MAX_SECRET_BYTES: + raise SecretConfigError( + f"DeepSeek API key file is unexpectedly large: {resolved}" + ) + try: + value = resolved.read_text(encoding="utf-8") + except (OSError, UnicodeError) as exc: + raise SecretConfigError( + f"cannot read DeepSeek API key file {resolved}: {exc}" + ) from exc + return _validate_secret_value(value, source=str(resolved)) + + +def _validate_secret_value(value: str, *, source: str) -> str: + normalized = value.strip() + if not normalized: + raise SecretConfigError(f"DeepSeek API key source is empty: {source}") + if "\n" in normalized or "\r" in normalized: + raise SecretConfigError( + f"DeepSeek API key source must contain exactly one line: {source}" + ) + if "\x00" in normalized: + raise SecretConfigError( + f"DeepSeek API key source contains invalid data: {source}" + ) + return normalized diff --git a/tests/evals/test_eval_cli.py b/tests/evals/test_eval_cli.py index 2105df5..5ec4df7 100644 --- a/tests/evals/test_eval_cli.py +++ b/tests/evals/test_eval_cli.py @@ -18,6 +18,14 @@ def test_eval_cli_has_two_task_selectors() -> None: assert case.editing_strategy == "replace-first" +def test_eval_cli_accepts_a_private_api_key_file() -> None: + args = build_parser().parse_args( + ["--swebench", "owner__repo-1", "--api-key-file", "secret.txt"] + ) + + assert str(args.api_key_file) == "secret.txt" + + def test_eval_cli_exposes_editing_strategy() -> None: args = build_parser().parse_args( ["--case", "case-dir", "--editing-strategy", "replace-first"] diff --git a/tests/scripts/test_eval_suite.py b/tests/scripts/test_eval_suite.py new file mode 100644 index 0000000..19af551 --- /dev/null +++ b/tests/scripts/test_eval_suite.py @@ -0,0 +1,293 @@ +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +SCRIPT_PATH = Path(__file__).resolve().parents[2] / "scripts/eval_suite.py" +SCRIPT_SPEC = importlib.util.spec_from_file_location("yada_eval_suite", SCRIPT_PATH) +assert SCRIPT_SPEC is not None and SCRIPT_SPEC.loader is not None +eval_suite = importlib.util.module_from_spec(SCRIPT_SPEC) +sys.modules[SCRIPT_SPEC.name] = eval_suite +SCRIPT_SPEC.loader.exec_module(eval_suite) + + +CANARY_INSTANCES = [ + "pytest-dev__pytest-10051", + "pytest-dev__pytest-10081", + "pytest-dev__pytest-10356", + "django__django-15987", + "sympy__sympy-19637", + "sphinx-doc__sphinx-9367", + "scikit-learn__scikit-learn-13439", + "pydata__xarray-6461", +] + + +def test_canary_manifest_is_versioned_and_contains_the_canary_eight() -> None: + manifest = eval_suite.load_manifest( + Path("benchmarks/suites/swebench-verified-canary-v1.json") + ) + + assert manifest.suite_id == "swebench-verified-canary-v1" + assert manifest.benchmark == "swebench-verified" + assert list(manifest.instances) == CANARY_INSTANCES + assert manifest.sha256.startswith("sha256:") + + +def test_semantic_patch_id_ignores_object_ids_and_hunk_coordinates() -> None: + first = """diff --git a/value.py b/value.py +index 1111111..2222222 100644 +--- a/value.py ++++ b/value.py +@@ -1,2 +1,2 @@ function +-VALUE = 1 ++VALUE = 2 + context +""" + second = """diff --git a/value.py b/value.py +index aaaaaaa..bbbbbbb 100644 +--- a/value.py ++++ b/value.py +@@ -20,2 +25,2 @@ function +-VALUE = 1 ++VALUE = 2 + context +""" + changed = second.replace("+VALUE = 2", "+VALUE = 3") + + assert eval_suite.semantic_patch_id(first) == eval_suite.semantic_patch_id(second) + assert eval_suite.semantic_patch_id(first) != eval_suite.semantic_patch_id(changed) + + +def test_runner_continues_summarizes_and_skips_completed_attempts( + tmp_path: Path, + monkeypatch, +) -> None: + manifest = _write_manifest(tmp_path, ["owner__one-1", "owner__two-2"]) + suite_dir = tmp_path / "suite-results" + secret = "sk-never-persist-this-value" + key_file = tmp_path / "deepseek_api_key" + key_file.write_text(secret + "\n", encoding="utf-8") + key_file.chmod(0o600) + monkeypatch.setenv("DEEPSEEK_API_KEY", secret) + monkeypatch.setattr(eval_suite, "git_head", lambda _: "abc123") + calls: list[list[str]] = [] + outcomes = ["resolved", "resolved", "unresolved", "missing"] + + def fake_run(command, *, cwd, check): + assert command[1:5] == ["-m", "yada", "eval", "--swebench"] + assert "--yes" in command + assert secret not in command + assert command[command.index("--api-key-file") + 1] == str(key_file) + calls.append(command) + outcome = outcomes[len(calls) - 1] + if outcome != "missing": + patch_variant = 1 if len(calls) == 1 else 20 + _write_result( + command, + status=outcome, + steps=len(calls), + tokens=100 * len(calls), + patch=_patch(patch_variant, value=2 if len(calls) < 3 else 3), + ) + return subprocess.CompletedProcess( + command, + 0 if outcome == "resolved" else 1 if outcome == "unresolved" else 2, + ) + + monkeypatch.setattr(eval_suite.subprocess, "run", fake_run) + + assert ( + eval_suite.main( + [ + "run", + str(manifest), + "--output-dir", + str(suite_dir), + "--repeat", + "2", + "--model", + "test-model", + "--api-key-file", + str(key_file), + "--max-steps", + "7", + "--wall-time", + "90", + "--max-output-tokens", + "2048", + "--python", + "test-python", + ] + ) + == 0 + ) + + summary_path = suite_dir / "summary.json" + markdown_path = suite_dir / "summary.md" + summary = json.loads(summary_path.read_text(encoding="utf-8")) + assert summary["counts"] == {"error": 1, "resolved": 2, "unresolved": 1} + assert summary["resolution_rate"] == 0.5 + assert summary["metrics"]["steps"] == {"max": 3, "median": 2, "min": 1} + assert summary["metrics"]["tokens"] == { + "max": 300, + "median": 200, + "min": 100, + } + first_instance, second_instance = summary["instances"] + assert first_instance["patches_converged"] is True + assert first_instance["metrics"]["steps"] == { + "max": 2, + "median": 1.5, + "min": 1, + } + assert second_instance["patches_converged"] is False + assert all( + attempt["result_path"] and not Path(attempt["result_path"]).is_absolute() + for attempt in first_instance["attempts"] + ) + assert "Development canary only" in markdown_path.read_text(encoding="utf-8") + assert secret not in "".join( + path.read_text(encoding="utf-8") for path in suite_dir.rglob("*.json") + ) + + json_before = summary_path.read_bytes() + markdown_before = markdown_path.read_bytes() + assert eval_suite.main(["run", str(manifest), "--resume", str(suite_dir)]) == 0 + assert len(calls) == 4 + assert summary_path.read_bytes() == json_before + assert markdown_path.read_bytes() == markdown_before + + +def test_resume_recovers_a_result_written_just_before_interruption( + tmp_path: Path, + monkeypatch, +) -> None: + manifest = _write_manifest(tmp_path, ["owner__repo-1"]) + suite_dir = tmp_path / "suite-results" + monkeypatch.setattr(eval_suite, "git_head", lambda _: "abc123") + calls = 0 + + def interrupted_run(command, *, cwd, check): + nonlocal calls + calls += 1 + _write_result( + command, + status="resolved", + steps=4, + tokens=123, + patch=_patch(1, value=2), + ) + raise KeyboardInterrupt + + monkeypatch.setattr(eval_suite.subprocess, "run", interrupted_run) + + assert ( + eval_suite.main(["run", str(manifest), "--output-dir", str(suite_dir)]) == 130 + ) + assert not list(suite_dir.rglob("attempt.json")) + + def must_not_run(*args, **kwargs): + raise AssertionError("a completed result must be recovered, not rerun") + + monkeypatch.setattr(eval_suite.subprocess, "run", must_not_run) + + assert eval_suite.main(["run", str(manifest), "--resume", str(suite_dir)]) == 0 + assert calls == 1 + markers = list(suite_dir.rglob("attempt.json")) + assert len(markers) == 1 + marker = json.loads(markers[0].read_text(encoding="utf-8")) + assert marker["status"] == "resolved" + assert marker["recovered"] is True + assert marker["return_code"] is None + + +def test_resume_rejects_a_changed_manifest(tmp_path: Path, monkeypatch) -> None: + manifest = _write_manifest(tmp_path, ["owner__repo-1"]) + suite_dir = tmp_path / "suite-results" + monkeypatch.setattr(eval_suite, "git_head", lambda _: "abc123") + + def fake_run(command, *, cwd, check): + _write_result( + command, + status="resolved", + steps=1, + tokens=10, + patch="", + ) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(eval_suite.subprocess, "run", fake_run) + assert eval_suite.main(["run", str(manifest), "--output-dir", str(suite_dir)]) == 0 + + manifest.write_text(manifest.read_text() + "\n", encoding="utf-8") + + assert eval_suite.main(["run", str(manifest), "--resume", str(suite_dir)]) == 2 + + +def _write_manifest(tmp_path: Path, instances: list[str]) -> Path: + path = tmp_path / "suite.json" + path.write_text( + json.dumps( + { + "schema_version": 1, + "suite_id": "test-suite-v1", + "benchmark": "swebench-verified", + "instances": instances, + } + ) + + "\n", + encoding="utf-8", + ) + return path + + +def _write_result( + command: list[str], + *, + status: str, + steps: int, + tokens: int, + patch: str, +) -> None: + result_path = Path(command[command.index("--output") + 1]) + artifacts = Path(command[command.index("--artifact-dir") + 1]) + artifacts.mkdir(parents=True, exist_ok=True) + trace = artifacts / "yada-trace.jsonl" + trace.write_text("{}\n", encoding="utf-8") + instance_id = command[command.index("--swebench") + 1] + result_path.write_text( + json.dumps( + { + "status": status, + "instance_id": instance_id, + "started_at": "2026-01-01T00:00:00+00:00", + "duration_ms": 500, + "error": "synthetic failure" if status == "error" else None, + "agent_run": { + "model": "test-model", + "steps": steps, + "usage": {"total_tokens": tokens}, + "duration_ms": steps * 1_000, + "patch": patch, + "trace_path": str(trace), + }, + } + ) + + "\n", + encoding="utf-8", + ) + + +def _patch(line: int, *, value: int) -> str: + return f"""diff --git a/value.py b/value.py +index 1111111..2222222 100644 +--- a/value.py ++++ b/value.py +@@ -{line} +{line} @@ +-VALUE = 1 ++VALUE = {value} +""" diff --git a/tests/test_cli.py b/tests/test_cli.py index 4999612..4dea5f1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -8,6 +8,8 @@ def test_direct_cli_exposes_editing_strategy() -> None: default = parser.parse_args(["Fix it"]) replace_first = parser.parse_args(["Fix it", "--editing-strategy", "replace-first"]) + key_file = parser.parse_args(["Fix it", "--api-key-file", "secret.txt"]) assert default.editing_strategy == "replace-first" assert replace_first.editing_strategy == "replace-first" + assert str(key_file.api_key_file) == "secret.txt" diff --git a/tests/test_secrets.py b/tests/test_secrets.py new file mode 100644 index 0000000..3856701 --- /dev/null +++ b/tests/test_secrets.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from yada.secrets import SecretConfigError, load_deepseek_api_key + + +def test_explicit_private_key_file_takes_precedence_over_environment( + tmp_path: Path, +) -> None: + path = tmp_path / "deepseek_api_key" + path.write_text("file-secret\n", encoding="utf-8") + path.chmod(0o600) + + value = load_deepseek_api_key( + path, + environment={"DEEPSEEK_API_KEY": "environment-secret"}, + ) + + assert value == "file-secret" + + +def test_default_config_file_precedes_legacy_environment(tmp_path: Path) -> None: + path = tmp_path / "yada" / "deepseek_api_key" + path.parent.mkdir() + path.write_text("config-secret\n", encoding="utf-8") + path.chmod(0o600) + + value = load_deepseek_api_key( + environment={ + "XDG_CONFIG_HOME": str(tmp_path), + "DEEPSEEK_API_KEY": "legacy-secret", + } + ) + + assert value == "config-secret" + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX permission bits are unavailable") +def test_key_file_rejects_group_or_other_access(tmp_path: Path) -> None: + path = tmp_path / "deepseek_api_key" + secret = "sk-must-not-appear-in-errors" + path.write_text(secret + "\n", encoding="utf-8") + path.chmod(0o644) + + with pytest.raises(SecretConfigError, match="chmod 600") as captured: + load_deepseek_api_key(path, environment={}) + + assert secret not in str(captured.value) + + +def test_configured_file_does_not_fall_back_when_missing(tmp_path: Path) -> None: + missing = tmp_path / "missing" + + with pytest.raises(SecretConfigError, match="cannot read"): + load_deepseek_api_key( + missing, + environment={"DEEPSEEK_API_KEY": "must-not-mask-file-errors"}, + ) + + +def test_legacy_environment_remains_a_compatibility_fallback(tmp_path: Path) -> None: + assert ( + load_deepseek_api_key( + environment={ + "XDG_CONFIG_HOME": str(tmp_path), + "DEEPSEEK_API_KEY": "legacy-secret", + } + ) + == "legacy-secret" + ) + + +def test_key_source_must_contain_exactly_one_nonempty_line(tmp_path: Path) -> None: + path = tmp_path / "deepseek_api_key" + path.write_text("first\nsecond\n", encoding="utf-8") + path.chmod(0o600) + + with pytest.raises(SecretConfigError, match="exactly one line"): + load_deepseek_api_key(path, environment={})