From 1db117b985f7a82a01e7dad5fb9a5966155c67b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 22:32:23 +0000 Subject: [PATCH 1/9] Run the agentic eval through Harbor, with a step gate and trace analysis The eval now runs evals/snae-build, a Harbor task inspired from lightcone-evals' port of this repository's eval, in one workflow: prepare (the engine wheel built from the checkout into the task image, the agent-skills bundles), an oracle gate, claude-code and codex legs with the skills installed through Harbor's --skill, and a report. The instruction is now the one line a user would type, so how to drive lc and astra has to come from the skills and the CLIs' help. The report gates on reward, breakage and mean agent steps against the median of the leg's last main runs, and always ships a trace analysis that names where the steps went and which change most plausibly caused it. agent-skills reuses the workflow through workflow_call. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Y9FLtiiMxVhhkvZjd9SQp4 --- .github/scripts/eval_report.py | 553 ++++++++++++++++++ .github/scripts/trace_digest.py | 116 ---- .github/workflows/eval.yml | 508 ++++++++++------ .gitignore | 3 + CLAUDE.md | 22 +- evals/prompt.md | 159 ----- evals/snae-build/environment/Dockerfile | 67 +++ .../environment/seed}/astra.yaml | 0 .../seed}/data/SCPUnion2.1_mu_vs_z.txt | 0 evals/snae-build/instruction.md | 3 + evals/snae-build/solution/astra.yaml | 96 +++ .../snae-build/solution/scripts/cosmology.py | 67 +++ evals/snae-build/solution/scripts/fit.py | 91 +++ .../solution/scripts/plot_hubble.py | 72 +++ .../solution/scripts/plot_residuals.py | 58 ++ evals/snae-build/solution/solve.sh | 45 ++ evals/snae-build/task.toml | 43 ++ evals/snae-build/tests/test.sh | 73 +++ evals/trace-analysis-prompt.md | 61 +- 19 files changed, 1571 insertions(+), 466 deletions(-) create mode 100644 .github/scripts/eval_report.py delete mode 100644 .github/scripts/trace_digest.py delete mode 100644 evals/prompt.md create mode 100644 evals/snae-build/environment/Dockerfile rename evals/{tasks/snae => snae-build/environment/seed}/astra.yaml (100%) rename evals/{tasks/snae => snae-build/environment/seed}/data/SCPUnion2.1_mu_vs_z.txt (100%) create mode 100644 evals/snae-build/instruction.md create mode 100644 evals/snae-build/solution/astra.yaml create mode 100644 evals/snae-build/solution/scripts/cosmology.py create mode 100644 evals/snae-build/solution/scripts/fit.py create mode 100644 evals/snae-build/solution/scripts/plot_hubble.py create mode 100644 evals/snae-build/solution/scripts/plot_residuals.py create mode 100755 evals/snae-build/solution/solve.sh create mode 100644 evals/snae-build/task.toml create mode 100755 evals/snae-build/tests/test.sh diff --git a/.github/scripts/eval_report.py b/.github/scripts/eval_report.py new file mode 100644 index 00000000..a783138e --- /dev/null +++ b/.github/scripts/eval_report.py @@ -0,0 +1,553 @@ +"""Report on a Harbor eval run: metrics, evidence, the step gate, the PR comment. + + eval_report.py summarize --jobs collected/ --out report/ [--baseline base/] + [--refs refs.json] [--changes changes.md] + [--mode main|pr] [--tolerance 0.25] + eval_report.py comment --out report/ + +``summarize`` reads every leg under ``--jobs`` (one directory per matrix leg, +holding ``leg.json`` and, unless the leg was skipped, Harbor's job output) and +writes into ``--out``: + +- ``results.json`` per-leg rewards, steps, tool calls, phases, cost; the gate +- ``history.json`` the baseline's history plus this run, on ``--mode main`` +- ``digest-.md`` the chronological trace of each trial (ATIF trajectory) +- ``evidence-.md`` failures, repeats and the phase table, deterministic +- ``analysis-input-.md`` everything the trace analyst is handed + +``comment`` assembles ``comment.md`` from those plus each ``analysis-.md`` +the analyst wrote. The gate's verdict is in ``results.json``; the workflow +fails the job on it once the comment is posted. + +Inspired from LightconeResearch/lightcone-bench (``summarize.py``, +``trajectory_metrics.py``, ``compare.py``, ``analyze_traces.py``), which reads +the same Harbor artifacts. Agent steps are counted as the bench counts them: +trajectory steps whose source is the agent. +""" + +from __future__ import annotations + +import argparse +import json +import re +import statistics +import sys +from collections import Counter +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +HISTORY_LENGTH = 10 +SNIPPET = 300 +ERROR_SNIPPET = 1500 +DIGEST_CAP = 120_000 +EVIDENCE_CAP = 40 + +# Harness tool name (lowercased) -> how the call is read. Anything unlisted is +# read by its arguments alone. Vocabularies differ per harness: claude-code +# `Bash`/`Read`/`Skill`, codex `exec_command`/`apply_patch`, opencode `bash`/`read`. +SHELL = { + "bash", + "shell", + "exec", + "exec_command", + "local_shell", + "run_command", + "execute_command", + "terminal", + "container.exec", +} +READ = { + "read", + "read_file", + "view", + "glob", + "grep", + "ls", + "list_dir", + "list_files", + "search_files", + "open_file", +} + +# Where a tool call's effort went, first match wins. Keyed on the call's +# command or path, so it reads the same under every harness. +PHASES = [ + ("skills", re.compile(r"SKILL\.md|/skills/")), + ("help", re.compile(r"--help\b|\s-h\b|\bastra info\b|\blc --version\b")), + ("compute", re.compile(r"\blc compute\b")), + ("build", re.compile(r"\blc (materialize|run|status)\b")), + ("spec", re.compile(r"astra\.yaml|universes/|\bastra\b")), + ("publication", re.compile(r"license|ro-crate", re.I)), + ("deps", re.compile(r"\buv (add|remove|lock|sync|pip)\b|pyproject\.toml|uv\.lock")), + ("scripts", re.compile(r"scripts/|\.py\b")), + ("git", re.compile(r"\bgit\b")), +] + +FAILURE = re.compile( + r"Traceback \(most recent call last\)|^Error:|^error:" + r"|\bexit(?:ed with)? (?:code|status):? ?[1-9]|command not found|Permission denied" + r"|EACCES|No such file or directory|sandbox" + r"|is not (?:up to date|current)|refus", + re.I | re.M, +) + + +def _load(path: Path) -> Any: + try: + return json.loads(path.read_text()) + except (OSError, ValueError): + return None + + +def _seconds(block: Any) -> float | None: + if not isinstance(block, dict): + return None + try: + start, end = ( + datetime.fromisoformat(str(block[k]).replace("Z", "+00:00")) + for k in ("started_at", "finished_at") + ) + except (KeyError, TypeError, ValueError): + return None + return (end - start).total_seconds() + + +def _text(value: Any) -> str: + """Flatten an ATIF content value (string, parts list, or dict) to text.""" + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, list): + return "\n".join(_text(part) for part in value) + if isinstance(value, dict): + for key in ("text", "content", "output", "stdout"): + if key in value: + return _text(value[key]) + return json.dumps(value) + + +def _observations(step: dict[str, Any]) -> dict[str, str]: + """Map each tool call id in a step to the text of its result.""" + obs = step.get("observation") + if isinstance(obs, str): + try: + obs = json.loads(obs) + except ValueError: + return {"": obs} + if not isinstance(obs, dict): + return {} + return { + str(r.get("source_call_id", "")): _text(r.get("content")) + for r in obs.get("results") or [] + if isinstance(r, dict) + } + + +def _call_text(call: dict[str, Any]) -> str: + """The command or path a tool call acted on, as one line.""" + args = call.get("arguments") + if isinstance(args, str): + try: + args = json.loads(args) + except ValueError: + return args + if not isinstance(args, dict): + return "" + parts = [] + for key in ( + "command", + "cmd", + "file_path", + "filePath", + "path", + "pattern", + "skill", + "input", + "patch", + ): + value = args.get(key) + if isinstance(value, list): + value = " ".join(map(str, value)) + if value: + parts.append(str(value)) + return " ".join(parts) or json.dumps(args) + + +def _phase(name: str, text: str) -> str: + if name == "skill": + return "skills" + for phase, pattern in PHASES: + if pattern.search(text): + return phase + return "explore" if name in READ else "other" + + +def _snip(text: str, limit: int) -> str: + text = text.strip().replace("```", "`​``") + return text if len(text) <= limit else text[:limit] + f"… [{len(text) - limit} more chars]" + + +def read_trial(trial_dir: Path) -> dict[str, Any]: + """Metrics, evidence and a digest for one Harbor trial directory.""" + result = _load(trial_dir / "result.json") or {} + verifier = result.get("verifier_result") or {} + rewards = verifier.get("rewards") if isinstance(verifier, dict) else None + exc = result.get("exception_info") + trial: dict[str, Any] = { + "trial": result.get("trial_name", trial_dir.name), + "rewards": rewards or {}, + "reward": (rewards or {}).get("reward"), + "exception": ( + f"{exc.get('exception_type', '?')}: {_snip(str(exc.get('exception_message', '')), 500)}" + if isinstance(exc, dict) + else None + ), + "agent_min": (lambda s: s / 60 if s is not None else None)( + _seconds(result.get("agent_execution")) + ), + "cost_usd": (result.get("agent_result") or {}).get("cost_usd"), + } + traj = _load(trial_dir / "agent" / "trajectory.json") + steps = [s for s in (traj or {}).get("steps") or [] if isinstance(s, dict)] + phases: Counter[str] = Counter() + commands: Counter[str] = Counter() + failures: list[str] = [] + digest = [f"## Trial `{trial['trial']}`\n"] + for step in steps: + if step.get("source") == "user": + continue + results = _observations(step) + head = _text(step.get("message")) or _text(step.get("reasoning_content")) + line = f"**step {step.get('step_id', '?')}**" + digest.append(f"{line} {_snip(head, SNIPPET)}" if head else line) + for call in step.get("tool_calls") or []: + if not isinstance(call, dict): + continue + name = str(call.get("function_name", "?")) + text = _call_text(call) + phases[_phase(name.lower(), text)] += 1 + if name.lower() in SHELL: + commands[text.strip()] += 1 + # A step-level observation with no call ids belongs to its first call + out = results.pop(str(call.get("tool_call_id", "")), None) or results.pop("", "") + failed = bool(FAILURE.search(out)) + digest.append(f"- `{name}` {_snip(text, SNIPPET)}") + if out: + digest.append( + " ```\n " + + _snip(out, ERROR_SNIPPET if failed else SNIPPET).replace("\n", "\n ") + + "\n ```" + ) + if failed: + failures.append( + f"- `{trial['trial']}` step {step.get('step_id', '?')} `{name}` " + f"`{_snip(text, 200)}`\n ```\n " + + _snip(out, 600).replace("\n", "\n ") + + "\n ```" + ) + totals = (traj or {}).get("final_metrics") or {} + trial.update( + { + "has_trajectory": bool(steps), + "steps": sum(1 for s in steps if s.get("source") == "agent") if steps else None, + "tool_calls": sum(phases.values()) if steps else None, + "phases": dict(phases), + "skills_used": phases.get("skills", 0) > 0, + "failures": len(failures), + "repeats": {c: n for c, n in commands.items() if n > 1}, + } + ) + if trial["cost_usd"] is None: + trial["cost_usd"] = totals.get("total_cost_usd") + trial["_failures"] = failures + trial["_digest"] = "\n".join(digest) + test_out = trial_dir / "verifier" / "test-stdout.txt" + if test_out.is_file(): + trial["_verifier"] = _snip(test_out.read_text(errors="replace"), 3000) + return trial + + +def _mean(values: list[Any]) -> float | None: + nums = [v for v in values if isinstance(v, (int, float))] + return statistics.mean(nums) if nums else None + + +def read_leg(leg_dir: Path) -> dict[str, Any]: + """Every trial of one matrix leg, plus the leg's own description.""" + meta = _load(leg_dir / "leg.json") or {} + trials = [ + read_trial(p.parent) + for p in sorted(leg_dir.rglob("result.json")) + if (_load(p) or {}).get("trial_name") + ] + phases: Counter[str] = Counter() + for t in trials: + phases.update(t["phases"]) + return { + "leg": meta.get("leg", leg_dir.name), + "agent": meta.get("agent", "?"), + "model": meta.get("model", "?"), + "skipped": bool(meta.get("skipped")), + "skip_reason": meta.get("skip_reason", ""), + "trials": trials, + "n": len(trials), + "errored": sum(1 for t in trials if t["exception"]), + "mean_reward": _mean([t["reward"] for t in trials]), + "mean_steps": _mean([t["steps"] for t in trials]), + "mean_tool_calls": _mean([t["tool_calls"] for t in trials]), + "mean_cost_usd": _mean([t["cost_usd"] for t in trials]), + "mean_agent_min": _mean([t["agent_min"] for t in trials]), + "phases": {k: v / max(len(trials), 1) for k, v in phases.items()}, + "skills_used": any(t["skills_used"] for t in trials), + } + + +def gate( + legs: list[dict[str, Any]], history: dict[str, Any], mode: str, tolerance: float +) -> dict[str, Any]: + """The verdict: reward and breakage always; steps against main's median off main.""" + reasons: list[str] = [] + notes: list[str] = [] + ran = [leg for leg in legs if not leg["skipped"]] + if not ran: + reasons.append("no agent leg ran (every leg was skipped)") + for leg in ran: + name = leg["leg"] + if not leg["n"]: + reasons.append(f"{name}: no trial results") + continue + if leg["errored"]: + reasons.append(f"{name}: {leg['errored']} trial(s) errored") + if leg["mean_reward"] is None or leg["mean_reward"] < 1.0: + reasons.append(f"{name}: mean reward {leg['mean_reward']} < 1.0") + past = [h["steps"] for h in history.get(name, []) if h.get("steps") is not None] + leg["baseline_steps"] = statistics.median(past) if past else None + leg["history"] = history.get(name, []) + if leg["baseline_steps"] is None: + notes.append(f"{name}: NEW — no main history to compare steps against") + elif leg["mean_steps"] is None: + reasons.append(f"{name}: no trajectory, steps cannot be compared") + else: + limit = leg["baseline_steps"] * (1 + tolerance) + leg["steps_limit"] = limit + if leg["mean_steps"] > limit and mode != "main": + reasons.append( + f"{name}: {leg['mean_steps']:.0f} steps exceeds main's median " + f"{leg['baseline_steps']:.0f} by more than {tolerance:.0%}" + ) + return {"ok": not reasons, "reasons": reasons, "notes": notes} + + +def _fmt(value: Any, spec: str = "{:.0f}") -> str: + return "—" if value is None else spec.format(value) + + +def summarize(args: argparse.Namespace) -> int: + out: Path = args.out + out.mkdir(parents=True, exist_ok=True) + refs = _load(args.refs) if args.refs else {} + legs = [read_leg(d) for d in sorted(p for p in args.jobs.iterdir() if p.is_dir())] + base = args.baseline + base_results = (_load(base / "results.json") if base else None) or {} + history = (_load(base / "history.json") if base else None) or {} + verdict = gate(legs, history, args.mode, args.tolerance) + changes = args.changes.read_text() if args.changes and args.changes.is_file() else "" + base_legs = {leg["leg"]: leg for leg in base_results.get("legs", [])} + + for leg in legs: + name = leg["leg"] + if leg["skipped"]: + continue + digest = "\n\n".join(t.pop("_digest") for t in leg["trials"]) + (out / f"digest-{name}.md").write_text(digest[:DIGEST_CAP]) + failures = [f for t in leg["trials"] for f in t.pop("_failures")] + verifier = [t.pop("_verifier") for t in leg["trials"] if "_verifier" in t] + before = base_legs.get(name, {}).get("phases", {}) + rows = ["| phase | this run | main | Δ |", "|---|---|---|---|"] + for phase in sorted(set(leg["phases"]) | set(before)): + now, then = leg["phases"].get(phase, 0), before.get(phase) + delta = "—" if then is None else f"{now - then:+.1f}" + rows.append(f"| {phase} | {now:.1f} | {_fmt(then, '{:.1f}')} | {delta} |") + repeats: Counter[str] = Counter() + for t in leg["trials"]: + repeats.update(t["repeats"]) + evidence = [ + f"### Evidence — {name}", + "", + "Tool calls by phase (mean per trial):", + "", + *rows, + "", + f"Skills loaded: {'yes' if leg['skills_used'] else '**no**'}", + "", + f"Failed or erroring tool calls ({len(failures)}):", + "", + *(failures[:EVIDENCE_CAP] or ["- none"]), + *( + [f"- … {len(failures) - EVIDENCE_CAP} more in the digest"] + if len(failures) > EVIDENCE_CAP + else [] + ), + "", + "Commands run more than once:", + "", + *([f"- {n}× `{_snip(c, 200)}`" for c, n in repeats.most_common(15)] or ["- none"]), + ] + if any(t["exception"] for t in leg["trials"]): + evidence += [ + "", + "Harness exceptions:", + "", + *(f"- `{t['trial']}`: {t['exception']}" for t in leg["trials"] if t["exception"]), + ] + if verifier: + evidence += ["", "Verifier output:", "", "```", *verifier, "```"] + (out / f"evidence-{name}.md").write_text("\n".join(evidence) + "\n") + + base_digest = base / f"digest-{name}.md" if base else None + summary = ( + f"leg {name} ({leg['agent']} / {leg['model']}): mean reward " + f"{_fmt(leg['mean_reward'], '{:.2f}')}, mean agent steps " + f"{_fmt(leg['mean_steps'])}, main's median " + f"{_fmt(leg.get('baseline_steps'))}, n={leg['n']}" + ) + (out / f"analysis-input-{name}.md").write_text( + "\n\n".join( + [ + "# Run summary", + summary, + "\n".join(evidence), + "# Change under test", + changes or "_not provided_", + "# This run's trace", + digest[:DIGEST_CAP], + "# Baseline trace (latest successful main run)", + base_digest.read_text()[:DIGEST_CAP] + if base_digest and base_digest.is_file() + else "_no baseline trace available_", + ] + ) + + "\n" + ) + + results = { + "refs": refs, + "mode": args.mode, + "tolerance": args.tolerance, + "legs": legs, + "gate": verdict, + } + (out / "results.json").write_text(json.dumps(results, indent=1, default=str) + "\n") + if args.mode == "main": + now = datetime.now(UTC).isoformat(timespec="seconds") + for leg in legs: + if leg["skipped"] or leg["mean_steps"] is None: + continue + entry = { + "at": now, + "sha": refs.get("sha", ""), + "steps": leg["mean_steps"], + "reward": leg["mean_reward"], + } + history[leg["leg"]] = (history.get(leg["leg"], []) + [entry])[-HISTORY_LENGTH:] + (out / "history.json").write_text(json.dumps(history, indent=1) + "\n") + elif base and (base / "history.json").is_file(): + # Carry main's history along so a downloaded PR artifact is self-explanatory. + (out / "history.json").write_text(json.dumps(history, indent=1) + "\n") + print("gate:", "ok" if verdict["ok"] else "; ".join(verdict["reasons"])) + return 0 + + +def comment(args: argparse.Namespace) -> int: + out: Path = args.out + results = _load(out / "results.json") or {} + verdict = results.get("gate", {}) + refs = results.get("refs", {}) + lines = [ + f"## {'✅' if verdict.get('ok') else '❌'} Eval", + "", + "", + "", + ] + if refs: + lines += [" · ".join(f"{k}: `{v}`" for k, v in refs.items() if v), ""] + lines += [ + "| Leg | Reward | Steps | main median | Tool calls | Cost | Agent min | Skills |", + "|---|---|---|---|---|---|---|---|", + ] + for leg in results.get("legs", []): + if leg["skipped"]: + lines.append(f"| {leg['leg']} | skipped: {leg['skip_reason']} | | | | | | |") + continue + steps = _fmt(leg["mean_steps"]) + if leg.get("baseline_steps") and leg["mean_steps"] is not None: + steps += f" ({(leg['mean_steps'] / leg['baseline_steps'] - 1):+.0%})" + lines.append( + f"| {leg['leg']} | {_fmt(leg['mean_reward'], '{:.2f}')} | {steps} " + f"| {_fmt(leg.get('baseline_steps'))} | {_fmt(leg['mean_tool_calls'])} " + f"| {_fmt(leg['mean_cost_usd'], '${:.2f}')} " + f"| {_fmt(leg['mean_agent_min'], '{:.1f}')} " + f"| {'yes' if leg['skills_used'] else 'no'} |" + ) + lines.append("") + for reason in verdict.get("reasons", []): + lines.append(f"- ❌ {reason}") + for note in verdict.get("notes", []): + lines.append(f"- ℹ️ {note}") + lines.append("") + for leg in results.get("legs", []): + if leg["skipped"]: + continue + name = leg["leg"] + analysis = out / f"analysis-{name}.md" + lines += [f"### {name}", ""] + if analysis.is_file() and analysis.read_text().strip(): + lines += [analysis.read_text().strip(), ""] + else: + lines += [ + "❌ _Trace analysis missing — see the evidence below and the " + "`eval-results` artifact._", + "", + ] + history = leg.get("history") or [] + if history: + trend = " → ".join(_fmt(h.get("steps")) for h in history) + lines += [f"Steps on main, oldest first: {trend}", ""] + evidence = out / f"evidence-{name}.md" + if evidence.is_file(): + lines += [ + "
Evidence (deterministic)", + "", + evidence.read_text(), + "
", + "", + ] + (out / "comment.md").write_text("\n".join(lines)) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + s = sub.add_parser("summarize") + s.add_argument("--jobs", type=Path, required=True) + s.add_argument("--out", type=Path, required=True) + s.add_argument("--baseline", type=Path) + s.add_argument("--refs", type=Path) + s.add_argument("--changes", type=Path) + s.add_argument("--mode", choices=["main", "pr"], default="pr") + s.add_argument("--tolerance", type=float, default=0.25) + c = sub.add_parser("comment") + c.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + if args.command == "comment": + return comment(args) + if args.baseline and not args.baseline.is_dir(): + args.baseline = None + return summarize(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/trace_digest.py b/.github/scripts/trace_digest.py deleted file mode 100644 index dfe7327b..00000000 --- a/.github/scripts/trace_digest.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Digest a `claude -p --output-format stream-json` transcript. - -Usage: trace_digest.py - -Writes a chronological, human-readable agent trace to and -prints shell-sourceable `key=value` run metrics on stdout (model, turns, -tool_calls, cost, duration). -""" - -import json -import sys -from collections import Counter -from pathlib import Path - -RESULT_SNIPPET = 300 -ERROR_SNIPPET = 1500 - - -def _fence(text: str) -> str: - """Neutralize backtick fences so embedded output can't break ours.""" - return text.replace("```", "`​``") - - -def main() -> None: - if len(sys.argv) != 3: - print("Usage: trace_digest.py ", file=sys.stderr) - sys.exit(1) - - transcript = Path(sys.argv[1]) - out_path = Path(sys.argv[2]) - - model = "" - metrics: dict = {} - tool_counts: Counter = Counter() - body: list[str] = [] - - for line in transcript.read_text().splitlines(): - try: - d = json.loads(line) - except json.JSONDecodeError: - continue - t = d.get("type") - - if t == "system" and d.get("subtype") == "init": - model = d.get("model", "") - - elif t == "assistant": - for c in d.get("message", {}).get("content", []): - if c.get("type") == "text" and c["text"].strip(): - body.append(f"**Claude:** {c['text'].strip()}\n") - elif c.get("type") == "tool_use": - name = c.get("name", "?") - tool_counts[name] += 1 - inp = c.get("input", {}) - detail = ( - inp.get("command") - or inp.get("file_path") - or inp.get("skill") - or json.dumps(inp)[:200] - ) - body.append(f"🔧 `{name}`\n\n```\n{_fence(str(detail))}\n```\n") - - elif t == "user": - for c in d.get("message", {}).get("content", []): - if isinstance(c, dict) and c.get("type") == "tool_result": - content = c.get("content", "") - if isinstance(content, list): - content = "\n".join( - x.get("text", "") for x in content if isinstance(x, dict) - ) - content = str(content).strip() - if c.get("is_error"): - body.append( - f"❌ **error result:**\n\n```\n{_fence(content[:ERROR_SNIPPET])}\n```\n" - ) - elif content: - # Collapse to one line so the whole snippet stays - # inside the blockquote - snippet = " ".join(content[:RESULT_SNIPPET].split()) - ellipsis = " …" if len(content) > RESULT_SNIPPET else "" - body.append(f"> {_fence(snippet)}{ellipsis}\n") - - elif t == "result": - metrics = d - - turns = metrics.get("num_turns", 0) - cost = metrics.get("total_cost_usd", 0.0) or 0.0 - duration_s = (metrics.get("duration_ms", 0) or 0) // 1000 - duration = f"{duration_s // 60}m{duration_s % 60:02d}s" - tool_calls = sum(tool_counts.values()) - tools_breakdown = ", ".join(f"{n} ×{c}" for n, c in tool_counts.most_common()) - - header = [ - "# Agent trace", - "", - f"- **Model:** {model or '?'}", - f"- **Turns:** {turns}", - f"- **Tool calls:** {tool_calls} ({tools_breakdown})", - f"- **Cost:** ${cost:.2f}", - f"- **Agent wall time:** {duration}", - f"- **Errored:** {metrics.get('is_error', '?')}", - "", - "---", - "", - ] - out_path.write_text("\n".join(header) + "\n".join(body)) - - print(f"model={model}") - print(f"turns={turns}") - print(f"tool_calls={tool_calls}") - print(f"cost={cost:.2f}") - print(f"duration={duration}") - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 949205c9..24e2e4e8 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -1,184 +1,373 @@ name: Eval -# Minimal agentic eval: scaffold a project from the eval seed on the -# runner, let Claude Code (with the astra plugin) build it using the -# branch's lc/astra, then check the results with astra validate and -# lc materialize --check. +# Agentic eval on Harbor: the task in evals/snae-build is run by real agent +# harnesses against the lightcone-cli and agent-skills under test, and every +# run explains itself from its trace. +# +# prepare build the engine wheel into the task, collect the skill bundles +# oracle the task's scripted solution must score 1.0 — no LLM involved +# agents one Harbor job per harness, the skills installed with --skill +# report rewards, agent steps against main's history, the trace analysis, +# the PR comment; red on a failed gate or a missing analysis +# +# The step gate: a leg whose mean agent steps exceed the median of its last +# runs on main by more than `tolerance` fails. Main runs record that history +# (in the eval-results artifact of the newest successful main run) and gate +# on reward only, so the reference follows main. +# +# Reuse from LightconeResearch/agent-skills, where the skills are the side +# under test and the stack is lightcone-cli main: +# +# on: +# pull_request: +# types: [opened, synchronize, reopened, ready_for_review] +# push: +# branches: [main] +# permissions: +# contents: read +# pull-requests: write +# actions: read +# jobs: +# eval: +# uses: LightconeResearch/lightcone-cli/.github/workflows/eval.yml@main +# secrets: inherit +# +# Whichever of the two repositories triggered the run is checked out at the +# triggering commit; the other at its input ref. Each repository keeps its +# own history, because the baseline is looked up in the triggering one. +# +# Limits: --skill installs SKILL.md bundles only, not the plugins' hooks +# (agent-skills' e2e-hooks workflow covers those); step counts compare within +# a leg only, since harness tool vocabularies differ; at attempts=1 the step +# comparison is coarse, which is what the median and the tolerance absorb. +# A leg whose secret is absent is skipped with a notice, and a run in which +# no leg ran fails. on: workflow_dispatch: + inputs: + stack_ref: + description: "lightcone-cli ref to evaluate when it is not the triggering repository" + type: string + default: main + skills_ref: + description: "agent-skills ref to install when it is not the triggering repository" + type: string + default: main + attempts: + description: "Trials per leg" + type: string + default: "1" + tolerance: + description: "Allowed fractional increase of agent steps over main's median" + type: string + default: "0.25" + workflow_call: + inputs: + stack_ref: + description: "lightcone-cli ref to evaluate when it is not the triggering repository" + type: string + default: main + skills_ref: + description: "agent-skills ref to install when it is not the triggering repository" + type: string + default: main + attempts: + description: "Trials per leg" + type: string + default: "1" + tolerance: + description: "Allowed fractional increase of agent steps over main's median" + type: string + default: "0.25" pull_request: - # `ready_for_review` beside the defaults, so flipping a draft to - # ready triggers the run the draft guard below skips. + # `ready_for_review` beside the defaults, so flipping a draft to ready + # triggers the run the draft guard below skips. types: [opened, synchronize, reopened, ready_for_review] + push: + branches: [main] # Only one eval per PR at a time — cancel in-progress runs concurrency: - group: eval-${{ github.event.pull_request.number || github.run_id }} + group: eval-${{ github.repository }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true permissions: contents: read pull-requests: write + actions: read + +env: + HARBOR_VERSION: 0.22.0 + STACK_REPO: LightconeResearch/lightcone-cli + SKILLS_REPO: LightconeResearch/agent-skills jobs: - eval: - if: github.event_name == 'workflow_dispatch' || github.event.pull_request.draft == false + prepare: + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false runs-on: ubuntu-latest - # Step-level timeout on the agent step (below) is what actually - # bounds the run: a job-level timeout would cancel the always() - # report/comment/artifact steps along with it. - timeout-minutes: 20 + timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - name: Check out lightcone-cli + uses: actions/checkout@v4 with: + repository: ${{ env.STACK_REPO }} + ref: ${{ github.repository == env.STACK_REPO && (github.event.pull_request.head.sha || github.sha) || inputs.stack_ref || 'main' }} + path: stack # Full history so hatch-vcs sees the tags and stamps a truthful - # dev version into the installed engine (and every manifest's - # lc_version) instead of a 0.1.dev fallback + # dev version into the wheel (and every manifest's lc_version), + # and so the change under test can be described + fetch-depth: 0 + + - name: Check out agent-skills + uses: actions/checkout@v4 + with: + repository: ${{ env.SKILLS_REPO }} + ref: ${{ github.repository == env.SKILLS_REPO && (github.event.pull_request.head.sha || github.sha) || inputs.skills_ref || 'main' }} + path: skills-src fetch-depth: 0 - name: Set up uv - # No python-version input: it would export an ambient UV_PYTHON, - # which lc's install-settings scrub then rightly warns about on - # every single invocation the agent makes. The interpreter is - # pinned per tool install below instead. uses: astral-sh/setup-uv@v6 + + - name: Assemble the eval inputs + env: + EVENT_REPO: ${{ github.repository }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + run: | + mkdir -p bundle/skills + cp -r stack/evals/snae-build bundle/task + cp stack/.github/scripts/eval_report.py stack/evals/trace-analysis-prompt.md bundle/ + # The engine under test, built from its checkout: the task image + # installs exactly this wheel + uv build --wheel --project stack -o bundle/task/environment/lc-wheel + + # Every built skill bundle, first of each name: plugins bundle + # their dependencies, so one skill can appear in several + for d in skills-src/plugins/*/skills/*/; do + name=$(basename "$d") + [ -f "${d}SKILL.md" ] && [ ! -e "bundle/skills/$name" ] && cp -r "$d" "bundle/skills/$name" + done + ls bundle/skills | grep -q . || { echo "::error::no skill bundles in $SKILLS_REPO"; exit 1; } + + case "${EVENT_REPO,,}" in + "${STACK_REPO,,}") changed=stack ;; + "${SKILLS_REPO,,}") changed=skills-src ;; + *) changed= ;; + esac + jq -n \ + --arg stack "$STACK_REPO@$(git -C stack rev-parse --short HEAD)" \ + --arg skills "$SKILLS_REPO@$(git -C skills-src rev-parse --short HEAD)" \ + --arg skill_set "$(ls bundle/skills | paste -sd, -)" \ + --arg sha "$(git -C "${changed:-stack}" rev-parse HEAD)" \ + '{stack: $stack, skills: $skills, skill_set: $skill_set, sha: $sha}' > bundle/refs.json + cat bundle/refs.json + + # What the analyst blames a regression on: the commits and files + # of the repository under test since its base + { + if [ -n "$changed" ] && [ -n "$BASE_SHA" ] \ + && git -C "$changed" cat-file -e "$BASE_SHA^{commit}" 2>/dev/null; then + echo "Repository under test: $EVENT_REPO" + echo + echo '```' + git -C "$changed" log --oneline "$BASE_SHA..HEAD" | head -50 + echo + git -C "$changed" diff --stat "$BASE_SHA...HEAD" | tail -60 + echo '```' + else + echo "No base commit to compare against for this event." + fi + } > bundle/changes.md + + - uses: actions/upload-artifact@v4 + with: + name: eval-inputs + path: bundle + if-no-files-found: error + + oracle: + needs: prepare + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/download-artifact@v4 + with: + name: eval-inputs + path: bundle + + - name: Set up uv + uses: astral-sh/setup-uv@v6 + + - name: Install Harbor + run: uv tool install "harbor==$HARBOR_VERSION" + + - name: Run the reference solution + run: harbor run -p bundle/task -a oracle -e docker -o jobs --job-name oracle -q -y + + - name: Require reward 1.0 + run: | + jq -e '[.stats.evals[] + | (.n_errors // 0) == 0 and (.metrics[0].reward // 0) == 1.0] + | length > 0 and all' jobs/oracle/result.json \ + || { jq .stats jobs/oracle/result.json; echo "::error::oracle gate failed"; exit 1; } + + - uses: actions/upload-artifact@v4 + if: always() with: - enable-cache: true - - - name: Install lightcone-cli + astra (uv tools) - # The engine is the host's uv tool — install the branch under - # test the way a user installs a release: `uv tool install`, - # which links `lc` and (via the mirrored entry points) the - # git-annex executables into ~/.local/bin. The commit under test - # is the checkout itself, stamped into the version by hatch-vcs. - # astra's executable belongs to astra-tools, so it is its own - # tool install, pinned to the version the branch depends on. No - # venv is activated: the agent's shell sees the tools exactly as - # an end user's would. + name: eval-oracle + path: jobs + if-no-files-found: warn + + agents: + needs: oracle + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - { leg: claude-code, agent: claude-code, model: claude-sonnet-5-5, secret_name: CLAUDE_CODE_OAUTH_TOKEN } + - { leg: codex, agent: codex, model: gpt-5.1-codex, secret_name: OPENAI_API_KEY } + steps: + - name: Gate on the leg's secret + id: gate + env: + LEG_KEY: ${{ secrets[matrix.secret_name] }} run: | - uv tool install --python 3.12 "$GITHUB_WORKSPACE" - uv tool install --python 3.12 "astra-tools==$(grep -oP 'astra-tools==\K[0-9][0-9.]*' pyproject.toml)" - echo "$HOME/.local/bin" >> "$GITHUB_PATH" + mkdir -p "jobs/${{ matrix.leg }}" + skipped=false + if [ -z "$LEG_KEY" ]; then + skipped=true + echo "::notice::Skipping ${{ matrix.leg }} — secret ${{ matrix.secret_name }} is not set" + fi + echo "skip=$skipped" >> "$GITHUB_OUTPUT" + jq -n --arg leg "${{ matrix.leg }}" --arg agent "${{ matrix.agent }}" \ + --arg model "${{ matrix.model }}" --argjson skipped "$skipped" \ + --arg reason "${{ matrix.secret_name }} not set" \ + '{leg: $leg, agent: $agent, model: $model, skipped: $skipped} + + (if $skipped then {skip_reason: $reason} else {} end)' \ + > "jobs/${{ matrix.leg }}/leg.json" + + - uses: actions/download-artifact@v4 + if: steps.gate.outputs.skip != 'true' + with: + name: eval-inputs + path: bundle + + - name: Set up uv + if: steps.gate.outputs.skip != 'true' + uses: astral-sh/setup-uv@v6 + + - name: Install Harbor + if: steps.gate.outputs.skip != 'true' + run: uv tool install "harbor==$HARBOR_VERSION" + + - name: Run the agent + if: steps.gate.outputs.skip != 'true' + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + CLAUDE_CODE_MAX_TURNS: "200" + run: | + skill_flags=() + for d in bundle/skills/*/; do skill_flags+=(--skill "${d%/}"); done + harbor run -p bundle/task \ + -a "${{ matrix.agent }}" -m "${{ matrix.model }}" \ + -e docker -k "${{ inputs.attempts || '1' }}" "${skill_flags[@]}" \ + -o "jobs/${{ matrix.leg }}" --job-name "${{ matrix.leg }}" -q -y - - name: Install Claude Code + astra plugin + - uses: actions/upload-artifact@v4 + if: always() + with: + name: eval-leg-${{ matrix.leg }} + path: jobs/${{ matrix.leg }} + if-no-files-found: warn + + report: + needs: [prepare, agents] + if: always() && needs.prepare.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/download-artifact@v4 + with: + name: eval-inputs + path: bundle + + - uses: actions/download-artifact@v4 + with: + pattern: eval-leg-* + path: collected + + - name: Fetch main's latest results + env: + GH_TOKEN: ${{ github.token }} + WORKFLOW: ${{ github.workflow }} + run: | + id=$(gh run list -R "$GITHUB_REPOSITORY" --workflow "$WORKFLOW" \ + --branch main --event push --status success --limit 1 \ + --json databaseId -q '.[0].databaseId' || true) + if [ -n "$id" ] && gh run download "$id" -R "$GITHUB_REPOSITORY" \ + -n eval-results -D baseline; then + echo "Baseline: run $id" + else + echo "::notice::No successful main run with eval-results yet — reporting without a baseline" + fi + + - name: Summarize + run: | + mode=pr + [ "$GITHUB_EVENT_NAME" = push ] && [ "$GITHUB_REF" = refs/heads/main ] && mode=main + python3 bundle/eval_report.py summarize --jobs collected --out report \ + --baseline baseline --refs bundle/refs.json --changes bundle/changes.md \ + --mode "$mode" --tolerance "${{ inputs.tolerance || '0.25' }}" + + - name: Install Claude Code run: | curl -fsSL https://claude.ai/install.sh | bash echo "$HOME/.local/bin" >> "$GITHUB_PATH" - export PATH="$HOME/.local/bin:$PATH" # Skip first-run onboarding in headless mode echo '{"hasCompletedOnboarding": true}' > "$HOME/.claude.json" - claude plugin marketplace add LightconeResearch/agent-skills - claude plugin install astra@lightcone-research - - name: Set up eval project - run: | - # Global git identity so any repo the agent commits in works - # (ephemeral runner — nothing to pollute) - git config --global user.name Eval - git config --global user.email eval@lightcone - project="$RUNNER_TEMP/eval-project" - echo "PROJECT=$project" >> "$GITHUB_ENV" - lc init "$project" - # Overlay the task seed (astra.yaml, data/) on the scaffold - cp -r evals/tasks/snae/. "$project"/ - cd "$project" - # Regenerate baseline.yaml from the task astra.yaml's defaults — - # the scaffolded one matches the boilerplate spec, not the task's - rm -f universes/baseline.yaml - astra universe generate -n baseline \ - -d 'Default configuration using standard practices' - # No scientific deps are pre-seeded — declaring what the - # recipes import (`uv add`) is part of what the eval measures - # lc materialize refuses a dirty tree — commit the seed so the - # agent starts from a clean one - git add -A - git commit -q -m 'Seed the eval task' - - - name: Run Claude Code - id: agent - timeout-minutes: 15 - working-directory: ${{ env.PROJECT }} + - name: Analyse the traces + id: analysis env: CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} run: | - set -o pipefail - # Transcript goes outside the project so the agent's git - # status stays clean; uploaded via the agent-trace artifact - claude -p "$(cat "$GITHUB_WORKSPACE/evals/prompt.md")" \ - --output-format stream-json --verbose \ - --dangerously-skip-permissions \ - --max-turns 200 \ - | tee "$RUNNER_TEMP/claude-transcript.jsonl" - - - name: Check results - id: check - working-directory: ${{ env.PROJECT }} - run: | - astra validate astra.yaml - lc status - # The gate: nothing failed, nothing left to make — and at least - # one output actually got materialized - lc materialize --check --json | jq -e \ - '.up_to_date and ((.current | length) > 0)' - # The publication view: declaring a license and materializing - # once more is part of the task - test -f ro-crate-metadata.json - - - name: Build report + # The analysis is part of the result: a leg without one fails the + # job, after the comment says so + status=ok + if [ -z "$CLAUDE_CODE_OAUTH_TOKEN" ]; then + echo "::error::CLAUDE_CODE_OAUTH_TOKEN is not set — no trace analysis" + echo "status=failed" >> "$GITHUB_OUTPUT" + exit 0 + fi + for input in report/analysis-input-*.md; do + [ -e "$input" ] || continue + leg=${input#report/analysis-input-}; leg=${leg%.md} + if ! timeout 600 claude -p "$(cat bundle/trace-analysis-prompt.md)" \ + < "$input" > "report/analysis-$leg.md" \ + || ! [ -s "report/analysis-$leg.md" ]; then + echo "::error::trace analysis failed for $leg" + rm -f "report/analysis-$leg.md" + status=failed + fi + done + echo "status=$status" >> "$GITHUB_OUTPUT" + + - name: Build the report if: always() - env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} run: | - transcript="$RUNNER_TEMP/claude-transcript.jsonl" - if [ -s "$transcript" ]; then - python3 .github/scripts/trace_digest.py \ - "$transcript" "$RUNNER_TEMP/agent-trace.md" > "$RUNNER_TEMP/metrics.env" - source "$RUNNER_TEMP/metrics.env" - # Claude's read of where the agent struggled — best-effort, - # the report still posts if this fails or times out - timeout 180 claude -p "$(cat evals/trace-analysis-prompt.md)" \ - < "$RUNNER_TEMP/agent-trace.md" \ - > "$RUNNER_TEMP/pain-points.md" \ - || echo "_Trace analysis unavailable._" > "$RUNNER_TEMP/pain-points.md" - fi + python3 bundle/eval_report.py comment --out report + cat report/comment.md >> "$GITHUB_STEP_SUMMARY" - icon="✅" - [ "${{ steps.check.outcome }}" = "success" ] || icon="❌" - { - echo "## $icon Eval" - echo "" - echo "" - echo "" - echo "| Metric | Value |" - echo "|--------|-------|" - echo "| Outputs check | ${{ steps.check.outcome }} |" - echo "| Agent run | ${{ steps.agent.outcome }} |" - echo "| Turns | ${turns:-?} |" - echo "| Tool calls | ${tool_calls:-?} |" - echo "| Cost | \$${cost:-?} |" - echo "| Agent wall time | ${duration:-?} |" - echo "| Model | ${model:-?} |" - echo "" - echo '
lc status' - echo "" - echo '```' - (cd "$PROJECT" && lc status) || echo "unavailable" - echo '```' - echo "" - echo '
' - echo "" - if [ -s "$RUNNER_TEMP/pain-points.md" ]; then - echo '
Confusion & pain points (Claude analysis)' - echo "" - cat "$RUNNER_TEMP/pain-points.md" - echo "" - echo '
' - echo "" - fi - echo "Full trace: \`agent-trace\` artifact on [this run](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})." - } > "$RUNNER_TEMP/comment.md" - cat "$RUNNER_TEMP/comment.md" >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: eval-results + path: report + if-no-files-found: warn - name: Comment on PR if: always() && github.event_name == 'pull_request' @@ -186,7 +375,12 @@ jobs: with: script: | const fs = require('fs'); - const body = fs.readFileSync(`${process.env.RUNNER_TEMP}/comment.md`, 'utf8'); + let body = fs.readFileSync('report/comment.md', 'utf8'); + // GitHub rejects comments over 65536 characters + if (body.length > 65000) { + body = body.slice(0, 65000) + '\n\n_Truncated — full report in the eval-results artifact._'; + } + body += `\n\nRun: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; const marker = ''; const { data: comments } = await github.rest.issues.listComments({ owner: context.repo.owner, @@ -210,28 +404,8 @@ jobs: }); } - - name: Upload agent trace - if: always() - uses: actions/upload-artifact@v4 - with: - name: agent-trace - path: | - ${{ runner.temp }}/claude-transcript.jsonl - ${{ runner.temp }}/agent-trace.md - ${{ runner.temp }}/pain-points.md - if-no-files-found: warn - - - name: Upload eval project - if: always() - uses: actions/upload-artifact@v4 - with: - name: eval-project - # include-hidden-files keeps the manifests, the run records in - # .git, and the annexed results; the venv and lc's per-run - # machine state are excluded - path: | - ${{ env.PROJECT }} - !${{ env.PROJECT }}/.venv/** - !${{ env.PROJECT }}/.lightcone/** - include-hidden-files: true - if-no-files-found: warn + - name: Gate + run: | + jq -r '.gate.reasons[]' report/results.json + jq -e '.gate.ok' report/results.json > /dev/null + [ "${{ steps.analysis.outputs.status }}" = ok ] diff --git a/.gitignore b/.gitignore index d763879e..fddabb8f 100644 --- a/.gitignore +++ b/.gitignore @@ -220,3 +220,6 @@ uv.lock # macOS .DS_Store + +# The engine wheel the eval workflow builds into the task image +evals/snae-build/environment/lc-wheel/ diff --git a/CLAUDE.md b/CLAUDE.md index 70b35fe2..96341ae1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -192,7 +192,7 @@ src/lightcone/ # namespace — NO __init__.py ├── __init__.py # loader; a renderer only where there is a value to decide └── files/*.tmpl # the templates themselves, as real files -evals/ # agentic eval seed: prompt.md + tasks// +evals/ # agentic eval: snae-build/ (a Harbor task) + the trace-analysis prompt tests/ # pytest — mirrors src/ ``` @@ -209,9 +209,21 @@ uv build # wheel + sdist (CI runs this only to publish) Test, lint and type-check are the whole loop, and they are what `.github/workflows/{tests,lint}.yml` run. There is deliberately no task runner in between — the pre-rebuild `justfile` was 90 lines of wrappers -around them. The other workflows are `eval.yml` (the agentic eval, -on dispatch or the `run-eval` PR label; re-trigger by re-adding the -label) and `pypi-publish.yaml`. +around them. The other workflows are `eval.yml` and `pypi-publish.yaml`. + +`eval.yml` is the agentic eval, on every non-draft PR, every push to +main and dispatch. It runs `evals/snae-build` through Harbor in four +jobs: `prepare` (the engine wheel built from the checkout into the task +image, the agent-skills bundles), `oracle` (the scripted solution must +score 1.0), `agents` (claude-code and codex, the skills installed with +`--skill`, a leg without its secret skipped with a notice) and `report` +(`.github/scripts/eval_report.py`, then one `claude -p` trace analysis +per leg). It fails on a reward below 1.0, an errored trial, a missing +trace analysis, or — off main — mean agent steps more than `tolerance` +over the median of that leg's last main runs, a history carried in the +newest successful main run's `eval-results` artifact. agent-skills +reuses it through `workflow_call` with the skills as the side under +test; the header comment holds the caller. ## Key Invariants (layer 1) @@ -712,7 +724,7 @@ must not discard edits made elsewhere while the graph was running. `definition_version = sha256(recipe ‖ canonical decisions ‖ format)` is what the spec says an output *is*. The format is framed in its own right rather than left to arrive through the rendered `{output}` path: a recipe -need not use that placeholder at all (`evals/tasks/snae` does not), and +need not use that placeholder at all (the `evals/snae-build` seed does not), and one that does not would keep its digest across a re-declared serialization — while the sidecar, named from the id alone, does not move either, so the manifest would go on describing an output at a path that diff --git a/evals/prompt.md b/evals/prompt.md deleted file mode 100644 index 87dd91d4..00000000 --- a/evals/prompt.md +++ /dev/null @@ -1,159 +0,0 @@ -Build the analysis specified in `astra.yaml` for universe `baseline`. - -## Toolchain - -This project is driven by two CLIs — use them rather than improvising: - -- `astra` is the spec layer. `astra info` summarizes `astra.yaml`; - `astra validate astra.yaml` checks it against the schema. If an `astra` - skill or plugin is available in your environment, load it before reading - or editing `astra.yaml` — it documents the full spec format. -- `lc` (lightcone-cli) is the execution layer: - - `lc materialize ` makes every output the spec declares, running each - recipe in dependency order and committing each result to git as it - lands, together with a provenance manifest. It refuses to start on - a dirty tree: commit your own edits first, with plain `git add` and - `git commit` — the project's git-annex filter handles large files - transparently, so never run a git-annex command yourself. - - `lc materialize ` (or `/`) narrows - a run to one output and whatever it depends on. Re-running is - idempotent: only what is stale gets remade — an output the spec now - defines differently, or one whose declared inputs changed. - - `lc status` reports each output as `current`, `stale`, or `behind`, - with the commit it was made at; `lc status --json` is the - machine-readable form. It always exits 0. The pass/fail gate is - `lc materialize --check`, which exits 1 while anything still needs - making. `--check` needs no cluster. - - `lc run -- ` runs an ad-hoc command in the project - environment under the same isolation a recipe gets — useful for - probing why a recipe would fail. Argv style, like `docker run` or - `uv run`: `lc run -- python scripts/fit.py --output /tmp/x`, never a - single quoted shell string; for shell syntax use - `lc run -- bash -c '...'`. - - Outputs land in `results/baseline/.`, each with a - `..manifest.json` manifest beside it, written and - committed by the engine. Never write into `results/` yourself: a - hand-placed file has no run record, and the engine detects the - foreign write and remakes the output. - - When a recipe fails, `lc materialize` reports which output failed - and why; fix the script or the spec, commit, and re-run. - -Allocate compute before running commands or recipes: `lc compute launch --wait`. -With no CPU/memory flags, this starts a cluster named `local` using all detected -usable CPUs and RAM on this machine and waits until it is ready. No configuration -is needed. GPUs still require explicit offers. Add `--name analysis` to choose -another name. Replace `` in these commands with the returned name -(normally `local`) or a full immutable ID from launch or status JSON (`--json`). -Reuse the cluster with `run` and `materialize`; neither creates compute automatically. - -Only one local cluster can run per user on this machine, even with different names -or catalogs. If one already exists, use the catalog identified in the refusal to -inspect `lc compute status` and reuse it rather than launching another. -`lc compute status --wait` waits for an -existing cluster. Launch's `--wait` defaults to a 300-second readiness timeout; -`--timeout SECONDS` overrides it. A waiting launch that fails reports the accepted -cluster ID and leaves the allocation unchanged: inspect it before retrying. -A local cluster ends after 30 minutes without task activity; running work keeps -it alive, and `--time` adds a hard lifetime that ends it even mid-run. After it -ends, launch again; the name `local` can be reused, but the immutable ID changes. - -Compute configuration is `~/.lightcone/compute.yaml`, or the file selected by -`LC_COMPUTE_CONFIG`. A catalog that lists its own local offers (`provider: local`) -uses those instead of the built-in one, which is how to set a smaller CPU/RAM -budget. Remote offers otherwise coexist with the default local offer. If -`allow_local: false` is configured, respect that policy: local launch and execution -are disabled. Inspect `lc compute resources` and supply both -`--cpus` and `--memory` to select a configured remote allocation; `--wait` works -there too. The no-resource shortcut never selects remote compute automatically. -Recognized NERSC login nodes refuse local compute automatically, even without a -catalog. Use Slurm or an interactive compute-node session; local compute remains -eligible on those compute nodes. Do not try to override the login-node guard. - -## Recipe template grammar - -A recipe's `command` is a template. The engine substitutes these -placeholders before invoking it: - -- `{output}` — the file the output is materialized to, - `results//.`, where `format` is the one the - output declares. Your script must write exactly that path, and nothing - else: an output is one file. The engine creates the directory; a recipe - that writes a directory there, or writes some other name, fails. -- `{inputs.}` — the named input's resolved path: an analysis-level - `Input`'s `source` (e.g. a file under `data/`), or, for an upstream - output, that output's own file — so your script opens it directly. -- `{inputs}` — space-separated paths of all declared inputs, in - declaration order. -- `{decisions.}` — the active option ID for the named decision in the - current universe (e.g. `nelder_mead`), which your script should accept - as an argparse choice. -- `{{` and `}}` emit literal braces. Format specs (`{x:>8}`) are rejected. - -Provenance is declared on the Output, not inside the recipe: every -`{inputs.}` / `{decisions.}` the command references must be listed -in that output's `inputs:` / `decisions:` lists, or validation fails. -Dependencies between outputs come from these `inputs:` declarations — that -is how the engine orders the build. - -## Environment - -Recipes run in the project's own locked environment (`pyproject.toml` + -`uv.lock` + `.venv`), sandboxed: the project tree is read-only apart from -the directory each recipe's output lands in, and only declared tools are -executable. - -- The project is managed by uv and starts with **no dependencies**. - Every package a recipe script imports must be declared before - materializing: run `uv add [ ...]` in the project - root (e.g. `uv add numpy scipy`). That updates `pyproject.toml`, - re-locks `uv.lock`, and syncs `.venv` in one step — commit all of it - along with your scripts, like any other edit. -- To remove a package use `uv remove `; to pin a version, - `uv add 'numpy>=2'`. Do **not** use plain `pip` or `uv pip install` — - an install that bypasses the lock reaches nothing a recipe sees. -- A sandbox denial names the path or tool that was denied and the - remedy — follow the remedy rather than working around the sandbox. - -## Build loop - -`astra.yaml` is the single source of truth: inputs, outputs, recipes, and -methodological decisions all live there — read it first. The seed spec is -deliberately incomplete: outputs declare no `format:`, recipe commands do -not yet pass their inputs, decisions, or output path, and outputs may be -missing entries in their `inputs:` / `decisions:` contracts. Completing the spec is part of -the task. For each output: - -1. Declare the output's `format:` — the file extension its artifact is - written with, without the leading dot (`png`, `csv`, `json`, …). lc - names the file from it and refuses a spec that omits it. -2. Complete the recipe `command` so it references `{output}` and the - `{inputs.}` / `{decisions.}` the computation needs, and - declare everything it references in that output's `inputs:` / - `decisions:` lists. -3. Write the script at the path the command names, parameterizing every - decision via argparse — never hardcode option values. -4. Commit your edits, then run `lc materialize ` (or - `lc materialize `) to build through the engine. - -Build iteratively from upstream outputs to downstream. `lc status` shows -where every output stands. - -## Publication - -Once every output is materialized, prepare the repository for -publication: - -1. Declare a license in `pyproject.toml`, as an SPDX expression under - `[project]` — e.g. `license = "CC-BY-4.0"`. Declaring one is what - turns publication on: from then on `lc materialize` also maintains - `ro-crate-metadata.json` at the project root, an RO-Crate view of - the project and its provenance. -2. Commit the edit, then run `lc materialize ` once more — nothing is - remade, but the crate document is generated and committed. - -You're done when `astra validate astra.yaml` and -`lc materialize --check` pass and `ro-crate-metadata.json` exists. -Release your allocation with `lc compute down ` when finished. - -Skip plan approval and interactive confirmations — this is an automated -eval run. diff --git a/evals/snae-build/environment/Dockerfile b/evals/snae-build/environment/Dockerfile new file mode 100644 index 00000000..6f6e4118 --- /dev/null +++ b/evals/snae-build/environment/Dockerfile @@ -0,0 +1,67 @@ +FROM python:3.12-slim + +# lc scrubs ambient UV_* settings from recipe environments, except these two: +# a shared managed interpreter and a shared package cache, seen identically +# by root (oracle, verifier) and the unprivileged agent user. +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONDONTWRITEBYTECODE=1 \ + UV_PYTHON_INSTALL_DIR=/opt/uv-python \ + UV_CACHE_DIR=/opt/uv-cache + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + git \ + jq \ + procps \ + ripgrep \ + && rm -rf /var/lib/apt/lists/* + +RUN python -m pip install --no-cache-dir uv==0.12.13 \ + && uv python install 3.12 + +# The engine under test is a wheel the eval workflow builds from its checkout +# into lc-wheel/ (gitignored), installed the way a user installs a release — +# `uv tool install`, which also links the git-annex executables — but into +# system-wide paths so every user sees the same tools. astra-tools is the +# version that wheel depends on. +COPY lc-wheel/ /opt/lc-wheel/ +RUN UV_TOOL_DIR=/opt/uv-tools UV_TOOL_BIN_DIR=/usr/local/bin \ + uv tool install --python 3.12 /opt/lc-wheel/lightcone_cli-*.whl \ + && UV_TOOL_DIR=/opt/uv-tools UV_TOOL_BIN_DIR=/usr/local/bin \ + uv tool install --python 3.12 \ + "astra-tools==$(/opt/uv-tools/lightcone-cli/bin/python -c 'import importlib.metadata as m; print(m.version("astra-tools"))')" + +# A git identity for whoever commits, and root reading the agent-owned repo +# in the verifier without git's dubious-ownership refusal. +RUN git config --system user.name "Lightcone Eval" \ + && git config --system user.email eval@lightcone \ + && git config --system init.defaultBranch main \ + && git config --system --add safe.directory '*' + +# `lc init`, the seed overlaid, the universe regenerated from the seed spec's +# defaults, then a clean seed commit (lc materialize refuses a dirty tree). +# No scientific dependencies are pre-seeded — declaring them (`uv add`) is +# part of what the task measures. +RUN lc init /root/snae-build +COPY seed/ /root/snae-build/ +RUN cd /root/snae-build \ + && rm -f universes/baseline.yaml \ + && astra universe generate -n baseline \ + -d 'Default configuration using standard practices' \ + && git add -A \ + && git commit -q -m 'Seed the eval task' + +# Warm the shared cache with the packages the recipes will plausibly need so +# a trial does not hinge on PyPI; the project itself stays dependency-free. +RUN uv init /tmp/warm \ + && cd /tmp/warm && uv add numpy scipy matplotlib \ + && rm -rf /tmp/warm + +# The harness runs the agent as an unprivileged user and the verifier as +# root: tools and interpreter readable by all, cache and project writable, +# and /root traversable (it is 0700 on this base image). +RUN chmod -R a+rX /opt/uv-tools /opt/uv-python \ + && chmod -R a+rwX /opt/uv-cache /root/snae-build \ + && chmod 711 /root + +WORKDIR /root/snae-build diff --git a/evals/tasks/snae/astra.yaml b/evals/snae-build/environment/seed/astra.yaml similarity index 100% rename from evals/tasks/snae/astra.yaml rename to evals/snae-build/environment/seed/astra.yaml diff --git a/evals/tasks/snae/data/SCPUnion2.1_mu_vs_z.txt b/evals/snae-build/environment/seed/data/SCPUnion2.1_mu_vs_z.txt similarity index 100% rename from evals/tasks/snae/data/SCPUnion2.1_mu_vs_z.txt rename to evals/snae-build/environment/seed/data/SCPUnion2.1_mu_vs_z.txt diff --git a/evals/snae-build/instruction.md b/evals/snae-build/instruction.md new file mode 100644 index 00000000..f0f063ce --- /dev/null +++ b/evals/snae-build/instruction.md @@ -0,0 +1,3 @@ +This is a Lightcone project. Build the analysis described in `astra.yaml` for the `baseline` universe, then get the project ready to publish. + +This is an automated run: nobody will answer questions or approve plans. Release any compute you allocate when you are done. diff --git a/evals/snae-build/solution/astra.yaml b/evals/snae-build/solution/astra.yaml new file mode 100644 index 00000000..6cdf5328 --- /dev/null +++ b/evals/snae-build/solution/astra.yaml @@ -0,0 +1,96 @@ +# ASTRA Analysis Specification +# Documentation: https://github.com/LightconeResearch/ASTRA + +version: "0.0.14" +name: "snae" +description: | + Fit the Union2.1 Type Ia supernova distance modulus vs redshift data + to a flat LCDM cosmological model with two free parameters (H0, Omega_L) + using maximum-likelihood (MAP) point estimation. This provides best-fit + cosmological parameters as a building block for a larger analysis. + +inputs: + - id: union21 + type: data + source: data/SCPUnion2.1_mu_vs_z.txt + description: "Union2.1 SN Ia compilation: 580 supernovae with redshift, distance modulus, and uncertainties" + +outputs: + - id: best_fit + type: metric + format: json + description: "Best-fit H0 and Omega_L from chi-squared minimization, with reduced chi-squared" + inputs: [union21] + decisions: [optimizer, error_model, redshift_cut] + recipe: + command: >- + python scripts/fit.py + --data {inputs.union21} + --output {output} + --optimizer {decisions.optimizer} + --error-model {decisions.error_model} + --redshift-cut {decisions.redshift_cut} + + - id: hubble_diagram + type: figure + format: png + description: "Hubble diagram: distance modulus vs redshift with best-fit model overlay" + inputs: [union21, best_fit] + recipe: + command: >- + python scripts/plot_hubble.py + --data {inputs.union21} + --fit {inputs.best_fit} + --output {output} + + - id: residuals + type: figure + format: png + description: "Residuals plot: data minus best-fit model vs redshift" + inputs: [union21, best_fit] + recipe: + command: >- + python scripts/plot_residuals.py + --data {inputs.union21} + --fit {inputs.best_fit} + --output {output} + +decisions: + optimizer: + label: "Optimizer" + tags: [inference] + rationale: "Nelder-Mead is robust and derivative-free; L-BFGS-B supports bounds natively" + default: nelder_mead + options: + nelder_mead: + label: "Nelder-Mead" + description: "Derivative-free simplex optimizer" + l_bfgs_b: + label: "L-BFGS-B" + description: "Bounded quasi-Newton optimizer" + + error_model: + label: "Error Treatment" + tags: [data] + rationale: "Union2.1 provides both statistical-only and systematics-included uncertainties; choice directly affects fit quality" + default: stat_and_sys + options: + stat_only: + label: "Statistical errors only" + description: "Use only the statistical distance modulus uncertainties" + stat_and_sys: + label: "Statistical + systematic errors" + description: "Include systematic uncertainty contributions" + + redshift_cut: + label: "Low-Redshift Cut" + tags: [data] + rationale: "SNe at z < 0.01 are dominated by peculiar velocity scatter rather than cosmological signal; standard practice varies on whether to exclude them" + default: z_min_001 + options: + none: + label: "No cut" + description: "Use all supernovae regardless of redshift" + z_min_001: + label: "z > 0.01" + description: "Exclude supernovae with z < 0.01 to mitigate peculiar velocity contamination" diff --git a/evals/snae-build/solution/scripts/cosmology.py b/evals/snae-build/solution/scripts/cosmology.py new file mode 100644 index 00000000..65227201 --- /dev/null +++ b/evals/snae-build/solution/scripts/cosmology.py @@ -0,0 +1,67 @@ +"""Shared numerical helpers for the Union2.1 flat-LCDM analysis.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +from scipy.integrate import quad + +C_KM_S = 299_792.458 +SYSTEMATIC_FLOOR_MAG = 0.03 + + +def load_union21(path: str | Path) -> tuple[np.ndarray, ...]: + """Load redshift, distance modulus, statistical error, and host probability.""" + redshift, distance_modulus, statistical_error, host_probability = np.loadtxt( + path, + comments="#", + usecols=(1, 2, 3, 4), + unpack=True, + ) + return redshift, distance_modulus, statistical_error, host_probability + + +def selection_mask(redshift: np.ndarray, redshift_cut: str) -> np.ndarray: + if redshift_cut == "none": + return np.ones(redshift.shape, dtype=bool) + if redshift_cut == "z_min_001": + return redshift > 0.01 + raise ValueError(f"Unknown redshift cut: {redshift_cut}") + + +def effective_error(statistical_error: np.ndarray, error_model: str) -> np.ndarray: + if error_model == "stat_only": + return statistical_error + if error_model == "stat_and_sys": + # The supplied table contains statistical errors and host-mass + # probabilities, but no systematic covariance matrix. A 0.03 mag + # irreducible floor is therefore used as a conservative approximation. + return np.hypot(statistical_error, SYSTEMATIC_FLOOR_MAG) + raise ValueError(f"Unknown error model: {error_model}") + + +def distance_modulus(redshift: np.ndarray, h0: float, omega_l: float) -> np.ndarray: + """Distance modulus for a spatially flat matter-plus-Lambda cosmology.""" + z = np.asarray(redshift, dtype=float) + omega_m = 1.0 - omega_l + + def inv_expansion_rate(z_value: float) -> float: + return 1.0 / np.sqrt(omega_m * (1.0 + z_value) ** 3 + omega_l) + + comoving_integral = np.fromiter( + (quad(inv_expansion_rate, 0.0, value, epsabs=1e-10, epsrel=1e-10)[0] for value in z), + dtype=float, + count=z.size, + ) + luminosity_distance_mpc = (C_KM_S / h0) * (1.0 + z) * comoving_integral + return 5.0 * np.log10(luminosity_distance_mpc) + 25.0 + + +def load_fit(path: str | Path) -> dict[str, object]: + fit_path = Path(path) + if fit_path.is_dir(): + fit_path = fit_path / "best_fit.json" + with fit_path.open(encoding="utf-8") as stream: + return json.load(stream) diff --git a/evals/snae-build/solution/scripts/fit.py b/evals/snae-build/solution/scripts/fit.py new file mode 100644 index 00000000..767b2b36 --- /dev/null +++ b/evals/snae-build/solution/scripts/fit.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Fit Union2.1 distance moduli with a two-parameter flat-LCDM model.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np +from cosmology import ( + SYSTEMATIC_FLOOR_MAG, + distance_modulus, + effective_error, + load_union21, + selection_mask, +) +from scipy.optimize import minimize + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--data", required=True) + parser.add_argument("--output", required=True, help="the output file, results//best_fit.json") + parser.add_argument("--optimizer", choices=("nelder_mead", "l_bfgs_b"), required=True) + parser.add_argument("--error-model", choices=("stat_only", "stat_and_sys"), required=True) + parser.add_argument("--redshift-cut", choices=("none", "z_min_001"), required=True) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + redshift, observed_mu, statistical_error, _ = load_union21(args.data) + keep = selection_mask(redshift, args.redshift_cut) + redshift = redshift[keep] + observed_mu = observed_mu[keep] + sigma_mu = effective_error(statistical_error[keep], args.error_model) + + def chi_squared(parameters: np.ndarray) -> float: + h0, omega_l = parameters + if not 40.0 <= h0 <= 100.0 or not 0.001 <= omega_l <= 0.999: + return float("inf") + model_mu = distance_modulus(redshift, h0, omega_l) + return float(np.sum(np.square((observed_mu - model_mu) / sigma_mu))) + + method = "Nelder-Mead" if args.optimizer == "nelder_mead" else "L-BFGS-B" + options: dict[str, float | int] + if method == "Nelder-Mead": + options = {"maxiter": 2000, "xatol": 1e-9, "fatol": 1e-7} + else: + options = {"maxiter": 2000, "ftol": 1e-12} + + result = minimize( + chi_squared, + x0=np.array([70.0, 0.7]), + method=method, + bounds=((40.0, 100.0), (0.001, 0.999)), + options=options, + ) + if not result.success: + raise RuntimeError(f"Optimization failed: {result.message}") + + h0, omega_l = (float(value) for value in result.x) + chi2 = float(result.fun) + degrees_of_freedom = int(redshift.size - 2) + summary = { + "H0_km_s_Mpc": h0, + "Omega_L": omega_l, + "Omega_m": 1.0 - omega_l, + "chi_squared": chi2, + "degrees_of_freedom": degrees_of_freedom, + "reduced_chi_squared": chi2 / degrees_of_freedom, + "n_supernovae": int(redshift.size), + "optimizer": args.optimizer, + "error_model": args.error_model, + "redshift_cut": args.redshift_cut, + "systematic_floor_mag": SYSTEMATIC_FLOOR_MAG if args.error_model == "stat_and_sys" else 0.0, + "optimization_evaluations": int(result.nfev), + "optimization_message": str(result.message), + } + + # An output is exactly one file — the path the engine passed as {output}. + output = Path(args.output) + with output.open("w", encoding="utf-8") as stream: + json.dump(summary, stream, indent=2, sort_keys=True) + stream.write("\n") + print(json.dumps(summary, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/evals/snae-build/solution/scripts/plot_hubble.py b/evals/snae-build/solution/scripts/plot_hubble.py new file mode 100644 index 00000000..d274f02b --- /dev/null +++ b/evals/snae-build/solution/scripts/plot_hubble.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Plot the Union2.1 Hubble diagram and the fitted flat-LCDM relation.""" + +from __future__ import annotations + +import argparse + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +from cosmology import distance_modulus, effective_error, load_fit, load_union21, selection_mask + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--data", required=True) + parser.add_argument("--fit", required=True, help="the best_fit output file (json)") + parser.add_argument("--output", required=True, help="the output file, results//hubble_diagram.png") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + fit = load_fit(args.fit) + redshift, observed_mu, statistical_error, _ = load_union21(args.data) + keep = selection_mask(redshift, str(fit["redshift_cut"])) + sigma_mu = effective_error(statistical_error, str(fit["error_model"])) + + model_redshift = np.linspace(float(redshift[keep].min()), float(redshift[keep].max()), 800) + model_mu = distance_modulus( + model_redshift, + float(fit["H0_km_s_Mpc"]), + float(fit["Omega_L"]), + ) + + figure, axis = plt.subplots(figsize=(8.2, 5.4), constrained_layout=True) + axis.errorbar( + redshift[keep], + observed_mu[keep], + yerr=sigma_mu[keep], + fmt=".", + color="#377eb8", + alpha=0.45, + markersize=3, + elinewidth=0.5, + label=f"Union2.1 ({keep.sum()} SNe Ia)", + ) + axis.plot( + model_redshift, + model_mu, + color="#e41a1c", + linewidth=2, + label=( + rf"Flat $\Lambda$CDM: $H_0={float(fit['H0_km_s_Mpc']):.2f}$ " + r"km s$^{-1}$ Mpc$^{-1}$, " + rf"$\Omega_\Lambda={float(fit['Omega_L']):.3f}$" + ), + ) + axis.set_xlabel("Redshift z") + axis.set_ylabel("Distance modulus μ") + axis.set_title("Union2.1 Hubble diagram") + axis.grid(alpha=0.2) + axis.legend(frameon=False) + + figure.savefig(args.output, dpi=180) + plt.close(figure) + + +if __name__ == "__main__": + main() diff --git a/evals/snae-build/solution/scripts/plot_residuals.py b/evals/snae-build/solution/scripts/plot_residuals.py new file mode 100644 index 00000000..1c67e639 --- /dev/null +++ b/evals/snae-build/solution/scripts/plot_residuals.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Plot Union2.1 distance-modulus residuals around the fitted model.""" + +from __future__ import annotations + +import argparse + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from cosmology import distance_modulus, effective_error, load_fit, load_union21, selection_mask + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--data", required=True) + parser.add_argument("--fit", required=True, help="the best_fit output file (json)") + parser.add_argument("--output", required=True, help="the output file, results//residuals.png") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + fit = load_fit(args.fit) + redshift, observed_mu, statistical_error, _ = load_union21(args.data) + keep = selection_mask(redshift, str(fit["redshift_cut"])) + sigma_mu = effective_error(statistical_error, str(fit["error_model"])) + model_mu = distance_modulus( + redshift[keep], + float(fit["H0_km_s_Mpc"]), + float(fit["Omega_L"]), + ) + residual = observed_mu[keep] - model_mu + + figure, axis = plt.subplots(figsize=(8.2, 4.6), constrained_layout=True) + axis.errorbar( + redshift[keep], + residual, + yerr=sigma_mu[keep], + fmt=".", + color="#4daf4a", + alpha=0.5, + markersize=3, + elinewidth=0.5, + ) + axis.axhline(0.0, color="#222222", linewidth=1.3) + axis.set_xlabel("Redshift z") + axis.set_ylabel("Observed − model μ") + axis.set_title("Union2.1 flat-ΛCDM residuals") + axis.grid(alpha=0.2) + + figure.savefig(args.output, dpi=180) + plt.close(figure) + + +if __name__ == "__main__": + main() diff --git a/evals/snae-build/solution/solve.sh b/evals/snae-build/solution/solve.sh new file mode 100755 index 00000000..7de8fb38 --- /dev/null +++ b/evals/snae-build/solution/solve.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Scripted reference solution (oracle): the build loop instruction.md asks +# for, done by hand. Runs as root in the seeded project; the verifier must +# score exactly 1.0 afterwards. +set -euo pipefail + +PROJECT="${PROJECT:-/root/snae-build}" +SOLUTION="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$PROJECT" + +# 1-2. Complete the spec: formats, templated recipe commands, and each +# output's inputs/decisions contract. +cp "$SOLUTION/astra.yaml" astra.yaml +astra validate astra.yaml + +# 3. Write the scripts the recipes name, decisions parameterized via argparse. +mkdir -p scripts +cp "$SOLUTION"/scripts/*.py scripts/ + +# The project starts with no dependencies; declare what the recipes import. +uv add numpy scipy matplotlib + +# 4. Commit, then build through the engine. Since lightcone-cli #226 the engine +# runs recipes on an explicitly allocated cluster: launch the default local +# one (all CPUs and RAM the container shows) and pass its name to every +# materialize; --check and status need no cluster. +git add -A +git commit -q -m "Complete the spec and write the recipes" +lc compute launch --wait +lc materialize local + +# Publication: declare a license, commit, materialize once more so the +# engine generates and commits ro-crate-metadata.json. +sed -i '/^\[project\]$/a license = "CC-BY-4.0"' pyproject.toml +git add -A +git commit -q -m "Declare a license for publication" +lc materialize local + +# The gate instruction.md states, checked here so a broken oracle fails loudly. +astra validate astra.yaml +lc materialize --check +test -f ro-crate-metadata.json + +# Release the allocation, as the instruction asks the agent to. +lc compute down local diff --git a/evals/snae-build/task.toml b/evals/snae-build/task.toml new file mode 100644 index 00000000..8f2156d4 --- /dev/null +++ b/evals/snae-build/task.toml @@ -0,0 +1,43 @@ +schema_version = "1.4" +artifacts = [] + +[task] +name = "lightcone/snae-build" +version = "0.3.0" +description = "Build a seeded ASTRA analysis end to end with lightcone-cli: complete the deliberately incomplete spec, write the recipe scripts, declare dependencies, materialize every output through the engine, and prepare the project for publication." +keywords = ["astronomy", "lightcone-cli", "astra", "build-from-scratch", "naive-usage", "provenance"] +[[task.authors]] +name = "dkn16" +email = "dkn16@foxmail.com" + +[metadata] +category = "natural-science" +difficulty = "medium" +# Inspired from LightconeResearch/lightcone-evals lightcone-cli/snae-build, +# itself a port of this repository's earlier eval. Installs the engine from a +# wheel the workflow builds out of the checkout under test, rather than a +# pinned git commit. +source_format = "lightcone-evals" + +[verifier] +timeout_sec = 600.0 +collect = [] + +[verifier.env] + +[agent] +timeout_sec = 1200.0 + +[environment] +network_mode = "public" +build_timeout_sec = 1800.0 +os = "linux" +cpus = 2 +memory_mb = 4096 +storage_mb = 10240 +gpus = 0 +mcp_servers = [] + +[environment.env] + +[solution.env] diff --git a/evals/snae-build/tests/test.sh b/evals/snae-build/tests/test.sh new file mode 100755 index 00000000..ce020526 --- /dev/null +++ b/evals/snae-build/tests/test.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Verifier (runs as root after the agent). Three deterministic dimensions, +# mirroring lightcone-cli's own eval gate (astra validate, lc materialize +# --check, ro-crate-metadata.json) with partial credit for outputs built: +# spec_valid 0.25 astra validate astra.yaml passes +# outputs_current 0.50 fraction of declared outputs `current` in lc status --json +# publication_ready 0.25 lc materialize --check is up to date with >=1 current +# output, and ro-crate-metadata.json exists +# Always exits 0; scoring problems are a 0.0, not a crash. +set +e + +PROJECT="${PROJECT:-/root/snae-build}" +LOGS="${LOGS_DIR:-/logs/verifier}" +mkdir -p "$LOGS" + +# The repository is owned by the agent user; git refuses to read another +# user's repo without this. +git config --global --add safe.directory '*' 2>/dev/null + +spec_valid=0.0 +outputs_current=0.0 +publication_ready=0.0 + +if cd "$PROJECT"; then + if astra validate astra.yaml > "$LOGS/astra-validate.txt" 2>&1; then + spec_valid=1.0 + fi + cat "$LOGS/astra-validate.txt" + + lc status > "$LOGS/lc-status.txt" 2>&1 + cat "$LOGS/lc-status.txt" + lc status --json > "$LOGS/lc-status.json" 2> "$LOGS/lc-status.err" + outputs_current=$(python3 - "$LOGS/lc-status.json" <<'PY' +import json, sys +try: + outputs = json.load(open(sys.argv[1])).get("outputs") or [] +except Exception: + outputs = [] +current = sum(1 for o in outputs if o.get("status") == "current") +print(f"{current / len(outputs):.4f}" if outputs else "0.0") +PY +) + + lc materialize --check --json > "$LOGS/lc-check.json" 2> "$LOGS/lc-check.err" + if python3 - "$LOGS/lc-check.json" <<'PY' +import json, sys +try: + report = json.load(open(sys.argv[1])) +except Exception: + sys.exit(1) +sys.exit(0 if report.get("up_to_date") and len(report.get("current") or []) > 0 else 1) +PY + then + if [ -f ro-crate-metadata.json ]; then + publication_ready=1.0 + fi + fi + + # Keep what explains the score in the artifacts. + cp astra.yaml "$LOGS/astra.yaml" 2>/dev/null || true + cp pyproject.toml "$LOGS/pyproject.toml" 2>/dev/null || true + cp ro-crate-metadata.json "$LOGS/ro-crate-metadata.json" 2>/dev/null || true + git log --oneline > "$LOGS/git-log.txt" 2>&1 || true + ls -laR results > "$LOGS/results-listing.txt" 2>&1 || true + ls -la scripts src > "$LOGS/scripts-listing.txt" 2>&1 || true +fi + +read -r reward < <(python3 -c "print(f'{0.25*$spec_valid + 0.5*$outputs_current + 0.25*$publication_ready:.4f}')") +printf '%s\n' "$reward" > "$LOGS/reward.txt" +printf '{"reward": %s, "spec_valid": %s, "outputs_current": %s, "publication_ready": %s}\n' \ + "$reward" "$spec_valid" "$outputs_current" "$publication_ready" > "$LOGS/reward.json" +cat "$LOGS/reward.json" +exit 0 diff --git a/evals/trace-analysis-prompt.md b/evals/trace-analysis-prompt.md index 120f4eb0..2aa63a2b 100644 --- a/evals/trace-analysis-prompt.md +++ b/evals/trace-analysis-prompt.md @@ -1,22 +1,45 @@ -You are reviewing the trace of an autonomous coding agent that was asked -to build a scientific analysis with the `astra` (spec) and `lc` -(execution) CLIs. The trace digest is provided on stdin: the agent's -remarks, its tool calls, and truncated results, in order. +You are reviewing one leg of an automated eval: an autonomous coding agent +was asked to build a scientific analysis with the `astra` (spec) and `lc` +(execution) CLIs, guided only by a one-line instruction, the Lightcone agent +skills installed into its harness, and the CLIs' own help. The input on stdin +has, in order: a run summary (reward, agent steps, main's median steps), the +deterministic evidence (tool calls by phase against main, failed tool calls, +repeated commands, harness exceptions, verifier output), the change under +test (commits and diffstat of the repository whose pull request this is), +this run's trace, and the trace of the latest successful main run of the +same leg. -Write a section titled exactly `### Confusion & pain points` containing -3-6 concise bullets. Look for and report: +Your job is to explain *why* this run went the way it did, so a reviewer can +act on it without opening the trace. Cite trace steps as `step N` and +evidence lines verbatim where it matters. Do not restate the task, do not +summarize a successful build, and do not pad. + +Write exactly these sections, as markdown: + +#### Confusion & pain points + +3-6 concise bullets. For each: what happened, its root cause, and the fix it +suggests when one is clear. Look for: - failed commands or errored tool results, and their root cause -- moments where the agent misunderstood the spec format, the CLI - surface, the sandbox, or the execution environment -- detours: reverse-engineering source code, probing the environment, - redoing work, or fixing things the harness should have provided -- workarounds the agent invented that hint at a product gap (these are - the most valuable — call out what lightcone-cli or the eval setup - should fix) - -Each bullet: what happened, then the underlying cause, with the fix it -suggests when one is clear. Do not pad — if the run was essentially -clean, say so in one bullet and only list what little friction existed. -Do not restate the task or summarize the successful build. Output only -the markdown section, no preamble. +- moments where the agent misunderstood the spec format, the CLI surface, + the sandbox, or the execution environment +- whether the skills were loaded and whether they told the agent what it + needed; where it fell back to `--help`, source reading or trial and error +- detours: reverse-engineering source code, probing the environment, redoing + work, or fixing things the harness should have provided +- workarounds the agent invented that hint at a product gap — call out what + lightcone-cli, the skills, or the eval setup should fix + +If the run was essentially clean, say so in one bullet. + +#### What changed vs main + +Only when a baseline trace is present. Compare the two runs: which phase the +step difference sits in (use the phase table), the first point where the two +traces diverge, and which commit or file in the change under test most +plausibly caused it — or say plainly that the difference looks like agent +variance, and why. If reward or breakage changed, explain that first. When +there is no baseline trace, write one line saying so. + +Output only these two sections, no preamble. From d27a8a5f5eac95f7cf9e4bc37b4c087c1556bb77 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Oct 2026 07:31:44 +0000 Subject: [PATCH 2/9] Fetch the eval tasks from lightcone-evals instead of vendoring them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tasks stay in LightconeResearch/lightcone-evals, in Harbor format. prepare fetches the requested set (a comma-separated `tasks` input, a folder meaning every task in it, at `tasks_ref`) and rewrites each task's `ARG LIGHTCONE_COMMIT=` line to the lightcone-cli commit under test — the contract lightcone-bench already uses — so no wheel is built and the same workflow serves agent-skills and lightcone-evals as callers. The report groups by leg and task, so steps, history and the gate are per task, and the analyst is handed the task's own instruction. lightcone-evals is private, so a REPO_READ_TOKEN secret is required. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Y9FLtiiMxVhhkvZjd9SQp4 --- .github/scripts/eval_report.py | 68 +- .github/workflows/eval.yml | 123 +++- .gitignore | 3 - CLAUDE.md | 29 +- evals/snae-build/environment/Dockerfile | 67 -- evals/snae-build/environment/seed/astra.yaml | 77 --- .../seed/data/SCPUnion2.1_mu_vs_z.txt | 585 ------------------ evals/snae-build/instruction.md | 3 - evals/snae-build/solution/astra.yaml | 96 --- .../snae-build/solution/scripts/cosmology.py | 67 -- evals/snae-build/solution/scripts/fit.py | 91 --- .../solution/scripts/plot_hubble.py | 72 --- .../solution/scripts/plot_residuals.py | 58 -- evals/snae-build/solution/solve.sh | 45 -- evals/snae-build/task.toml | 43 -- evals/snae-build/tests/test.sh | 73 --- evals/trace-analysis-prompt.md | 18 +- 17 files changed, 162 insertions(+), 1356 deletions(-) delete mode 100644 evals/snae-build/environment/Dockerfile delete mode 100644 evals/snae-build/environment/seed/astra.yaml delete mode 100644 evals/snae-build/environment/seed/data/SCPUnion2.1_mu_vs_z.txt delete mode 100644 evals/snae-build/instruction.md delete mode 100644 evals/snae-build/solution/astra.yaml delete mode 100644 evals/snae-build/solution/scripts/cosmology.py delete mode 100644 evals/snae-build/solution/scripts/fit.py delete mode 100644 evals/snae-build/solution/scripts/plot_hubble.py delete mode 100644 evals/snae-build/solution/scripts/plot_residuals.py delete mode 100755 evals/snae-build/solution/solve.sh delete mode 100644 evals/snae-build/task.toml delete mode 100755 evals/snae-build/tests/test.sh diff --git a/.github/scripts/eval_report.py b/.github/scripts/eval_report.py index a783138e..77207e2a 100644 --- a/.github/scripts/eval_report.py +++ b/.github/scripts/eval_report.py @@ -197,6 +197,8 @@ def read_trial(trial_dir: Path) -> dict[str, Any]: exc = result.get("exception_info") trial: dict[str, Any] = { "trial": result.get("trial_name", trial_dir.name), + "task": Path(str((result.get("task_id") or {}).get("path") or "")).name + or str(result.get("task_name") or "?").split("/")[-1], "rewards": rewards or {}, "reward": (rewards or {}).get("reward"), "exception": ( @@ -274,34 +276,49 @@ def _mean(values: list[Any]) -> float | None: return statistics.mean(nums) if nums else None -def read_leg(leg_dir: Path) -> dict[str, Any]: - """Every trial of one matrix leg, plus the leg's own description.""" +def read_leg(leg_dir: Path) -> list[dict[str, Any]]: + """One group per task a matrix leg ran, keyed ``.``. + + Steps, history and the gate are per group: two tasks' step counts say + nothing about each other. A skipped leg is one group with no task. + """ meta = _load(leg_dir / "leg.json") or {} + leg = meta.get("leg", leg_dir.name) trials = [ read_trial(p.parent) for p in sorted(leg_dir.rglob("result.json")) if (_load(p) or {}).get("trial_name") ] - phases: Counter[str] = Counter() + by_task: dict[str, list[dict[str, Any]]] = {} for t in trials: - phases.update(t["phases"]) - return { - "leg": meta.get("leg", leg_dir.name), - "agent": meta.get("agent", "?"), - "model": meta.get("model", "?"), - "skipped": bool(meta.get("skipped")), - "skip_reason": meta.get("skip_reason", ""), - "trials": trials, - "n": len(trials), - "errored": sum(1 for t in trials if t["exception"]), - "mean_reward": _mean([t["reward"] for t in trials]), - "mean_steps": _mean([t["steps"] for t in trials]), - "mean_tool_calls": _mean([t["tool_calls"] for t in trials]), - "mean_cost_usd": _mean([t["cost_usd"] for t in trials]), - "mean_agent_min": _mean([t["agent_min"] for t in trials]), - "phases": {k: v / max(len(trials), 1) for k, v in phases.items()}, - "skills_used": any(t["skills_used"] for t in trials), - } + by_task.setdefault(t["task"], []).append(t) + groups = [] + for task, ts in sorted(by_task.items()) or [("", [])]: + phases: Counter[str] = Counter() + for t in ts: + phases.update(t["phases"]) + groups.append( + { + "leg": f"{leg}.{task}" if task else leg, + "harness_leg": leg, + "task": task, + "agent": meta.get("agent", "?"), + "model": meta.get("model", "?"), + "skipped": bool(meta.get("skipped")), + "skip_reason": meta.get("skip_reason", ""), + "trials": ts, + "n": len(ts), + "errored": sum(1 for t in ts if t["exception"]), + "mean_reward": _mean([t["reward"] for t in ts]), + "mean_steps": _mean([t["steps"] for t in ts]), + "mean_tool_calls": _mean([t["tool_calls"] for t in ts]), + "mean_cost_usd": _mean([t["cost_usd"] for t in ts]), + "mean_agent_min": _mean([t["agent_min"] for t in ts]), + "phases": {k: v / max(len(ts), 1) for k, v in phases.items()}, + "skills_used": any(t["skills_used"] for t in ts), + } + ) + return groups def gate( @@ -348,7 +365,7 @@ def summarize(args: argparse.Namespace) -> int: out: Path = args.out out.mkdir(parents=True, exist_ok=True) refs = _load(args.refs) if args.refs else {} - legs = [read_leg(d) for d in sorted(p for p in args.jobs.iterdir() if p.is_dir())] + legs = [g for d in sorted(p for p in args.jobs.iterdir() if p.is_dir()) for g in read_leg(d)] base = args.baseline base_results = (_load(base / "results.json") if base else None) or {} history = (_load(base / "history.json") if base else None) or {} @@ -407,6 +424,10 @@ def summarize(args: argparse.Namespace) -> int: (out / f"evidence-{name}.md").write_text("\n".join(evidence) + "\n") base_digest = base / f"digest-{name}.md" if base else None + task_file = args.tasks / leg["task"] / "instruction.md" if args.tasks else None + instruction = ( + task_file.read_text() if task_file and task_file.is_file() else "_not provided_" + ) summary = ( f"leg {name} ({leg['agent']} / {leg['model']}): mean reward " f"{_fmt(leg['mean_reward'], '{:.2f}')}, mean agent steps " @@ -416,6 +437,8 @@ def summarize(args: argparse.Namespace) -> int: (out / f"analysis-input-{name}.md").write_text( "\n\n".join( [ + "# Task instruction", + instruction, "# Run summary", summary, "\n".join(evidence), @@ -537,6 +560,7 @@ def main() -> int: s.add_argument("--baseline", type=Path) s.add_argument("--refs", type=Path) s.add_argument("--changes", type=Path) + s.add_argument("--tasks", type=Path) s.add_argument("--mode", choices=["main", "pr"], default="pr") s.add_argument("--tolerance", type=float, default=0.25) c = sub.add_parser("comment") diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 24e2e4e8..8f944bf3 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -1,22 +1,29 @@ name: Eval -# Agentic eval on Harbor: the task in evals/snae-build is run by real agent -# harnesses against the lightcone-cli and agent-skills under test, and every -# run explains itself from its trace. +# Agentic eval on Harbor: tasks from LightconeResearch/lightcone-evals are run +# by real agent harnesses against the lightcone-cli and agent-skills under +# test, and every run explains itself from its trace. # -# prepare build the engine wheel into the task, collect the skill bundles -# oracle the task's scripted solution must score 1.0 — no LLM involved +# prepare fetch the tasks and pin each image to the lightcone-cli commit +# under test, collect the skill bundles +# oracle every task's scripted solution must score 1.0 — no LLM involved # agents one Harbor job per harness, the skills installed with --skill # report rewards, agent steps against main's history, the trace analysis, # the PR comment; red on a failed gate or a missing analysis # -# The step gate: a leg whose mean agent steps exceed the median of its last -# runs on main by more than `tolerance` fails. Main runs record that history -# (in the eval-results artifact of the newest successful main run) and gate -# on reward only, so the reference follows main. +# The tasks stay in lightcone-evals, in Harbor format. Each image installs +# lightcone-cli from git at its `ARG LIGHTCONE_COMMIT=` line, which prepare +# rewrites to the commit under test (lightcone-bench's contract too). +# lightcone-evals is private: a REPO_READ_TOKEN secret with contents:read on +# it is required. # -# Reuse from LightconeResearch/agent-skills, where the skills are the side -# under test and the stack is lightcone-cli main: +# The step gate: a leg whose mean agent steps on a task exceed the median of +# its last runs on main by more than `tolerance` fails. Main runs record that +# history (in the eval-results artifact of the newest successful main run) +# and gate on reward only, so the reference follows main. +# +# Reuse from LightconeResearch/agent-skills (or lightcone-evals), where that +# repository is the side under test and the others are at their input refs: # # on: # pull_request: @@ -32,16 +39,16 @@ name: Eval # uses: LightconeResearch/lightcone-cli/.github/workflows/eval.yml@main # secrets: inherit # -# Whichever of the two repositories triggered the run is checked out at the -# triggering commit; the other at its input ref. Each repository keeps its -# own history, because the baseline is looked up in the triggering one. +# Whichever of the three repositories triggered the run is checked out at the +# triggering commit; the others at their input refs. Each repository keeps +# its own history, because the baseline is looked up in the triggering one. # # Limits: --skill installs SKILL.md bundles only, not the plugins' hooks # (agent-skills' e2e-hooks workflow covers those); step counts compare within -# a leg only, since harness tool vocabularies differ; at attempts=1 the step -# comparison is coarse, which is what the median and the tolerance absorb. -# A leg whose secret is absent is skipped with a notice, and a run in which -# no leg ran fails. +# a leg and task only, since harness tool vocabularies differ; at attempts=1 +# the step comparison is coarse, which is what the median and the tolerance +# absorb. A leg whose secret is absent is skipped with a notice, and a run in +# which no leg ran fails. on: workflow_dispatch: @@ -54,6 +61,14 @@ on: description: "agent-skills ref to install when it is not the triggering repository" type: string default: main + tasks: + description: "Comma-separated task paths in lightcone-evals; a folder means every task in it" + type: string + default: lightcone-cli/snae-build + tasks_ref: + description: "lightcone-evals ref to fetch the tasks from when it is not the triggering repository" + type: string + default: main attempts: description: "Trials per leg" type: string @@ -72,6 +87,14 @@ on: description: "agent-skills ref to install when it is not the triggering repository" type: string default: main + tasks: + description: "Comma-separated task paths in lightcone-evals; a folder means every task in it" + type: string + default: lightcone-cli/snae-build + tasks_ref: + description: "lightcone-evals ref to fetch the tasks from when it is not the triggering repository" + type: string + default: main attempts: description: "Trials per leg" type: string @@ -101,6 +124,7 @@ env: HARBOR_VERSION: 0.22.0 STACK_REPO: LightconeResearch/lightcone-cli SKILLS_REPO: LightconeResearch/agent-skills + TASKS_REPO: LightconeResearch/lightcone-evals jobs: prepare: @@ -114,9 +138,7 @@ jobs: repository: ${{ env.STACK_REPO }} ref: ${{ github.repository == env.STACK_REPO && (github.event.pull_request.head.sha || github.sha) || inputs.stack_ref || 'main' }} path: stack - # Full history so hatch-vcs sees the tags and stamps a truthful - # dev version into the wheel (and every manifest's lc_version), - # and so the change under test can be described + # Full history, so the change under test can be described fetch-depth: 0 - name: Check out agent-skills @@ -127,20 +149,53 @@ jobs: path: skills-src fetch-depth: 0 - - name: Set up uv - uses: astral-sh/setup-uv@v6 + - name: Require the lightcone-evals token + env: + REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN }} + run: | + [ -n "$REPO_READ_TOKEN" ] || { + echo "::error::REPO_READ_TOKEN is not set — it needs contents:read on $TASKS_REPO, which is private" + exit 1 + } + + - name: Check out lightcone-evals + uses: actions/checkout@v4 + with: + repository: ${{ env.TASKS_REPO }} + ref: ${{ github.repository == env.TASKS_REPO && (github.event.pull_request.head.sha || github.sha) || inputs.tasks_ref || 'main' }} + token: ${{ secrets.REPO_READ_TOKEN }} + path: tasks-src + fetch-depth: 0 - name: Assemble the eval inputs env: EVENT_REPO: ${{ github.repository }} BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + TASKS: ${{ inputs.tasks || 'lightcone-cli/snae-build' }} run: | - mkdir -p bundle/skills - cp -r stack/evals/snae-build bundle/task + mkdir -p bundle/tasks bundle/skills cp stack/.github/scripts/eval_report.py stack/evals/trace-analysis-prompt.md bundle/ - # The engine under test, built from its checkout: the task image - # installs exactly this wheel - uv build --wheel --project stack -o bundle/task/environment/lc-wheel + + # The requested tasks, side by side: to Harbor a folder of tasks is + # one job. Each image installs lightcone-cli at its + # `ARG LIGHTCONE_COMMIT=` line, pinned here to the commit under test + stack_sha=$(git -C stack rev-parse HEAD) + IFS=', ' read -r -a entries <<< "$TASKS" + for entry in "${entries[@]}"; do + [ -n "$entry" ] || continue + src="tasks-src/${entry%/}" + if [ -f "$src/task.toml" ]; then found=("$src"); else found=("$src"/*/task.toml); found=("${found[@]%/task.toml}"); fi + for t in "${found[@]}"; do + [ -f "$t/task.toml" ] || { echo "::error::'$entry' is neither a task nor a folder of tasks in $TASKS_REPO"; exit 1; } + name=$(basename "$t") + [ ! -e "bundle/tasks/$name" ] || { echo "::error::two tasks named '$name'"; exit 1; } + grep -q '^ARG LIGHTCONE_COMMIT=' "$t/environment/Dockerfile" \ + || { echo "::error::task $name does not pin lightcone-cli with ARG LIGHTCONE_COMMIT="; exit 1; } + cp -r "$t" "bundle/tasks/$name" + sed -i "s|^ARG LIGHTCONE_COMMIT=.*|ARG LIGHTCONE_COMMIT=$stack_sha|" "bundle/tasks/$name/environment/Dockerfile" + done + done + ls bundle/tasks | grep -q . || { echo "::error::no tasks selected"; exit 1; } # Every built skill bundle, first of each name: plugins bundle # their dependencies, so one skill can appear in several @@ -153,14 +208,17 @@ jobs: case "${EVENT_REPO,,}" in "${STACK_REPO,,}") changed=stack ;; "${SKILLS_REPO,,}") changed=skills-src ;; + "${TASKS_REPO,,}") changed=tasks-src ;; *) changed= ;; esac jq -n \ --arg stack "$STACK_REPO@$(git -C stack rev-parse --short HEAD)" \ --arg skills "$SKILLS_REPO@$(git -C skills-src rev-parse --short HEAD)" \ + --arg tasks "$TASKS_REPO@$(git -C tasks-src rev-parse --short HEAD)" \ + --arg task_set "$(ls bundle/tasks | paste -sd, -)" \ --arg skill_set "$(ls bundle/skills | paste -sd, -)" \ --arg sha "$(git -C "${changed:-stack}" rev-parse HEAD)" \ - '{stack: $stack, skills: $skills, skill_set: $skill_set, sha: $sha}' > bundle/refs.json + '{stack: $stack, skills: $skills, tasks: $tasks, task_set: $task_set, skill_set: $skill_set, sha: $sha}' > bundle/refs.json cat bundle/refs.json # What the analyst blames a regression on: the commits and files @@ -189,7 +247,7 @@ jobs: oracle: needs: prepare runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 45 steps: - uses: actions/download-artifact@v4 with: @@ -203,7 +261,7 @@ jobs: run: uv tool install "harbor==$HARBOR_VERSION" - name: Run the reference solution - run: harbor run -p bundle/task -a oracle -e docker -o jobs --job-name oracle -q -y + run: harbor run -p bundle/tasks -a oracle -e docker -o jobs --job-name oracle -q -y - name: Require reward 1.0 run: | @@ -272,7 +330,7 @@ jobs: run: | skill_flags=() for d in bundle/skills/*/; do skill_flags+=(--skill "${d%/}"); done - harbor run -p bundle/task \ + harbor run -p bundle/tasks \ -a "${{ matrix.agent }}" -m "${{ matrix.model }}" \ -e docker -k "${{ inputs.attempts || '1' }}" "${skill_flags[@]}" \ -o "jobs/${{ matrix.leg }}" --job-name "${{ matrix.leg }}" -q -y @@ -321,6 +379,7 @@ jobs: [ "$GITHUB_EVENT_NAME" = push ] && [ "$GITHUB_REF" = refs/heads/main ] && mode=main python3 bundle/eval_report.py summarize --jobs collected --out report \ --baseline baseline --refs bundle/refs.json --changes bundle/changes.md \ + --tasks bundle/tasks \ --mode "$mode" --tolerance "${{ inputs.tolerance || '0.25' }}" - name: Install Claude Code diff --git a/.gitignore b/.gitignore index fddabb8f..d763879e 100644 --- a/.gitignore +++ b/.gitignore @@ -220,6 +220,3 @@ uv.lock # macOS .DS_Store - -# The engine wheel the eval workflow builds into the task image -evals/snae-build/environment/lc-wheel/ diff --git a/CLAUDE.md b/CLAUDE.md index 96341ae1..c22fee70 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -192,7 +192,7 @@ src/lightcone/ # namespace — NO __init__.py ├── __init__.py # loader; a renderer only where there is a value to decide └── files/*.tmpl # the templates themselves, as real files -evals/ # agentic eval: snae-build/ (a Harbor task) + the trace-analysis prompt +evals/ # the eval's trace-analysis prompt (the tasks live in lightcone-evals) tests/ # pytest — mirrors src/ ``` @@ -212,17 +212,20 @@ runner in between — the pre-rebuild `justfile` was 90 lines of wrappers around them. The other workflows are `eval.yml` and `pypi-publish.yaml`. `eval.yml` is the agentic eval, on every non-draft PR, every push to -main and dispatch. It runs `evals/snae-build` through Harbor in four -jobs: `prepare` (the engine wheel built from the checkout into the task -image, the agent-skills bundles), `oracle` (the scripted solution must -score 1.0), `agents` (claude-code and codex, the skills installed with -`--skill`, a leg without its secret skipped with a notice) and `report` -(`.github/scripts/eval_report.py`, then one `claude -p` trace analysis -per leg). It fails on a reward below 1.0, an errored trial, a missing -trace analysis, or — off main — mean agent steps more than `tolerance` -over the median of that leg's last main runs, a history carried in the -newest successful main run's `eval-results` artifact. agent-skills -reuses it through `workflow_call` with the skills as the side under +main and dispatch. It runs Harbor tasks fetched from +`LightconeResearch/lightcone-evals` (`lightcone-cli/snae-build` by +default; private, so a `REPO_READ_TOKEN` secret is required) in four +jobs: `prepare` (each task's `ARG LIGHTCONE_COMMIT=` rewritten to the +lightcone-cli commit under test, the agent-skills bundles), `oracle` +(every scripted solution must score 1.0), `agents` (claude-code and +codex, the skills installed with `--skill`, a leg without its secret +skipped with a notice) and `report` (`.github/scripts/eval_report.py`, +then one `claude -p` trace analysis per leg and task). It fails on a +reward below 1.0, an errored trial, a missing trace analysis, or — off +main — mean agent steps more than `tolerance` over the median of that +leg's last main runs on the task, a history carried in the newest +successful main run's `eval-results` artifact. agent-skills and +lightcone-evals reuse it through `workflow_call`, each as the side under test; the header comment holds the caller. ## Key Invariants (layer 1) @@ -724,7 +727,7 @@ must not discard edits made elsewhere while the graph was running. `definition_version = sha256(recipe ‖ canonical decisions ‖ format)` is what the spec says an output *is*. The format is framed in its own right rather than left to arrive through the rendered `{output}` path: a recipe -need not use that placeholder at all (the `evals/snae-build` seed does not), and +need not use that placeholder at all (the eval's `snae-build` seed does not), and one that does not would keep its digest across a re-declared serialization — while the sidecar, named from the id alone, does not move either, so the manifest would go on describing an output at a path that diff --git a/evals/snae-build/environment/Dockerfile b/evals/snae-build/environment/Dockerfile deleted file mode 100644 index 6f6e4118..00000000 --- a/evals/snae-build/environment/Dockerfile +++ /dev/null @@ -1,67 +0,0 @@ -FROM python:3.12-slim - -# lc scrubs ambient UV_* settings from recipe environments, except these two: -# a shared managed interpreter and a shared package cache, seen identically -# by root (oracle, verifier) and the unprivileged agent user. -ENV DEBIAN_FRONTEND=noninteractive \ - PYTHONDONTWRITEBYTECODE=1 \ - UV_PYTHON_INSTALL_DIR=/opt/uv-python \ - UV_CACHE_DIR=/opt/uv-cache - -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates \ - git \ - jq \ - procps \ - ripgrep \ - && rm -rf /var/lib/apt/lists/* - -RUN python -m pip install --no-cache-dir uv==0.12.13 \ - && uv python install 3.12 - -# The engine under test is a wheel the eval workflow builds from its checkout -# into lc-wheel/ (gitignored), installed the way a user installs a release — -# `uv tool install`, which also links the git-annex executables — but into -# system-wide paths so every user sees the same tools. astra-tools is the -# version that wheel depends on. -COPY lc-wheel/ /opt/lc-wheel/ -RUN UV_TOOL_DIR=/opt/uv-tools UV_TOOL_BIN_DIR=/usr/local/bin \ - uv tool install --python 3.12 /opt/lc-wheel/lightcone_cli-*.whl \ - && UV_TOOL_DIR=/opt/uv-tools UV_TOOL_BIN_DIR=/usr/local/bin \ - uv tool install --python 3.12 \ - "astra-tools==$(/opt/uv-tools/lightcone-cli/bin/python -c 'import importlib.metadata as m; print(m.version("astra-tools"))')" - -# A git identity for whoever commits, and root reading the agent-owned repo -# in the verifier without git's dubious-ownership refusal. -RUN git config --system user.name "Lightcone Eval" \ - && git config --system user.email eval@lightcone \ - && git config --system init.defaultBranch main \ - && git config --system --add safe.directory '*' - -# `lc init`, the seed overlaid, the universe regenerated from the seed spec's -# defaults, then a clean seed commit (lc materialize refuses a dirty tree). -# No scientific dependencies are pre-seeded — declaring them (`uv add`) is -# part of what the task measures. -RUN lc init /root/snae-build -COPY seed/ /root/snae-build/ -RUN cd /root/snae-build \ - && rm -f universes/baseline.yaml \ - && astra universe generate -n baseline \ - -d 'Default configuration using standard practices' \ - && git add -A \ - && git commit -q -m 'Seed the eval task' - -# Warm the shared cache with the packages the recipes will plausibly need so -# a trial does not hinge on PyPI; the project itself stays dependency-free. -RUN uv init /tmp/warm \ - && cd /tmp/warm && uv add numpy scipy matplotlib \ - && rm -rf /tmp/warm - -# The harness runs the agent as an unprivileged user and the verifier as -# root: tools and interpreter readable by all, cache and project writable, -# and /root traversable (it is 0700 on this base image). -RUN chmod -R a+rX /opt/uv-tools /opt/uv-python \ - && chmod -R a+rwX /opt/uv-cache /root/snae-build \ - && chmod 711 /root - -WORKDIR /root/snae-build diff --git a/evals/snae-build/environment/seed/astra.yaml b/evals/snae-build/environment/seed/astra.yaml deleted file mode 100644 index cca7363a..00000000 --- a/evals/snae-build/environment/seed/astra.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# ASTRA Analysis Specification -# Documentation: https://github.com/LightconeResearch/ASTRA - -version: "0.0.14" -name: "snae" -description: | - Fit the Union2.1 Type Ia supernova distance modulus vs redshift data - to a flat LCDM cosmological model with two free parameters (H0, Omega_L) - using maximum-likelihood (MAP) point estimation. This provides best-fit - cosmological parameters as a building block for a larger analysis. - -inputs: - - id: union21 - type: data - source: data/SCPUnion2.1_mu_vs_z.txt - description: "Union2.1 SN Ia compilation: 580 supernovae with redshift, distance modulus, and uncertainties" - -outputs: - - id: best_fit - type: metric - description: "Best-fit H0 and Omega_L from chi-squared minimization, with reduced chi-squared" - recipe: - command: python scripts/fit.py - - - id: hubble_diagram - type: figure - description: "Hubble diagram: distance modulus vs redshift with best-fit model overlay" - inputs: [best_fit] - recipe: - command: python scripts/plot_hubble.py - - - id: residuals - type: figure - description: "Residuals plot: data minus best-fit model vs redshift" - inputs: [best_fit] - recipe: - command: python scripts/plot_residuals.py - -decisions: - optimizer: - label: "Optimizer" - tags: [inference] - rationale: "Nelder-Mead is robust and derivative-free; L-BFGS-B supports bounds natively" - default: nelder_mead - options: - nelder_mead: - label: "Nelder-Mead" - description: "Derivative-free simplex optimizer" - l_bfgs_b: - label: "L-BFGS-B" - description: "Bounded quasi-Newton optimizer" - - error_model: - label: "Error Treatment" - tags: [data] - rationale: "Union2.1 provides both statistical-only and systematics-included uncertainties; choice directly affects fit quality" - default: stat_and_sys - options: - stat_only: - label: "Statistical errors only" - description: "Use only the statistical distance modulus uncertainties" - stat_and_sys: - label: "Statistical + systematic errors" - description: "Include systematic uncertainty contributions" - - redshift_cut: - label: "Low-Redshift Cut" - tags: [data] - rationale: "SNe at z < 0.01 are dominated by peculiar velocity scatter rather than cosmological signal; standard practice varies on whether to exclude them" - default: z_min_001 - options: - none: - label: "No cut" - description: "Use all supernovae regardless of redshift" - z_min_001: - label: "z > 0.01" - description: "Exclude supernovae with z < 0.01 to mitigate peculiar velocity contamination" diff --git a/evals/snae-build/environment/seed/data/SCPUnion2.1_mu_vs_z.txt b/evals/snae-build/environment/seed/data/SCPUnion2.1_mu_vs_z.txt deleted file mode 100644 index b3d48e97..00000000 --- a/evals/snae-build/environment/seed/data/SCPUnion2.1_mu_vs_z.txt +++ /dev/null @@ -1,585 +0,0 @@ -# alpha 0.121851859725 -# beta 2.46569277393 -# delta -0.0363405630486 -# M(h=0.7, statistical only) -19.3182761161 -# M(h=0.7, with systematics) -19.3081547178 -1993ah 0.028488 35.3465833928 0.223905932998 0.128418942246 -1993ag 0.050043 36.6823679154 0.166828851413 0.128418942246 -1993o 0.052926 36.8176912545 0.1557559148 0.128418942246 -1993b 0.070086 37.4467365424 0.158466934433 0.128418942246 -1992bs 0.062668 37.4834093505 0.156099434739 0.128418942246 -1992br 0.087589 38.2290570494 0.187745679272 0.128418942246 -1992bp 0.078577 37.4881622607 0.155635656185 0.128418942246 -1992bo 0.017227 34.6543699503 0.199337179559 0.128418942246 -1992bl 0.042233 36.3364595483 0.167174042338 0.128418942246 -1992bh 0.045295 36.6402721756 0.164981248644 0.128418942246 -1992bg 0.03648 35.9053219652 0.170174952845 0.128418942246 -1992bc 0.019599 34.5852174312 0.184691219687 0.128418942246 -1992aq 0.100915 38.4567455954 0.167333481677 0.128418942246 -1992ag 0.027342 35.085765693 0.175510835947 0.128418942246 -1992ae 0.074605 37.5881157565 0.15977086456 0.128418942246 -1992p 0.026489 35.4806851993 0.19131226974 0.001870261557 -1990af 0.049922 36.5669734706 0.162303819627 0.128418942246 -1990o 0.030604 35.5502377594 0.173295444142 0.128418942246 -2001cz 0.016345641 34.0440277752 0.142912931364 0.128418942246 -2001cn 0.0154363 33.9409483971 0.148694109648 0.128418942246 -2001ba 0.030529 35.5992457186 0.088750663709 0.128418942246 -2000ca 0.024525 35.0581706649 0.102438504345 0.128418942246 -2000bh 0.023953 34.9687103775 0.107041197195 0.128418942246 -1999gp 0.026038 35.3672620682 0.108499792159 0.128418942246 -1993ac 0.048948 36.7315973987 0.172547618731 0.0 -1994m 0.024314 35.1094950645 0.181662706307 6.1999960721e-11 -1994s 0.015166 34.1016666152 0.215239340732 0.009354993721 -1994t 0.03572 35.9605406407 0.171186987363 0.79821525066 -1995ac 0.048818 36.3820107761 0.160299265066 5.14460000156e-08 -1995ak 0.0219800059146 34.8529733574 0.187544764358 0.128418942246 -1996ab 0.1244 39.0447885112 0.164268687505 0.128418942246 -1996bl 0.036 35.8210170957 0.167885249541 0.128418942246 -1996bo 0.016321 34.0174211141 0.204965074268 0.128418942246 -1996bv 0.01673 34.2263371692 0.209464670311 0.128418942246 -1996c 0.0275 35.6497059082 0.176364688958 0.218251065653 -2000fa 0.021793 34.9737868712 0.232449382954 0.128418942246 -2000dk 0.01645 34.1812962858 0.250896560248 0.128418942246 -2000cn 0.023208 35.0855427174 0.231479604833 0.128418942246 -2000cf 0.036457 36.1342331256 0.217628588382 0.128418942246 -1999gd 0.019264 34.9526137319 0.24063381695 0.128418942246 -1999ek 0.017605 34.3437956957 0.271650644819 0.128418942246 -1999cc 0.031528 35.7287687764 0.225362488845 0.0 -1998eg 0.023536 35.1695990888 0.234698201629 0.128418942246 -1998ef 0.016743 34.0027278026 0.248385125526 0.128418942246 -1998dx 0.05371 36.4764384949 0.221700260552 0.128418942246 -1998co 0.016991 34.3787718073 0.309889810973 0.128418942246 -1998ab 0.027865 35.0933783283 0.222426416634 0.00016313449 -1998v 0.017173 34.2606714573 0.248216009365 0.128418942246 -1997dg 0.029955 35.9722578275 0.224368551768 0.128418942246 -1997y 0.016559 34.3438338071 0.251177496491 0.018543670198 -1999aa 0.015 34.1635038861 0.161452856549 0.128418942246 -1999ao 0.0544 36.9544354105 0.0860953525286 0.128418942246 -1999ar 0.1561 39.2292540222 0.0841441236726 0.128418942246 -1999aw 0.0393 36.3343950149 0.0991005283825 0.128418942246 -1999bi 0.1241 38.8220333958 0.111614246562 0.128418942246 -1999bm 0.1441 38.8360422985 0.156777491835 0.128418942246 -1999bn 0.1299 38.979185469 0.12944495155 0.128418942246 -1999bp 0.0784 37.6822404534 0.0872168046336 0.128418942246 -2001ah 0.0583 37.0326301671 0.206256669369 0.0 -2001ay 0.0309 35.9294728801 0.183689296262 0.128418942246 -2001az 0.0406 36.3656351257 0.172317344841 0.128418942246 -2001bf 0.0152 34.0169042989 0.21507123117 0.128418942246 -2001cp 0.0224 34.9470872022 0.239011369258 0.128418942246 -2001da 0.016 34.1740153938 0.221115374688 0.137610159633 -2001eh 0.0362 35.9868705998 0.171742204068 0.128418942246 -2001g 0.0173 34.2497347985 0.215508499624 0.128418942246 -2001ie 0.0312 35.6268097646 0.180216697419 0.0 -2001n 0.0221 34.9115497582 0.189960682839 2.64313000042e-07 -2001v 0.016 33.824608888 0.207959703185 0.0 -2002bf 0.0249 34.8037070766 0.193523924165 4.231227e-06 -2002ck 0.0303 35.6282613469 0.17674738626 4.96800001137e-09 -2002de 0.0283 35.5202699542 0.180124445838 0.128418942246 -2002do 0.0152 34.2583697939 0.241474862259 0.128418942246 -2002g 0.0345 35.9782444842 0.211562474249 9.21700005208e-09 -2002hd 0.036 35.679259682 0.179022122406 0.128418942246 -2002he 0.0248 35.2561775084 0.184605302796 0.128418942246 -2002hu 0.0292 35.9925667846 0.174820107635 0.128418942246 -2002hw 0.0163 34.453252912 0.212891042717 0.128418942246 -2002jy 0.0187 35.0482998472 0.197196789214 0.012399792616 -2002kf 0.0195 34.75691168 0.197526188093 0.128418942246 -2003ch 0.0256 35.6847217932 0.186173891969 0.128418942246 -2003cq 0.0337 35.8436933159 0.179737453597 0.0 -2003ic 0.0546 36.6095585146 0.176655181089 0.0 -2003it 0.024 35.1761857794 0.195512674436 0.128418942246 -2003iv 0.0336 36.0053945507 0.18852047363 0.128418942246 -2003kc 0.0341 35.8419046981 0.175222461707 0.128418942246 -2003u 0.0261 35.3604177873 0.192469561714 0.004314509643 -2003w 0.0211 34.6607182426 0.188393367669 0.128418942246 -2004as 0.0321 35.8959945722 0.173845159832 0.128418942246 -2004bg 0.0221 34.9221291018 0.189014756295 0.128418942246 -2004l 0.0334 35.8799403015 0.1757727334 9.24477300002e-06 -2005eu 0.0341 35.9425408413 0.172834591542 0.128418942246 -2005hf 0.0421 36.4004650275 0.169930417471 0.128418942246 -2005hj 0.0576 37.0802566571 0.161852672221 0.996265053657 -2005ls 0.0205 34.6172800695 0.191412743444 0.128418942246 -2005lz 0.0402 36.3745330785 0.17063734958 0.128418942246 -2005mc 0.026 35.3812932276 0.183933665117 1.7670300001e-07 -2005ms 0.0259 35.4159686901 0.178489658823 0.033789980906 -2006ac 0.0239 35.0349741436 0.181392417208 1.09587499997e-06 -2006al 0.069 37.566047978 0.176163639895 3.07778750001e-05 -2006an 0.0651 37.3066907976 0.162808584338 1.0 -2006ar 0.0229 35.1968671677 0.185459075264 0.089659819877 -2006az 0.0315 35.651136343 0.170993523745 0.0 -2006bq 0.0215 34.9327363282 0.187040817965 1.1733000016e-08 -2006br 0.0255 35.7201731359 0.196167302112 0.128418942246 -2006bt 0.0325 35.8130914151 0.170394735896 0.0 -2006bu 0.0843 38.0518311208 0.200253645227 0.128418942246 -2006bw 0.0308 35.6289808082 0.178370444318 0.128418942246 -2006cc 0.0327 36.0941672133 0.16949230504 0.128418942246 -2006cf 0.0423 36.392825896 0.171582196702 0.0 -2006cj 0.0684 37.7311969845 0.170591643132 0.0 -2006cm 0.0153 34.7071857289 0.213242896074 0.128418942246 -2006cp 0.0233 34.8821286745 0.183424685088 0.011101109848 -2006cq 0.0491 36.7301358821 0.175843223164 0.0 -2006cz 0.0425 35.9281018139 0.195296013078 0.128418942246 -2006ej 0.0192 34.7366784514 0.197842881265 8.36000046966e-10 -2006en 0.0308 35.7791362444 0.173310004596 0.128418942246 -2006et 0.0212 34.8471345313 0.19152972211 0.128418942246 -2006gj 0.0277 35.7050037726 0.183298735051 0.128418942246 -2006gr 0.0335 35.9738325393 0.170490251613 0.128418942246 -2006kf 0.0208 34.7954563657 0.201629405401 0.128418942246 -2006le 0.0173 34.2300660446 0.220994474003 0.128418942246 -2006mo 0.036 36.146298861 0.177000363965 0.128418942246 -2006mp 0.0233 35.1983574195 0.185738575456 0.128418942246 -2006oa 0.0589 37.1116475131 0.16568235078 1.0 -2006ob 0.0583 37.0582881494 0.16808968138 0.0 -2006on 0.0688 37.4866356466 0.198025808769 0.027111473488 -2006os 0.0321 35.6480768335 0.175025422739 0.128418942246 -2006ot 0.0522 36.6743146094 0.189117803003 0.128418942246 -2006qo 0.0308 35.5935042223 0.172493084423 0.128418942246 -2006s 0.0329 35.9417685428 0.170393175896 0.00076391263 -2006sr 0.023 35.0706425048 0.186848384979 0.128418942246 -2006td 0.015 34.3797728047 0.217348534701 0.128418942246 -2006te 0.0321 35.8703618342 0.1737410077 0.000101920284 -2007ae 0.0643 37.1755022179 0.166008010074 0.128418942246 -2007ai 0.032 35.8266913419 0.193104691662 0.128418942246 -2007au 0.0209 34.7028984995 0.19422693515 0.128418942246 -2007bc 0.0219 34.8489904808 0.188387184264 3.80834000002e-07 -2007bd 0.032 35.5884973241 0.173446310936 7.81762000024e-07 -2007ca 0.0151 34.5258752881 0.213682160531 0.128418942246 -2007ci 0.0192 34.4915623332 0.197091647943 0.006396224841 -2007co 0.0266 35.3234935897 0.177716836428 0.128418942246 -2007cp 0.0377 35.7952144325 0.294106560612 0.128418942246 -2007cq 0.0247 34.9136928277 0.181621011598 0.128418942246 -2007f 0.0242 35.1893805829 0.180174916343 0.435402819564 -2007o 0.0366 35.9712906539 0.168963687947 0.128418942246 -2007qe 0.0229 35.1386260702 0.183018039949 0.128418942246 -2007r 0.0312 35.8828585622 0.173762463024 0.0 -2007s 0.015 34.1114109643 0.213388869977 0.128418942246 -2008af 0.0341 35.770474764 0.177569140796 0.0 -2008bf 0.0251 34.9482752367 0.179222700336 0.0 -2008l 0.0189 34.3747016599 0.209571630196 0.10474237468 -2004ef 0.029802136901 35.4709217104 0.122226817874 0.128418942246 -2004gc 0.0321340168614 35.3781278853 0.127512090116 0.128418942246 -2004gs 0.0275687259109 35.4754718132 0.128117983088 0.128418942246 -2004gu 0.0469673346156 36.4944134465 0.110309142172 0.128418942246 -2005a 0.0183152320362 34.3717548996 0.160464926406 0.128418942246 -2005ag 0.0800481440021 37.6857093353 0.101968492144 0.128418942246 -2005bg 0.0241852990802 35.052324275 0.133368622118 0.128418942246 -2005bo 0.0150270427632 33.9501915023 0.178117024083 0.128418942246 -2005eq 0.0283960268116 35.5481484694 0.125198881719 9.90000303958e-11 -2005hc 0.0449766726113 36.5503954158 0.108342547225 0.930467219184 -2005iq 0.0329123710881 35.9678520173 0.118114324121 0.128418942246 -2005ir 0.0753501119815 37.5800899168 0.102274672969 0.996080571775 -2005ki 0.0203747245299 34.6564438473 0.145603349944 0.000769867316 -2005m 0.0229711675782 35.133882951 0.135640926041 0.128418942246 -2005na 0.0268091974805 35.2601505395 0.127879595703 0.128418942246 -2006ax 0.0179312833626 34.3498200027 0.155413940574 0.128418942246 -2006eq 0.0483921953954 36.6858343824 0.119215628214 0.128418942246 -2006py 0.0566833670752 36.9636230671 0.110070937939 0.128418942246 -2005kt 0.0638640836964 37.3159416121 0.119614965162 0.551671793295 -10106 0.146290296068 39.5590095488 0.120248735474 0.551671793295 -2005ez 0.129278206634 38.9180009489 0.134402615626 2.58499999184e-09 -2005lk 0.102715033935 38.4872595888 0.119892508421 0.551671793295 -2005ll 0.242504680476 40.1304465537 0.150915056663 0.551671793295 -2005lf 0.29840927363 41.0624002336 0.21581859647 0.551671793295 -2005ku 0.0437189114895 36.3865660158 0.127214292529 0.551671793295 -2005ml 0.113042644637 38.5514541276 0.119237437717 0.551671793295 -2005jb 0.256475742725 40.6108439003 0.156286225391 0.551671793295 -2005fc 0.295585550063 41.1370597018 0.215064876219 0.551671793295 -2005mm 0.380359487336 41.657498659 0.217910527631 0.551671793295 -2005ln 0.145668547261 39.056715703 0.125750126249 0.551671793295 -2005mo 0.273454768884 40.7271582441 0.163668270948 0.551671793295 -2005lo 0.297518833561 40.7722050136 0.205185826917 0.551671793295 -2005lq 0.378965802039 41.5814889023 0.197094007682 0.551671793295 -1166 0.380416514286 41.2723405299 0.198558434394 0.551671793295 -2005lp 0.301755029535 41.4938104959 0.246761480943 0.551671793295 -2005mq 0.348345020835 41.3077915312 0.217774889061 0.551671793295 -2005ff 0.0856894593607 37.9974134061 0.116365280048 0.004916236676 -2005fd 0.260586108317 40.5231676777 0.141595051803 0.551671793295 -2005fe 0.215543320953 40.2965294236 0.140207657342 0.551671793295 -2005fh 0.117625328764 38.5794185081 0.113609962836 7.41867000054e-07 -2005gj 0.18221823956 39.5938188438 0.119334463704 0.551671793295 -1688 0.357507357032 41.3288394045 0.202587682493 0.551671793295 -2005fj 0.141787999126 39.2738113329 0.119521982693 0.551671793295 -2005fo 0.260533476953 40.8081341413 0.148279348857 0.551671793295 -2005fl 0.232781107182 40.2045290903 0.15178836784 0.551671793295 -2005fm 0.151857894665 39.155788359 0.113847311938 0.989888621753 -2005fn 0.0939086318905 38.1728205141 0.117952906116 0.551671793295 -2005fr 0.286618706522 41.0384076468 0.133800275646 0.551671793295 -2005fy 0.194316512142 39.9615200415 0.126373844923 0.551671793295 -2005ey 0.147025138268 39.3035839668 0.111657296738 4.30000479668e-11 -2005fp 0.2115869815 40.5603236924 0.159324733885 0.551671793295 -2005ft 0.180119779968 39.6385777035 0.115560254718 0.551671793295 -2005fi 0.263491026592 40.7673226085 0.127293080139 0.551671793295 -2005fu 0.192149980044 40.0392178782 0.121087443202 0.001229231543 -2005fs 0.338802609134 41.3053718801 0.170908367946 0.551671793295 -2005fv 0.117277362889 38.7459338248 0.113702596154 0.551671793295 -2005fw 0.1424046524 39.1164772511 0.113661193986 0.830402042657 -2005fa 0.160861854636 39.3191758892 0.11864489981 9.9997787828e-13 -2005fx 0.288418344585 40.8427853587 0.149598782119 0.551671793295 -2005fz 0.12282889997 38.7997897008 0.12106543913 0.551671793295 -2005go 0.263647950642 40.5481756685 0.139823620366 0.551671793295 -2005gp 0.126473161901 38.7104691761 0.119200794649 0.586055835562 -2005ga 0.172742230556 39.5029361421 0.113458015916 0.551671793295 -2005gc 0.163795894987 39.3945630608 0.113134872307 1.0 -2005gs 0.249511054792 40.7809243328 0.132323047466 0.551671793295 -2005gh 0.257740303509 40.6514783957 0.135891055781 0.551671793295 -2005hn 0.106712340103 38.6366669013 0.117621429285 0.867576933443 -2005gd 0.159889937748 39.3505845262 0.114747057769 0.999968604262 -2005ge 0.204979685349 40.0245002346 0.123526942937 0.551671793295 -2005gr 0.244378876797 40.2019303938 0.124309900698 0.551671793295 -2005gf 0.248508130676 40.2721746838 0.125645589328 0.551671793295 -2005gg 0.228528473881 40.2544336386 0.12201137132 0.551671793295 -2005gb 0.0858546441437 37.9524441212 0.112589853288 1.0 -2005ho 0.0618357646938 37.1309944806 0.115680071342 0.987710988495 -2005gt 0.277853422619 40.8380084515 0.168994207516 0.551671793295 -2005gw 0.275440197435 40.743391802 0.142941615821 0.551671793295 -4064 0.155247328221 39.3173204059 0.115415755865 0.551671793295 -2005gu 0.330512448683 41.2279390018 0.156723683034 0.551671793295 -2005gv 0.361934308832 41.3844533517 0.166441914984 0.551671793295 -2005gy 0.33063462833 41.0522598891 0.152699339448 0.551671793295 -2005gx 0.144621085972 39.290829773 0.112727024487 1.0 -2005gq 0.389288787688 41.6026284289 0.200867575788 0.551671793295 -2005hp 0.173910056566 39.5319411053 0.128967775815 0.551671793295 -2005hs 0.300312695802 40.8468481928 0.15150098613 0.551671793295 -2005hr 0.116348502599 38.7461381651 0.112296117135 1.0 -2005hu 0.218585189041 40.1990974567 0.117113950953 0.551671793295 -2005hx 0.119671538006 38.7555381178 0.110740033634 0.551671793295 -2005hy 0.154632097239 39.3239502295 0.112047393667 0.999997454964 -2005hw 0.408319092164 41.840293526 0.206369618511 0.551671793295 -2005hv 0.177600694977 40.070675851 0.12481415503 1.0 -2005ia 0.25066763111 40.7456030264 0.120970959661 0.551671793295 -2005jz 0.251740186424 40.5417217599 0.118440110682 0.551671793295 -5737 0.391599213441 41.5087271022 0.171564315895 0.551671793295 -2005hz 0.128726735467 38.8642253068 0.109513340257 0.551671793295 -2005hq 0.399601334976 41.7288003127 0.205132433748 0.551671793295 -2005ic 0.309492645411 41.0181924732 0.134960171498 0.551671793295 -2005is 0.170628396749 39.474900116 0.116376625267 0.551671793295 -2005ie 0.278924676328 40.7678297655 0.130157044183 0.551671793295 -2005it 0.308580866071 40.9076217053 0.177652782589 0.551671793295 -2005ht 0.18581244671 39.8240975844 0.120667757481 0.551671793295 -2005if 0.0664403123402 37.3721050548 0.1175824014 0.543563084394 -2005ka 0.316429844695 41.3950733806 0.192866122248 0.551671793295 -2005ih 0.257497888297 40.5941645935 0.131393591319 0.551671793295 -2005iv 0.298777444088 41.0293579818 0.154536935815 0.551671793295 -2005jy 0.270434442827 40.668983836 0.145477928824 0.551671793295 -2005ig 0.27945473261 40.5012843665 0.138724732909 0.551671793295 -2005ii 0.292469756082 40.919048654 0.141400626542 0.551671793295 -2005jk 0.188853175387 39.7920374862 0.119741860505 7.8208394e-05 -2005ix 0.265762479518 40.4981499217 0.124803942242 0.551671793295 -2005ij 0.124273528772 38.719986901 0.111134858876 0.029911743888 -2005id 0.182548913397 39.7840215634 0.114083136974 0.986226043013 -2005jd 0.312883364158 41.0144487771 0.139506514174 0.551671793295 -2005ik 0.309547337158 41.1925567238 0.154012332803 0.551671793295 -2005iu 0.0890194292939 37.8284529286 0.117432705446 0.551671793295 -2005iy 0.402459619216 41.9111467578 0.211278493994 0.551671793295 -2005iz 0.200611719928 39.8523918219 0.132867364384 1.0 -2005ja 0.326396483203 40.972304176 0.144388065833 0.551671793295 -2005jc 0.211629598493 39.9801047509 0.115235180715 0.551671793295 -2005jl 0.17968564077 39.7209599451 0.116984880342 1.0 -2005je 0.0931494026855 38.280913725 0.114453830019 0.551671793295 -2005jg 0.30240162108 41.033200098 0.134910116135 0.551671793295 -2005jh 0.108638265524 38.6505001979 0.119489654857 0.551671793295 -2005ed 0.0856961170652 37.9937917107 0.117971237872 0.551671793295 -2005jm 0.202608679897 39.9677592554 0.128807331127 1.0 -2005kn 0.196716068579 39.9244173787 0.134983153598 0.551671793295 -2005ei 0.12668798909 38.9152084322 0.123937308071 0.551671793295 -2005ji 0.214568258658 40.2018496659 0.11698744578 0.551671793295 -2005jn 0.320446988861 41.102785135 0.134380311167 0.551671793295 -2005jo 0.218347445403 40.2208498377 0.124421502953 0.551671793295 -2005eg 0.189706566951 39.9395625109 0.119682792229 0.551671793295 -2005ex 0.0929368181997 38.1458300404 0.118950355954 0.551671793295 -2005jw 0.379662293575 41.4330618932 0.147219740299 0.551671793295 -2005jp 0.210938395061 39.983937734 0.120818770577 0.551671793295 -2005jj 0.366602898789 41.8598572351 0.198404429556 0.551671793295 -2005jv 0.420926821103 42.1335480049 0.202321001311 0.551671793295 -2005ju 0.258028270562 40.6446788667 0.137108498782 0.551671793295 -2005ko 0.183568404877 39.7107958033 0.123775734742 0.551671793295 -2005mi 0.212548765094 40.0875932773 0.13157696281 0.551671793295 -2005jt 0.360034211989 41.1573004359 0.175048186524 0.551671793295 -2005mh 0.393974478272 41.7811534223 0.150707319622 0.551671793295 -2005kp 0.11471262087 38.669095116 0.113372328428 1.0 -2005ld 0.143705907 39.1857706544 0.116311967547 0.372920925377 -2005le 0.252486060029 40.5184210801 0.145050195669 0.551671793295 -2005kq 0.387297107465 41.8924653292 0.18642986619 0.551671793295 -2005lg 0.348583857544 41.3009410359 0.177371887871 0.551671793295 -2005li 0.255490620022 40.4859446261 0.150967373278 0.551671793295 -2005lh 0.216582821954 40.3602717031 0.152479838933 0.551671793295 -1996e 0.43 41.3188581715 0.357827626062 0.551671793295 -1996h 0.62 43.2279620233 0.390344663066 0.551671793295 -1996i 0.57 42.4891103591 0.388952633173 0.551671793295 -1996j 0.3 40.9627951522 0.314865571912 0.551671793295 -1996k 0.38 42.063176534 0.327833114422 0.551671793295 -1996u 0.43 42.3985415069 0.458327873977 0.551671793295 -1995ao 0.24 40.7421915815 0.402844159873 0.551671793295 -1997ce 0.44 42.0495332736 0.319794878908 0.551671793295 -1997cj 0.5 42.3630032623 0.317337166801 0.551671793295 -1997ck 0.97 42.8212038325 0.805210302782 0.551671793295 -1995k 0.479 42.3517577625 0.358136299792 0.551671793295 -1997ap 0.83 43.5424034366 0.471046842772 0.551671793295 -1997am 0.416 42.4236170407 0.556342333092 0.551671793295 -1997aj 0.581 42.063423278 0.505033960368 0.551671793295 -1997ai 0.45 41.8319918495 0.453701962845 0.551671793295 -1997af 0.579 43.1863633442 0.643297806625 0.551671793295 -1997ac 0.32 41.2446034516 0.420998113276 0.551671793295 -1997r 0.657 42.9752487173 0.658728481435 0.551671793295 -1997q 0.43 41.7661368295 0.608328916424 0.551671793295 -1997p 0.472 41.9691632564 0.514472850425 0.551671793295 -1997o 0.374 43.1851146804 0.923198713714 0.551671793295 -1997n 0.18 40.1895173161 0.445447560867 0.551671793295 -1997l 0.55 44.3434675429 1.00681218041 0.551671793295 -1997k 0.592 44.1523548256 0.717963493004 0.551671793295 -1997i 0.172 39.3022260063 0.429709120635 0.551671793295 -1997h 0.526 41.9560808598 0.513679185483 0.551671793295 -1997g 0.763 44.4724245942 0.898163121417 0.551671793295 -1997f 0.58 43.3051250139 0.519962251252 0.551671793295 -1996cn 0.43 41.8019360973 0.458804990654 0.551671793295 -1996cm 0.45 42.2709731768 0.589947886789 0.551671793295 -1996ck 0.656 43.1448591059 0.627202505766 0.551671793295 -1996ci 0.495 42.1189081538 0.443653473108 0.551671793295 -1996cg 0.49 41.7874349383 0.446484505947 0.551671793295 -1996cf 0.57 42.6714667124 0.470382088569 0.551671793295 -1995ba 0.388 42.2076140366 0.465017827284 0.551671793295 -1995az 0.45 42.4024477254 0.508290617768 0.551671793295 -1995ay 0.48 42.1599485926 0.521420422571 0.551671793295 -1995ax 0.615 42.542636608 0.556222743429 0.551671793295 -1995aw 0.4 42.3127671586 0.477589020966 0.551671793295 -1995at 0.655 42.3150852482 0.497081624743 0.551671793295 -1995as 0.498 42.9837202371 0.642406504117 0.551671793295 -1995ar 0.465 41.8251633017 0.600030636232 0.551671793295 -1995aq 0.453 42.8212084765 0.524912951846 0.551671793295 -1994g 0.425 41.2021312228 0.489149082777 0.551671793295 -2002ad 0.514 42.7861936213 0.493735911043 0.551671793295 -2002ab 0.423 41.5655258816 0.243579000627 0.551671793295 -2002x 0.859 44.0929026019 0.296965045086 0.551671793295 -2001kd 0.936 43.3071058635 0.705981399741 0.551671793295 -2001jp 0.528 42.4543495693 0.24429067529 0.551671793295 -2001jm 0.978 43.5000721647 0.292647301809 0.551671793295 -2001jh 0.885 44.1843333063 0.286230184051 0.551671793295 -2001jf 0.815 44.0744016007 0.728921730286 0.551671793295 -2001jb 0.698 43.768728686 0.433480217975 0.551671793295 -2001iy 0.568 42.7092496229 0.289546896784 0.551671793295 -2001ix 0.711 43.5665495646 0.366725181041 0.551671793295 -2001iw 0.3396 41.0839733838 0.226527941211 0.551671793295 -2001iv 0.3965 41.4879207897 0.202374999057 0.551671793295 -2001hy 0.812 43.6535732762 0.379016341937 0.551671793295 -2001hx 0.799 43.3788294006 0.233174228134 0.551671793295 -2001hu 0.882 43.3750389271 0.583987119465 0.551671793295 -2001hs 0.833 43.6880978478 0.520913245917 0.551671793295 -2001fs 0.874 43.295379241 0.388089303157 0.551671793295 -2001fo 0.772 43.5099938279 0.218834770025 0.551671793295 -1999dr 0.178 39.454790821 0.235746048095 0.551671793295 -1999du 0.26 40.8233657709 0.199851735971 0.551671793295 -1999dv 0.186 39.7121679254 0.189043139487 0.551671793295 -1999dx 0.269 40.7761617315 0.25640063853 0.551671793295 -1999dy 0.215 40.3780576732 0.193933997767 0.551671793295 -2000fr 0.543 42.4788058506 0.0981414342081 0.551671793295 -1998bi 0.75 43.242998687 0.138318457637 0.551671793295 -1998be 0.64 42.7642472749 0.18880937743 0.551671793295 -1998ba 0.43 42.1841242053 0.148171058429 0.551671793295 -1998ay 0.64 43.1643806938 0.193749016644 0.551671793295 -1998ax 0.497 42.3245530903 0.167980194286 0.551671793295 -1998aw 0.44 42.0106475331 0.107484807666 0.551671793295 -1998as 0.355 41.3423115755 0.20146843402 0.551671793295 -1997ez 0.78 43.6008071703 0.171318833305 0.551671793295 -1997eq 0.54 42.4246212672 0.111551880919 0.551671793295 -1997ek 0.86 43.9216606703 0.172411965167 0.551671793295 -03D4au 0.468 42.5489037371 0.1639147706 0.551671793295 -04D4bk 0.84 43.8731031943 0.222348502282 0.910256228198 -04D3nr 0.96 43.6132116082 0.275514631381 0.551671793295 -04D3lu 0.8218 43.8153639621 0.21320698395 0.0 -04D3ki 0.93 43.5524295782 0.289918379066 0.551671793295 -04D3gt 0.451 41.7966744145 0.143055821894 0.551671793295 -04D3do 0.61 42.904965712 0.156248253322 0.0 -04D3cp 0.83 44.0506599335 0.208835355956 0.019019116792 -04D2gp 0.707 43.2824256201 0.256253916455 0.999999011725 -04D2fp 0.415 41.8767704643 0.141093702805 0.125506643627 -04D1ag 0.557 42.5570112027 0.157456031152 0.551671793295 -03D4fd 0.791 43.5768811197 0.201343679161 0.424378352473 -03D4cz 0.695 43.2117042607 0.20544919439 0.048213467444 -03D4at 0.633 43.0498635302 0.169957137727 1.0 -03D3bh 0.2486 40.6111853429 0.154515955385 0.551671793295 -03D3af 0.532 42.5658156596 0.166651053217 0.551671793295 -03D1fc 0.331 41.0784700501 0.137704766569 0.0 -03D1bp 0.346 41.3646268615 0.15221324645 9.00002294912e-12 -04D4dw 0.961 44.2642016168 0.340056560686 9.6749703e-05 -04D4an 0.613 42.9937246636 0.165096720419 0.000122102116 -04D3nh 0.3402 41.3280915278 0.135546197344 1.0 -04D3lp 0.983 44.1572843671 0.43499764107 1.0 -04D3is 0.71 43.0220388098 0.184280954678 0.551671793295 -04D3fq 0.73 43.27699552 0.194894052413 1.0 -04D3df 0.47 42.1345594858 0.15323960172 0.551671793295 -04D3co 0.62 43.0092526047 0.169564685824 0.964610252373 -04D2gc 0.521 42.1801294521 0.160996803083 0.998949555959 -04D2cf 0.369 41.6340230154 0.152644121885 7.25190000272e-08 -03D4gl 0.571 42.3991620661 0.174223963618 0.999999999363 -03D4dy 0.604 42.526976481 0.15968704969 0.991658804539 -03D4cy 0.9271 43.9404978712 0.286707554506 0.846559054956 -03D4ag 0.285 40.8542434728 0.136594130276 5.69999603073e-11 -03D3ba 0.2912 40.8390642775 0.146624503178 0.005691513851 -03D1gt 0.548 42.2962985563 0.18837089341 0.551671793295 -03D1ew 0.868 43.4932874061 0.24609516526 0.921233644992 -03D1ax 0.496 42.2146760288 0.152814539841 0.0 -04D4dm 0.811 43.4045360216 0.210619526441 1.0 -04D3oe 0.756 43.8139921904 0.192775955331 0.551671793295 -04D3nc 0.817 43.6515637361 0.210286863572 1.0 -04D3ks 0.752 43.3018298804 0.207071436502 0.999999999949 -04D3hn 0.5516 42.3031837333 0.149254872779 0.0 -04D3fk 0.3578 41.4349095934 0.136096151868 0.0027167408 -04D3dd 1.01 44.0124745362 0.375672417143 0.999785771595 -04D2ja 0.741 43.7172522874 0.223286859678 0.0 -04D2gb 0.43 41.8109207973 0.15108214332 0.001759266231 -04D1ak 0.526 42.4002106795 0.16694309723 0.551671793295 -03D4gg 0.592 42.5773227618 0.172394047168 0.0 -03D4di 0.905 43.6387400062 0.25746754741 0.860692933969 -03D4cx 0.949 43.4449098532 0.284630783131 9.00002294912e-12 -03D3cd 0.4607 42.0716665463 0.18126245549 0.999972076003 -03D3ay 0.3709 41.6692043983 0.154834006045 0.000892345443 -03D1fq 0.8 43.7140601053 0.215165244047 5.37990000327e-08 -03D1co 0.679 43.4574063402 0.195969077309 0.999996816858 -03D1aw 0.5817 42.6720406074 0.159803369147 1.0 -04D4bq 0.55 42.2756470464 0.15520464155 0.916905651559 -04D3ny 0.81 43.3698924946 0.212685692189 0.399751945497 -04D3ml 0.95 43.9766075066 0.291243768847 0.620797641982 -04D3kr 0.3373 41.2944909521 0.137240588972 1.0 -04D3gx 0.91 44.305516423 0.267241049423 0.551671793295 -04D3ez 0.263 40.6346721186 0.134567832845 0.0 -04D3cy 0.643 43.0112669877 0.166369771753 0.474555440866 -04D2iu 0.691 43.0887669301 0.253842244922 0.551671793295 -04D2fs 0.357 41.4255279641 0.137721476533 0.99999988436 -04D1aj 0.721 43.1756783825 0.187179852377 0.551671793295 -03D4gf 0.581 42.7423233406 0.155462517843 1.0 -03D4dh 0.6268 42.7586126154 0.157227804978 0.999411415446 -03D4cn 0.818 43.3928941721 0.26755880574 0.551671793295 -03D3cc 0.4627 42.0422849962 0.14756880162 0.551671793295 -03D3aw 0.449 42.0230683496 0.157063020734 0.0 -03D1fl 0.688 43.0539860347 0.165777265258 0.551671793295 -03D1cm 0.87 44.2313042047 0.278817290376 0.551671793295 -03D1au 0.5043 42.3382006953 0.149577507475 0.999977235075 -b010 0.591 43.2091059677 0.306006112419 0.551671793295 -b013 0.426 41.7615774796 0.22492067105 0.551671793295 -b016 0.329 41.3402567282 0.269807401411 0.551671793295 -d058 0.583 42.4011253659 0.28648393048 0.551671793295 -d084 0.519 43.071509714 0.314462491184 0.551671793295 -d085 0.401 41.6954244654 0.229308722315 0.551671793295 -d086 0.205 39.9037694347 0.217272003785 0.551671793295 -d087 0.34 41.2131787728 0.228190770173 0.551671793295 -d089 0.436 41.8800161997 0.21653394713 0.551671793295 -d093 0.363 41.551975686 0.208173297091 0.551671793295 -d097 0.436 41.9257693002 0.228262759927 0.551671793295 -d117 0.309 41.1737957091 0.235823875951 0.551671793295 -d149 0.342 41.3845981768 0.21481032067 0.551671793295 -e020 0.159 39.4163640953 0.242628007812 0.551671793295 -e029 0.332 41.2554783013 0.239903276269 0.551671793295 -e108 0.469 42.3544812112 0.28315240581 0.551671793295 -e132 0.239 40.2591287799 0.204167929675 0.551671793295 -e136 0.352 41.423779352 0.224191540842 0.551671793295 -e138 0.612 42.813940944 0.283023414891 0.551671793295 -e140 0.631 42.3773831209 0.244974123167 0.551671793295 -e147 0.645 42.8169731662 0.234273129747 0.551671793295 -e148 0.429 41.8900388123 0.215471389689 0.551671793295 -e149 0.497 42.0757738154 0.221637712856 0.551671793295 -f011 0.539 42.2266632084 0.241797816352 0.551671793295 -f041 0.561 42.8719504017 0.306521908685 0.551671793295 -f076 0.41 41.3474933283 0.256981142148 0.551671793295 -f096 0.412 41.4237121884 0.312056381676 0.551671793295 -f216 0.599 42.7434648961 0.382165112455 0.551671793295 -f231 0.619 43.056018637 0.247822481956 0.551671793295 -f235 0.422 41.7283094576 0.230713845858 0.551671793295 -f244 0.54 42.5111635186 0.282945441834 0.551671793295 -f308 0.401 42.5546203602 0.369893982362 0.551671793295 -g005 0.218 40.0754074212 0.222222737823 0.551671793295 -g050 0.633 42.2016724827 0.259877853306 0.551671793295 -g052 0.383 41.6420339038 0.251010174803 0.551671793295 -g055 0.302 41.3107982848 0.272346550274 0.551671793295 -g097 0.34 41.0800432546 0.231193988299 0.551671793295 -g120 0.51 41.8806853688 0.226762461825 0.551671793295 -g133 0.421 42.1360316298 0.322004955245 0.551671793295 -g142 0.399 41.4880817606 0.292952212036 0.551671793295 -g160 0.493 42.1469146695 0.2639575465 0.551671793295 -g240 0.687 42.9963850785 0.285188798534 0.551671793295 -h300 0.687 42.8348351217 0.272537782393 0.551671793295 -h319 0.495 42.2486633669 0.232347965963 0.551671793295 -h323 0.603 42.6462047067 0.278628902676 0.551671793295 -h342 0.421 42.1796277538 0.233773507359 0.551671793295 -h359 0.348 41.5896262003 0.217425302485 0.551671793295 -h363 0.213 40.109041203 0.22335261095 0.551671793295 -h364 0.344 41.1728711166 0.209772839036 0.551671793295 -k396 0.271 40.5323020705 0.21526700069 0.551671793295 -k411 0.564 42.3728917486 0.292022400672 0.551671793295 -k425 0.274 40.7246506476 0.208670920573 0.551671793295 -k429 0.181 39.6828885172 0.227088514354 0.551671793295 -k430 0.582 43.1639102562 0.325207917744 0.551671793295 -k441 0.68 42.9040344577 0.290951926367 0.551671793295 -k448 0.401 41.9356323131 0.332838743195 0.551671793295 -k485 0.416 41.5575369467 0.303252121943 0.551671793295 -m027 0.286 41.2111308437 0.25814462102 0.551671793295 -m034 0.562 43.0505225945 0.33321544156 0.551671793295 -m043 0.266 40.3907073912 0.250227167194 0.551671793295 -m062 0.314 41.2319680355 0.242363893683 0.551671793295 -m138 0.581 43.668679767 0.391236930244 0.551671793295 -m158 0.463 41.947753754 0.26646445376 0.551671793295 -m193 0.341 40.9915138593 0.224504319999 0.551671793295 -n256 0.631 42.8818252933 0.234369669625 0.551671793295 -n258 0.522 42.6773303466 0.24715086884 0.551671793295 -n263 0.368 41.4676952372 0.207180966902 0.551671793295 -n278 0.309 40.8607746451 0.214863256477 0.551671793295 -n285 0.528 42.3696918414 0.302654151443 0.551671793295 -n404 0.216 40.4049089433 0.21521177477 0.551671793295 -p455 0.284 40.8353825838 0.206616021304 0.551671793295 -p524 0.508 42.1942707319 0.231767202806 0.551671793295 -p528 0.781 43.4372295515 0.321348684213 0.551671793295 -p534 0.613 42.6167095942 0.260214469673 0.551671793295 -1999fw 0.278 40.5662405695 0.198452987813 0.551671793295 -1999fn 0.477 42.0543351554 0.183599933891 0.551671793295 -1999fm 0.95 43.6383176075 0.297754710705 0.551671793295 -1999fk 1.057 44.1506301001 0.238584685235 0.551671793295 -1999fj 0.816 43.6910828816 0.457863421823 0.551671793295 -1999ff 0.455 42.3239322721 0.229627540013 0.551671793295 -04Eag 1.02 44.3629247027 0.220963762065 0.551671793295 -04Gre 1.14 44.322957093 0.228701029542 0.551671793295 -04Man 0.854 43.6084642861 0.225569905838 0.551671793295 -04Mcg 1.37 45.049713349 0.262748449529 0.551671793295 -04Omb 0.975 44.3337254974 0.21278060312 0.551671793295 -04Pat 0.97 44.4626016974 0.291454456047 0.551671793295 -04Rak 0.74 43.2875467038 0.198402487067 0.551671793295 -04Sas 1.39 44.8762343403 0.250631965185 0.551671793295 -04Yow 0.46 42.1484496113 0.210095497723 0.551671793295 -05Fer 1.02 44.1640655276 0.232339527378 0.551671793295 -05Gab 1.12 44.5144054871 0.224079094547 0.551671793295 -05Lan 1.23 45.0206757228 0.235028400564 0.551671793295 -05Red 1.19 44.3628316562 0.247108113742 0.551671793295 -05Spo 0.839 43.3980657237 0.222124147444 0.551671793295 -05Str 1.01 44.9121378293 0.33374861699 0.551671793295 -05Zwi 0.521 42.3782445565 0.199332155448 0.551671793295 -2002dc 0.475 42.1048814028 0.255958316646 0.551671793295 -2002dd 0.95 43.884150294 0.235845124417 0.551671793295 -2002fw 1.3 45.0162581016 0.242719167821 0.551671793295 -2002hp 1.305 44.740169328 0.25954011501 0.551671793295 -2002kc 0.216 40.5560465985 0.244099860655 0.551671793295 -2002kd 0.735 43.0918403533 0.200575809439 0.551671793295 -2002ki 1.14 44.1969521333 0.368403293803 0.551671793295 -2003aj 1.307 45.4107441086 0.314696709568 0.551671793295 -2003az 1.265 44.944110841 0.235469959716 0.551671793295 -2003bd 0.67 43.1432095538 0.209654285426 0.551671793295 -2003be 0.64 42.9242754337 0.280114332269 0.551671793295 -2003dy 1.34 45.0675055813 0.275015093127 0.551671793295 -2003eq 0.84 43.5143043773 0.208697053243 0.551671793295 -2003XX 0.935 43.5401972359 0.227616538358 0.551671793295 -2001cw 0.953 44.2736202267 0.954717441018 0.551671793295 -2001gn 1.124 44.5675191814 0.197263047621 0.551671793295 -2001go 0.552 42.5109342761 0.103414502954 0.551671793295 -2001gq 0.671 42.9820070011 0.12054552758 0.551671793295 -2001gy 0.511 42.3736604669 0.0890617545983 0.551671793295 -2001hb 1.03 44.2400933572 0.141254202601 0.551671793295 -A-004 1.192 44.4587516 0.200258071902 0.551671793295 -C-000 1.092 44.0078477189 0.246511864633 0.551671793295 -C-001 0.974 43.8341432564 0.174238940337 0.551671793295 -F-012 1.11 44.6253358523 0.447832632795 0.551671793295 -G-004 1.35 44.8270654771 0.185705412075 0.551671793295 -H-003 0.85 43.4942573455 0.171796430655 0.551671793295 -H-005 1.241 44.5817029608 0.478371053196 0.0 -K-000 1.414 44.8037661387 0.346181483252 0.0 -N-033 1.188 44.6076425498 0.500544949291 0.551671793295 -D-000 1.017 44.2939707709 0.171046154322 1.4752251e-05 -D-006 1.315 44.9713577744 0.187508839367 0.551671793295 -P-009 0.821 43.6409387868 0.194013591972 0.551671793295 -R-012 1.215 45.2465209495 0.560317445151 0.000147652666 -Z-005 0.623 42.5145239973 0.241428134977 0.551671793295 diff --git a/evals/snae-build/instruction.md b/evals/snae-build/instruction.md deleted file mode 100644 index f0f063ce..00000000 --- a/evals/snae-build/instruction.md +++ /dev/null @@ -1,3 +0,0 @@ -This is a Lightcone project. Build the analysis described in `astra.yaml` for the `baseline` universe, then get the project ready to publish. - -This is an automated run: nobody will answer questions or approve plans. Release any compute you allocate when you are done. diff --git a/evals/snae-build/solution/astra.yaml b/evals/snae-build/solution/astra.yaml deleted file mode 100644 index 6cdf5328..00000000 --- a/evals/snae-build/solution/astra.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# ASTRA Analysis Specification -# Documentation: https://github.com/LightconeResearch/ASTRA - -version: "0.0.14" -name: "snae" -description: | - Fit the Union2.1 Type Ia supernova distance modulus vs redshift data - to a flat LCDM cosmological model with two free parameters (H0, Omega_L) - using maximum-likelihood (MAP) point estimation. This provides best-fit - cosmological parameters as a building block for a larger analysis. - -inputs: - - id: union21 - type: data - source: data/SCPUnion2.1_mu_vs_z.txt - description: "Union2.1 SN Ia compilation: 580 supernovae with redshift, distance modulus, and uncertainties" - -outputs: - - id: best_fit - type: metric - format: json - description: "Best-fit H0 and Omega_L from chi-squared minimization, with reduced chi-squared" - inputs: [union21] - decisions: [optimizer, error_model, redshift_cut] - recipe: - command: >- - python scripts/fit.py - --data {inputs.union21} - --output {output} - --optimizer {decisions.optimizer} - --error-model {decisions.error_model} - --redshift-cut {decisions.redshift_cut} - - - id: hubble_diagram - type: figure - format: png - description: "Hubble diagram: distance modulus vs redshift with best-fit model overlay" - inputs: [union21, best_fit] - recipe: - command: >- - python scripts/plot_hubble.py - --data {inputs.union21} - --fit {inputs.best_fit} - --output {output} - - - id: residuals - type: figure - format: png - description: "Residuals plot: data minus best-fit model vs redshift" - inputs: [union21, best_fit] - recipe: - command: >- - python scripts/plot_residuals.py - --data {inputs.union21} - --fit {inputs.best_fit} - --output {output} - -decisions: - optimizer: - label: "Optimizer" - tags: [inference] - rationale: "Nelder-Mead is robust and derivative-free; L-BFGS-B supports bounds natively" - default: nelder_mead - options: - nelder_mead: - label: "Nelder-Mead" - description: "Derivative-free simplex optimizer" - l_bfgs_b: - label: "L-BFGS-B" - description: "Bounded quasi-Newton optimizer" - - error_model: - label: "Error Treatment" - tags: [data] - rationale: "Union2.1 provides both statistical-only and systematics-included uncertainties; choice directly affects fit quality" - default: stat_and_sys - options: - stat_only: - label: "Statistical errors only" - description: "Use only the statistical distance modulus uncertainties" - stat_and_sys: - label: "Statistical + systematic errors" - description: "Include systematic uncertainty contributions" - - redshift_cut: - label: "Low-Redshift Cut" - tags: [data] - rationale: "SNe at z < 0.01 are dominated by peculiar velocity scatter rather than cosmological signal; standard practice varies on whether to exclude them" - default: z_min_001 - options: - none: - label: "No cut" - description: "Use all supernovae regardless of redshift" - z_min_001: - label: "z > 0.01" - description: "Exclude supernovae with z < 0.01 to mitigate peculiar velocity contamination" diff --git a/evals/snae-build/solution/scripts/cosmology.py b/evals/snae-build/solution/scripts/cosmology.py deleted file mode 100644 index 65227201..00000000 --- a/evals/snae-build/solution/scripts/cosmology.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Shared numerical helpers for the Union2.1 flat-LCDM analysis.""" - -from __future__ import annotations - -import json -from pathlib import Path - -import numpy as np -from scipy.integrate import quad - -C_KM_S = 299_792.458 -SYSTEMATIC_FLOOR_MAG = 0.03 - - -def load_union21(path: str | Path) -> tuple[np.ndarray, ...]: - """Load redshift, distance modulus, statistical error, and host probability.""" - redshift, distance_modulus, statistical_error, host_probability = np.loadtxt( - path, - comments="#", - usecols=(1, 2, 3, 4), - unpack=True, - ) - return redshift, distance_modulus, statistical_error, host_probability - - -def selection_mask(redshift: np.ndarray, redshift_cut: str) -> np.ndarray: - if redshift_cut == "none": - return np.ones(redshift.shape, dtype=bool) - if redshift_cut == "z_min_001": - return redshift > 0.01 - raise ValueError(f"Unknown redshift cut: {redshift_cut}") - - -def effective_error(statistical_error: np.ndarray, error_model: str) -> np.ndarray: - if error_model == "stat_only": - return statistical_error - if error_model == "stat_and_sys": - # The supplied table contains statistical errors and host-mass - # probabilities, but no systematic covariance matrix. A 0.03 mag - # irreducible floor is therefore used as a conservative approximation. - return np.hypot(statistical_error, SYSTEMATIC_FLOOR_MAG) - raise ValueError(f"Unknown error model: {error_model}") - - -def distance_modulus(redshift: np.ndarray, h0: float, omega_l: float) -> np.ndarray: - """Distance modulus for a spatially flat matter-plus-Lambda cosmology.""" - z = np.asarray(redshift, dtype=float) - omega_m = 1.0 - omega_l - - def inv_expansion_rate(z_value: float) -> float: - return 1.0 / np.sqrt(omega_m * (1.0 + z_value) ** 3 + omega_l) - - comoving_integral = np.fromiter( - (quad(inv_expansion_rate, 0.0, value, epsabs=1e-10, epsrel=1e-10)[0] for value in z), - dtype=float, - count=z.size, - ) - luminosity_distance_mpc = (C_KM_S / h0) * (1.0 + z) * comoving_integral - return 5.0 * np.log10(luminosity_distance_mpc) + 25.0 - - -def load_fit(path: str | Path) -> dict[str, object]: - fit_path = Path(path) - if fit_path.is_dir(): - fit_path = fit_path / "best_fit.json" - with fit_path.open(encoding="utf-8") as stream: - return json.load(stream) diff --git a/evals/snae-build/solution/scripts/fit.py b/evals/snae-build/solution/scripts/fit.py deleted file mode 100644 index 767b2b36..00000000 --- a/evals/snae-build/solution/scripts/fit.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 -"""Fit Union2.1 distance moduli with a two-parameter flat-LCDM model.""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path - -import numpy as np -from cosmology import ( - SYSTEMATIC_FLOOR_MAG, - distance_modulus, - effective_error, - load_union21, - selection_mask, -) -from scipy.optimize import minimize - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--data", required=True) - parser.add_argument("--output", required=True, help="the output file, results//best_fit.json") - parser.add_argument("--optimizer", choices=("nelder_mead", "l_bfgs_b"), required=True) - parser.add_argument("--error-model", choices=("stat_only", "stat_and_sys"), required=True) - parser.add_argument("--redshift-cut", choices=("none", "z_min_001"), required=True) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - redshift, observed_mu, statistical_error, _ = load_union21(args.data) - keep = selection_mask(redshift, args.redshift_cut) - redshift = redshift[keep] - observed_mu = observed_mu[keep] - sigma_mu = effective_error(statistical_error[keep], args.error_model) - - def chi_squared(parameters: np.ndarray) -> float: - h0, omega_l = parameters - if not 40.0 <= h0 <= 100.0 or not 0.001 <= omega_l <= 0.999: - return float("inf") - model_mu = distance_modulus(redshift, h0, omega_l) - return float(np.sum(np.square((observed_mu - model_mu) / sigma_mu))) - - method = "Nelder-Mead" if args.optimizer == "nelder_mead" else "L-BFGS-B" - options: dict[str, float | int] - if method == "Nelder-Mead": - options = {"maxiter": 2000, "xatol": 1e-9, "fatol": 1e-7} - else: - options = {"maxiter": 2000, "ftol": 1e-12} - - result = minimize( - chi_squared, - x0=np.array([70.0, 0.7]), - method=method, - bounds=((40.0, 100.0), (0.001, 0.999)), - options=options, - ) - if not result.success: - raise RuntimeError(f"Optimization failed: {result.message}") - - h0, omega_l = (float(value) for value in result.x) - chi2 = float(result.fun) - degrees_of_freedom = int(redshift.size - 2) - summary = { - "H0_km_s_Mpc": h0, - "Omega_L": omega_l, - "Omega_m": 1.0 - omega_l, - "chi_squared": chi2, - "degrees_of_freedom": degrees_of_freedom, - "reduced_chi_squared": chi2 / degrees_of_freedom, - "n_supernovae": int(redshift.size), - "optimizer": args.optimizer, - "error_model": args.error_model, - "redshift_cut": args.redshift_cut, - "systematic_floor_mag": SYSTEMATIC_FLOOR_MAG if args.error_model == "stat_and_sys" else 0.0, - "optimization_evaluations": int(result.nfev), - "optimization_message": str(result.message), - } - - # An output is exactly one file — the path the engine passed as {output}. - output = Path(args.output) - with output.open("w", encoding="utf-8") as stream: - json.dump(summary, stream, indent=2, sort_keys=True) - stream.write("\n") - print(json.dumps(summary, sort_keys=True)) - - -if __name__ == "__main__": - main() diff --git a/evals/snae-build/solution/scripts/plot_hubble.py b/evals/snae-build/solution/scripts/plot_hubble.py deleted file mode 100644 index d274f02b..00000000 --- a/evals/snae-build/solution/scripts/plot_hubble.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 -"""Plot the Union2.1 Hubble diagram and the fitted flat-LCDM relation.""" - -from __future__ import annotations - -import argparse - -import matplotlib - -matplotlib.use("Agg") -import matplotlib.pyplot as plt -import numpy as np -from cosmology import distance_modulus, effective_error, load_fit, load_union21, selection_mask - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--data", required=True) - parser.add_argument("--fit", required=True, help="the best_fit output file (json)") - parser.add_argument("--output", required=True, help="the output file, results//hubble_diagram.png") - return parser.parse_args() - - -def main() -> None: - args = parse_args() - fit = load_fit(args.fit) - redshift, observed_mu, statistical_error, _ = load_union21(args.data) - keep = selection_mask(redshift, str(fit["redshift_cut"])) - sigma_mu = effective_error(statistical_error, str(fit["error_model"])) - - model_redshift = np.linspace(float(redshift[keep].min()), float(redshift[keep].max()), 800) - model_mu = distance_modulus( - model_redshift, - float(fit["H0_km_s_Mpc"]), - float(fit["Omega_L"]), - ) - - figure, axis = plt.subplots(figsize=(8.2, 5.4), constrained_layout=True) - axis.errorbar( - redshift[keep], - observed_mu[keep], - yerr=sigma_mu[keep], - fmt=".", - color="#377eb8", - alpha=0.45, - markersize=3, - elinewidth=0.5, - label=f"Union2.1 ({keep.sum()} SNe Ia)", - ) - axis.plot( - model_redshift, - model_mu, - color="#e41a1c", - linewidth=2, - label=( - rf"Flat $\Lambda$CDM: $H_0={float(fit['H0_km_s_Mpc']):.2f}$ " - r"km s$^{-1}$ Mpc$^{-1}$, " - rf"$\Omega_\Lambda={float(fit['Omega_L']):.3f}$" - ), - ) - axis.set_xlabel("Redshift z") - axis.set_ylabel("Distance modulus μ") - axis.set_title("Union2.1 Hubble diagram") - axis.grid(alpha=0.2) - axis.legend(frameon=False) - - figure.savefig(args.output, dpi=180) - plt.close(figure) - - -if __name__ == "__main__": - main() diff --git a/evals/snae-build/solution/scripts/plot_residuals.py b/evals/snae-build/solution/scripts/plot_residuals.py deleted file mode 100644 index 1c67e639..00000000 --- a/evals/snae-build/solution/scripts/plot_residuals.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 -"""Plot Union2.1 distance-modulus residuals around the fitted model.""" - -from __future__ import annotations - -import argparse - -import matplotlib - -matplotlib.use("Agg") -import matplotlib.pyplot as plt -from cosmology import distance_modulus, effective_error, load_fit, load_union21, selection_mask - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--data", required=True) - parser.add_argument("--fit", required=True, help="the best_fit output file (json)") - parser.add_argument("--output", required=True, help="the output file, results//residuals.png") - return parser.parse_args() - - -def main() -> None: - args = parse_args() - fit = load_fit(args.fit) - redshift, observed_mu, statistical_error, _ = load_union21(args.data) - keep = selection_mask(redshift, str(fit["redshift_cut"])) - sigma_mu = effective_error(statistical_error, str(fit["error_model"])) - model_mu = distance_modulus( - redshift[keep], - float(fit["H0_km_s_Mpc"]), - float(fit["Omega_L"]), - ) - residual = observed_mu[keep] - model_mu - - figure, axis = plt.subplots(figsize=(8.2, 4.6), constrained_layout=True) - axis.errorbar( - redshift[keep], - residual, - yerr=sigma_mu[keep], - fmt=".", - color="#4daf4a", - alpha=0.5, - markersize=3, - elinewidth=0.5, - ) - axis.axhline(0.0, color="#222222", linewidth=1.3) - axis.set_xlabel("Redshift z") - axis.set_ylabel("Observed − model μ") - axis.set_title("Union2.1 flat-ΛCDM residuals") - axis.grid(alpha=0.2) - - figure.savefig(args.output, dpi=180) - plt.close(figure) - - -if __name__ == "__main__": - main() diff --git a/evals/snae-build/solution/solve.sh b/evals/snae-build/solution/solve.sh deleted file mode 100755 index 7de8fb38..00000000 --- a/evals/snae-build/solution/solve.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -# Scripted reference solution (oracle): the build loop instruction.md asks -# for, done by hand. Runs as root in the seeded project; the verifier must -# score exactly 1.0 afterwards. -set -euo pipefail - -PROJECT="${PROJECT:-/root/snae-build}" -SOLUTION="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$PROJECT" - -# 1-2. Complete the spec: formats, templated recipe commands, and each -# output's inputs/decisions contract. -cp "$SOLUTION/astra.yaml" astra.yaml -astra validate astra.yaml - -# 3. Write the scripts the recipes name, decisions parameterized via argparse. -mkdir -p scripts -cp "$SOLUTION"/scripts/*.py scripts/ - -# The project starts with no dependencies; declare what the recipes import. -uv add numpy scipy matplotlib - -# 4. Commit, then build through the engine. Since lightcone-cli #226 the engine -# runs recipes on an explicitly allocated cluster: launch the default local -# one (all CPUs and RAM the container shows) and pass its name to every -# materialize; --check and status need no cluster. -git add -A -git commit -q -m "Complete the spec and write the recipes" -lc compute launch --wait -lc materialize local - -# Publication: declare a license, commit, materialize once more so the -# engine generates and commits ro-crate-metadata.json. -sed -i '/^\[project\]$/a license = "CC-BY-4.0"' pyproject.toml -git add -A -git commit -q -m "Declare a license for publication" -lc materialize local - -# The gate instruction.md states, checked here so a broken oracle fails loudly. -astra validate astra.yaml -lc materialize --check -test -f ro-crate-metadata.json - -# Release the allocation, as the instruction asks the agent to. -lc compute down local diff --git a/evals/snae-build/task.toml b/evals/snae-build/task.toml deleted file mode 100644 index 8f2156d4..00000000 --- a/evals/snae-build/task.toml +++ /dev/null @@ -1,43 +0,0 @@ -schema_version = "1.4" -artifacts = [] - -[task] -name = "lightcone/snae-build" -version = "0.3.0" -description = "Build a seeded ASTRA analysis end to end with lightcone-cli: complete the deliberately incomplete spec, write the recipe scripts, declare dependencies, materialize every output through the engine, and prepare the project for publication." -keywords = ["astronomy", "lightcone-cli", "astra", "build-from-scratch", "naive-usage", "provenance"] -[[task.authors]] -name = "dkn16" -email = "dkn16@foxmail.com" - -[metadata] -category = "natural-science" -difficulty = "medium" -# Inspired from LightconeResearch/lightcone-evals lightcone-cli/snae-build, -# itself a port of this repository's earlier eval. Installs the engine from a -# wheel the workflow builds out of the checkout under test, rather than a -# pinned git commit. -source_format = "lightcone-evals" - -[verifier] -timeout_sec = 600.0 -collect = [] - -[verifier.env] - -[agent] -timeout_sec = 1200.0 - -[environment] -network_mode = "public" -build_timeout_sec = 1800.0 -os = "linux" -cpus = 2 -memory_mb = 4096 -storage_mb = 10240 -gpus = 0 -mcp_servers = [] - -[environment.env] - -[solution.env] diff --git a/evals/snae-build/tests/test.sh b/evals/snae-build/tests/test.sh deleted file mode 100755 index ce020526..00000000 --- a/evals/snae-build/tests/test.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -# Verifier (runs as root after the agent). Three deterministic dimensions, -# mirroring lightcone-cli's own eval gate (astra validate, lc materialize -# --check, ro-crate-metadata.json) with partial credit for outputs built: -# spec_valid 0.25 astra validate astra.yaml passes -# outputs_current 0.50 fraction of declared outputs `current` in lc status --json -# publication_ready 0.25 lc materialize --check is up to date with >=1 current -# output, and ro-crate-metadata.json exists -# Always exits 0; scoring problems are a 0.0, not a crash. -set +e - -PROJECT="${PROJECT:-/root/snae-build}" -LOGS="${LOGS_DIR:-/logs/verifier}" -mkdir -p "$LOGS" - -# The repository is owned by the agent user; git refuses to read another -# user's repo without this. -git config --global --add safe.directory '*' 2>/dev/null - -spec_valid=0.0 -outputs_current=0.0 -publication_ready=0.0 - -if cd "$PROJECT"; then - if astra validate astra.yaml > "$LOGS/astra-validate.txt" 2>&1; then - spec_valid=1.0 - fi - cat "$LOGS/astra-validate.txt" - - lc status > "$LOGS/lc-status.txt" 2>&1 - cat "$LOGS/lc-status.txt" - lc status --json > "$LOGS/lc-status.json" 2> "$LOGS/lc-status.err" - outputs_current=$(python3 - "$LOGS/lc-status.json" <<'PY' -import json, sys -try: - outputs = json.load(open(sys.argv[1])).get("outputs") or [] -except Exception: - outputs = [] -current = sum(1 for o in outputs if o.get("status") == "current") -print(f"{current / len(outputs):.4f}" if outputs else "0.0") -PY -) - - lc materialize --check --json > "$LOGS/lc-check.json" 2> "$LOGS/lc-check.err" - if python3 - "$LOGS/lc-check.json" <<'PY' -import json, sys -try: - report = json.load(open(sys.argv[1])) -except Exception: - sys.exit(1) -sys.exit(0 if report.get("up_to_date") and len(report.get("current") or []) > 0 else 1) -PY - then - if [ -f ro-crate-metadata.json ]; then - publication_ready=1.0 - fi - fi - - # Keep what explains the score in the artifacts. - cp astra.yaml "$LOGS/astra.yaml" 2>/dev/null || true - cp pyproject.toml "$LOGS/pyproject.toml" 2>/dev/null || true - cp ro-crate-metadata.json "$LOGS/ro-crate-metadata.json" 2>/dev/null || true - git log --oneline > "$LOGS/git-log.txt" 2>&1 || true - ls -laR results > "$LOGS/results-listing.txt" 2>&1 || true - ls -la scripts src > "$LOGS/scripts-listing.txt" 2>&1 || true -fi - -read -r reward < <(python3 -c "print(f'{0.25*$spec_valid + 0.5*$outputs_current + 0.25*$publication_ready:.4f}')") -printf '%s\n' "$reward" > "$LOGS/reward.txt" -printf '{"reward": %s, "spec_valid": %s, "outputs_current": %s, "publication_ready": %s}\n' \ - "$reward" "$spec_valid" "$outputs_current" "$publication_ready" > "$LOGS/reward.json" -cat "$LOGS/reward.json" -exit 0 diff --git a/evals/trace-analysis-prompt.md b/evals/trace-analysis-prompt.md index 2aa63a2b..cc8dda15 100644 --- a/evals/trace-analysis-prompt.md +++ b/evals/trace-analysis-prompt.md @@ -1,13 +1,13 @@ You are reviewing one leg of an automated eval: an autonomous coding agent -was asked to build a scientific analysis with the `astra` (spec) and `lc` -(execution) CLIs, guided only by a one-line instruction, the Lightcone agent -skills installed into its harness, and the CLIs' own help. The input on stdin -has, in order: a run summary (reward, agent steps, main's median steps), the -deterministic evidence (tool calls by phase against main, failed tool calls, -repeated commands, harness exceptions, verifier output), the change under -test (commits and diffstat of the repository whose pull request this is), -this run's trace, and the trace of the latest successful main run of the -same leg. +was given the task instruction below and had to carry it out with the +`astra` (spec) and `lc` (execution) CLIs, guided only by that instruction, +the Lightcone agent skills installed into its harness, and the CLIs' own +help. The input on stdin has, in order: the task instruction, a run summary +(reward, agent steps, main's median steps), the deterministic evidence (tool +calls by phase against main, failed tool calls, repeated commands, harness +exceptions, verifier output), the change under test (commits and diffstat of +the repository whose pull request this is), this run's trace, and the trace +of the latest successful main run of the same leg and task. Your job is to explain *why* this run went the way it did, so a reviewer can act on it without opening the trace. Cite trace steps as `step N` and From 585c52966c97bd11de59c61adf9f47461d150cb8 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Oct 2026 13:11:00 +0000 Subject: [PATCH 3/9] Cut the eval back to what a reviewer can read in a few minutes The report script loses its heuristics: the phase classifier, the failure regex (it flagged every `sandbox:` header line as a failure), repeated-command counts and the per-task grouping. The trace analyst reads the digest and is handed only the exact facts: reward, steps, harness exceptions and the verifier's output. The step gate waits for three main runs and needs both 25% and three steps over the median, so one noisy trial cannot fail a PR. An oracle failure is now reported as one, rather than as every leg skipped. The workflow runs one task, drops lightcone-evals as a caller and the attempts/tolerance/tasks inputs, and fetches history only for the repository under test. The cost column goes: under OAuth it is an API price nobody pays. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Y9FLtiiMxVhhkvZjd9SQp4 --- .github/scripts/eval_report.py | 578 +++++++++------------------------ .github/workflows/eval.yml | 370 ++++++--------------- CLAUDE.md | 31 +- evals/trace-analysis-prompt.md | 16 +- 4 files changed, 272 insertions(+), 723 deletions(-) diff --git a/.github/scripts/eval_report.py b/.github/scripts/eval_report.py index 77207e2a..69806154 100644 --- a/.github/scripts/eval_report.py +++ b/.github/scripts/eval_report.py @@ -1,96 +1,40 @@ -"""Report on a Harbor eval run: metrics, evidence, the step gate, the PR comment. +"""Report on a Harbor eval run: the gate, the trace digests, the PR comment. - eval_report.py summarize --jobs collected/ --out report/ [--baseline base/] - [--refs refs.json] [--changes changes.md] - [--mode main|pr] [--tolerance 0.25] + eval_report.py summarize --jobs collected/ --out report/ --task bundle/task + [--baseline base/] [--refs refs.json] + [--changes changes.md] [--oracle success] + [--mode main|pr] eval_report.py comment --out report/ -``summarize`` reads every leg under ``--jobs`` (one directory per matrix leg, -holding ``leg.json`` and, unless the leg was skipped, Harbor's job output) and -writes into ``--out``: +``summarize`` reads one directory per matrix leg (``leg.json``, plus +Harbor's job output unless the leg was skipped) and writes ``results.json``, +``history.json`` (main runs only), and per leg a trace digest and the input +the trace analyst is handed. ``comment`` assembles ``comment.md`` from those +and each ``analysis-.md`` the analyst wrote. -- ``results.json`` per-leg rewards, steps, tool calls, phases, cost; the gate -- ``history.json`` the baseline's history plus this run, on ``--mode main`` -- ``digest-.md`` the chronological trace of each trial (ATIF trajectory) -- ``evidence-.md`` failures, repeats and the phase table, deterministic -- ``analysis-input-.md`` everything the trace analyst is handed - -``comment`` assembles ``comment.md`` from those plus each ``analysis-.md`` -the analyst wrote. The gate's verdict is in ``results.json``; the workflow -fails the job on it once the comment is posted. - -Inspired from LightconeResearch/lightcone-bench (``summarize.py``, -``trajectory_metrics.py``, ``compare.py``, ``analyze_traces.py``), which reads -the same Harbor artifacts. Agent steps are counted as the bench counts them: -trajectory steps whose source is the agent. +Steps are counted as lightcone-bench counts them (inspired from its +``trajectory_metrics.py``): trajectory steps whose source is the agent. """ from __future__ import annotations import argparse import json -import re import statistics import sys -from collections import Counter from datetime import UTC, datetime from pathlib import Path from typing import Any +# The step gate: off main, a leg fails when its mean steps exceed the median of +# its last main runs by more than TOLERANCE *and* by at least MIN_SLACK steps, +# once at least MIN_HISTORY main runs exist. One trial per leg is noisy. +TOLERANCE = 0.25 +MIN_SLACK = 3 +MIN_HISTORY = 3 HISTORY_LENGTH = 10 -SNIPPET = 300 -ERROR_SNIPPET = 1500 +SNIPPET = 800 DIGEST_CAP = 120_000 -EVIDENCE_CAP = 40 - -# Harness tool name (lowercased) -> how the call is read. Anything unlisted is -# read by its arguments alone. Vocabularies differ per harness: claude-code -# `Bash`/`Read`/`Skill`, codex `exec_command`/`apply_patch`, opencode `bash`/`read`. -SHELL = { - "bash", - "shell", - "exec", - "exec_command", - "local_shell", - "run_command", - "execute_command", - "terminal", - "container.exec", -} -READ = { - "read", - "read_file", - "view", - "glob", - "grep", - "ls", - "list_dir", - "list_files", - "search_files", - "open_file", -} - -# Where a tool call's effort went, first match wins. Keyed on the call's -# command or path, so it reads the same under every harness. -PHASES = [ - ("skills", re.compile(r"SKILL\.md|/skills/")), - ("help", re.compile(r"--help\b|\s-h\b|\bastra info\b|\blc --version\b")), - ("compute", re.compile(r"\blc compute\b")), - ("build", re.compile(r"\blc (materialize|run|status)\b")), - ("spec", re.compile(r"astra\.yaml|universes/|\bastra\b")), - ("publication", re.compile(r"license|ro-crate", re.I)), - ("deps", re.compile(r"\buv (add|remove|lock|sync|pip)\b|pyproject\.toml|uv\.lock")), - ("scripts", re.compile(r"scripts/|\.py\b")), - ("git", re.compile(r"\bgit\b")), -] - -FAILURE = re.compile( - r"Traceback \(most recent call last\)|^Error:|^error:" - r"|\bexit(?:ed with)? (?:code|status):? ?[1-9]|command not found|Permission denied" - r"|EACCES|No such file or directory|sandbox" - r"|is not (?:up to date|current)|refus", - re.I | re.M, -) def _load(path: Path) -> Any: @@ -100,175 +44,77 @@ def _load(path: Path) -> Any: return None -def _seconds(block: Any) -> float | None: - if not isinstance(block, dict): - return None - try: - start, end = ( - datetime.fromisoformat(str(block[k]).replace("Z", "+00:00")) - for k in ("started_at", "finished_at") - ) - except (KeyError, TypeError, ValueError): - return None - return (end - start).total_seconds() - - def _text(value: Any) -> str: """Flatten an ATIF content value (string, parts list, or dict) to text.""" - if value is None: - return "" - if isinstance(value, str): - return value + if value is None or isinstance(value, str): + return value or "" if isinstance(value, list): return "\n".join(_text(part) for part in value) if isinstance(value, dict): - for key in ("text", "content", "output", "stdout"): + for key in ("text", "content", "output"): if key in value: return _text(value[key]) return json.dumps(value) -def _observations(step: dict[str, Any]) -> dict[str, str]: - """Map each tool call id in a step to the text of its result.""" - obs = step.get("observation") - if isinstance(obs, str): - try: - obs = json.loads(obs) - except ValueError: - return {"": obs} - if not isinstance(obs, dict): - return {} - return { - str(r.get("source_call_id", "")): _text(r.get("content")) - for r in obs.get("results") or [] - if isinstance(r, dict) - } - - -def _call_text(call: dict[str, Any]) -> str: - """The command or path a tool call acted on, as one line.""" - args = call.get("arguments") - if isinstance(args, str): - try: - args = json.loads(args) - except ValueError: - return args - if not isinstance(args, dict): - return "" - parts = [] - for key in ( - "command", - "cmd", - "file_path", - "filePath", - "path", - "pattern", - "skill", - "input", - "patch", - ): - value = args.get(key) - if isinstance(value, list): - value = " ".join(map(str, value)) - if value: - parts.append(str(value)) - return " ".join(parts) or json.dumps(args) - - -def _phase(name: str, text: str) -> str: - if name == "skill": - return "skills" - for phase, pattern in PHASES: - if pattern.search(text): - return phase - return "explore" if name in READ else "other" - - -def _snip(text: str, limit: int) -> str: +def _snip(text: str, limit: int = SNIPPET) -> str: text = text.strip().replace("```", "`​``") return text if len(text) <= limit else text[:limit] + f"… [{len(text) - limit} more chars]" def read_trial(trial_dir: Path) -> dict[str, Any]: - """Metrics, evidence and a digest for one Harbor trial directory.""" + """Reward, exception, steps and a chronological digest for one trial.""" result = _load(trial_dir / "result.json") or {} - verifier = result.get("verifier_result") or {} - rewards = verifier.get("rewards") if isinstance(verifier, dict) else None + rewards = (result.get("verifier_result") or {}).get("rewards") or {} exc = result.get("exception_info") - trial: dict[str, Any] = { - "trial": result.get("trial_name", trial_dir.name), - "task": Path(str((result.get("task_id") or {}).get("path") or "")).name - or str(result.get("task_name") or "?").split("/")[-1], - "rewards": rewards or {}, - "reward": (rewards or {}).get("reward"), - "exception": ( - f"{exc.get('exception_type', '?')}: {_snip(str(exc.get('exception_message', '')), 500)}" - if isinstance(exc, dict) - else None - ), - "agent_min": (lambda s: s / 60 if s is not None else None)( - _seconds(result.get("agent_execution")) - ), - "cost_usd": (result.get("agent_result") or {}).get("cost_usd"), - } - traj = _load(trial_dir / "agent" / "trajectory.json") - steps = [s for s in (traj or {}).get("steps") or [] if isinstance(s, dict)] - phases: Counter[str] = Counter() - commands: Counter[str] = Counter() - failures: list[str] = [] - digest = [f"## Trial `{trial['trial']}`\n"] + timing = result.get("agent_execution") or {} + try: + start, end = (datetime.fromisoformat(timing[k]) for k in ("started_at", "finished_at")) + minutes: float | None = (end - start).total_seconds() / 60 + except (KeyError, TypeError, ValueError): + minutes = None + + steps = [s for s in (_load(trial_dir / "agent" / "trajectory.json") or {}).get("steps", [])] + digest = [f"## Trial `{result.get('trial_name', trial_dir.name)}`"] + skills_used = False for step in steps: if step.get("source") == "user": continue - results = _observations(step) + obs = step.get("observation") + if isinstance(obs, str): + try: + obs = json.loads(obs) + except ValueError: + obs = {"results": [{"content": obs}]} + results = { + r.get("source_call_id", ""): _text(r.get("content")) + for r in (obs or {}).get("results", []) + } head = _text(step.get("message")) or _text(step.get("reasoning_content")) - line = f"**step {step.get('step_id', '?')}**" - digest.append(f"{line} {_snip(head, SNIPPET)}" if head else line) + digest.append(f"**step {step.get('step_id', '?')}** {_snip(head, 300)}".rstrip()) for call in step.get("tool_calls") or []: - if not isinstance(call, dict): - continue name = str(call.get("function_name", "?")) - text = _call_text(call) - phases[_phase(name.lower(), text)] += 1 - if name.lower() in SHELL: - commands[text.strip()] += 1 - # A step-level observation with no call ids belongs to its first call - out = results.pop(str(call.get("tool_call_id", "")), None) or results.pop("", "") - failed = bool(FAILURE.search(out)) - digest.append(f"- `{name}` {_snip(text, SNIPPET)}") + args = call.get("arguments") + text = json.dumps(args) if not isinstance(args, str) else args + skills_used |= name.lower() == "skill" or "SKILL.md" in text + # An observation with no call ids belongs to the step's first call + out = results.pop(call.get("tool_call_id", ""), None) or results.pop("", "") + digest.append(f"- `{name}` {_snip(text, 300)}") if out: - digest.append( - " ```\n " - + _snip(out, ERROR_SNIPPET if failed else SNIPPET).replace("\n", "\n ") - + "\n ```" - ) - if failed: - failures.append( - f"- `{trial['trial']}` step {step.get('step_id', '?')} `{name}` " - f"`{_snip(text, 200)}`\n ```\n " - + _snip(out, 600).replace("\n", "\n ") - + "\n ```" - ) - totals = (traj or {}).get("final_metrics") or {} - trial.update( - { - "has_trajectory": bool(steps), - "steps": sum(1 for s in steps if s.get("source") == "agent") if steps else None, - "tool_calls": sum(phases.values()) if steps else None, - "phases": dict(phases), - "skills_used": phases.get("skills", 0) > 0, - "failures": len(failures), - "repeats": {c: n for c, n in commands.items() if n > 1}, - } - ) - if trial["cost_usd"] is None: - trial["cost_usd"] = totals.get("total_cost_usd") - trial["_failures"] = failures - trial["_digest"] = "\n".join(digest) - test_out = trial_dir / "verifier" / "test-stdout.txt" - if test_out.is_file(): - trial["_verifier"] = _snip(test_out.read_text(errors="replace"), 3000) - return trial + digest.append(" ```\n " + _snip(out).replace("\n", "\n ") + "\n ```") + + verifier = trial_dir / "verifier" / "test-stdout.txt" + return { + "reward": rewards.get("reward"), + "exception": f"{exc.get('exception_type')}: {_snip(str(exc.get('exception_message')), 500)}" + if isinstance(exc, dict) + else None, + "steps": sum(1 for s in steps if s.get("source") == "agent") if steps else None, + "minutes": minutes, + "skills_used": skills_used, + "digest": "\n".join(digest), + "verifier": _snip(verifier.read_text(errors="replace"), 3000) if verifier.is_file() else "", + } def _mean(values: list[Any]) -> float | None: @@ -276,59 +122,36 @@ def _mean(values: list[Any]) -> float | None: return statistics.mean(nums) if nums else None -def read_leg(leg_dir: Path) -> list[dict[str, Any]]: - """One group per task a matrix leg ran, keyed ``.``. - - Steps, history and the gate are per group: two tasks' step counts say - nothing about each other. A skipped leg is one group with no task. - """ +def read_leg(leg_dir: Path) -> dict[str, Any]: + """Every trial of one matrix leg, summarized.""" meta = _load(leg_dir / "leg.json") or {} - leg = meta.get("leg", leg_dir.name) trials = [ read_trial(p.parent) for p in sorted(leg_dir.rglob("result.json")) if (_load(p) or {}).get("trial_name") ] - by_task: dict[str, list[dict[str, Any]]] = {} - for t in trials: - by_task.setdefault(t["task"], []).append(t) - groups = [] - for task, ts in sorted(by_task.items()) or [("", [])]: - phases: Counter[str] = Counter() - for t in ts: - phases.update(t["phases"]) - groups.append( - { - "leg": f"{leg}.{task}" if task else leg, - "harness_leg": leg, - "task": task, - "agent": meta.get("agent", "?"), - "model": meta.get("model", "?"), - "skipped": bool(meta.get("skipped")), - "skip_reason": meta.get("skip_reason", ""), - "trials": ts, - "n": len(ts), - "errored": sum(1 for t in ts if t["exception"]), - "mean_reward": _mean([t["reward"] for t in ts]), - "mean_steps": _mean([t["steps"] for t in ts]), - "mean_tool_calls": _mean([t["tool_calls"] for t in ts]), - "mean_cost_usd": _mean([t["cost_usd"] for t in ts]), - "mean_agent_min": _mean([t["agent_min"] for t in ts]), - "phases": {k: v / max(len(ts), 1) for k, v in phases.items()}, - "skills_used": any(t["skills_used"] for t in ts), - } - ) - return groups + return { + **meta, + "leg": meta.get("leg", leg_dir.name), + "trials": trials, + "n": len(trials), + "errored": sum(1 for t in trials if t["exception"]), + "reward": _mean([t["reward"] for t in trials]), + "steps": _mean([t["steps"] for t in trials]), + "minutes": _mean([t["minutes"] for t in trials]), + "skills_used": any(t["skills_used"] for t in trials), + } def gate( - legs: list[dict[str, Any]], history: dict[str, Any], mode: str, tolerance: float + legs: list[dict[str, Any]], history: dict[str, Any], oracle: str, mode: str ) -> dict[str, Any]: - """The verdict: reward and breakage always; steps against main's median off main.""" - reasons: list[str] = [] - notes: list[str] = [] - ran = [leg for leg in legs if not leg["skipped"]] - if not ran: + """Reward and breakage always; steps against main's median off main.""" + reasons, notes = [], [] + if oracle != "success": + reasons.append(f"reference solution: {oracle} — the task or the stack is broken") + ran = [leg for leg in legs if not leg.get("skipped")] + if oracle == "success" and not ran: reasons.append("no agent leg ran (every leg was skipped)") for leg in ran: name = leg["leg"] @@ -337,23 +160,23 @@ def gate( continue if leg["errored"]: reasons.append(f"{name}: {leg['errored']} trial(s) errored") - if leg["mean_reward"] is None or leg["mean_reward"] < 1.0: - reasons.append(f"{name}: mean reward {leg['mean_reward']} < 1.0") - past = [h["steps"] for h in history.get(name, []) if h.get("steps") is not None] - leg["baseline_steps"] = statistics.median(past) if past else None - leg["history"] = history.get(name, []) - if leg["baseline_steps"] is None: - notes.append(f"{name}: NEW — no main history to compare steps against") - elif leg["mean_steps"] is None: + if leg["reward"] is None or leg["reward"] < 1.0: + reasons.append(f"{name}: mean reward {leg['reward']} < 1.0") + past = [h["steps"] for h in history.get(name, [])] + leg["history"] = past + leg["median"] = statistics.median(past) if past else None + if len(past) < MIN_HISTORY: + notes.append(f"{name}: {len(past)} main run(s) on record, steps not gated yet") + elif leg["steps"] is None: reasons.append(f"{name}: no trajectory, steps cannot be compared") - else: - limit = leg["baseline_steps"] * (1 + tolerance) - leg["steps_limit"] = limit - if leg["mean_steps"] > limit and mode != "main": - reasons.append( - f"{name}: {leg['mean_steps']:.0f} steps exceeds main's median " - f"{leg['baseline_steps']:.0f} by more than {tolerance:.0%}" - ) + elif ( + mode != "main" + and leg["steps"] > leg["median"] * (1 + TOLERANCE) + and leg["steps"] - leg["median"] >= MIN_SLACK + ): + reasons.append( + f"{name}: {leg['steps']:.0f} steps against main's median {leg['median']:.0f}" + ) return {"ok": not reasons, "reasons": reasons, "notes": notes} @@ -364,120 +187,63 @@ def _fmt(value: Any, spec: str = "{:.0f}") -> str: def summarize(args: argparse.Namespace) -> int: out: Path = args.out out.mkdir(parents=True, exist_ok=True) - refs = _load(args.refs) if args.refs else {} - legs = [g for d in sorted(p for p in args.jobs.iterdir() if p.is_dir()) for g in read_leg(d)] - base = args.baseline - base_results = (_load(base / "results.json") if base else None) or {} + base: Path | None = args.baseline if args.baseline and args.baseline.is_dir() else None history = (_load(base / "history.json") if base else None) or {} - verdict = gate(legs, history, args.mode, args.tolerance) + legs = [read_leg(d) for d in sorted(p for p in args.jobs.iterdir() if p.is_dir())] + verdict = gate(legs, history, args.oracle, args.mode) + instruction = (args.task / "instruction.md").read_text() changes = args.changes.read_text() if args.changes and args.changes.is_file() else "" - base_legs = {leg["leg"]: leg for leg in base_results.get("legs", [])} for leg in legs: name = leg["leg"] - if leg["skipped"]: + if leg.get("skipped") or not leg["n"]: continue - digest = "\n\n".join(t.pop("_digest") for t in leg["trials"]) - (out / f"digest-{name}.md").write_text(digest[:DIGEST_CAP]) - failures = [f for t in leg["trials"] for f in t.pop("_failures")] - verifier = [t.pop("_verifier") for t in leg["trials"] if "_verifier" in t] - before = base_legs.get(name, {}).get("phases", {}) - rows = ["| phase | this run | main | Δ |", "|---|---|---|---|"] - for phase in sorted(set(leg["phases"]) | set(before)): - now, then = leg["phases"].get(phase, 0), before.get(phase) - delta = "—" if then is None else f"{now - then:+.1f}" - rows.append(f"| {phase} | {now:.1f} | {_fmt(then, '{:.1f}')} | {delta} |") - repeats: Counter[str] = Counter() - for t in leg["trials"]: - repeats.update(t["repeats"]) - evidence = [ - f"### Evidence — {name}", - "", - "Tool calls by phase (mean per trial):", - "", - *rows, - "", - f"Skills loaded: {'yes' if leg['skills_used'] else '**no**'}", - "", - f"Failed or erroring tool calls ({len(failures)}):", - "", - *(failures[:EVIDENCE_CAP] or ["- none"]), + digest = "\n\n".join(t.pop("digest") for t in leg["trials"])[:DIGEST_CAP] + (out / f"digest-{name}.md").write_text(digest) + base_digest = base / f"digest-{name}.md" if base else None + facts = [ + f"Reward {_fmt(leg['reward'], '{:.2f}')}, {_fmt(leg['steps'])} agent steps " + f"(main's median: {_fmt(leg.get('median'))}), {leg['n']} trial(s).", + *(f"Harness exception: {t['exception']}" for t in leg["trials"] if t["exception"]), *( - [f"- … {len(failures) - EVIDENCE_CAP} more in the digest"] - if len(failures) > EVIDENCE_CAP - else [] + f"Verifier output:\n```\n{t['verifier']}\n```" + for t in leg["trials"] + if t["verifier"] ), - "", - "Commands run more than once:", - "", - *([f"- {n}× `{_snip(c, 200)}`" for c, n in repeats.most_common(15)] or ["- none"]), ] - if any(t["exception"] for t in leg["trials"]): - evidence += [ - "", - "Harness exceptions:", - "", - *(f"- `{t['trial']}`: {t['exception']}" for t in leg["trials"] if t["exception"]), - ] - if verifier: - evidence += ["", "Verifier output:", "", "```", *verifier, "```"] - (out / f"evidence-{name}.md").write_text("\n".join(evidence) + "\n") - - base_digest = base / f"digest-{name}.md" if base else None - task_file = args.tasks / leg["task"] / "instruction.md" if args.tasks else None - instruction = ( - task_file.read_text() if task_file and task_file.is_file() else "_not provided_" - ) - summary = ( - f"leg {name} ({leg['agent']} / {leg['model']}): mean reward " - f"{_fmt(leg['mean_reward'], '{:.2f}')}, mean agent steps " - f"{_fmt(leg['mean_steps'])}, main's median " - f"{_fmt(leg.get('baseline_steps'))}, n={leg['n']}" - ) (out / f"analysis-input-{name}.md").write_text( "\n\n".join( [ "# Task instruction", instruction, "# Run summary", - summary, - "\n".join(evidence), + *facts, "# Change under test", changes or "_not provided_", "# This run's trace", - digest[:DIGEST_CAP], + digest, "# Baseline trace (latest successful main run)", - base_digest.read_text()[:DIGEST_CAP] - if base_digest and base_digest.is_file() - else "_no baseline trace available_", + base_digest.read_text() if base_digest and base_digest.is_file() else "_none_", ] ) + "\n" ) + for t in leg["trials"]: + t.pop("verifier") - results = { - "refs": refs, - "mode": args.mode, - "tolerance": args.tolerance, - "legs": legs, - "gate": verdict, - } - (out / "results.json").write_text(json.dumps(results, indent=1, default=str) + "\n") + refs = (_load(args.refs) if args.refs else None) or {} + (out / "results.json").write_text( + json.dumps({"refs": refs, "legs": legs, "gate": verdict}, indent=1) + "\n" + ) if args.mode == "main": - now = datetime.now(UTC).isoformat(timespec="seconds") for leg in legs: - if leg["skipped"] or leg["mean_steps"] is None: - continue - entry = { - "at": now, - "sha": refs.get("sha", ""), - "steps": leg["mean_steps"], - "reward": leg["mean_reward"], - } - history[leg["leg"]] = (history.get(leg["leg"], []) + [entry])[-HISTORY_LENGTH:] - (out / "history.json").write_text(json.dumps(history, indent=1) + "\n") - elif base and (base / "history.json").is_file(): - # Carry main's history along so a downloaded PR artifact is self-explanatory. + if leg["steps"] is not None: + entry = { + "at": datetime.now(UTC).isoformat(timespec="seconds"), + "sha": refs.get("sha", ""), + "steps": leg["steps"], + } + history[leg["leg"]] = (history.get(leg["leg"], []) + [entry])[-HISTORY_LENGTH:] (out / "history.json").write_text(json.dumps(history, indent=1) + "\n") print("gate:", "ok" if verdict["ok"] else "; ".join(verdict["reasons"])) return 0 @@ -487,66 +253,35 @@ def comment(args: argparse.Namespace) -> int: out: Path = args.out results = _load(out / "results.json") or {} verdict = results.get("gate", {}) - refs = results.get("refs", {}) lines = [ f"## {'✅' if verdict.get('ok') else '❌'} Eval", "", "", "", - ] - if refs: - lines += [" · ".join(f"{k}: `{v}`" for k, v in refs.items() if v), ""] - lines += [ - "| Leg | Reward | Steps | main median | Tool calls | Cost | Agent min | Skills |", - "|---|---|---|---|---|---|---|---|", + " · ".join(f"{k}: `{v}`" for k, v in results.get("refs", {}).items() if v), + "", + "| Leg | Reward | Agent steps | main median | Agent min | Skills loaded |", + "|---|---|---|---|---|---|", ] for leg in results.get("legs", []): - if leg["skipped"]: - lines.append(f"| {leg['leg']} | skipped: {leg['skip_reason']} | | | | | | |") + if leg.get("skipped"): + lines.append(f"| {leg['leg']} | skipped: {leg.get('skip_reason', '')} | | | | |") continue - steps = _fmt(leg["mean_steps"]) - if leg.get("baseline_steps") and leg["mean_steps"] is not None: - steps += f" ({(leg['mean_steps'] / leg['baseline_steps'] - 1):+.0%})" lines.append( - f"| {leg['leg']} | {_fmt(leg['mean_reward'], '{:.2f}')} | {steps} " - f"| {_fmt(leg.get('baseline_steps'))} | {_fmt(leg['mean_tool_calls'])} " - f"| {_fmt(leg['mean_cost_usd'], '${:.2f}')} " - f"| {_fmt(leg['mean_agent_min'], '{:.1f}')} " - f"| {'yes' if leg['skills_used'] else 'no'} |" + f"| {leg['leg']} | {_fmt(leg['reward'], '{:.2f}')} | {_fmt(leg['steps'])} " + f"| {_fmt(leg.get('median'))} | {_fmt(leg['minutes'], '{:.1f}')} " + f"| {'yes' if leg['skills_used'] else '**no**'} |" ) - lines.append("") - for reason in verdict.get("reasons", []): - lines.append(f"- ❌ {reason}") - for note in verdict.get("notes", []): - lines.append(f"- ℹ️ {note}") - lines.append("") + lines += ["", *(f"- ❌ {r}" for r in verdict.get("reasons", []))] + lines += [*(f"- ℹ️ {n}" for n in verdict.get("notes", [])), ""] for leg in results.get("legs", []): - if leg["skipped"]: + if leg.get("skipped") or not leg["n"]: continue - name = leg["leg"] - analysis = out / f"analysis-{name}.md" - lines += [f"### {name}", ""] - if analysis.is_file() and analysis.read_text().strip(): - lines += [analysis.read_text().strip(), ""] - else: - lines += [ - "❌ _Trace analysis missing — see the evidence below and the " - "`eval-results` artifact._", - "", - ] - history = leg.get("history") or [] - if history: - trend = " → ".join(_fmt(h.get("steps")) for h in history) - lines += [f"Steps on main, oldest first: {trend}", ""] - evidence = out / f"evidence-{name}.md" - if evidence.is_file(): - lines += [ - "
Evidence (deterministic)", - "", - evidence.read_text(), - "
", - "", - ] + analysis = out / f"analysis-{leg['leg']}.md" + text = analysis.read_text().strip() if analysis.is_file() else "" + lines += [f"### {leg['leg']}", "", text or "❌ _Trace analysis missing._", ""] + if leg.get("history"): + lines += ["Steps on main, oldest first: " + " → ".join(map(_fmt, leg["history"])), ""] (out / "comment.md").write_text("\n".join(lines)) return 0 @@ -557,20 +292,15 @@ def main() -> int: s = sub.add_parser("summarize") s.add_argument("--jobs", type=Path, required=True) s.add_argument("--out", type=Path, required=True) + s.add_argument("--task", type=Path, required=True) s.add_argument("--baseline", type=Path) s.add_argument("--refs", type=Path) s.add_argument("--changes", type=Path) - s.add_argument("--tasks", type=Path) + s.add_argument("--oracle", default="success") s.add_argument("--mode", choices=["main", "pr"], default="pr") - s.add_argument("--tolerance", type=float, default=0.25) - c = sub.add_parser("comment") - c.add_argument("--out", type=Path, required=True) + sub.add_parser("comment").add_argument("--out", type=Path, required=True) args = parser.parse_args() - if args.command == "comment": - return comment(args) - if args.baseline and not args.baseline.is_dir(): - args.baseline = None - return summarize(args) + return summarize(args) if args.command == "summarize" else comment(args) if __name__ == "__main__": diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 8f944bf3..7c44fde5 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -1,108 +1,39 @@ name: Eval -# Agentic eval on Harbor: tasks from LightconeResearch/lightcone-evals are run -# by real agent harnesses against the lightcone-cli and agent-skills under -# test, and every run explains itself from its trace. +# Agentic eval on Harbor. A task from LightconeResearch/lightcone-evals runs +# against the lightcone-cli and agent-skills under test: prepare pins the +# task image to the lightcone-cli commit and collects the skills, oracle +# requires the scripted solution to score 1.0, agents runs each harness with +# the skills installed, and report gates on reward, breakage and agent steps +# against main's history, then posts a trace analysis. # -# prepare fetch the tasks and pin each image to the lightcone-cli commit -# under test, collect the skill bundles -# oracle every task's scripted solution must score 1.0 — no LLM involved -# agents one Harbor job per harness, the skills installed with --skill -# report rewards, agent steps against main's history, the trace analysis, -# the PR comment; red on a failed gate or a missing analysis +# agent-skills reuses it, with its own PR as the side under test: # -# The tasks stay in lightcone-evals, in Harbor format. Each image installs -# lightcone-cli from git at its `ARG LIGHTCONE_COMMIT=` line, which prepare -# rewrites to the commit under test (lightcone-bench's contract too). -# lightcone-evals is private: a REPO_READ_TOKEN secret with contents:read on -# it is required. -# -# The step gate: a leg whose mean agent steps on a task exceed the median of -# its last runs on main by more than `tolerance` fails. Main runs record that -# history (in the eval-results artifact of the newest successful main run) -# and gate on reward only, so the reference follows main. -# -# Reuse from LightconeResearch/agent-skills (or lightcone-evals), where that -# repository is the side under test and the others are at their input refs: -# -# on: -# pull_request: -# types: [opened, synchronize, reopened, ready_for_review] -# push: -# branches: [main] -# permissions: -# contents: read -# pull-requests: write -# actions: read # jobs: # eval: # uses: LightconeResearch/lightcone-cli/.github/workflows/eval.yml@main # secrets: inherit # -# Whichever of the three repositories triggered the run is checked out at the -# triggering commit; the others at their input refs. Each repository keeps -# its own history, because the baseline is looked up in the triggering one. -# -# Limits: --skill installs SKILL.md bundles only, not the plugins' hooks -# (agent-skills' e2e-hooks workflow covers those); step counts compare within -# a leg and task only, since harness tool vocabularies differ; at attempts=1 -# the step comparison is coarse, which is what the median and the tolerance -# absorb. A leg whose secret is absent is skipped with a notice, and a run in -# which no leg ran fails. +# The caller needs `pull-requests: write` and `actions: read`, the same +# triggers as below, and a REPO_READ_TOKEN that can read lightcone-evals. on: workflow_dispatch: inputs: stack_ref: - description: "lightcone-cli ref to evaluate when it is not the triggering repository" - type: string + description: "lightcone-cli ref, when it is not the triggering repository" default: main skills_ref: - description: "agent-skills ref to install when it is not the triggering repository" - type: string + description: "agent-skills ref, when it is not the triggering repository" default: main - tasks: - description: "Comma-separated task paths in lightcone-evals; a folder means every task in it" - type: string - default: lightcone-cli/snae-build tasks_ref: - description: "lightcone-evals ref to fetch the tasks from when it is not the triggering repository" - type: string + description: "lightcone-evals ref to fetch the task from" default: main - attempts: - description: "Trials per leg" - type: string - default: "1" - tolerance: - description: "Allowed fractional increase of agent steps over main's median" - type: string - default: "0.25" workflow_call: inputs: - stack_ref: - description: "lightcone-cli ref to evaluate when it is not the triggering repository" - type: string - default: main - skills_ref: - description: "agent-skills ref to install when it is not the triggering repository" - type: string - default: main - tasks: - description: "Comma-separated task paths in lightcone-evals; a folder means every task in it" - type: string - default: lightcone-cli/snae-build - tasks_ref: - description: "lightcone-evals ref to fetch the tasks from when it is not the triggering repository" - type: string - default: main - attempts: - description: "Trials per leg" - type: string - default: "1" - tolerance: - description: "Allowed fractional increase of agent steps over main's median" - type: string - default: "0.25" + stack_ref: { type: string, default: main } + skills_ref: { type: string, default: main } + tasks_ref: { type: string, default: main } pull_request: # `ready_for_review` beside the defaults, so flipping a draft to ready # triggers the run the draft guard below skips. @@ -122,9 +53,9 @@ permissions: env: HARBOR_VERSION: 0.22.0 + TASK: lightcone-cli/snae-build STACK_REPO: LightconeResearch/lightcone-cli SKILLS_REPO: LightconeResearch/agent-skills - TASKS_REPO: LightconeResearch/lightcone-evals jobs: prepare: @@ -132,117 +63,72 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - name: Check out lightcone-cli - uses: actions/checkout@v4 + # The triggering repository at its commit, with the history that + # describes the change; the other at its input ref + - uses: actions/checkout@v4 with: repository: ${{ env.STACK_REPO }} ref: ${{ github.repository == env.STACK_REPO && (github.event.pull_request.head.sha || github.sha) || inputs.stack_ref || 'main' }} + fetch-depth: ${{ github.repository == env.STACK_REPO && '0' || '1' }} path: stack - # Full history, so the change under test can be described - fetch-depth: 0 - - name: Check out agent-skills - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: repository: ${{ env.SKILLS_REPO }} ref: ${{ github.repository == env.SKILLS_REPO && (github.event.pull_request.head.sha || github.sha) || inputs.skills_ref || 'main' }} - path: skills-src - fetch-depth: 0 - - - name: Require the lightcone-evals token - env: - REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN }} - run: | - [ -n "$REPO_READ_TOKEN" ] || { - echo "::error::REPO_READ_TOKEN is not set — it needs contents:read on $TASKS_REPO, which is private" - exit 1 - } + fetch-depth: ${{ github.repository == env.SKILLS_REPO && '0' || '1' }} + path: skills - - name: Check out lightcone-evals + - name: Check out lightcone-evals (private) uses: actions/checkout@v4 with: - repository: ${{ env.TASKS_REPO }} - ref: ${{ github.repository == env.TASKS_REPO && (github.event.pull_request.head.sha || github.sha) || inputs.tasks_ref || 'main' }} + repository: LightconeResearch/lightcone-evals + ref: ${{ inputs.tasks_ref || 'main' }} token: ${{ secrets.REPO_READ_TOKEN }} - path: tasks-src - fetch-depth: 0 + path: evals - name: Assemble the eval inputs env: - EVENT_REPO: ${{ github.repository }} BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} - TASKS: ${{ inputs.tasks || 'lightcone-cli/snae-build' }} run: | - mkdir -p bundle/tasks bundle/skills + mkdir -p bundle/skills cp stack/.github/scripts/eval_report.py stack/evals/trace-analysis-prompt.md bundle/ - # The requested tasks, side by side: to Harbor a folder of tasks is - # one job. Each image installs lightcone-cli at its - # `ARG LIGHTCONE_COMMIT=` line, pinned here to the commit under test + # The task image installs lightcone-cli at its ARG LIGHTCONE_COMMIT= + cp -r "evals/$TASK" bundle/task stack_sha=$(git -C stack rev-parse HEAD) - IFS=', ' read -r -a entries <<< "$TASKS" - for entry in "${entries[@]}"; do - [ -n "$entry" ] || continue - src="tasks-src/${entry%/}" - if [ -f "$src/task.toml" ]; then found=("$src"); else found=("$src"/*/task.toml); found=("${found[@]%/task.toml}"); fi - for t in "${found[@]}"; do - [ -f "$t/task.toml" ] || { echo "::error::'$entry' is neither a task nor a folder of tasks in $TASKS_REPO"; exit 1; } - name=$(basename "$t") - [ ! -e "bundle/tasks/$name" ] || { echo "::error::two tasks named '$name'"; exit 1; } - grep -q '^ARG LIGHTCONE_COMMIT=' "$t/environment/Dockerfile" \ - || { echo "::error::task $name does not pin lightcone-cli with ARG LIGHTCONE_COMMIT="; exit 1; } - cp -r "$t" "bundle/tasks/$name" - sed -i "s|^ARG LIGHTCONE_COMMIT=.*|ARG LIGHTCONE_COMMIT=$stack_sha|" "bundle/tasks/$name/environment/Dockerfile" - done - done - ls bundle/tasks | grep -q . || { echo "::error::no tasks selected"; exit 1; } + grep -q '^ARG LIGHTCONE_COMMIT=' bundle/task/environment/Dockerfile \ + || { echo "::error::$TASK does not pin lightcone-cli with ARG LIGHTCONE_COMMIT="; exit 1; } + sed -i "s|^ARG LIGHTCONE_COMMIT=.*|ARG LIGHTCONE_COMMIT=$stack_sha|" bundle/task/environment/Dockerfile - # Every built skill bundle, first of each name: plugins bundle - # their dependencies, so one skill can appear in several - for d in skills-src/plugins/*/skills/*/; do + # The built skill bundles, first of each name (plugins bundle + # their dependencies, so one skill can appear twice) + for d in skills/plugins/*/skills/*/; do name=$(basename "$d") - [ -f "${d}SKILL.md" ] && [ ! -e "bundle/skills/$name" ] && cp -r "$d" "bundle/skills/$name" + [ -e "bundle/skills/$name" ] || cp -r "$d" "bundle/skills/$name" done - ls bundle/skills | grep -q . || { echo "::error::no skill bundles in $SKILLS_REPO"; exit 1; } - - case "${EVENT_REPO,,}" in - "${STACK_REPO,,}") changed=stack ;; - "${SKILLS_REPO,,}") changed=skills-src ;; - "${TASKS_REPO,,}") changed=tasks-src ;; - *) changed= ;; - esac - jq -n \ - --arg stack "$STACK_REPO@$(git -C stack rev-parse --short HEAD)" \ - --arg skills "$SKILLS_REPO@$(git -C skills-src rev-parse --short HEAD)" \ - --arg tasks "$TASKS_REPO@$(git -C tasks-src rev-parse --short HEAD)" \ - --arg task_set "$(ls bundle/tasks | paste -sd, -)" \ - --arg skill_set "$(ls bundle/skills | paste -sd, -)" \ - --arg sha "$(git -C "${changed:-stack}" rev-parse HEAD)" \ - '{stack: $stack, skills: $skills, tasks: $tasks, task_set: $task_set, skill_set: $skill_set, sha: $sha}' > bundle/refs.json - cat bundle/refs.json - # What the analyst blames a regression on: the commits and files - # of the repository under test since its base - { - if [ -n "$changed" ] && [ -n "$BASE_SHA" ] \ - && git -C "$changed" cat-file -e "$BASE_SHA^{commit}" 2>/dev/null; then - echo "Repository under test: $EVENT_REPO" - echo - echo '```' + changed=stack + [ "${GITHUB_REPOSITORY,,}" = "${SKILLS_REPO,,}" ] && changed=skills + jq -n --arg stack "$(git -C stack rev-parse --short HEAD)" \ + --arg skills "$(git -C skills rev-parse --short HEAD)" \ + --arg tasks "$(git -C evals rev-parse --short HEAD)" \ + --arg sha "$(git -C "$changed" rev-parse HEAD)" \ + '{"lightcone-cli": $stack, "agent-skills": $skills, "lightcone-evals": $tasks, sha: $sha}' \ + | tee bundle/refs.json + + # What the analyst can blame a change in behaviour on + if [ -n "$BASE_SHA" ] && git -C "$changed" cat-file -e "$BASE_SHA^{commit}" 2>/dev/null; then + { echo '```' git -C "$changed" log --oneline "$BASE_SHA..HEAD" | head -50 - echo - git -C "$changed" diff --stat "$BASE_SHA...HEAD" | tail -60 - echo '```' - else - echo "No base commit to compare against for this event." - fi - } > bundle/changes.md + git -C "$changed" diff --stat "$BASE_SHA...HEAD" | tail -40 + echo '```'; } > bundle/changes.md + fi - uses: actions/upload-artifact@v4 with: name: eval-inputs path: bundle - if-no-files-found: error oracle: needs: prepare @@ -253,29 +139,20 @@ jobs: with: name: eval-inputs path: bundle - - - name: Set up uv - uses: astral-sh/setup-uv@v6 - - - name: Install Harbor - run: uv tool install "harbor==$HARBOR_VERSION" - + - uses: astral-sh/setup-uv@v6 + - run: uv tool install "harbor==$HARBOR_VERSION" - name: Run the reference solution - run: harbor run -p bundle/tasks -a oracle -e docker -o jobs --job-name oracle -q -y - + run: harbor run -p bundle/task -a oracle -e docker -o jobs --job-name oracle -q -y - name: Require reward 1.0 run: | - jq -e '[.stats.evals[] - | (.n_errors // 0) == 0 and (.metrics[0].reward // 0) == 1.0] + jq -e '[.stats.evals[] | (.n_errors // 0) == 0 and (.metrics[0].reward // 0) == 1.0] | length > 0 and all' jobs/oracle/result.json \ - || { jq .stats jobs/oracle/result.json; echo "::error::oracle gate failed"; exit 1; } - + || { jq .stats jobs/oracle/result.json; exit 1; } - uses: actions/upload-artifact@v4 if: always() with: name: eval-oracle path: jobs - if-no-files-found: warn agents: needs: oracle @@ -285,65 +162,48 @@ jobs: fail-fast: false matrix: include: - - { leg: claude-code, agent: claude-code, model: claude-sonnet-5-5, secret_name: CLAUDE_CODE_OAUTH_TOKEN } - - { leg: codex, agent: codex, model: gpt-5.1-codex, secret_name: OPENAI_API_KEY } + - { leg: claude-code, model: claude-sonnet-5-5, secret: CLAUDE_CODE_OAUTH_TOKEN } + - { leg: codex, model: gpt-5.1-codex, secret: OPENAI_API_KEY } steps: - - name: Gate on the leg's secret + - name: Skip without the leg's secret id: gate env: - LEG_KEY: ${{ secrets[matrix.secret_name] }} + KEY: ${{ secrets[matrix.secret] }} run: | mkdir -p "jobs/${{ matrix.leg }}" skipped=false - if [ -z "$LEG_KEY" ]; then - skipped=true - echo "::notice::Skipping ${{ matrix.leg }} — secret ${{ matrix.secret_name }} is not set" - fi + [ -n "$KEY" ] || { skipped=true; echo "::notice::Skipping ${{ matrix.leg }}: ${{ matrix.secret }} is not set"; } echo "skip=$skipped" >> "$GITHUB_OUTPUT" - jq -n --arg leg "${{ matrix.leg }}" --arg agent "${{ matrix.agent }}" \ - --arg model "${{ matrix.model }}" --argjson skipped "$skipped" \ - --arg reason "${{ matrix.secret_name }} not set" \ - '{leg: $leg, agent: $agent, model: $model, skipped: $skipped} - + (if $skipped then {skip_reason: $reason} else {} end)' \ - > "jobs/${{ matrix.leg }}/leg.json" - + jq -n --arg leg "${{ matrix.leg }}" --argjson skipped "$skipped" \ + --arg reason "${{ matrix.secret }} not set" \ + '{leg: $leg, skipped: $skipped, skip_reason: $reason}' > "jobs/${{ matrix.leg }}/leg.json" - uses: actions/download-artifact@v4 - if: steps.gate.outputs.skip != 'true' + if: steps.gate.outputs.skip == 'false' with: name: eval-inputs path: bundle - - - name: Set up uv - if: steps.gate.outputs.skip != 'true' - uses: astral-sh/setup-uv@v6 - - - name: Install Harbor - if: steps.gate.outputs.skip != 'true' + - uses: astral-sh/setup-uv@v6 + if: steps.gate.outputs.skip == 'false' + - if: steps.gate.outputs.skip == 'false' run: uv tool install "harbor==$HARBOR_VERSION" - - name: Run the agent - if: steps.gate.outputs.skip != 'true' + if: steps.gate.outputs.skip == 'false' env: CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} CLAUDE_CODE_MAX_TURNS: "200" run: | - skill_flags=() - for d in bundle/skills/*/; do skill_flags+=(--skill "${d%/}"); done - harbor run -p bundle/tasks \ - -a "${{ matrix.agent }}" -m "${{ matrix.model }}" \ - -e docker -k "${{ inputs.attempts || '1' }}" "${skill_flags[@]}" \ - -o "jobs/${{ matrix.leg }}" --job-name "${{ matrix.leg }}" -q -y - + skills=(); for d in bundle/skills/*/; do skills+=(--skill "${d%/}"); done + harbor run -p bundle/task -a "${{ matrix.leg }}" -m "${{ matrix.model }}" "${skills[@]}" \ + -e docker -o "jobs/${{ matrix.leg }}" --job-name "${{ matrix.leg }}" -q -y - uses: actions/upload-artifact@v4 if: always() with: name: eval-leg-${{ matrix.leg }} path: jobs/${{ matrix.leg }} - if-no-files-found: warn report: - needs: [prepare, agents] + needs: [prepare, oracle, agents] if: always() && needs.prepare.result == 'success' runs-on: ubuntu-latest timeout-minutes: 20 @@ -352,66 +212,44 @@ jobs: with: name: eval-inputs path: bundle - - uses: actions/download-artifact@v4 with: pattern: eval-leg-* path: collected - - name: Fetch main's latest results + - name: Fetch the history from main's latest successful run env: GH_TOKEN: ${{ github.token }} - WORKFLOW: ${{ github.workflow }} run: | - id=$(gh run list -R "$GITHUB_REPOSITORY" --workflow "$WORKFLOW" \ - --branch main --event push --status success --limit 1 \ - --json databaseId -q '.[0].databaseId' || true) - if [ -n "$id" ] && gh run download "$id" -R "$GITHUB_REPOSITORY" \ - -n eval-results -D baseline; then - echo "Baseline: run $id" - else - echo "::notice::No successful main run with eval-results yet — reporting without a baseline" - fi + id=$(gh run list -R "$GITHUB_REPOSITORY" --workflow "$GITHUB_WORKFLOW" --branch main \ + --event push --status success --limit 1 --json databaseId -q '.[0].databaseId' || true) + [ -n "$id" ] && gh run download "$id" -R "$GITHUB_REPOSITORY" -n eval-results -D baseline \ + || echo "::notice::No main history yet" - name: Summarize run: | mode=pr - [ "$GITHUB_EVENT_NAME" = push ] && [ "$GITHUB_REF" = refs/heads/main ] && mode=main - python3 bundle/eval_report.py summarize --jobs collected --out report \ + [ "$GITHUB_REF" = refs/heads/main ] && [ "$GITHUB_EVENT_NAME" = push ] && mode=main + mkdir -p collected + python3 bundle/eval_report.py summarize --jobs collected --out report --task bundle/task \ --baseline baseline --refs bundle/refs.json --changes bundle/changes.md \ - --tasks bundle/tasks \ - --mode "$mode" --tolerance "${{ inputs.tolerance || '0.25' }}" - - - name: Install Claude Code - run: | - curl -fsSL https://claude.ai/install.sh | bash - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - # Skip first-run onboarding in headless mode - echo '{"hasCompletedOnboarding": true}' > "$HOME/.claude.json" + --oracle "${{ needs.oracle.result }}" --mode "$mode" - name: Analyse the traces id: analysis env: CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} run: | - # The analysis is part of the result: a leg without one fails the - # job, after the comment says so + # The analysis is part of the result: a leg without one fails the job + ls report/analysis-input-*.md >/dev/null 2>&1 || exit 0 + curl -fsSL https://claude.ai/install.sh | bash + echo '{"hasCompletedOnboarding": true}' > "$HOME/.claude.json" status=ok - if [ -z "$CLAUDE_CODE_OAUTH_TOKEN" ]; then - echo "::error::CLAUDE_CODE_OAUTH_TOKEN is not set — no trace analysis" - echo "status=failed" >> "$GITHUB_OUTPUT" - exit 0 - fi for input in report/analysis-input-*.md; do - [ -e "$input" ] || continue leg=${input#report/analysis-input-}; leg=${leg%.md} - if ! timeout 600 claude -p "$(cat bundle/trace-analysis-prompt.md)" \ - < "$input" > "report/analysis-$leg.md" \ - || ! [ -s "report/analysis-$leg.md" ]; then - echo "::error::trace analysis failed for $leg" - rm -f "report/analysis-$leg.md" - status=failed - fi + timeout 600 "$HOME/.local/bin/claude" -p "$(cat bundle/trace-analysis-prompt.md)" \ + < "$input" > "report/analysis-$leg.md" && [ -s "report/analysis-$leg.md" ] \ + || { echo "::error::trace analysis failed for $leg"; status=failed; } done echo "status=$status" >> "$GITHUB_OUTPUT" @@ -426,7 +264,6 @@ jobs: with: name: eval-results path: report - if-no-files-found: warn - name: Comment on PR if: always() && github.event_name == 'pull_request' @@ -434,37 +271,22 @@ jobs: with: script: | const fs = require('fs'); - let body = fs.readFileSync('report/comment.md', 'utf8'); // GitHub rejects comments over 65536 characters - if (body.length > 65000) { - body = body.slice(0, 65000) + '\n\n_Truncated — full report in the eval-results artifact._'; - } - body += `\n\nRun: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const body = fs.readFileSync('report/comment.md', 'utf8').slice(0, 65000) + + `\n\nRun: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; const marker = ''; const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, + ...context.repo, issue_number: context.issue.number, }); const existing = comments.find(c => c.body.includes(marker)); if (existing) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: existing.id, - body, - }); + await github.rest.issues.updateComment({ ...context.repo, comment_id: existing.id, body }); } else { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body, - }); + await github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, body }); } - name: Gate run: | jq -r '.gate.reasons[]' report/results.json jq -e '.gate.ok' report/results.json > /dev/null - [ "${{ steps.analysis.outputs.status }}" = ok ] + [ "${{ steps.analysis.outputs.status }}" != failed ] diff --git a/CLAUDE.md b/CLAUDE.md index c22fee70..7b12ea21 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -192,7 +192,7 @@ src/lightcone/ # namespace — NO __init__.py ├── __init__.py # loader; a renderer only where there is a value to decide └── files/*.tmpl # the templates themselves, as real files -evals/ # the eval's trace-analysis prompt (the tasks live in lightcone-evals) +evals/ # the eval's trace-analysis prompt (the task lives in lightcone-evals) tests/ # pytest — mirrors src/ ``` @@ -212,21 +212,20 @@ runner in between — the pre-rebuild `justfile` was 90 lines of wrappers around them. The other workflows are `eval.yml` and `pypi-publish.yaml`. `eval.yml` is the agentic eval, on every non-draft PR, every push to -main and dispatch. It runs Harbor tasks fetched from -`LightconeResearch/lightcone-evals` (`lightcone-cli/snae-build` by -default; private, so a `REPO_READ_TOKEN` secret is required) in four -jobs: `prepare` (each task's `ARG LIGHTCONE_COMMIT=` rewritten to the -lightcone-cli commit under test, the agent-skills bundles), `oracle` -(every scripted solution must score 1.0), `agents` (claude-code and -codex, the skills installed with `--skill`, a leg without its secret -skipped with a notice) and `report` (`.github/scripts/eval_report.py`, -then one `claude -p` trace analysis per leg and task). It fails on a -reward below 1.0, an errored trial, a missing trace analysis, or — off -main — mean agent steps more than `tolerance` over the median of that -leg's last main runs on the task, a history carried in the newest -successful main run's `eval-results` artifact. agent-skills and -lightcone-evals reuse it through `workflow_call`, each as the side under -test; the header comment holds the caller. +main and dispatch. It runs the Harbor task `lightcone-cli/snae-build` +from the private `LightconeResearch/lightcone-evals` (a `REPO_READ_TOKEN` +secret reads it): `prepare` rewrites the task's `ARG LIGHTCONE_COMMIT=` +to the lightcone-cli commit under test and collects the agent-skills +bundles, `oracle` requires the scripted solution to score 1.0, `agents` +runs claude-code and codex with the skills installed through `--skill` +(a leg without its secret is skipped), and `report` +(`.github/scripts/eval_report.py`, then one `claude -p` trace analysis +per leg) fails on a reward below 1.0, an errored trial, a missing +analysis, or — off main — agent steps more than 25% and 3 steps over the +median of the leg's last main runs, once three are on record. That +history travels in the newest successful main run's `eval-results` +artifact. agent-skills reuses the workflow through `workflow_call`, with +its own PR as the side under test; the header comment holds the caller. ## Key Invariants (layer 1) diff --git a/evals/trace-analysis-prompt.md b/evals/trace-analysis-prompt.md index cc8dda15..92aec668 100644 --- a/evals/trace-analysis-prompt.md +++ b/evals/trace-analysis-prompt.md @@ -3,11 +3,10 @@ was given the task instruction below and had to carry it out with the `astra` (spec) and `lc` (execution) CLIs, guided only by that instruction, the Lightcone agent skills installed into its harness, and the CLIs' own help. The input on stdin has, in order: the task instruction, a run summary -(reward, agent steps, main's median steps), the deterministic evidence (tool -calls by phase against main, failed tool calls, repeated commands, harness -exceptions, verifier output), the change under test (commits and diffstat of -the repository whose pull request this is), this run's trace, and the trace -of the latest successful main run of the same leg and task. +(reward, agent steps against main's median, any harness exception, the +verifier's output), the change under test (commits and diffstat of the +repository whose pull request this is), this run's trace, and the trace of +the latest successful main run of the same leg. Your job is to explain *why* this run went the way it did, so a reviewer can act on it without opening the trace. Cite trace steps as `step N` and @@ -35,10 +34,9 @@ If the run was essentially clean, say so in one bullet. #### What changed vs main -Only when a baseline trace is present. Compare the two runs: which phase the -step difference sits in (use the phase table), the first point where the two -traces diverge, and which commit or file in the change under test most -plausibly caused it — or say plainly that the difference looks like agent +Only when a baseline trace is present. Compare the two runs: where the +extra or missing steps are, the first point where the two traces diverge, +and which commit or file in the change under test most plausibly caused it — or say plainly that the difference looks like agent variance, and why. If reward or breakage changed, explain that first. When there is no baseline trace, write one line saying so. From 4dd8b4e0651a3bf805b430e90491d542e882197d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Oct 2026 16:37:47 +0000 Subject: [PATCH 4/9] Fold the eval report into the workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit prepare copied eval_report.py and the analysis prompt out of the lightcone-cli checkout at stack_ref, while the workflow ran from the ref its caller pinned — so a stack_ref older than the script broke the run, and agent-skills could get a script that did not match its workflow. eval.yml is now one self-contained file. The numbers and the gate are jq over each trial's result.json and ATIF trajectory, as the original eval did. The trace digest is gone: the analyst reads the raw trajectories (this run's and main's) itself, with Read and Grep only, and the prompt is inline. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Y9FLtiiMxVhhkvZjd9SQp4 --- .github/scripts/eval_report.py | 307 --------------------------------- .github/workflows/eval.yml | 140 +++++++++++++-- CLAUDE.md | 12 +- evals/trace-analysis-prompt.md | 43 ----- 4 files changed, 132 insertions(+), 370 deletions(-) delete mode 100644 .github/scripts/eval_report.py delete mode 100644 evals/trace-analysis-prompt.md diff --git a/.github/scripts/eval_report.py b/.github/scripts/eval_report.py deleted file mode 100644 index 69806154..00000000 --- a/.github/scripts/eval_report.py +++ /dev/null @@ -1,307 +0,0 @@ -"""Report on a Harbor eval run: the gate, the trace digests, the PR comment. - - eval_report.py summarize --jobs collected/ --out report/ --task bundle/task - [--baseline base/] [--refs refs.json] - [--changes changes.md] [--oracle success] - [--mode main|pr] - eval_report.py comment --out report/ - -``summarize`` reads one directory per matrix leg (``leg.json``, plus -Harbor's job output unless the leg was skipped) and writes ``results.json``, -``history.json`` (main runs only), and per leg a trace digest and the input -the trace analyst is handed. ``comment`` assembles ``comment.md`` from those -and each ``analysis-.md`` the analyst wrote. - -Steps are counted as lightcone-bench counts them (inspired from its -``trajectory_metrics.py``): trajectory steps whose source is the agent. -""" - -from __future__ import annotations - -import argparse -import json -import statistics -import sys -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -# The step gate: off main, a leg fails when its mean steps exceed the median of -# its last main runs by more than TOLERANCE *and* by at least MIN_SLACK steps, -# once at least MIN_HISTORY main runs exist. One trial per leg is noisy. -TOLERANCE = 0.25 -MIN_SLACK = 3 -MIN_HISTORY = 3 -HISTORY_LENGTH = 10 -SNIPPET = 800 -DIGEST_CAP = 120_000 - - -def _load(path: Path) -> Any: - try: - return json.loads(path.read_text()) - except (OSError, ValueError): - return None - - -def _text(value: Any) -> str: - """Flatten an ATIF content value (string, parts list, or dict) to text.""" - if value is None or isinstance(value, str): - return value or "" - if isinstance(value, list): - return "\n".join(_text(part) for part in value) - if isinstance(value, dict): - for key in ("text", "content", "output"): - if key in value: - return _text(value[key]) - return json.dumps(value) - - -def _snip(text: str, limit: int = SNIPPET) -> str: - text = text.strip().replace("```", "`​``") - return text if len(text) <= limit else text[:limit] + f"… [{len(text) - limit} more chars]" - - -def read_trial(trial_dir: Path) -> dict[str, Any]: - """Reward, exception, steps and a chronological digest for one trial.""" - result = _load(trial_dir / "result.json") or {} - rewards = (result.get("verifier_result") or {}).get("rewards") or {} - exc = result.get("exception_info") - timing = result.get("agent_execution") or {} - try: - start, end = (datetime.fromisoformat(timing[k]) for k in ("started_at", "finished_at")) - minutes: float | None = (end - start).total_seconds() / 60 - except (KeyError, TypeError, ValueError): - minutes = None - - steps = [s for s in (_load(trial_dir / "agent" / "trajectory.json") or {}).get("steps", [])] - digest = [f"## Trial `{result.get('trial_name', trial_dir.name)}`"] - skills_used = False - for step in steps: - if step.get("source") == "user": - continue - obs = step.get("observation") - if isinstance(obs, str): - try: - obs = json.loads(obs) - except ValueError: - obs = {"results": [{"content": obs}]} - results = { - r.get("source_call_id", ""): _text(r.get("content")) - for r in (obs or {}).get("results", []) - } - head = _text(step.get("message")) or _text(step.get("reasoning_content")) - digest.append(f"**step {step.get('step_id', '?')}** {_snip(head, 300)}".rstrip()) - for call in step.get("tool_calls") or []: - name = str(call.get("function_name", "?")) - args = call.get("arguments") - text = json.dumps(args) if not isinstance(args, str) else args - skills_used |= name.lower() == "skill" or "SKILL.md" in text - # An observation with no call ids belongs to the step's first call - out = results.pop(call.get("tool_call_id", ""), None) or results.pop("", "") - digest.append(f"- `{name}` {_snip(text, 300)}") - if out: - digest.append(" ```\n " + _snip(out).replace("\n", "\n ") + "\n ```") - - verifier = trial_dir / "verifier" / "test-stdout.txt" - return { - "reward": rewards.get("reward"), - "exception": f"{exc.get('exception_type')}: {_snip(str(exc.get('exception_message')), 500)}" - if isinstance(exc, dict) - else None, - "steps": sum(1 for s in steps if s.get("source") == "agent") if steps else None, - "minutes": minutes, - "skills_used": skills_used, - "digest": "\n".join(digest), - "verifier": _snip(verifier.read_text(errors="replace"), 3000) if verifier.is_file() else "", - } - - -def _mean(values: list[Any]) -> float | None: - nums = [v for v in values if isinstance(v, (int, float))] - return statistics.mean(nums) if nums else None - - -def read_leg(leg_dir: Path) -> dict[str, Any]: - """Every trial of one matrix leg, summarized.""" - meta = _load(leg_dir / "leg.json") or {} - trials = [ - read_trial(p.parent) - for p in sorted(leg_dir.rglob("result.json")) - if (_load(p) or {}).get("trial_name") - ] - return { - **meta, - "leg": meta.get("leg", leg_dir.name), - "trials": trials, - "n": len(trials), - "errored": sum(1 for t in trials if t["exception"]), - "reward": _mean([t["reward"] for t in trials]), - "steps": _mean([t["steps"] for t in trials]), - "minutes": _mean([t["minutes"] for t in trials]), - "skills_used": any(t["skills_used"] for t in trials), - } - - -def gate( - legs: list[dict[str, Any]], history: dict[str, Any], oracle: str, mode: str -) -> dict[str, Any]: - """Reward and breakage always; steps against main's median off main.""" - reasons, notes = [], [] - if oracle != "success": - reasons.append(f"reference solution: {oracle} — the task or the stack is broken") - ran = [leg for leg in legs if not leg.get("skipped")] - if oracle == "success" and not ran: - reasons.append("no agent leg ran (every leg was skipped)") - for leg in ran: - name = leg["leg"] - if not leg["n"]: - reasons.append(f"{name}: no trial results") - continue - if leg["errored"]: - reasons.append(f"{name}: {leg['errored']} trial(s) errored") - if leg["reward"] is None or leg["reward"] < 1.0: - reasons.append(f"{name}: mean reward {leg['reward']} < 1.0") - past = [h["steps"] for h in history.get(name, [])] - leg["history"] = past - leg["median"] = statistics.median(past) if past else None - if len(past) < MIN_HISTORY: - notes.append(f"{name}: {len(past)} main run(s) on record, steps not gated yet") - elif leg["steps"] is None: - reasons.append(f"{name}: no trajectory, steps cannot be compared") - elif ( - mode != "main" - and leg["steps"] > leg["median"] * (1 + TOLERANCE) - and leg["steps"] - leg["median"] >= MIN_SLACK - ): - reasons.append( - f"{name}: {leg['steps']:.0f} steps against main's median {leg['median']:.0f}" - ) - return {"ok": not reasons, "reasons": reasons, "notes": notes} - - -def _fmt(value: Any, spec: str = "{:.0f}") -> str: - return "—" if value is None else spec.format(value) - - -def summarize(args: argparse.Namespace) -> int: - out: Path = args.out - out.mkdir(parents=True, exist_ok=True) - base: Path | None = args.baseline if args.baseline and args.baseline.is_dir() else None - history = (_load(base / "history.json") if base else None) or {} - legs = [read_leg(d) for d in sorted(p for p in args.jobs.iterdir() if p.is_dir())] - verdict = gate(legs, history, args.oracle, args.mode) - instruction = (args.task / "instruction.md").read_text() - changes = args.changes.read_text() if args.changes and args.changes.is_file() else "" - - for leg in legs: - name = leg["leg"] - if leg.get("skipped") or not leg["n"]: - continue - digest = "\n\n".join(t.pop("digest") for t in leg["trials"])[:DIGEST_CAP] - (out / f"digest-{name}.md").write_text(digest) - base_digest = base / f"digest-{name}.md" if base else None - facts = [ - f"Reward {_fmt(leg['reward'], '{:.2f}')}, {_fmt(leg['steps'])} agent steps " - f"(main's median: {_fmt(leg.get('median'))}), {leg['n']} trial(s).", - *(f"Harness exception: {t['exception']}" for t in leg["trials"] if t["exception"]), - *( - f"Verifier output:\n```\n{t['verifier']}\n```" - for t in leg["trials"] - if t["verifier"] - ), - ] - (out / f"analysis-input-{name}.md").write_text( - "\n\n".join( - [ - "# Task instruction", - instruction, - "# Run summary", - *facts, - "# Change under test", - changes or "_not provided_", - "# This run's trace", - digest, - "# Baseline trace (latest successful main run)", - base_digest.read_text() if base_digest and base_digest.is_file() else "_none_", - ] - ) - + "\n" - ) - for t in leg["trials"]: - t.pop("verifier") - - refs = (_load(args.refs) if args.refs else None) or {} - (out / "results.json").write_text( - json.dumps({"refs": refs, "legs": legs, "gate": verdict}, indent=1) + "\n" - ) - if args.mode == "main": - for leg in legs: - if leg["steps"] is not None: - entry = { - "at": datetime.now(UTC).isoformat(timespec="seconds"), - "sha": refs.get("sha", ""), - "steps": leg["steps"], - } - history[leg["leg"]] = (history.get(leg["leg"], []) + [entry])[-HISTORY_LENGTH:] - (out / "history.json").write_text(json.dumps(history, indent=1) + "\n") - print("gate:", "ok" if verdict["ok"] else "; ".join(verdict["reasons"])) - return 0 - - -def comment(args: argparse.Namespace) -> int: - out: Path = args.out - results = _load(out / "results.json") or {} - verdict = results.get("gate", {}) - lines = [ - f"## {'✅' if verdict.get('ok') else '❌'} Eval", - "", - "", - "", - " · ".join(f"{k}: `{v}`" for k, v in results.get("refs", {}).items() if v), - "", - "| Leg | Reward | Agent steps | main median | Agent min | Skills loaded |", - "|---|---|---|---|---|---|", - ] - for leg in results.get("legs", []): - if leg.get("skipped"): - lines.append(f"| {leg['leg']} | skipped: {leg.get('skip_reason', '')} | | | | |") - continue - lines.append( - f"| {leg['leg']} | {_fmt(leg['reward'], '{:.2f}')} | {_fmt(leg['steps'])} " - f"| {_fmt(leg.get('median'))} | {_fmt(leg['minutes'], '{:.1f}')} " - f"| {'yes' if leg['skills_used'] else '**no**'} |" - ) - lines += ["", *(f"- ❌ {r}" for r in verdict.get("reasons", []))] - lines += [*(f"- ℹ️ {n}" for n in verdict.get("notes", [])), ""] - for leg in results.get("legs", []): - if leg.get("skipped") or not leg["n"]: - continue - analysis = out / f"analysis-{leg['leg']}.md" - text = analysis.read_text().strip() if analysis.is_file() else "" - lines += [f"### {leg['leg']}", "", text or "❌ _Trace analysis missing._", ""] - if leg.get("history"): - lines += ["Steps on main, oldest first: " + " → ".join(map(_fmt, leg["history"])), ""] - (out / "comment.md").write_text("\n".join(lines)) - return 0 - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - sub = parser.add_subparsers(dest="command", required=True) - s = sub.add_parser("summarize") - s.add_argument("--jobs", type=Path, required=True) - s.add_argument("--out", type=Path, required=True) - s.add_argument("--task", type=Path, required=True) - s.add_argument("--baseline", type=Path) - s.add_argument("--refs", type=Path) - s.add_argument("--changes", type=Path) - s.add_argument("--oracle", default="success") - s.add_argument("--mode", choices=["main", "pr"], default="pr") - sub.add_parser("comment").add_argument("--out", type=Path, required=True) - args = parser.parse_args() - return summarize(args) if args.command == "summarize" else comment(args) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 7c44fde5..d565c86e 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -92,7 +92,6 @@ jobs: BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} run: | mkdir -p bundle/skills - cp stack/.github/scripts/eval_report.py stack/evals/trace-analysis-prompt.md bundle/ # The task image installs lightcone-cli at its ARG LIGHTCONE_COMMIT= cp -r "evals/$TASK" bundle/task @@ -226,14 +225,71 @@ jobs: [ -n "$id" ] && gh run download "$id" -R "$GITHUB_REPOSITORY" -n eval-results -D baseline \ || echo "::notice::No main history yet" - - name: Summarize + - name: Summarize and gate + env: + ORACLE: ${{ needs.oracle.result }} run: | + shopt -s nullglob mode=pr [ "$GITHUB_REF" = refs/heads/main ] && [ "$GITHUB_EVENT_NAME" = push ] && mode=main - mkdir -p collected - python3 bundle/eval_report.py summarize --jobs collected --out report --task bundle/task \ - --baseline baseline --refs bundle/refs.json --changes bundle/changes.md \ - --oracle "${{ needs.oracle.result }}" --mode "$mode" + mkdir -p report rows + [ -f baseline/history.json ] || { mkdir -p baseline; echo '{}' > baseline/history.json; } + + # One row per leg, from its trial's result.json and ATIF trajectory + for d in collected/eval-leg-*/; do + leg=$(jq -r .leg "$d/leg.json") + if [ "$(jq -r .skipped "$d/leg.json")" = true ]; then + jq '{leg, skipped: true, reason: .skip_reason}' "$d/leg.json" > "rows/$leg.json" + continue + fi + trial=$(dirname "$(ls "$d"*/*/result.json 2>/dev/null | head -1)") + traj="$trial/agent/trajectory.json" + [ -f "$traj" ] && cp "$traj" "report/trace-$leg.json" || echo '{}' > "report/trace-$leg.json" + jq -n --arg leg "$leg" --arg trial "$trial" \ + --slurpfile r "$trial/result.json" --slurpfile a "report/trace-$leg.json" ' + ($a[0].steps // []) as $steps + | {leg: $leg, skipped: false, trial: $trial, + reward: $r[0].verifier_result.rewards.reward, + exception: ($r[0].exception_info + | if . then "\(.exception_type): \(.exception_message | tostring | .[:300])" else null end), + steps: ([$steps[] | select(.source == "agent")] | if length > 0 then length else null end), + skills: any($steps[].tool_calls[]?; + (.function_name | ascii_downcase) == "skill" or (.arguments | tostring | test("SKILL\\.md")))}' \ + > "rows/$leg.json" 2>/dev/null \ + || jq -n --arg leg "$leg" '{leg: $leg, skipped: false, reward: null, exception: "no trial result", steps: null}' > "rows/$leg.json" + done + + # The gate: the oracle, reward and breakage always; off main, steps + # more than 25% and 3 over the median of the leg's main history, + # once three main runs are on record (one trial per leg is noisy) + jq -s --arg oracle "$ORACLE" --arg mode "$mode" --slurpfile h baseline/history.json \ + --slurpfile refs bundle/refs.json ' + def median: sort | if length % 2 == 1 then .[length / 2 | floor] + else (.[length / 2 - 1] + .[length / 2]) / 2 end; + map(if .skipped then . else + .history = [$h[0][.leg][]?.steps] + | .median = (if (.history | length) > 0 then (.history | median) else null end) end) + | {refs: $refs[0], legs: ., reasons: ( + (if $oracle != "success" then ["reference solution: \($oracle) — the task or the stack is broken"] + elif all(.[]; .skipped) then ["no agent leg ran"] else [] end) + + [.[] | select(.skipped | not) + | (if .exception then "\(.leg): errored — \(.exception)" else empty end), + (if (.reward // 0) < 1 then "\(.leg): reward \(.reward) < 1.0" else empty end), + (if $mode == "pr" and (.history | length) >= 3 then + if .steps == null then "\(.leg): no trajectory, steps cannot be compared" + elif .steps > .median * 1.25 and .steps - .median >= 3 + then "\(.leg): \(.steps) agent steps against a main median of \(.median)" + else empty end + else empty end)])}' rows/*.json < /dev/null > report/results.json + jq -r '.reasons[]' report/results.json + + # Main runs append to the history the next run compares against + if [ "$mode" = main ]; then + jq --slurpfile h baseline/history.json --arg sha "$(jq -r .sha bundle/refs.json)" ' + reduce (.legs[] | select(.steps != null)) as $l ($h[0]; + .[$l.leg] = ((.[$l.leg] // []) + [{sha: $sha, steps: $l.steps}])[-10:])' \ + report/results.json > report/history.json + fi - name: Analyse the traces id: analysis @@ -241,14 +297,59 @@ jobs: CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} run: | # The analysis is part of the result: a leg without one fails the job - ls report/analysis-input-*.md >/dev/null 2>&1 || exit 0 + legs=$(jq -r '.legs[] | select(.skipped | not) | select(.trial) | .leg' report/results.json) + [ -n "$legs" ] || exit 0 curl -fsSL https://claude.ai/install.sh | bash echo '{"hasCompletedOnboarding": true}' > "$HOME/.claude.json" + cat > prompt.md <<'EOF' + You are reviewing one leg of an automated eval: an autonomous coding + agent was given a task instruction and had to carry it out with the + `astra` (spec) and `lc` (execution) CLIs, guided only by that + instruction, the Lightcone agent skills installed into its harness, and + the CLIs' own help. The message lists the files to read: the + instruction, this run's ATIF trajectory (JSON: `steps`, each with + `tool_calls` and an `observation`), the verifier's output, the change + under test, and, when present, the trajectory of the latest successful + main run of the same leg. Read them before answering. + + Explain *why* this run went the way it did, so a reviewer can act on it + without opening the trace. Cite trajectory steps as `step N` and quote + output verbatim where it matters. Do not restate the task, do not + summarize a successful build, and do not pad. Write exactly these two + markdown sections and nothing else: + + #### Confusion & pain points + + 3-6 concise bullets: what happened, its root cause, and the fix it + suggests when one is clear. Look for failed commands and their cause; + misunderstandings of the spec format, the CLI, the sandbox or the + environment; whether the skills were loaded and told the agent what it + needed, or it fell back to `--help`, source reading or trial and error; + detours; and workarounds that hint at a gap lightcone-cli, the skills or + the eval setup should fix. If the run was essentially clean, say so in + one bullet. + + #### What changed vs main + + Only with a main trajectory: where the extra or missing steps are, the + first point where the two runs diverge, and which commit or file in the + change under test most plausibly caused it — or say plainly that it + looks like agent variance, and why. If reward or breakage changed, + explain that first. Without a main trajectory, write one line saying so. + EOF status=ok - for input in report/analysis-input-*.md; do - leg=${input#report/analysis-input-}; leg=${leg%.md} - timeout 600 "$HOME/.local/bin/claude" -p "$(cat bundle/trace-analysis-prompt.md)" \ - < "$input" > "report/analysis-$leg.md" && [ -s "report/analysis-$leg.md" ] \ + for leg in $legs; do + trial=$(jq -r --arg l "$leg" '.legs[] | select(.leg == $l) | .trial' report/results.json) + { + echo "Leg: $leg. Summary: $(jq -c --arg l "$leg" '.legs[] | select(.leg == $l) | del(.trial)' report/results.json)" + echo "- instruction: bundle/task/instruction.md" + echo "- this run's trajectory: report/trace-$leg.json" + echo "- verifier output: $trial/verifier/test-stdout.txt" + [ -f bundle/changes.md ] && echo "- change under test: bundle/changes.md" + [ -f "baseline/trace-$leg.json" ] && echo "- main's trajectory: baseline/trace-$leg.json" + } > "input-$leg.txt" + timeout 900 "$HOME/.local/bin/claude" -p "$(cat prompt.md)" --allowedTools Read Grep \ + < "input-$leg.txt" > "report/analysis-$leg.md" && [ -s "report/analysis-$leg.md" ] \ || { echo "::error::trace analysis failed for $leg"; status=failed; } done echo "status=$status" >> "$GITHUB_OUTPUT" @@ -256,7 +357,19 @@ jobs: - name: Build the report if: always() run: | - python3 bundle/eval_report.py comment --out report + { + jq -r '"## \(if (.reasons | length) == 0 then "✅" else "❌" end) Eval\n\n\n", + (.refs | to_entries | map("\(.key): `\(.value)`") | join(" · ")), "", + "| Leg | Reward | Agent steps | main median (runs) | Skills loaded |", "|---|---|---|---|---|", + (.legs[] | if .skipped then "| \(.leg) | skipped: \(.reason) | | | |" + else "| \(.leg) | \(.reward // "—") | \(.steps // "—") | \(.median // "—") (\(.history | length)) | \(if .skills then "yes" else "**no**" end) |" end), + "", (.reasons[] | "- ❌ \(.)"), ""' report/results.json + for f in report/analysis-*.md; do + [ -e "$f" ] || continue + leg=${f#report/analysis-}; echo "### ${leg%.md}"; echo; cat "$f"; echo + done + echo "Steps are gated once three main runs are on record; trajectories are in the \`eval-results\` artifact." + } > report/comment.md cat report/comment.md >> "$GITHUB_STEP_SUMMARY" - uses: actions/upload-artifact@v4 @@ -287,6 +400,5 @@ jobs: - name: Gate run: | - jq -r '.gate.reasons[]' report/results.json - jq -e '.gate.ok' report/results.json > /dev/null + jq -e '.reasons | length == 0' report/results.json > /dev/null [ "${{ steps.analysis.outputs.status }}" != failed ] diff --git a/CLAUDE.md b/CLAUDE.md index 7b12ea21..538c1ca8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -192,7 +192,6 @@ src/lightcone/ # namespace — NO __init__.py ├── __init__.py # loader; a renderer only where there is a value to decide └── files/*.tmpl # the templates themselves, as real files -evals/ # the eval's trace-analysis prompt (the task lives in lightcone-evals) tests/ # pytest — mirrors src/ ``` @@ -218,14 +217,15 @@ secret reads it): `prepare` rewrites the task's `ARG LIGHTCONE_COMMIT=` to the lightcone-cli commit under test and collects the agent-skills bundles, `oracle` requires the scripted solution to score 1.0, `agents` runs claude-code and codex with the skills installed through `--skill` -(a leg without its secret is skipped), and `report` -(`.github/scripts/eval_report.py`, then one `claude -p` trace analysis -per leg) fails on a reward below 1.0, an errored trial, a missing +(a leg without its secret is skipped), and `report` (jq over each +trial's result and ATIF trajectory, then one `claude -p` per leg that +reads the trajectories itself) fails on a reward below 1.0, an errored trial, a missing analysis, or — off main — agent steps more than 25% and 3 steps over the median of the leg's last main runs, once three are on record. That history travels in the newest successful main run's `eval-results` -artifact. agent-skills reuses the workflow through `workflow_call`, with -its own PR as the side under test; the header comment holds the caller. +artifact. The workflow is one self-contained file, so agent-skills reuses +it through `workflow_call` with its own PR as the side under test (the +header comment holds the caller) and nothing it runs can drift from it. ## Key Invariants (layer 1) diff --git a/evals/trace-analysis-prompt.md b/evals/trace-analysis-prompt.md deleted file mode 100644 index 92aec668..00000000 --- a/evals/trace-analysis-prompt.md +++ /dev/null @@ -1,43 +0,0 @@ -You are reviewing one leg of an automated eval: an autonomous coding agent -was given the task instruction below and had to carry it out with the -`astra` (spec) and `lc` (execution) CLIs, guided only by that instruction, -the Lightcone agent skills installed into its harness, and the CLIs' own -help. The input on stdin has, in order: the task instruction, a run summary -(reward, agent steps against main's median, any harness exception, the -verifier's output), the change under test (commits and diffstat of the -repository whose pull request this is), this run's trace, and the trace of -the latest successful main run of the same leg. - -Your job is to explain *why* this run went the way it did, so a reviewer can -act on it without opening the trace. Cite trace steps as `step N` and -evidence lines verbatim where it matters. Do not restate the task, do not -summarize a successful build, and do not pad. - -Write exactly these sections, as markdown: - -#### Confusion & pain points - -3-6 concise bullets. For each: what happened, its root cause, and the fix it -suggests when one is clear. Look for: - -- failed commands or errored tool results, and their root cause -- moments where the agent misunderstood the spec format, the CLI surface, - the sandbox, or the execution environment -- whether the skills were loaded and whether they told the agent what it - needed; where it fell back to `--help`, source reading or trial and error -- detours: reverse-engineering source code, probing the environment, redoing - work, or fixing things the harness should have provided -- workarounds the agent invented that hint at a product gap — call out what - lightcone-cli, the skills, or the eval setup should fix - -If the run was essentially clean, say so in one bullet. - -#### What changed vs main - -Only when a baseline trace is present. Compare the two runs: where the -extra or missing steps are, the first point where the two traces diverge, -and which commit or file in the change under test most plausibly caused it — or say plainly that the difference looks like agent -variance, and why. If reward or breakage changed, explain that first. When -there is no baseline trace, write one line saying so. - -Output only these two sections, no preamble. From 1809700845dee9aaaa15b9a76d967b8df4a4f88b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Oct 2026 16:59:08 +0000 Subject: [PATCH 5/9] Eval: a legs input and a require_skills gate Callers choose their harness legs as a JSON list of Harbor agents, each naming the secret it needs and optional extra variables; lightcone-cli runs one claude-code leg by default. require_skills fails a leg that never loaded a skill, for agent-skills' checks. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Y9FLtiiMxVhhkvZjd9SQp4 --- .github/workflows/eval.yml | 71 +++++++++++++++++++++++++++++--------- CLAUDE.md | 6 ++-- 2 files changed, 59 insertions(+), 18 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index d565c86e..954713c2 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -7,15 +7,27 @@ name: Eval # the skills installed, and report gates on reward, breakage and agent steps # against main's history, then posts a trace analysis. # -# agent-skills reuses it, with its own PR as the side under test: +# Each leg is one Harbor agent: `legs` is a JSON list of {name, agent, +# model, secret, env?}, where `secret` names the secret exported under its +# own name and `env` adds variables (e.g. ANTHROPIC_BASE_URL). The default +# is one claude-code leg. `require_skills` fails a leg that never loaded a +# skill. agent-skills reuses the workflow, with its own PR as the side under +# test: # # jobs: # eval: # uses: LightconeResearch/lightcone-cli/.github/workflows/eval.yml@main # secrets: inherit +# with: +# require_skills: true +# legs: >- +# [{"name": "claude-code", "agent": "claude-code", "model": "claude-sonnet-5-5", "secret": "CLAUDE_CODE_OAUTH_TOKEN"}, +# {"name": "opencode", "agent": "opencode", "model": "deepseek/deepseek-flash", "secret": "DEEPSEEK_API_KEY"}] # # The caller needs `pull-requests: write` and `actions: read`, the same # triggers as below, and a REPO_READ_TOKEN that can read lightcone-evals. +# --skill installs the skill bundles only; plugin installs and hooks are +# agent-skills' e2e-hooks workflow's to test. on: workflow_dispatch: @@ -29,11 +41,20 @@ on: tasks_ref: description: "lightcone-evals ref to fetch the task from" default: main + legs: + description: "JSON list of legs (see the header); empty means one claude-code leg" + default: "" + require_skills: + description: "Fail a leg that never loaded a skill" + type: boolean + default: false workflow_call: inputs: stack_ref: { type: string, default: main } skills_ref: { type: string, default: main } tasks_ref: { type: string, default: main } + legs: { type: string, default: "" } + require_skills: { type: boolean, default: false } pull_request: # `ready_for_review` beside the defaults, so flipping a draft to ready # triggers the run the draft guard below skips. @@ -56,13 +77,27 @@ env: TASK: lightcone-cli/snae-build STACK_REPO: LightconeResearch/lightcone-cli SKILLS_REPO: LightconeResearch/agent-skills + DEFAULT_LEGS: '[{"name": "claude-code", "agent": "claude-code", "model": "claude-sonnet-5-5", "secret": "CLAUDE_CODE_OAUTH_TOKEN"}]' jobs: prepare: if: github.event_name != 'pull_request' || github.event.pull_request.draft == false runs-on: ubuntu-latest timeout-minutes: 10 + outputs: + legs: ${{ steps.legs.outputs.legs }} steps: + - name: Resolve the legs + id: legs + env: + LEGS: ${{ inputs.legs }} + run: | + legs=$(jq -c . <<< "${LEGS:-$DEFAULT_LEGS}") \ + && jq -e 'type == "array" and length > 0 + and all(.[]; (.name, .agent, .model, .secret) | type == "string")' <<< "$legs" > /dev/null \ + || { echo "::error::legs must be a JSON list of {name, agent, model, secret, env?}"; exit 1; } + echo "legs=$legs" >> "$GITHUB_OUTPUT" + # The triggering repository at its commit, with the history that # describes the change; the other at its input ref - uses: actions/checkout@v4 @@ -154,28 +189,26 @@ jobs: path: jobs agents: - needs: oracle + needs: [prepare, oracle] runs-on: ubuntu-latest timeout-minutes: 60 strategy: fail-fast: false matrix: - include: - - { leg: claude-code, model: claude-sonnet-5-5, secret: CLAUDE_CODE_OAUTH_TOKEN } - - { leg: codex, model: gpt-5.1-codex, secret: OPENAI_API_KEY } + include: ${{ fromJSON(needs.prepare.outputs.legs) }} steps: - name: Skip without the leg's secret id: gate env: KEY: ${{ secrets[matrix.secret] }} run: | - mkdir -p "jobs/${{ matrix.leg }}" + mkdir -p "jobs/${{ matrix.name }}" skipped=false - [ -n "$KEY" ] || { skipped=true; echo "::notice::Skipping ${{ matrix.leg }}: ${{ matrix.secret }} is not set"; } + [ -n "$KEY" ] || { skipped=true; echo "::notice::Skipping ${{ matrix.name }}: ${{ matrix.secret }} is not set"; } echo "skip=$skipped" >> "$GITHUB_OUTPUT" - jq -n --arg leg "${{ matrix.leg }}" --argjson skipped "$skipped" \ + jq -n --arg leg "${{ matrix.name }}" --argjson skipped "$skipped" \ --arg reason "${{ matrix.secret }} not set" \ - '{leg: $leg, skipped: $skipped, skip_reason: $reason}' > "jobs/${{ matrix.leg }}/leg.json" + '{leg: $leg, skipped: $skipped, skip_reason: $reason}' > "jobs/${{ matrix.name }}/leg.json" - uses: actions/download-artifact@v4 if: steps.gate.outputs.skip == 'false' with: @@ -188,18 +221,21 @@ jobs: - name: Run the agent if: steps.gate.outputs.skip == 'false' env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + KEY: ${{ secrets[matrix.secret] }} + LEG_ENV: ${{ toJSON(matrix.env) }} CLAUDE_CODE_MAX_TURNS: "200" run: | + export "${{ matrix.secret }}=$KEY" + while IFS= read -r kv; do export "$kv"; done \ + < <(jq -r '(. // {}) | to_entries[] | "\(.key)=\(.value)"' <<< "$LEG_ENV") skills=(); for d in bundle/skills/*/; do skills+=(--skill "${d%/}"); done - harbor run -p bundle/task -a "${{ matrix.leg }}" -m "${{ matrix.model }}" "${skills[@]}" \ - -e docker -o "jobs/${{ matrix.leg }}" --job-name "${{ matrix.leg }}" -q -y + harbor run -p bundle/task -a "${{ matrix.agent }}" -m "${{ matrix.model }}" "${skills[@]}" \ + -e docker -o "jobs/${{ matrix.name }}" --job-name "${{ matrix.name }}" -q -y - uses: actions/upload-artifact@v4 if: always() with: - name: eval-leg-${{ matrix.leg }} - path: jobs/${{ matrix.leg }} + name: eval-leg-${{ matrix.name }} + path: jobs/${{ matrix.name }} report: needs: [prepare, oracle, agents] @@ -228,6 +264,7 @@ jobs: - name: Summarize and gate env: ORACLE: ${{ needs.oracle.result }} + REQUIRE_SKILLS: ${{ inputs.require_skills || false }} run: | shopt -s nullglob mode=pr @@ -262,7 +299,8 @@ jobs: # The gate: the oracle, reward and breakage always; off main, steps # more than 25% and 3 over the median of the leg's main history, # once three main runs are on record (one trial per leg is noisy) - jq -s --arg oracle "$ORACLE" --arg mode "$mode" --slurpfile h baseline/history.json \ + jq -s --arg oracle "$ORACLE" --arg mode "$mode" --argjson skills "$REQUIRE_SKILLS" \ + --slurpfile h baseline/history.json \ --slurpfile refs bundle/refs.json ' def median: sort | if length % 2 == 1 then .[length / 2 | floor] else (.[length / 2 - 1] + .[length / 2]) / 2 end; @@ -275,6 +313,7 @@ jobs: + [.[] | select(.skipped | not) | (if .exception then "\(.leg): errored — \(.exception)" else empty end), (if (.reward // 0) < 1 then "\(.leg): reward \(.reward) < 1.0" else empty end), + (if $skills and (.skills | not) then "\(.leg): no skill was loaded" else empty end), (if $mode == "pr" and (.history | length) >= 3 then if .steps == null then "\(.leg): no trajectory, steps cannot be compared" elif .steps > .median * 1.25 and .steps - .median >= 3 diff --git a/CLAUDE.md b/CLAUDE.md index 538c1ca8..bf861f72 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -216,11 +216,13 @@ from the private `LightconeResearch/lightcone-evals` (a `REPO_READ_TOKEN` secret reads it): `prepare` rewrites the task's `ARG LIGHTCONE_COMMIT=` to the lightcone-cli commit under test and collects the agent-skills bundles, `oracle` requires the scripted solution to score 1.0, `agents` -runs claude-code and codex with the skills installed through `--skill` +runs each leg of the `legs` input (a JSON list of Harbor agents; one +claude-code leg by default) with the skills installed through `--skill` (a leg without its secret is skipped), and `report` (jq over each trial's result and ATIF trajectory, then one `claude -p` per leg that reads the trajectories itself) fails on a reward below 1.0, an errored trial, a missing -analysis, or — off main — agent steps more than 25% and 3 steps over the +analysis, a leg that loaded no skill when `require_skills` is set, or — +off main — agent steps more than 25% and 3 steps over the median of the leg's last main runs, once three are on record. That history travels in the newest successful main run's `eval-results` artifact. The workflow is one self-contained file, so agent-skills reuses From 8b7e88026b96f00993504e0f5305323ded250816 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 08:14:36 +0000 Subject: [PATCH 6/9] Eval: read lightcone-evals without a token, now that it is public Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Y9FLtiiMxVhhkvZjd9SQp4 --- .github/workflows/eval.yml | 7 +++---- CLAUDE.md | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 954713c2..991f7326 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -24,8 +24,8 @@ name: Eval # [{"name": "claude-code", "agent": "claude-code", "model": "claude-sonnet-5-5", "secret": "CLAUDE_CODE_OAUTH_TOKEN"}, # {"name": "opencode", "agent": "opencode", "model": "deepseek/deepseek-flash", "secret": "DEEPSEEK_API_KEY"}] # -# The caller needs `pull-requests: write` and `actions: read`, the same -# triggers as below, and a REPO_READ_TOKEN that can read lightcone-evals. +# The caller needs `pull-requests: write` and `actions: read`, and the same +# triggers as below. # --skill installs the skill bundles only; plugin installs and hooks are # agent-skills' e2e-hooks workflow's to test. @@ -114,12 +114,11 @@ jobs: fetch-depth: ${{ github.repository == env.SKILLS_REPO && '0' || '1' }} path: skills - - name: Check out lightcone-evals (private) + - name: Check out lightcone-evals uses: actions/checkout@v4 with: repository: LightconeResearch/lightcone-evals ref: ${{ inputs.tasks_ref || 'main' }} - token: ${{ secrets.REPO_READ_TOKEN }} path: evals - name: Assemble the eval inputs diff --git a/CLAUDE.md b/CLAUDE.md index bf861f72..b1577883 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -212,8 +212,7 @@ around them. The other workflows are `eval.yml` and `pypi-publish.yaml`. `eval.yml` is the agentic eval, on every non-draft PR, every push to main and dispatch. It runs the Harbor task `lightcone-cli/snae-build` -from the private `LightconeResearch/lightcone-evals` (a `REPO_READ_TOKEN` -secret reads it): `prepare` rewrites the task's `ARG LIGHTCONE_COMMIT=` +from `LightconeResearch/lightcone-evals`: `prepare` rewrites the task's `ARG LIGHTCONE_COMMIT=` to the lightcone-cli commit under test and collects the agent-skills bundles, `oracle` requires the scripted solution to score 1.0, `agents` runs each leg of the `legs` input (a JSON list of Harbor agents; one From 7ef4ea4ed2173d4423b44090a727aa121227d0ac Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:56:43 +0000 Subject: [PATCH 7/9] Eval: call the workflow hosted in lightcone-evals The workflow moved beside the task it runs, so this repository keeps a caller that passes the commit under test and its secrets. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Y9FLtiiMxVhhkvZjd9SQp4 --- .github/workflows/eval.yml | 439 ++----------------------------------- CLAUDE.md | 22 +- 2 files changed, 19 insertions(+), 442 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 991f7326..e05f1751 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -1,442 +1,29 @@ name: Eval -# Agentic eval on Harbor. A task from LightconeResearch/lightcone-evals runs -# against the lightcone-cli and agent-skills under test: prepare pins the -# task image to the lightcone-cli commit and collects the skills, oracle -# requires the scripted solution to score 1.0, agents runs each harness with -# the skills installed, and report gates on reward, breakage and agent steps -# against main's history, then posts a trace analysis. -# -# Each leg is one Harbor agent: `legs` is a JSON list of {name, agent, -# model, secret, env?}, where `secret` names the secret exported under its -# own name and `env` adds variables (e.g. ANTHROPIC_BASE_URL). The default -# is one claude-code leg. `require_skills` fails a leg that never loaded a -# skill. agent-skills reuses the workflow, with its own PR as the side under -# test: -# -# jobs: -# eval: -# uses: LightconeResearch/lightcone-cli/.github/workflows/eval.yml@main -# secrets: inherit -# with: -# require_skills: true -# legs: >- -# [{"name": "claude-code", "agent": "claude-code", "model": "claude-sonnet-5-5", "secret": "CLAUDE_CODE_OAUTH_TOKEN"}, -# {"name": "opencode", "agent": "opencode", "model": "deepseek/deepseek-flash", "secret": "DEEPSEEK_API_KEY"}] -# -# The caller needs `pull-requests: write` and `actions: read`, and the same -# triggers as below. -# --skill installs the skill bundles only; plugin installs and hooks are -# agent-skills' e2e-hooks workflow's to test. +# The agentic eval, hosted in LightconeResearch/lightcone-evals beside the +# task it runs; see that workflow's header and README. This repository is +# the side under test: its commit pins the task image, its secrets run the +# legs, and its main runs are the baseline its PRs are compared against. on: - workflow_dispatch: - inputs: - stack_ref: - description: "lightcone-cli ref, when it is not the triggering repository" - default: main - skills_ref: - description: "agent-skills ref, when it is not the triggering repository" - default: main - tasks_ref: - description: "lightcone-evals ref to fetch the task from" - default: main - legs: - description: "JSON list of legs (see the header); empty means one claude-code leg" - default: "" - require_skills: - description: "Fail a leg that never loaded a skill" - type: boolean - default: false - workflow_call: - inputs: - stack_ref: { type: string, default: main } - skills_ref: { type: string, default: main } - tasks_ref: { type: string, default: main } - legs: { type: string, default: "" } - require_skills: { type: boolean, default: false } pull_request: # `ready_for_review` beside the defaults, so flipping a draft to ready - # triggers the run the draft guard below skips. + # triggers the run the draft guard skips types: [opened, synchronize, reopened, ready_for_review] push: branches: [main] - -# Only one eval per PR at a time — cancel in-progress runs -concurrency: - group: eval-${{ github.repository }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: true + workflow_dispatch: permissions: contents: read pull-requests: write actions: read -env: - HARBOR_VERSION: 0.22.0 - TASK: lightcone-cli/snae-build - STACK_REPO: LightconeResearch/lightcone-cli - SKILLS_REPO: LightconeResearch/agent-skills - DEFAULT_LEGS: '[{"name": "claude-code", "agent": "claude-code", "model": "claude-sonnet-5-5", "secret": "CLAUDE_CODE_OAUTH_TOKEN"}]' - jobs: - prepare: - if: github.event_name != 'pull_request' || github.event.pull_request.draft == false - runs-on: ubuntu-latest - timeout-minutes: 10 - outputs: - legs: ${{ steps.legs.outputs.legs }} - steps: - - name: Resolve the legs - id: legs - env: - LEGS: ${{ inputs.legs }} - run: | - legs=$(jq -c . <<< "${LEGS:-$DEFAULT_LEGS}") \ - && jq -e 'type == "array" and length > 0 - and all(.[]; (.name, .agent, .model, .secret) | type == "string")' <<< "$legs" > /dev/null \ - || { echo "::error::legs must be a JSON list of {name, agent, model, secret, env?}"; exit 1; } - echo "legs=$legs" >> "$GITHUB_OUTPUT" - - # The triggering repository at its commit, with the history that - # describes the change; the other at its input ref - - uses: actions/checkout@v4 - with: - repository: ${{ env.STACK_REPO }} - ref: ${{ github.repository == env.STACK_REPO && (github.event.pull_request.head.sha || github.sha) || inputs.stack_ref || 'main' }} - fetch-depth: ${{ github.repository == env.STACK_REPO && '0' || '1' }} - path: stack - - - uses: actions/checkout@v4 - with: - repository: ${{ env.SKILLS_REPO }} - ref: ${{ github.repository == env.SKILLS_REPO && (github.event.pull_request.head.sha || github.sha) || inputs.skills_ref || 'main' }} - fetch-depth: ${{ github.repository == env.SKILLS_REPO && '0' || '1' }} - path: skills - - - name: Check out lightcone-evals - uses: actions/checkout@v4 - with: - repository: LightconeResearch/lightcone-evals - ref: ${{ inputs.tasks_ref || 'main' }} - path: evals - - - name: Assemble the eval inputs - env: - BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} - run: | - mkdir -p bundle/skills - - # The task image installs lightcone-cli at its ARG LIGHTCONE_COMMIT= - cp -r "evals/$TASK" bundle/task - stack_sha=$(git -C stack rev-parse HEAD) - grep -q '^ARG LIGHTCONE_COMMIT=' bundle/task/environment/Dockerfile \ - || { echo "::error::$TASK does not pin lightcone-cli with ARG LIGHTCONE_COMMIT="; exit 1; } - sed -i "s|^ARG LIGHTCONE_COMMIT=.*|ARG LIGHTCONE_COMMIT=$stack_sha|" bundle/task/environment/Dockerfile - - # The built skill bundles, first of each name (plugins bundle - # their dependencies, so one skill can appear twice) - for d in skills/plugins/*/skills/*/; do - name=$(basename "$d") - [ -e "bundle/skills/$name" ] || cp -r "$d" "bundle/skills/$name" - done - - changed=stack - [ "${GITHUB_REPOSITORY,,}" = "${SKILLS_REPO,,}" ] && changed=skills - jq -n --arg stack "$(git -C stack rev-parse --short HEAD)" \ - --arg skills "$(git -C skills rev-parse --short HEAD)" \ - --arg tasks "$(git -C evals rev-parse --short HEAD)" \ - --arg sha "$(git -C "$changed" rev-parse HEAD)" \ - '{"lightcone-cli": $stack, "agent-skills": $skills, "lightcone-evals": $tasks, sha: $sha}' \ - | tee bundle/refs.json - - # What the analyst can blame a change in behaviour on - if [ -n "$BASE_SHA" ] && git -C "$changed" cat-file -e "$BASE_SHA^{commit}" 2>/dev/null; then - { echo '```' - git -C "$changed" log --oneline "$BASE_SHA..HEAD" | head -50 - git -C "$changed" diff --stat "$BASE_SHA...HEAD" | tail -40 - echo '```'; } > bundle/changes.md - fi - - - uses: actions/upload-artifact@v4 - with: - name: eval-inputs - path: bundle - - oracle: - needs: prepare - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - uses: actions/download-artifact@v4 - with: - name: eval-inputs - path: bundle - - uses: astral-sh/setup-uv@v6 - - run: uv tool install "harbor==$HARBOR_VERSION" - - name: Run the reference solution - run: harbor run -p bundle/task -a oracle -e docker -o jobs --job-name oracle -q -y - - name: Require reward 1.0 - run: | - jq -e '[.stats.evals[] | (.n_errors // 0) == 0 and (.metrics[0].reward // 0) == 1.0] - | length > 0 and all' jobs/oracle/result.json \ - || { jq .stats jobs/oracle/result.json; exit 1; } - - uses: actions/upload-artifact@v4 - if: always() - with: - name: eval-oracle - path: jobs - - agents: - needs: [prepare, oracle] - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.prepare.outputs.legs) }} - steps: - - name: Skip without the leg's secret - id: gate - env: - KEY: ${{ secrets[matrix.secret] }} - run: | - mkdir -p "jobs/${{ matrix.name }}" - skipped=false - [ -n "$KEY" ] || { skipped=true; echo "::notice::Skipping ${{ matrix.name }}: ${{ matrix.secret }} is not set"; } - echo "skip=$skipped" >> "$GITHUB_OUTPUT" - jq -n --arg leg "${{ matrix.name }}" --argjson skipped "$skipped" \ - --arg reason "${{ matrix.secret }} not set" \ - '{leg: $leg, skipped: $skipped, skip_reason: $reason}' > "jobs/${{ matrix.name }}/leg.json" - - uses: actions/download-artifact@v4 - if: steps.gate.outputs.skip == 'false' - with: - name: eval-inputs - path: bundle - - uses: astral-sh/setup-uv@v6 - if: steps.gate.outputs.skip == 'false' - - if: steps.gate.outputs.skip == 'false' - run: uv tool install "harbor==$HARBOR_VERSION" - - name: Run the agent - if: steps.gate.outputs.skip == 'false' - env: - KEY: ${{ secrets[matrix.secret] }} - LEG_ENV: ${{ toJSON(matrix.env) }} - CLAUDE_CODE_MAX_TURNS: "200" - run: | - export "${{ matrix.secret }}=$KEY" - while IFS= read -r kv; do export "$kv"; done \ - < <(jq -r '(. // {}) | to_entries[] | "\(.key)=\(.value)"' <<< "$LEG_ENV") - skills=(); for d in bundle/skills/*/; do skills+=(--skill "${d%/}"); done - harbor run -p bundle/task -a "${{ matrix.agent }}" -m "${{ matrix.model }}" "${skills[@]}" \ - -e docker -o "jobs/${{ matrix.name }}" --job-name "${{ matrix.name }}" -q -y - - uses: actions/upload-artifact@v4 - if: always() - with: - name: eval-leg-${{ matrix.name }} - path: jobs/${{ matrix.name }} - - report: - needs: [prepare, oracle, agents] - if: always() && needs.prepare.result == 'success' - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/download-artifact@v4 - with: - name: eval-inputs - path: bundle - - uses: actions/download-artifact@v4 - with: - pattern: eval-leg-* - path: collected - - - name: Fetch the history from main's latest successful run - env: - GH_TOKEN: ${{ github.token }} - run: | - id=$(gh run list -R "$GITHUB_REPOSITORY" --workflow "$GITHUB_WORKFLOW" --branch main \ - --event push --status success --limit 1 --json databaseId -q '.[0].databaseId' || true) - [ -n "$id" ] && gh run download "$id" -R "$GITHUB_REPOSITORY" -n eval-results -D baseline \ - || echo "::notice::No main history yet" - - - name: Summarize and gate - env: - ORACLE: ${{ needs.oracle.result }} - REQUIRE_SKILLS: ${{ inputs.require_skills || false }} - run: | - shopt -s nullglob - mode=pr - [ "$GITHUB_REF" = refs/heads/main ] && [ "$GITHUB_EVENT_NAME" = push ] && mode=main - mkdir -p report rows - [ -f baseline/history.json ] || { mkdir -p baseline; echo '{}' > baseline/history.json; } - - # One row per leg, from its trial's result.json and ATIF trajectory - for d in collected/eval-leg-*/; do - leg=$(jq -r .leg "$d/leg.json") - if [ "$(jq -r .skipped "$d/leg.json")" = true ]; then - jq '{leg, skipped: true, reason: .skip_reason}' "$d/leg.json" > "rows/$leg.json" - continue - fi - trial=$(dirname "$(ls "$d"*/*/result.json 2>/dev/null | head -1)") - traj="$trial/agent/trajectory.json" - [ -f "$traj" ] && cp "$traj" "report/trace-$leg.json" || echo '{}' > "report/trace-$leg.json" - jq -n --arg leg "$leg" --arg trial "$trial" \ - --slurpfile r "$trial/result.json" --slurpfile a "report/trace-$leg.json" ' - ($a[0].steps // []) as $steps - | {leg: $leg, skipped: false, trial: $trial, - reward: $r[0].verifier_result.rewards.reward, - exception: ($r[0].exception_info - | if . then "\(.exception_type): \(.exception_message | tostring | .[:300])" else null end), - steps: ([$steps[] | select(.source == "agent")] | if length > 0 then length else null end), - skills: any($steps[].tool_calls[]?; - (.function_name | ascii_downcase) == "skill" or (.arguments | tostring | test("SKILL\\.md")))}' \ - > "rows/$leg.json" 2>/dev/null \ - || jq -n --arg leg "$leg" '{leg: $leg, skipped: false, reward: null, exception: "no trial result", steps: null}' > "rows/$leg.json" - done - - # The gate: the oracle, reward and breakage always; off main, steps - # more than 25% and 3 over the median of the leg's main history, - # once three main runs are on record (one trial per leg is noisy) - jq -s --arg oracle "$ORACLE" --arg mode "$mode" --argjson skills "$REQUIRE_SKILLS" \ - --slurpfile h baseline/history.json \ - --slurpfile refs bundle/refs.json ' - def median: sort | if length % 2 == 1 then .[length / 2 | floor] - else (.[length / 2 - 1] + .[length / 2]) / 2 end; - map(if .skipped then . else - .history = [$h[0][.leg][]?.steps] - | .median = (if (.history | length) > 0 then (.history | median) else null end) end) - | {refs: $refs[0], legs: ., reasons: ( - (if $oracle != "success" then ["reference solution: \($oracle) — the task or the stack is broken"] - elif all(.[]; .skipped) then ["no agent leg ran"] else [] end) - + [.[] | select(.skipped | not) - | (if .exception then "\(.leg): errored — \(.exception)" else empty end), - (if (.reward // 0) < 1 then "\(.leg): reward \(.reward) < 1.0" else empty end), - (if $skills and (.skills | not) then "\(.leg): no skill was loaded" else empty end), - (if $mode == "pr" and (.history | length) >= 3 then - if .steps == null then "\(.leg): no trajectory, steps cannot be compared" - elif .steps > .median * 1.25 and .steps - .median >= 3 - then "\(.leg): \(.steps) agent steps against a main median of \(.median)" - else empty end - else empty end)])}' rows/*.json < /dev/null > report/results.json - jq -r '.reasons[]' report/results.json - - # Main runs append to the history the next run compares against - if [ "$mode" = main ]; then - jq --slurpfile h baseline/history.json --arg sha "$(jq -r .sha bundle/refs.json)" ' - reduce (.legs[] | select(.steps != null)) as $l ($h[0]; - .[$l.leg] = ((.[$l.leg] // []) + [{sha: $sha, steps: $l.steps}])[-10:])' \ - report/results.json > report/history.json - fi - - - name: Analyse the traces - id: analysis - env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - run: | - # The analysis is part of the result: a leg without one fails the job - legs=$(jq -r '.legs[] | select(.skipped | not) | select(.trial) | .leg' report/results.json) - [ -n "$legs" ] || exit 0 - curl -fsSL https://claude.ai/install.sh | bash - echo '{"hasCompletedOnboarding": true}' > "$HOME/.claude.json" - cat > prompt.md <<'EOF' - You are reviewing one leg of an automated eval: an autonomous coding - agent was given a task instruction and had to carry it out with the - `astra` (spec) and `lc` (execution) CLIs, guided only by that - instruction, the Lightcone agent skills installed into its harness, and - the CLIs' own help. The message lists the files to read: the - instruction, this run's ATIF trajectory (JSON: `steps`, each with - `tool_calls` and an `observation`), the verifier's output, the change - under test, and, when present, the trajectory of the latest successful - main run of the same leg. Read them before answering. - - Explain *why* this run went the way it did, so a reviewer can act on it - without opening the trace. Cite trajectory steps as `step N` and quote - output verbatim where it matters. Do not restate the task, do not - summarize a successful build, and do not pad. Write exactly these two - markdown sections and nothing else: - - #### Confusion & pain points - - 3-6 concise bullets: what happened, its root cause, and the fix it - suggests when one is clear. Look for failed commands and their cause; - misunderstandings of the spec format, the CLI, the sandbox or the - environment; whether the skills were loaded and told the agent what it - needed, or it fell back to `--help`, source reading or trial and error; - detours; and workarounds that hint at a gap lightcone-cli, the skills or - the eval setup should fix. If the run was essentially clean, say so in - one bullet. - - #### What changed vs main - - Only with a main trajectory: where the extra or missing steps are, the - first point where the two runs diverge, and which commit or file in the - change under test most plausibly caused it — or say plainly that it - looks like agent variance, and why. If reward or breakage changed, - explain that first. Without a main trajectory, write one line saying so. - EOF - status=ok - for leg in $legs; do - trial=$(jq -r --arg l "$leg" '.legs[] | select(.leg == $l) | .trial' report/results.json) - { - echo "Leg: $leg. Summary: $(jq -c --arg l "$leg" '.legs[] | select(.leg == $l) | del(.trial)' report/results.json)" - echo "- instruction: bundle/task/instruction.md" - echo "- this run's trajectory: report/trace-$leg.json" - echo "- verifier output: $trial/verifier/test-stdout.txt" - [ -f bundle/changes.md ] && echo "- change under test: bundle/changes.md" - [ -f "baseline/trace-$leg.json" ] && echo "- main's trajectory: baseline/trace-$leg.json" - } > "input-$leg.txt" - timeout 900 "$HOME/.local/bin/claude" -p "$(cat prompt.md)" --allowedTools Read Grep \ - < "input-$leg.txt" > "report/analysis-$leg.md" && [ -s "report/analysis-$leg.md" ] \ - || { echo "::error::trace analysis failed for $leg"; status=failed; } - done - echo "status=$status" >> "$GITHUB_OUTPUT" - - - name: Build the report - if: always() - run: | - { - jq -r '"## \(if (.reasons | length) == 0 then "✅" else "❌" end) Eval\n\n\n", - (.refs | to_entries | map("\(.key): `\(.value)`") | join(" · ")), "", - "| Leg | Reward | Agent steps | main median (runs) | Skills loaded |", "|---|---|---|---|---|", - (.legs[] | if .skipped then "| \(.leg) | skipped: \(.reason) | | | |" - else "| \(.leg) | \(.reward // "—") | \(.steps // "—") | \(.median // "—") (\(.history | length)) | \(if .skills then "yes" else "**no**" end) |" end), - "", (.reasons[] | "- ❌ \(.)"), ""' report/results.json - for f in report/analysis-*.md; do - [ -e "$f" ] || continue - leg=${f#report/analysis-}; echo "### ${leg%.md}"; echo; cat "$f"; echo - done - echo "Steps are gated once three main runs are on record; trajectories are in the \`eval-results\` artifact." - } > report/comment.md - cat report/comment.md >> "$GITHUB_STEP_SUMMARY" - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: eval-results - path: report - - - name: Comment on PR - if: always() && github.event_name == 'pull_request' - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - // GitHub rejects comments over 65536 characters - const body = fs.readFileSync('report/comment.md', 'utf8').slice(0, 65000) - + `\n\nRun: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const marker = ''; - const { data: comments } = await github.rest.issues.listComments({ - ...context.repo, issue_number: context.issue.number, - }); - const existing = comments.find(c => c.body.includes(marker)); - if (existing) { - await github.rest.issues.updateComment({ ...context.repo, comment_id: existing.id, body }); - } else { - await github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, body }); - } - - - name: Gate - run: | - jq -e '.reasons | length == 0' report/results.json > /dev/null - [ "${{ steps.analysis.outputs.status }}" != failed ] + eval: + # TODO: switch to @main once LightconeResearch/lightcone-evals#5 merges. + uses: LightconeResearch/lightcone-evals/.github/workflows/eval.yml@hosted-eval + secrets: inherit + with: + stack_ref: ${{ github.event.pull_request.head.sha || github.sha }} + tasks_ref: hosted-eval diff --git a/CLAUDE.md b/CLAUDE.md index b1577883..2c1675a9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -211,22 +211,12 @@ runner in between — the pre-rebuild `justfile` was 90 lines of wrappers around them. The other workflows are `eval.yml` and `pypi-publish.yaml`. `eval.yml` is the agentic eval, on every non-draft PR, every push to -main and dispatch. It runs the Harbor task `lightcone-cli/snae-build` -from `LightconeResearch/lightcone-evals`: `prepare` rewrites the task's `ARG LIGHTCONE_COMMIT=` -to the lightcone-cli commit under test and collects the agent-skills -bundles, `oracle` requires the scripted solution to score 1.0, `agents` -runs each leg of the `legs` input (a JSON list of Harbor agents; one -claude-code leg by default) with the skills installed through `--skill` -(a leg without its secret is skipped), and `report` (jq over each -trial's result and ATIF trajectory, then one `claude -p` per leg that -reads the trajectories itself) fails on a reward below 1.0, an errored trial, a missing -analysis, a leg that loaded no skill when `require_skills` is set, or — -off main — agent steps more than 25% and 3 steps over the -median of the leg's last main runs, once three are on record. That -history travels in the newest successful main run's `eval-results` -artifact. The workflow is one self-contained file, so agent-skills reuses -it through `workflow_call` with its own PR as the side under test (the -header comment holds the caller) and nothing it runs can drift from it. +main and dispatch. It is a caller only: the workflow lives in +`LightconeResearch/lightcone-evals` beside the task it runs (its README +holds the contract), and this repository hands it the commit under test +as `stack_ref` and its own secrets. A PR fails on a reward below 1.0, a +failed oracle, an errored trial or analysis, or agent steps more than 25% +and 3 over the median of this repository's last main runs. ## Key Invariants (layer 1) From 4f48c2401f257f9423589466b5cd081b929358cf Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 17:30:38 +0000 Subject: [PATCH 8/9] Eval: no step gate, so no actions: read The hosted workflow no longer reads main's history. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Y9FLtiiMxVhhkvZjd9SQp4 --- .github/workflows/eval.yml | 5 ++--- CLAUDE.md | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index e05f1751..658b3168 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -2,8 +2,8 @@ name: Eval # The agentic eval, hosted in LightconeResearch/lightcone-evals beside the # task it runs; see that workflow's header and README. This repository is -# the side under test: its commit pins the task image, its secrets run the -# legs, and its main runs are the baseline its PRs are compared against. +# the side under test: its commit pins the task image and its secrets run +# the legs. on: pull_request: @@ -17,7 +17,6 @@ on: permissions: contents: read pull-requests: write - actions: read jobs: eval: diff --git a/CLAUDE.md b/CLAUDE.md index 2c1675a9..7da3f6df 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -215,8 +215,7 @@ main and dispatch. It is a caller only: the workflow lives in `LightconeResearch/lightcone-evals` beside the task it runs (its README holds the contract), and this repository hands it the commit under test as `stack_ref` and its own secrets. A PR fails on a reward below 1.0, a -failed oracle, an errored trial or analysis, or agent steps more than 25% -and 3 over the median of this repository's last main runs. +failed oracle, or an errored trial or analysis. ## Key Invariants (layer 1) From 6919e8b7a238199fee233478520f26a28a2b15c2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 21:37:20 +0000 Subject: [PATCH 9/9] Eval: call the hosted workflow at main LightconeResearch/lightcone-evals#5 is merged, so the caller follows its main branch for both the workflow and the task. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Y9FLtiiMxVhhkvZjd9SQp4 --- .github/workflows/eval.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 658b3168..49b5723b 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -20,9 +20,7 @@ permissions: jobs: eval: - # TODO: switch to @main once LightconeResearch/lightcone-evals#5 merges. - uses: LightconeResearch/lightcone-evals/.github/workflows/eval.yml@hosted-eval + uses: LightconeResearch/lightcone-evals/.github/workflows/eval.yml@main secrets: inherit with: stack_ref: ${{ github.event.pull_request.head.sha || github.sha }} - tasks_ref: hosted-eval