diff --git a/.github/scripts/trace_digest.py b/.github/scripts/trace_digest.py deleted file mode 100644 index dfe7327b..00000000 --- a/.github/scripts/trace_digest.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Digest a `claude -p --output-format stream-json` transcript. - -Usage: trace_digest.py - -Writes a chronological, human-readable agent trace to and -prints shell-sourceable `key=value` run metrics on stdout (model, turns, -tool_calls, cost, duration). -""" - -import json -import sys -from collections import Counter -from pathlib import Path - -RESULT_SNIPPET = 300 -ERROR_SNIPPET = 1500 - - -def _fence(text: str) -> str: - """Neutralize backtick fences so embedded output can't break ours.""" - return text.replace("```", "`​``") - - -def main() -> None: - if len(sys.argv) != 3: - print("Usage: trace_digest.py ", file=sys.stderr) - sys.exit(1) - - transcript = Path(sys.argv[1]) - out_path = Path(sys.argv[2]) - - model = "" - metrics: dict = {} - tool_counts: Counter = Counter() - body: list[str] = [] - - for line in transcript.read_text().splitlines(): - try: - d = json.loads(line) - except json.JSONDecodeError: - continue - t = d.get("type") - - if t == "system" and d.get("subtype") == "init": - model = d.get("model", "") - - elif t == "assistant": - for c in d.get("message", {}).get("content", []): - if c.get("type") == "text" and c["text"].strip(): - body.append(f"**Claude:** {c['text'].strip()}\n") - elif c.get("type") == "tool_use": - name = c.get("name", "?") - tool_counts[name] += 1 - inp = c.get("input", {}) - detail = ( - inp.get("command") - or inp.get("file_path") - or inp.get("skill") - or json.dumps(inp)[:200] - ) - body.append(f"🔧 `{name}`\n\n```\n{_fence(str(detail))}\n```\n") - - elif t == "user": - for c in d.get("message", {}).get("content", []): - if isinstance(c, dict) and c.get("type") == "tool_result": - content = c.get("content", "") - if isinstance(content, list): - content = "\n".join( - x.get("text", "") for x in content if isinstance(x, dict) - ) - content = str(content).strip() - if c.get("is_error"): - body.append( - f"❌ **error result:**\n\n```\n{_fence(content[:ERROR_SNIPPET])}\n```\n" - ) - elif content: - # Collapse to one line so the whole snippet stays - # inside the blockquote - snippet = " ".join(content[:RESULT_SNIPPET].split()) - ellipsis = " …" if len(content) > RESULT_SNIPPET else "" - body.append(f"> {_fence(snippet)}{ellipsis}\n") - - elif t == "result": - metrics = d - - turns = metrics.get("num_turns", 0) - cost = metrics.get("total_cost_usd", 0.0) or 0.0 - duration_s = (metrics.get("duration_ms", 0) or 0) // 1000 - duration = f"{duration_s // 60}m{duration_s % 60:02d}s" - tool_calls = sum(tool_counts.values()) - tools_breakdown = ", ".join(f"{n} ×{c}" for n, c in tool_counts.most_common()) - - header = [ - "# Agent trace", - "", - f"- **Model:** {model or '?'}", - f"- **Turns:** {turns}", - f"- **Tool calls:** {tool_calls} ({tools_breakdown})", - f"- **Cost:** ${cost:.2f}", - f"- **Agent wall time:** {duration}", - f"- **Errored:** {metrics.get('is_error', '?')}", - "", - "---", - "", - ] - out_path.write_text("\n".join(header) + "\n".join(body)) - - print(f"model={model}") - print(f"turns={turns}") - print(f"tool_calls={tool_calls}") - print(f"cost={cost:.2f}") - print(f"duration={duration}") - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 949205c9..49b5723b 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -1,21 +1,18 @@ name: Eval -# Minimal agentic eval: scaffold a project from the eval seed on the -# runner, let Claude Code (with the astra plugin) build it using the -# branch's lc/astra, then check the results with astra validate and -# lc materialize --check. +# The agentic eval, hosted in LightconeResearch/lightcone-evals beside the +# task it runs; see that workflow's header and README. This repository is +# the side under test: its commit pins the task image and its secrets run +# the legs. on: - workflow_dispatch: pull_request: - # `ready_for_review` beside the defaults, so flipping a draft to - # ready triggers the run the draft guard below skips. + # `ready_for_review` beside the defaults, so flipping a draft to ready + # triggers the run the draft guard skips types: [opened, synchronize, reopened, ready_for_review] - -# Only one eval per PR at a time — cancel in-progress runs -concurrency: - group: eval-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: true + push: + branches: [main] + workflow_dispatch: permissions: contents: read @@ -23,215 +20,7 @@ permissions: jobs: eval: - if: github.event_name == 'workflow_dispatch' || github.event.pull_request.draft == false - runs-on: ubuntu-latest - # Step-level timeout on the agent step (below) is what actually - # bounds the run: a job-level timeout would cancel the always() - # report/comment/artifact steps along with it. - timeout-minutes: 20 - steps: - - uses: actions/checkout@v4 - with: - # Full history so hatch-vcs sees the tags and stamps a truthful - # dev version into the installed engine (and every manifest's - # lc_version) instead of a 0.1.dev fallback - fetch-depth: 0 - - - name: Set up uv - # No python-version input: it would export an ambient UV_PYTHON, - # which lc's install-settings scrub then rightly warns about on - # every single invocation the agent makes. The interpreter is - # pinned per tool install below instead. - uses: astral-sh/setup-uv@v6 - with: - enable-cache: true - - - name: Install lightcone-cli + astra (uv tools) - # The engine is the host's uv tool — install the branch under - # test the way a user installs a release: `uv tool install`, - # which links `lc` and (via the mirrored entry points) the - # git-annex executables into ~/.local/bin. The commit under test - # is the checkout itself, stamped into the version by hatch-vcs. - # astra's executable belongs to astra-tools, so it is its own - # tool install, pinned to the version the branch depends on. No - # venv is activated: the agent's shell sees the tools exactly as - # an end user's would. - run: | - uv tool install --python 3.12 "$GITHUB_WORKSPACE" - uv tool install --python 3.12 "astra-tools==$(grep -oP 'astra-tools==\K[0-9][0-9.]*' pyproject.toml)" - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - - name: Install Claude Code + astra plugin - run: | - curl -fsSL https://claude.ai/install.sh | bash - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - export PATH="$HOME/.local/bin:$PATH" - # Skip first-run onboarding in headless mode - echo '{"hasCompletedOnboarding": true}' > "$HOME/.claude.json" - claude plugin marketplace add LightconeResearch/agent-skills - claude plugin install astra@lightcone-research - - - name: Set up eval project - run: | - # Global git identity so any repo the agent commits in works - # (ephemeral runner — nothing to pollute) - git config --global user.name Eval - git config --global user.email eval@lightcone - project="$RUNNER_TEMP/eval-project" - echo "PROJECT=$project" >> "$GITHUB_ENV" - lc init "$project" - # Overlay the task seed (astra.yaml, data/) on the scaffold - cp -r evals/tasks/snae/. "$project"/ - cd "$project" - # Regenerate baseline.yaml from the task astra.yaml's defaults — - # the scaffolded one matches the boilerplate spec, not the task's - rm -f universes/baseline.yaml - astra universe generate -n baseline \ - -d 'Default configuration using standard practices' - # No scientific deps are pre-seeded — declaring what the - # recipes import (`uv add`) is part of what the eval measures - # lc materialize refuses a dirty tree — commit the seed so the - # agent starts from a clean one - git add -A - git commit -q -m 'Seed the eval task' - - - name: Run Claude Code - id: agent - timeout-minutes: 15 - working-directory: ${{ env.PROJECT }} - env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - run: | - set -o pipefail - # Transcript goes outside the project so the agent's git - # status stays clean; uploaded via the agent-trace artifact - claude -p "$(cat "$GITHUB_WORKSPACE/evals/prompt.md")" \ - --output-format stream-json --verbose \ - --dangerously-skip-permissions \ - --max-turns 200 \ - | tee "$RUNNER_TEMP/claude-transcript.jsonl" - - - name: Check results - id: check - working-directory: ${{ env.PROJECT }} - run: | - astra validate astra.yaml - lc status - # The gate: nothing failed, nothing left to make — and at least - # one output actually got materialized - lc materialize --check --json | jq -e \ - '.up_to_date and ((.current | length) > 0)' - # The publication view: declaring a license and materializing - # once more is part of the task - test -f ro-crate-metadata.json - - - name: Build report - if: always() - env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - run: | - transcript="$RUNNER_TEMP/claude-transcript.jsonl" - if [ -s "$transcript" ]; then - python3 .github/scripts/trace_digest.py \ - "$transcript" "$RUNNER_TEMP/agent-trace.md" > "$RUNNER_TEMP/metrics.env" - source "$RUNNER_TEMP/metrics.env" - # Claude's read of where the agent struggled — best-effort, - # the report still posts if this fails or times out - timeout 180 claude -p "$(cat evals/trace-analysis-prompt.md)" \ - < "$RUNNER_TEMP/agent-trace.md" \ - > "$RUNNER_TEMP/pain-points.md" \ - || echo "_Trace analysis unavailable._" > "$RUNNER_TEMP/pain-points.md" - fi - - icon="✅" - [ "${{ steps.check.outcome }}" = "success" ] || icon="❌" - { - echo "## $icon Eval" - echo "" - echo "" - echo "" - echo "| Metric | Value |" - echo "|--------|-------|" - echo "| Outputs check | ${{ steps.check.outcome }} |" - echo "| Agent run | ${{ steps.agent.outcome }} |" - echo "| Turns | ${turns:-?} |" - echo "| Tool calls | ${tool_calls:-?} |" - echo "| Cost | \$${cost:-?} |" - echo "| Agent wall time | ${duration:-?} |" - echo "| Model | ${model:-?} |" - echo "" - echo '
lc status' - echo "" - echo '```' - (cd "$PROJECT" && lc status) || echo "unavailable" - echo '```' - echo "" - echo '
' - echo "" - if [ -s "$RUNNER_TEMP/pain-points.md" ]; then - echo '
Confusion & pain points (Claude analysis)' - echo "" - cat "$RUNNER_TEMP/pain-points.md" - echo "" - echo '
' - echo "" - fi - echo "Full trace: \`agent-trace\` artifact on [this run](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})." - } > "$RUNNER_TEMP/comment.md" - cat "$RUNNER_TEMP/comment.md" >> "$GITHUB_STEP_SUMMARY" - - - name: Comment on PR - if: always() && github.event_name == 'pull_request' - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - const body = fs.readFileSync(`${process.env.RUNNER_TEMP}/comment.md`, 'utf8'); - const marker = ''; - const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); - const existing = comments.find(c => c.body.includes(marker)); - if (existing) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: existing.id, - body, - }); - } else { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body, - }); - } - - - name: Upload agent trace - if: always() - uses: actions/upload-artifact@v4 - with: - name: agent-trace - path: | - ${{ runner.temp }}/claude-transcript.jsonl - ${{ runner.temp }}/agent-trace.md - ${{ runner.temp }}/pain-points.md - if-no-files-found: warn - - - name: Upload eval project - if: always() - uses: actions/upload-artifact@v4 - with: - name: eval-project - # include-hidden-files keeps the manifests, the run records in - # .git, and the annexed results; the venv and lc's per-run - # machine state are excluded - path: | - ${{ env.PROJECT }} - !${{ env.PROJECT }}/.venv/** - !${{ env.PROJECT }}/.lightcone/** - include-hidden-files: true - if-no-files-found: warn + uses: LightconeResearch/lightcone-evals/.github/workflows/eval.yml@main + secrets: inherit + with: + stack_ref: ${{ github.event.pull_request.head.sha || github.sha }} diff --git a/CLAUDE.md b/CLAUDE.md index 70b35fe2..7da3f6df 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -192,7 +192,6 @@ src/lightcone/ # namespace — NO __init__.py ├── __init__.py # loader; a renderer only where there is a value to decide └── files/*.tmpl # the templates themselves, as real files -evals/ # agentic eval seed: prompt.md + tasks// tests/ # pytest — mirrors src/ ``` @@ -209,9 +208,14 @@ uv build # wheel + sdist (CI runs this only to publish) Test, lint and type-check are the whole loop, and they are what `.github/workflows/{tests,lint}.yml` run. There is deliberately no task runner in between — the pre-rebuild `justfile` was 90 lines of wrappers -around them. The other workflows are `eval.yml` (the agentic eval, -on dispatch or the `run-eval` PR label; re-trigger by re-adding the -label) and `pypi-publish.yaml`. +around them. The other workflows are `eval.yml` and `pypi-publish.yaml`. + +`eval.yml` is the agentic eval, on every non-draft PR, every push to +main and dispatch. It is a caller only: the workflow lives in +`LightconeResearch/lightcone-evals` beside the task it runs (its README +holds the contract), and this repository hands it the commit under test +as `stack_ref` and its own secrets. A PR fails on a reward below 1.0, a +failed oracle, or an errored trial or analysis. ## Key Invariants (layer 1) @@ -712,7 +716,7 @@ must not discard edits made elsewhere while the graph was running. `definition_version = sha256(recipe ‖ canonical decisions ‖ format)` is what the spec says an output *is*. The format is framed in its own right rather than left to arrive through the rendered `{output}` path: a recipe -need not use that placeholder at all (`evals/tasks/snae` does not), and +need not use that placeholder at all (the eval's `snae-build` seed does not), and one that does not would keep its digest across a re-declared serialization — while the sidecar, named from the id alone, does not move either, so the manifest would go on describing an output at a path that diff --git a/evals/prompt.md b/evals/prompt.md deleted file mode 100644 index 87dd91d4..00000000 --- a/evals/prompt.md +++ /dev/null @@ -1,159 +0,0 @@ -Build the analysis specified in `astra.yaml` for universe `baseline`. - -## Toolchain - -This project is driven by two CLIs — use them rather than improvising: - -- `astra` is the spec layer. `astra info` summarizes `astra.yaml`; - `astra validate astra.yaml` checks it against the schema. If an `astra` - skill or plugin is available in your environment, load it before reading - or editing `astra.yaml` — it documents the full spec format. -- `lc` (lightcone-cli) is the execution layer: - - `lc materialize ` makes every output the spec declares, running each - recipe in dependency order and committing each result to git as it - lands, together with a provenance manifest. It refuses to start on - a dirty tree: commit your own edits first, with plain `git add` and - `git commit` — the project's git-annex filter handles large files - transparently, so never run a git-annex command yourself. - - `lc materialize ` (or `/`) narrows - a run to one output and whatever it depends on. Re-running is - idempotent: only what is stale gets remade — an output the spec now - defines differently, or one whose declared inputs changed. - - `lc status` reports each output as `current`, `stale`, or `behind`, - with the commit it was made at; `lc status --json` is the - machine-readable form. It always exits 0. The pass/fail gate is - `lc materialize --check`, which exits 1 while anything still needs - making. `--check` needs no cluster. - - `lc run -- ` runs an ad-hoc command in the project - environment under the same isolation a recipe gets — useful for - probing why a recipe would fail. Argv style, like `docker run` or - `uv run`: `lc run -- python scripts/fit.py --output /tmp/x`, never a - single quoted shell string; for shell syntax use - `lc run -- bash -c '...'`. - - Outputs land in `results/baseline/.`, each with a - `..manifest.json` manifest beside it, written and - committed by the engine. Never write into `results/` yourself: a - hand-placed file has no run record, and the engine detects the - foreign write and remakes the output. - - When a recipe fails, `lc materialize` reports which output failed - and why; fix the script or the spec, commit, and re-run. - -Allocate compute before running commands or recipes: `lc compute launch --wait`. -With no CPU/memory flags, this starts a cluster named `local` using all detected -usable CPUs and RAM on this machine and waits until it is ready. No configuration -is needed. GPUs still require explicit offers. Add `--name analysis` to choose -another name. Replace `` in these commands with the returned name -(normally `local`) or a full immutable ID from launch or status JSON (`--json`). -Reuse the cluster with `run` and `materialize`; neither creates compute automatically. - -Only one local cluster can run per user on this machine, even with different names -or catalogs. If one already exists, use the catalog identified in the refusal to -inspect `lc compute status` and reuse it rather than launching another. -`lc compute status --wait` waits for an -existing cluster. Launch's `--wait` defaults to a 300-second readiness timeout; -`--timeout SECONDS` overrides it. A waiting launch that fails reports the accepted -cluster ID and leaves the allocation unchanged: inspect it before retrying. -A local cluster ends after 30 minutes without task activity; running work keeps -it alive, and `--time` adds a hard lifetime that ends it even mid-run. After it -ends, launch again; the name `local` can be reused, but the immutable ID changes. - -Compute configuration is `~/.lightcone/compute.yaml`, or the file selected by -`LC_COMPUTE_CONFIG`. A catalog that lists its own local offers (`provider: local`) -uses those instead of the built-in one, which is how to set a smaller CPU/RAM -budget. Remote offers otherwise coexist with the default local offer. If -`allow_local: false` is configured, respect that policy: local launch and execution -are disabled. Inspect `lc compute resources` and supply both -`--cpus` and `--memory` to select a configured remote allocation; `--wait` works -there too. The no-resource shortcut never selects remote compute automatically. -Recognized NERSC login nodes refuse local compute automatically, even without a -catalog. Use Slurm or an interactive compute-node session; local compute remains -eligible on those compute nodes. Do not try to override the login-node guard. - -## Recipe template grammar - -A recipe's `command` is a template. The engine substitutes these -placeholders before invoking it: - -- `{output}` — the file the output is materialized to, - `results//.`, where `format` is the one the - output declares. Your script must write exactly that path, and nothing - else: an output is one file. The engine creates the directory; a recipe - that writes a directory there, or writes some other name, fails. -- `{inputs.}` — the named input's resolved path: an analysis-level - `Input`'s `source` (e.g. a file under `data/`), or, for an upstream - output, that output's own file — so your script opens it directly. -- `{inputs}` — space-separated paths of all declared inputs, in - declaration order. -- `{decisions.}` — the active option ID for the named decision in the - current universe (e.g. `nelder_mead`), which your script should accept - as an argparse choice. -- `{{` and `}}` emit literal braces. Format specs (`{x:>8}`) are rejected. - -Provenance is declared on the Output, not inside the recipe: every -`{inputs.}` / `{decisions.}` the command references must be listed -in that output's `inputs:` / `decisions:` lists, or validation fails. -Dependencies between outputs come from these `inputs:` declarations — that -is how the engine orders the build. - -## Environment - -Recipes run in the project's own locked environment (`pyproject.toml` + -`uv.lock` + `.venv`), sandboxed: the project tree is read-only apart from -the directory each recipe's output lands in, and only declared tools are -executable. - -- The project is managed by uv and starts with **no dependencies**. - Every package a recipe script imports must be declared before - materializing: run `uv add [ ...]` in the project - root (e.g. `uv add numpy scipy`). That updates `pyproject.toml`, - re-locks `uv.lock`, and syncs `.venv` in one step — commit all of it - along with your scripts, like any other edit. -- To remove a package use `uv remove `; to pin a version, - `uv add 'numpy>=2'`. Do **not** use plain `pip` or `uv pip install` — - an install that bypasses the lock reaches nothing a recipe sees. -- A sandbox denial names the path or tool that was denied and the - remedy — follow the remedy rather than working around the sandbox. - -## Build loop - -`astra.yaml` is the single source of truth: inputs, outputs, recipes, and -methodological decisions all live there — read it first. The seed spec is -deliberately incomplete: outputs declare no `format:`, recipe commands do -not yet pass their inputs, decisions, or output path, and outputs may be -missing entries in their `inputs:` / `decisions:` contracts. Completing the spec is part of -the task. For each output: - -1. Declare the output's `format:` — the file extension its artifact is - written with, without the leading dot (`png`, `csv`, `json`, …). lc - names the file from it and refuses a spec that omits it. -2. Complete the recipe `command` so it references `{output}` and the - `{inputs.}` / `{decisions.}` the computation needs, and - declare everything it references in that output's `inputs:` / - `decisions:` lists. -3. Write the script at the path the command names, parameterizing every - decision via argparse — never hardcode option values. -4. Commit your edits, then run `lc materialize ` (or - `lc materialize `) to build through the engine. - -Build iteratively from upstream outputs to downstream. `lc status` shows -where every output stands. - -## Publication - -Once every output is materialized, prepare the repository for -publication: - -1. Declare a license in `pyproject.toml`, as an SPDX expression under - `[project]` — e.g. `license = "CC-BY-4.0"`. Declaring one is what - turns publication on: from then on `lc materialize` also maintains - `ro-crate-metadata.json` at the project root, an RO-Crate view of - the project and its provenance. -2. Commit the edit, then run `lc materialize ` once more — nothing is - remade, but the crate document is generated and committed. - -You're done when `astra validate astra.yaml` and -`lc materialize --check` pass and `ro-crate-metadata.json` exists. -Release your allocation with `lc compute down ` when finished. - -Skip plan approval and interactive confirmations — this is an automated -eval run. diff --git a/evals/tasks/snae/astra.yaml b/evals/tasks/snae/astra.yaml deleted file mode 100644 index cca7363a..00000000 --- a/evals/tasks/snae/astra.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# ASTRA Analysis Specification -# Documentation: https://github.com/LightconeResearch/ASTRA - -version: "0.0.14" -name: "snae" -description: | - Fit the Union2.1 Type Ia supernova distance modulus vs redshift data - to a flat LCDM cosmological model with two free parameters (H0, Omega_L) - using maximum-likelihood (MAP) point estimation. This provides best-fit - cosmological parameters as a building block for a larger analysis. - -inputs: - - id: union21 - type: data - source: data/SCPUnion2.1_mu_vs_z.txt - description: "Union2.1 SN Ia compilation: 580 supernovae with redshift, distance modulus, and uncertainties" - -outputs: - - id: best_fit - type: metric - description: "Best-fit H0 and Omega_L from chi-squared minimization, with reduced chi-squared" - recipe: - command: python scripts/fit.py - - - id: hubble_diagram - type: figure - description: "Hubble diagram: distance modulus vs redshift with best-fit model overlay" - inputs: [best_fit] - recipe: - command: python scripts/plot_hubble.py - - - id: residuals - type: figure - description: "Residuals plot: data minus best-fit model vs redshift" - inputs: [best_fit] - recipe: - command: python scripts/plot_residuals.py - -decisions: - optimizer: - label: "Optimizer" - tags: [inference] - rationale: "Nelder-Mead is robust and derivative-free; L-BFGS-B supports bounds natively" - default: nelder_mead - options: - nelder_mead: - label: "Nelder-Mead" - description: "Derivative-free simplex optimizer" - l_bfgs_b: - label: "L-BFGS-B" - description: "Bounded quasi-Newton optimizer" - - error_model: - label: "Error Treatment" - tags: [data] - rationale: "Union2.1 provides both statistical-only and systematics-included uncertainties; choice directly affects fit quality" - default: stat_and_sys - options: - stat_only: - label: "Statistical errors only" - description: "Use only the statistical distance modulus uncertainties" - stat_and_sys: - label: "Statistical + systematic errors" - description: "Include systematic uncertainty contributions" - - redshift_cut: - label: "Low-Redshift Cut" - tags: [data] - rationale: "SNe at z < 0.01 are dominated by peculiar velocity scatter rather than cosmological signal; standard practice varies on whether to exclude them" - default: z_min_001 - options: - none: - label: "No cut" - description: "Use all supernovae regardless of redshift" - z_min_001: - label: "z > 0.01" - description: "Exclude supernovae with z < 0.01 to mitigate peculiar velocity contamination" diff --git a/evals/tasks/snae/data/SCPUnion2.1_mu_vs_z.txt b/evals/tasks/snae/data/SCPUnion2.1_mu_vs_z.txt deleted file mode 100644 index b3d48e97..00000000 --- a/evals/tasks/snae/data/SCPUnion2.1_mu_vs_z.txt +++ /dev/null @@ -1,585 +0,0 @@ -# alpha 0.121851859725 -# beta 2.46569277393 -# delta -0.0363405630486 -# M(h=0.7, statistical only) -19.3182761161 -# M(h=0.7, with systematics) -19.3081547178 -1993ah 0.028488 35.3465833928 0.223905932998 0.128418942246 -1993ag 0.050043 36.6823679154 0.166828851413 0.128418942246 -1993o 0.052926 36.8176912545 0.1557559148 0.128418942246 -1993b 0.070086 37.4467365424 0.158466934433 0.128418942246 -1992bs 0.062668 37.4834093505 0.156099434739 0.128418942246 -1992br 0.087589 38.2290570494 0.187745679272 0.128418942246 -1992bp 0.078577 37.4881622607 0.155635656185 0.128418942246 -1992bo 0.017227 34.6543699503 0.199337179559 0.128418942246 -1992bl 0.042233 36.3364595483 0.167174042338 0.128418942246 -1992bh 0.045295 36.6402721756 0.164981248644 0.128418942246 -1992bg 0.03648 35.9053219652 0.170174952845 0.128418942246 -1992bc 0.019599 34.5852174312 0.184691219687 0.128418942246 -1992aq 0.100915 38.4567455954 0.167333481677 0.128418942246 -1992ag 0.027342 35.085765693 0.175510835947 0.128418942246 -1992ae 0.074605 37.5881157565 0.15977086456 0.128418942246 -1992p 0.026489 35.4806851993 0.19131226974 0.001870261557 -1990af 0.049922 36.5669734706 0.162303819627 0.128418942246 -1990o 0.030604 35.5502377594 0.173295444142 0.128418942246 -2001cz 0.016345641 34.0440277752 0.142912931364 0.128418942246 -2001cn 0.0154363 33.9409483971 0.148694109648 0.128418942246 -2001ba 0.030529 35.5992457186 0.088750663709 0.128418942246 -2000ca 0.024525 35.0581706649 0.102438504345 0.128418942246 -2000bh 0.023953 34.9687103775 0.107041197195 0.128418942246 -1999gp 0.026038 35.3672620682 0.108499792159 0.128418942246 -1993ac 0.048948 36.7315973987 0.172547618731 0.0 -1994m 0.024314 35.1094950645 0.181662706307 6.1999960721e-11 -1994s 0.015166 34.1016666152 0.215239340732 0.009354993721 -1994t 0.03572 35.9605406407 0.171186987363 0.79821525066 -1995ac 0.048818 36.3820107761 0.160299265066 5.14460000156e-08 -1995ak 0.0219800059146 34.8529733574 0.187544764358 0.128418942246 -1996ab 0.1244 39.0447885112 0.164268687505 0.128418942246 -1996bl 0.036 35.8210170957 0.167885249541 0.128418942246 -1996bo 0.016321 34.0174211141 0.204965074268 0.128418942246 -1996bv 0.01673 34.2263371692 0.209464670311 0.128418942246 -1996c 0.0275 35.6497059082 0.176364688958 0.218251065653 -2000fa 0.021793 34.9737868712 0.232449382954 0.128418942246 -2000dk 0.01645 34.1812962858 0.250896560248 0.128418942246 -2000cn 0.023208 35.0855427174 0.231479604833 0.128418942246 -2000cf 0.036457 36.1342331256 0.217628588382 0.128418942246 -1999gd 0.019264 34.9526137319 0.24063381695 0.128418942246 -1999ek 0.017605 34.3437956957 0.271650644819 0.128418942246 -1999cc 0.031528 35.7287687764 0.225362488845 0.0 -1998eg 0.023536 35.1695990888 0.234698201629 0.128418942246 -1998ef 0.016743 34.0027278026 0.248385125526 0.128418942246 -1998dx 0.05371 36.4764384949 0.221700260552 0.128418942246 -1998co 0.016991 34.3787718073 0.309889810973 0.128418942246 -1998ab 0.027865 35.0933783283 0.222426416634 0.00016313449 -1998v 0.017173 34.2606714573 0.248216009365 0.128418942246 -1997dg 0.029955 35.9722578275 0.224368551768 0.128418942246 -1997y 0.016559 34.3438338071 0.251177496491 0.018543670198 -1999aa 0.015 34.1635038861 0.161452856549 0.128418942246 -1999ao 0.0544 36.9544354105 0.0860953525286 0.128418942246 -1999ar 0.1561 39.2292540222 0.0841441236726 0.128418942246 -1999aw 0.0393 36.3343950149 0.0991005283825 0.128418942246 -1999bi 0.1241 38.8220333958 0.111614246562 0.128418942246 -1999bm 0.1441 38.8360422985 0.156777491835 0.128418942246 -1999bn 0.1299 38.979185469 0.12944495155 0.128418942246 -1999bp 0.0784 37.6822404534 0.0872168046336 0.128418942246 -2001ah 0.0583 37.0326301671 0.206256669369 0.0 -2001ay 0.0309 35.9294728801 0.183689296262 0.128418942246 -2001az 0.0406 36.3656351257 0.172317344841 0.128418942246 -2001bf 0.0152 34.0169042989 0.21507123117 0.128418942246 -2001cp 0.0224 34.9470872022 0.239011369258 0.128418942246 -2001da 0.016 34.1740153938 0.221115374688 0.137610159633 -2001eh 0.0362 35.9868705998 0.171742204068 0.128418942246 -2001g 0.0173 34.2497347985 0.215508499624 0.128418942246 -2001ie 0.0312 35.6268097646 0.180216697419 0.0 -2001n 0.0221 34.9115497582 0.189960682839 2.64313000042e-07 -2001v 0.016 33.824608888 0.207959703185 0.0 -2002bf 0.0249 34.8037070766 0.193523924165 4.231227e-06 -2002ck 0.0303 35.6282613469 0.17674738626 4.96800001137e-09 -2002de 0.0283 35.5202699542 0.180124445838 0.128418942246 -2002do 0.0152 34.2583697939 0.241474862259 0.128418942246 -2002g 0.0345 35.9782444842 0.211562474249 9.21700005208e-09 -2002hd 0.036 35.679259682 0.179022122406 0.128418942246 -2002he 0.0248 35.2561775084 0.184605302796 0.128418942246 -2002hu 0.0292 35.9925667846 0.174820107635 0.128418942246 -2002hw 0.0163 34.453252912 0.212891042717 0.128418942246 -2002jy 0.0187 35.0482998472 0.197196789214 0.012399792616 -2002kf 0.0195 34.75691168 0.197526188093 0.128418942246 -2003ch 0.0256 35.6847217932 0.186173891969 0.128418942246 -2003cq 0.0337 35.8436933159 0.179737453597 0.0 -2003ic 0.0546 36.6095585146 0.176655181089 0.0 -2003it 0.024 35.1761857794 0.195512674436 0.128418942246 -2003iv 0.0336 36.0053945507 0.18852047363 0.128418942246 -2003kc 0.0341 35.8419046981 0.175222461707 0.128418942246 -2003u 0.0261 35.3604177873 0.192469561714 0.004314509643 -2003w 0.0211 34.6607182426 0.188393367669 0.128418942246 -2004as 0.0321 35.8959945722 0.173845159832 0.128418942246 -2004bg 0.0221 34.9221291018 0.189014756295 0.128418942246 -2004l 0.0334 35.8799403015 0.1757727334 9.24477300002e-06 -2005eu 0.0341 35.9425408413 0.172834591542 0.128418942246 -2005hf 0.0421 36.4004650275 0.169930417471 0.128418942246 -2005hj 0.0576 37.0802566571 0.161852672221 0.996265053657 -2005ls 0.0205 34.6172800695 0.191412743444 0.128418942246 -2005lz 0.0402 36.3745330785 0.17063734958 0.128418942246 -2005mc 0.026 35.3812932276 0.183933665117 1.7670300001e-07 -2005ms 0.0259 35.4159686901 0.178489658823 0.033789980906 -2006ac 0.0239 35.0349741436 0.181392417208 1.09587499997e-06 -2006al 0.069 37.566047978 0.176163639895 3.07778750001e-05 -2006an 0.0651 37.3066907976 0.162808584338 1.0 -2006ar 0.0229 35.1968671677 0.185459075264 0.089659819877 -2006az 0.0315 35.651136343 0.170993523745 0.0 -2006bq 0.0215 34.9327363282 0.187040817965 1.1733000016e-08 -2006br 0.0255 35.7201731359 0.196167302112 0.128418942246 -2006bt 0.0325 35.8130914151 0.170394735896 0.0 -2006bu 0.0843 38.0518311208 0.200253645227 0.128418942246 -2006bw 0.0308 35.6289808082 0.178370444318 0.128418942246 -2006cc 0.0327 36.0941672133 0.16949230504 0.128418942246 -2006cf 0.0423 36.392825896 0.171582196702 0.0 -2006cj 0.0684 37.7311969845 0.170591643132 0.0 -2006cm 0.0153 34.7071857289 0.213242896074 0.128418942246 -2006cp 0.0233 34.8821286745 0.183424685088 0.011101109848 -2006cq 0.0491 36.7301358821 0.175843223164 0.0 -2006cz 0.0425 35.9281018139 0.195296013078 0.128418942246 -2006ej 0.0192 34.7366784514 0.197842881265 8.36000046966e-10 -2006en 0.0308 35.7791362444 0.173310004596 0.128418942246 -2006et 0.0212 34.8471345313 0.19152972211 0.128418942246 -2006gj 0.0277 35.7050037726 0.183298735051 0.128418942246 -2006gr 0.0335 35.9738325393 0.170490251613 0.128418942246 -2006kf 0.0208 34.7954563657 0.201629405401 0.128418942246 -2006le 0.0173 34.2300660446 0.220994474003 0.128418942246 -2006mo 0.036 36.146298861 0.177000363965 0.128418942246 -2006mp 0.0233 35.1983574195 0.185738575456 0.128418942246 -2006oa 0.0589 37.1116475131 0.16568235078 1.0 -2006ob 0.0583 37.0582881494 0.16808968138 0.0 -2006on 0.0688 37.4866356466 0.198025808769 0.027111473488 -2006os 0.0321 35.6480768335 0.175025422739 0.128418942246 -2006ot 0.0522 36.6743146094 0.189117803003 0.128418942246 -2006qo 0.0308 35.5935042223 0.172493084423 0.128418942246 -2006s 0.0329 35.9417685428 0.170393175896 0.00076391263 -2006sr 0.023 35.0706425048 0.186848384979 0.128418942246 -2006td 0.015 34.3797728047 0.217348534701 0.128418942246 -2006te 0.0321 35.8703618342 0.1737410077 0.000101920284 -2007ae 0.0643 37.1755022179 0.166008010074 0.128418942246 -2007ai 0.032 35.8266913419 0.193104691662 0.128418942246 -2007au 0.0209 34.7028984995 0.19422693515 0.128418942246 -2007bc 0.0219 34.8489904808 0.188387184264 3.80834000002e-07 -2007bd 0.032 35.5884973241 0.173446310936 7.81762000024e-07 -2007ca 0.0151 34.5258752881 0.213682160531 0.128418942246 -2007ci 0.0192 34.4915623332 0.197091647943 0.006396224841 -2007co 0.0266 35.3234935897 0.177716836428 0.128418942246 -2007cp 0.0377 35.7952144325 0.294106560612 0.128418942246 -2007cq 0.0247 34.9136928277 0.181621011598 0.128418942246 -2007f 0.0242 35.1893805829 0.180174916343 0.435402819564 -2007o 0.0366 35.9712906539 0.168963687947 0.128418942246 -2007qe 0.0229 35.1386260702 0.183018039949 0.128418942246 -2007r 0.0312 35.8828585622 0.173762463024 0.0 -2007s 0.015 34.1114109643 0.213388869977 0.128418942246 -2008af 0.0341 35.770474764 0.177569140796 0.0 -2008bf 0.0251 34.9482752367 0.179222700336 0.0 -2008l 0.0189 34.3747016599 0.209571630196 0.10474237468 -2004ef 0.029802136901 35.4709217104 0.122226817874 0.128418942246 -2004gc 0.0321340168614 35.3781278853 0.127512090116 0.128418942246 -2004gs 0.0275687259109 35.4754718132 0.128117983088 0.128418942246 -2004gu 0.0469673346156 36.4944134465 0.110309142172 0.128418942246 -2005a 0.0183152320362 34.3717548996 0.160464926406 0.128418942246 -2005ag 0.0800481440021 37.6857093353 0.101968492144 0.128418942246 -2005bg 0.0241852990802 35.052324275 0.133368622118 0.128418942246 -2005bo 0.0150270427632 33.9501915023 0.178117024083 0.128418942246 -2005eq 0.0283960268116 35.5481484694 0.125198881719 9.90000303958e-11 -2005hc 0.0449766726113 36.5503954158 0.108342547225 0.930467219184 -2005iq 0.0329123710881 35.9678520173 0.118114324121 0.128418942246 -2005ir 0.0753501119815 37.5800899168 0.102274672969 0.996080571775 -2005ki 0.0203747245299 34.6564438473 0.145603349944 0.000769867316 -2005m 0.0229711675782 35.133882951 0.135640926041 0.128418942246 -2005na 0.0268091974805 35.2601505395 0.127879595703 0.128418942246 -2006ax 0.0179312833626 34.3498200027 0.155413940574 0.128418942246 -2006eq 0.0483921953954 36.6858343824 0.119215628214 0.128418942246 -2006py 0.0566833670752 36.9636230671 0.110070937939 0.128418942246 -2005kt 0.0638640836964 37.3159416121 0.119614965162 0.551671793295 -10106 0.146290296068 39.5590095488 0.120248735474 0.551671793295 -2005ez 0.129278206634 38.9180009489 0.134402615626 2.58499999184e-09 -2005lk 0.102715033935 38.4872595888 0.119892508421 0.551671793295 -2005ll 0.242504680476 40.1304465537 0.150915056663 0.551671793295 -2005lf 0.29840927363 41.0624002336 0.21581859647 0.551671793295 -2005ku 0.0437189114895 36.3865660158 0.127214292529 0.551671793295 -2005ml 0.113042644637 38.5514541276 0.119237437717 0.551671793295 -2005jb 0.256475742725 40.6108439003 0.156286225391 0.551671793295 -2005fc 0.295585550063 41.1370597018 0.215064876219 0.551671793295 -2005mm 0.380359487336 41.657498659 0.217910527631 0.551671793295 -2005ln 0.145668547261 39.056715703 0.125750126249 0.551671793295 -2005mo 0.273454768884 40.7271582441 0.163668270948 0.551671793295 -2005lo 0.297518833561 40.7722050136 0.205185826917 0.551671793295 -2005lq 0.378965802039 41.5814889023 0.197094007682 0.551671793295 -1166 0.380416514286 41.2723405299 0.198558434394 0.551671793295 -2005lp 0.301755029535 41.4938104959 0.246761480943 0.551671793295 -2005mq 0.348345020835 41.3077915312 0.217774889061 0.551671793295 -2005ff 0.0856894593607 37.9974134061 0.116365280048 0.004916236676 -2005fd 0.260586108317 40.5231676777 0.141595051803 0.551671793295 -2005fe 0.215543320953 40.2965294236 0.140207657342 0.551671793295 -2005fh 0.117625328764 38.5794185081 0.113609962836 7.41867000054e-07 -2005gj 0.18221823956 39.5938188438 0.119334463704 0.551671793295 -1688 0.357507357032 41.3288394045 0.202587682493 0.551671793295 -2005fj 0.141787999126 39.2738113329 0.119521982693 0.551671793295 -2005fo 0.260533476953 40.8081341413 0.148279348857 0.551671793295 -2005fl 0.232781107182 40.2045290903 0.15178836784 0.551671793295 -2005fm 0.151857894665 39.155788359 0.113847311938 0.989888621753 -2005fn 0.0939086318905 38.1728205141 0.117952906116 0.551671793295 -2005fr 0.286618706522 41.0384076468 0.133800275646 0.551671793295 -2005fy 0.194316512142 39.9615200415 0.126373844923 0.551671793295 -2005ey 0.147025138268 39.3035839668 0.111657296738 4.30000479668e-11 -2005fp 0.2115869815 40.5603236924 0.159324733885 0.551671793295 -2005ft 0.180119779968 39.6385777035 0.115560254718 0.551671793295 -2005fi 0.263491026592 40.7673226085 0.127293080139 0.551671793295 -2005fu 0.192149980044 40.0392178782 0.121087443202 0.001229231543 -2005fs 0.338802609134 41.3053718801 0.170908367946 0.551671793295 -2005fv 0.117277362889 38.7459338248 0.113702596154 0.551671793295 -2005fw 0.1424046524 39.1164772511 0.113661193986 0.830402042657 -2005fa 0.160861854636 39.3191758892 0.11864489981 9.9997787828e-13 -2005fx 0.288418344585 40.8427853587 0.149598782119 0.551671793295 -2005fz 0.12282889997 38.7997897008 0.12106543913 0.551671793295 -2005go 0.263647950642 40.5481756685 0.139823620366 0.551671793295 -2005gp 0.126473161901 38.7104691761 0.119200794649 0.586055835562 -2005ga 0.172742230556 39.5029361421 0.113458015916 0.551671793295 -2005gc 0.163795894987 39.3945630608 0.113134872307 1.0 -2005gs 0.249511054792 40.7809243328 0.132323047466 0.551671793295 -2005gh 0.257740303509 40.6514783957 0.135891055781 0.551671793295 -2005hn 0.106712340103 38.6366669013 0.117621429285 0.867576933443 -2005gd 0.159889937748 39.3505845262 0.114747057769 0.999968604262 -2005ge 0.204979685349 40.0245002346 0.123526942937 0.551671793295 -2005gr 0.244378876797 40.2019303938 0.124309900698 0.551671793295 -2005gf 0.248508130676 40.2721746838 0.125645589328 0.551671793295 -2005gg 0.228528473881 40.2544336386 0.12201137132 0.551671793295 -2005gb 0.0858546441437 37.9524441212 0.112589853288 1.0 -2005ho 0.0618357646938 37.1309944806 0.115680071342 0.987710988495 -2005gt 0.277853422619 40.8380084515 0.168994207516 0.551671793295 -2005gw 0.275440197435 40.743391802 0.142941615821 0.551671793295 -4064 0.155247328221 39.3173204059 0.115415755865 0.551671793295 -2005gu 0.330512448683 41.2279390018 0.156723683034 0.551671793295 -2005gv 0.361934308832 41.3844533517 0.166441914984 0.551671793295 -2005gy 0.33063462833 41.0522598891 0.152699339448 0.551671793295 -2005gx 0.144621085972 39.290829773 0.112727024487 1.0 -2005gq 0.389288787688 41.6026284289 0.200867575788 0.551671793295 -2005hp 0.173910056566 39.5319411053 0.128967775815 0.551671793295 -2005hs 0.300312695802 40.8468481928 0.15150098613 0.551671793295 -2005hr 0.116348502599 38.7461381651 0.112296117135 1.0 -2005hu 0.218585189041 40.1990974567 0.117113950953 0.551671793295 -2005hx 0.119671538006 38.7555381178 0.110740033634 0.551671793295 -2005hy 0.154632097239 39.3239502295 0.112047393667 0.999997454964 -2005hw 0.408319092164 41.840293526 0.206369618511 0.551671793295 -2005hv 0.177600694977 40.070675851 0.12481415503 1.0 -2005ia 0.25066763111 40.7456030264 0.120970959661 0.551671793295 -2005jz 0.251740186424 40.5417217599 0.118440110682 0.551671793295 -5737 0.391599213441 41.5087271022 0.171564315895 0.551671793295 -2005hz 0.128726735467 38.8642253068 0.109513340257 0.551671793295 -2005hq 0.399601334976 41.7288003127 0.205132433748 0.551671793295 -2005ic 0.309492645411 41.0181924732 0.134960171498 0.551671793295 -2005is 0.170628396749 39.474900116 0.116376625267 0.551671793295 -2005ie 0.278924676328 40.7678297655 0.130157044183 0.551671793295 -2005it 0.308580866071 40.9076217053 0.177652782589 0.551671793295 -2005ht 0.18581244671 39.8240975844 0.120667757481 0.551671793295 -2005if 0.0664403123402 37.3721050548 0.1175824014 0.543563084394 -2005ka 0.316429844695 41.3950733806 0.192866122248 0.551671793295 -2005ih 0.257497888297 40.5941645935 0.131393591319 0.551671793295 -2005iv 0.298777444088 41.0293579818 0.154536935815 0.551671793295 -2005jy 0.270434442827 40.668983836 0.145477928824 0.551671793295 -2005ig 0.27945473261 40.5012843665 0.138724732909 0.551671793295 -2005ii 0.292469756082 40.919048654 0.141400626542 0.551671793295 -2005jk 0.188853175387 39.7920374862 0.119741860505 7.8208394e-05 -2005ix 0.265762479518 40.4981499217 0.124803942242 0.551671793295 -2005ij 0.124273528772 38.719986901 0.111134858876 0.029911743888 -2005id 0.182548913397 39.7840215634 0.114083136974 0.986226043013 -2005jd 0.312883364158 41.0144487771 0.139506514174 0.551671793295 -2005ik 0.309547337158 41.1925567238 0.154012332803 0.551671793295 -2005iu 0.0890194292939 37.8284529286 0.117432705446 0.551671793295 -2005iy 0.402459619216 41.9111467578 0.211278493994 0.551671793295 -2005iz 0.200611719928 39.8523918219 0.132867364384 1.0 -2005ja 0.326396483203 40.972304176 0.144388065833 0.551671793295 -2005jc 0.211629598493 39.9801047509 0.115235180715 0.551671793295 -2005jl 0.17968564077 39.7209599451 0.116984880342 1.0 -2005je 0.0931494026855 38.280913725 0.114453830019 0.551671793295 -2005jg 0.30240162108 41.033200098 0.134910116135 0.551671793295 -2005jh 0.108638265524 38.6505001979 0.119489654857 0.551671793295 -2005ed 0.0856961170652 37.9937917107 0.117971237872 0.551671793295 -2005jm 0.202608679897 39.9677592554 0.128807331127 1.0 -2005kn 0.196716068579 39.9244173787 0.134983153598 0.551671793295 -2005ei 0.12668798909 38.9152084322 0.123937308071 0.551671793295 -2005ji 0.214568258658 40.2018496659 0.11698744578 0.551671793295 -2005jn 0.320446988861 41.102785135 0.134380311167 0.551671793295 -2005jo 0.218347445403 40.2208498377 0.124421502953 0.551671793295 -2005eg 0.189706566951 39.9395625109 0.119682792229 0.551671793295 -2005ex 0.0929368181997 38.1458300404 0.118950355954 0.551671793295 -2005jw 0.379662293575 41.4330618932 0.147219740299 0.551671793295 -2005jp 0.210938395061 39.983937734 0.120818770577 0.551671793295 -2005jj 0.366602898789 41.8598572351 0.198404429556 0.551671793295 -2005jv 0.420926821103 42.1335480049 0.202321001311 0.551671793295 -2005ju 0.258028270562 40.6446788667 0.137108498782 0.551671793295 -2005ko 0.183568404877 39.7107958033 0.123775734742 0.551671793295 -2005mi 0.212548765094 40.0875932773 0.13157696281 0.551671793295 -2005jt 0.360034211989 41.1573004359 0.175048186524 0.551671793295 -2005mh 0.393974478272 41.7811534223 0.150707319622 0.551671793295 -2005kp 0.11471262087 38.669095116 0.113372328428 1.0 -2005ld 0.143705907 39.1857706544 0.116311967547 0.372920925377 -2005le 0.252486060029 40.5184210801 0.145050195669 0.551671793295 -2005kq 0.387297107465 41.8924653292 0.18642986619 0.551671793295 -2005lg 0.348583857544 41.3009410359 0.177371887871 0.551671793295 -2005li 0.255490620022 40.4859446261 0.150967373278 0.551671793295 -2005lh 0.216582821954 40.3602717031 0.152479838933 0.551671793295 -1996e 0.43 41.3188581715 0.357827626062 0.551671793295 -1996h 0.62 43.2279620233 0.390344663066 0.551671793295 -1996i 0.57 42.4891103591 0.388952633173 0.551671793295 -1996j 0.3 40.9627951522 0.314865571912 0.551671793295 -1996k 0.38 42.063176534 0.327833114422 0.551671793295 -1996u 0.43 42.3985415069 0.458327873977 0.551671793295 -1995ao 0.24 40.7421915815 0.402844159873 0.551671793295 -1997ce 0.44 42.0495332736 0.319794878908 0.551671793295 -1997cj 0.5 42.3630032623 0.317337166801 0.551671793295 -1997ck 0.97 42.8212038325 0.805210302782 0.551671793295 -1995k 0.479 42.3517577625 0.358136299792 0.551671793295 -1997ap 0.83 43.5424034366 0.471046842772 0.551671793295 -1997am 0.416 42.4236170407 0.556342333092 0.551671793295 -1997aj 0.581 42.063423278 0.505033960368 0.551671793295 -1997ai 0.45 41.8319918495 0.453701962845 0.551671793295 -1997af 0.579 43.1863633442 0.643297806625 0.551671793295 -1997ac 0.32 41.2446034516 0.420998113276 0.551671793295 -1997r 0.657 42.9752487173 0.658728481435 0.551671793295 -1997q 0.43 41.7661368295 0.608328916424 0.551671793295 -1997p 0.472 41.9691632564 0.514472850425 0.551671793295 -1997o 0.374 43.1851146804 0.923198713714 0.551671793295 -1997n 0.18 40.1895173161 0.445447560867 0.551671793295 -1997l 0.55 44.3434675429 1.00681218041 0.551671793295 -1997k 0.592 44.1523548256 0.717963493004 0.551671793295 -1997i 0.172 39.3022260063 0.429709120635 0.551671793295 -1997h 0.526 41.9560808598 0.513679185483 0.551671793295 -1997g 0.763 44.4724245942 0.898163121417 0.551671793295 -1997f 0.58 43.3051250139 0.519962251252 0.551671793295 -1996cn 0.43 41.8019360973 0.458804990654 0.551671793295 -1996cm 0.45 42.2709731768 0.589947886789 0.551671793295 -1996ck 0.656 43.1448591059 0.627202505766 0.551671793295 -1996ci 0.495 42.1189081538 0.443653473108 0.551671793295 -1996cg 0.49 41.7874349383 0.446484505947 0.551671793295 -1996cf 0.57 42.6714667124 0.470382088569 0.551671793295 -1995ba 0.388 42.2076140366 0.465017827284 0.551671793295 -1995az 0.45 42.4024477254 0.508290617768 0.551671793295 -1995ay 0.48 42.1599485926 0.521420422571 0.551671793295 -1995ax 0.615 42.542636608 0.556222743429 0.551671793295 -1995aw 0.4 42.3127671586 0.477589020966 0.551671793295 -1995at 0.655 42.3150852482 0.497081624743 0.551671793295 -1995as 0.498 42.9837202371 0.642406504117 0.551671793295 -1995ar 0.465 41.8251633017 0.600030636232 0.551671793295 -1995aq 0.453 42.8212084765 0.524912951846 0.551671793295 -1994g 0.425 41.2021312228 0.489149082777 0.551671793295 -2002ad 0.514 42.7861936213 0.493735911043 0.551671793295 -2002ab 0.423 41.5655258816 0.243579000627 0.551671793295 -2002x 0.859 44.0929026019 0.296965045086 0.551671793295 -2001kd 0.936 43.3071058635 0.705981399741 0.551671793295 -2001jp 0.528 42.4543495693 0.24429067529 0.551671793295 -2001jm 0.978 43.5000721647 0.292647301809 0.551671793295 -2001jh 0.885 44.1843333063 0.286230184051 0.551671793295 -2001jf 0.815 44.0744016007 0.728921730286 0.551671793295 -2001jb 0.698 43.768728686 0.433480217975 0.551671793295 -2001iy 0.568 42.7092496229 0.289546896784 0.551671793295 -2001ix 0.711 43.5665495646 0.366725181041 0.551671793295 -2001iw 0.3396 41.0839733838 0.226527941211 0.551671793295 -2001iv 0.3965 41.4879207897 0.202374999057 0.551671793295 -2001hy 0.812 43.6535732762 0.379016341937 0.551671793295 -2001hx 0.799 43.3788294006 0.233174228134 0.551671793295 -2001hu 0.882 43.3750389271 0.583987119465 0.551671793295 -2001hs 0.833 43.6880978478 0.520913245917 0.551671793295 -2001fs 0.874 43.295379241 0.388089303157 0.551671793295 -2001fo 0.772 43.5099938279 0.218834770025 0.551671793295 -1999dr 0.178 39.454790821 0.235746048095 0.551671793295 -1999du 0.26 40.8233657709 0.199851735971 0.551671793295 -1999dv 0.186 39.7121679254 0.189043139487 0.551671793295 -1999dx 0.269 40.7761617315 0.25640063853 0.551671793295 -1999dy 0.215 40.3780576732 0.193933997767 0.551671793295 -2000fr 0.543 42.4788058506 0.0981414342081 0.551671793295 -1998bi 0.75 43.242998687 0.138318457637 0.551671793295 -1998be 0.64 42.7642472749 0.18880937743 0.551671793295 -1998ba 0.43 42.1841242053 0.148171058429 0.551671793295 -1998ay 0.64 43.1643806938 0.193749016644 0.551671793295 -1998ax 0.497 42.3245530903 0.167980194286 0.551671793295 -1998aw 0.44 42.0106475331 0.107484807666 0.551671793295 -1998as 0.355 41.3423115755 0.20146843402 0.551671793295 -1997ez 0.78 43.6008071703 0.171318833305 0.551671793295 -1997eq 0.54 42.4246212672 0.111551880919 0.551671793295 -1997ek 0.86 43.9216606703 0.172411965167 0.551671793295 -03D4au 0.468 42.5489037371 0.1639147706 0.551671793295 -04D4bk 0.84 43.8731031943 0.222348502282 0.910256228198 -04D3nr 0.96 43.6132116082 0.275514631381 0.551671793295 -04D3lu 0.8218 43.8153639621 0.21320698395 0.0 -04D3ki 0.93 43.5524295782 0.289918379066 0.551671793295 -04D3gt 0.451 41.7966744145 0.143055821894 0.551671793295 -04D3do 0.61 42.904965712 0.156248253322 0.0 -04D3cp 0.83 44.0506599335 0.208835355956 0.019019116792 -04D2gp 0.707 43.2824256201 0.256253916455 0.999999011725 -04D2fp 0.415 41.8767704643 0.141093702805 0.125506643627 -04D1ag 0.557 42.5570112027 0.157456031152 0.551671793295 -03D4fd 0.791 43.5768811197 0.201343679161 0.424378352473 -03D4cz 0.695 43.2117042607 0.20544919439 0.048213467444 -03D4at 0.633 43.0498635302 0.169957137727 1.0 -03D3bh 0.2486 40.6111853429 0.154515955385 0.551671793295 -03D3af 0.532 42.5658156596 0.166651053217 0.551671793295 -03D1fc 0.331 41.0784700501 0.137704766569 0.0 -03D1bp 0.346 41.3646268615 0.15221324645 9.00002294912e-12 -04D4dw 0.961 44.2642016168 0.340056560686 9.6749703e-05 -04D4an 0.613 42.9937246636 0.165096720419 0.000122102116 -04D3nh 0.3402 41.3280915278 0.135546197344 1.0 -04D3lp 0.983 44.1572843671 0.43499764107 1.0 -04D3is 0.71 43.0220388098 0.184280954678 0.551671793295 -04D3fq 0.73 43.27699552 0.194894052413 1.0 -04D3df 0.47 42.1345594858 0.15323960172 0.551671793295 -04D3co 0.62 43.0092526047 0.169564685824 0.964610252373 -04D2gc 0.521 42.1801294521 0.160996803083 0.998949555959 -04D2cf 0.369 41.6340230154 0.152644121885 7.25190000272e-08 -03D4gl 0.571 42.3991620661 0.174223963618 0.999999999363 -03D4dy 0.604 42.526976481 0.15968704969 0.991658804539 -03D4cy 0.9271 43.9404978712 0.286707554506 0.846559054956 -03D4ag 0.285 40.8542434728 0.136594130276 5.69999603073e-11 -03D3ba 0.2912 40.8390642775 0.146624503178 0.005691513851 -03D1gt 0.548 42.2962985563 0.18837089341 0.551671793295 -03D1ew 0.868 43.4932874061 0.24609516526 0.921233644992 -03D1ax 0.496 42.2146760288 0.152814539841 0.0 -04D4dm 0.811 43.4045360216 0.210619526441 1.0 -04D3oe 0.756 43.8139921904 0.192775955331 0.551671793295 -04D3nc 0.817 43.6515637361 0.210286863572 1.0 -04D3ks 0.752 43.3018298804 0.207071436502 0.999999999949 -04D3hn 0.5516 42.3031837333 0.149254872779 0.0 -04D3fk 0.3578 41.4349095934 0.136096151868 0.0027167408 -04D3dd 1.01 44.0124745362 0.375672417143 0.999785771595 -04D2ja 0.741 43.7172522874 0.223286859678 0.0 -04D2gb 0.43 41.8109207973 0.15108214332 0.001759266231 -04D1ak 0.526 42.4002106795 0.16694309723 0.551671793295 -03D4gg 0.592 42.5773227618 0.172394047168 0.0 -03D4di 0.905 43.6387400062 0.25746754741 0.860692933969 -03D4cx 0.949 43.4449098532 0.284630783131 9.00002294912e-12 -03D3cd 0.4607 42.0716665463 0.18126245549 0.999972076003 -03D3ay 0.3709 41.6692043983 0.154834006045 0.000892345443 -03D1fq 0.8 43.7140601053 0.215165244047 5.37990000327e-08 -03D1co 0.679 43.4574063402 0.195969077309 0.999996816858 -03D1aw 0.5817 42.6720406074 0.159803369147 1.0 -04D4bq 0.55 42.2756470464 0.15520464155 0.916905651559 -04D3ny 0.81 43.3698924946 0.212685692189 0.399751945497 -04D3ml 0.95 43.9766075066 0.291243768847 0.620797641982 -04D3kr 0.3373 41.2944909521 0.137240588972 1.0 -04D3gx 0.91 44.305516423 0.267241049423 0.551671793295 -04D3ez 0.263 40.6346721186 0.134567832845 0.0 -04D3cy 0.643 43.0112669877 0.166369771753 0.474555440866 -04D2iu 0.691 43.0887669301 0.253842244922 0.551671793295 -04D2fs 0.357 41.4255279641 0.137721476533 0.99999988436 -04D1aj 0.721 43.1756783825 0.187179852377 0.551671793295 -03D4gf 0.581 42.7423233406 0.155462517843 1.0 -03D4dh 0.6268 42.7586126154 0.157227804978 0.999411415446 -03D4cn 0.818 43.3928941721 0.26755880574 0.551671793295 -03D3cc 0.4627 42.0422849962 0.14756880162 0.551671793295 -03D3aw 0.449 42.0230683496 0.157063020734 0.0 -03D1fl 0.688 43.0539860347 0.165777265258 0.551671793295 -03D1cm 0.87 44.2313042047 0.278817290376 0.551671793295 -03D1au 0.5043 42.3382006953 0.149577507475 0.999977235075 -b010 0.591 43.2091059677 0.306006112419 0.551671793295 -b013 0.426 41.7615774796 0.22492067105 0.551671793295 -b016 0.329 41.3402567282 0.269807401411 0.551671793295 -d058 0.583 42.4011253659 0.28648393048 0.551671793295 -d084 0.519 43.071509714 0.314462491184 0.551671793295 -d085 0.401 41.6954244654 0.229308722315 0.551671793295 -d086 0.205 39.9037694347 0.217272003785 0.551671793295 -d087 0.34 41.2131787728 0.228190770173 0.551671793295 -d089 0.436 41.8800161997 0.21653394713 0.551671793295 -d093 0.363 41.551975686 0.208173297091 0.551671793295 -d097 0.436 41.9257693002 0.228262759927 0.551671793295 -d117 0.309 41.1737957091 0.235823875951 0.551671793295 -d149 0.342 41.3845981768 0.21481032067 0.551671793295 -e020 0.159 39.4163640953 0.242628007812 0.551671793295 -e029 0.332 41.2554783013 0.239903276269 0.551671793295 -e108 0.469 42.3544812112 0.28315240581 0.551671793295 -e132 0.239 40.2591287799 0.204167929675 0.551671793295 -e136 0.352 41.423779352 0.224191540842 0.551671793295 -e138 0.612 42.813940944 0.283023414891 0.551671793295 -e140 0.631 42.3773831209 0.244974123167 0.551671793295 -e147 0.645 42.8169731662 0.234273129747 0.551671793295 -e148 0.429 41.8900388123 0.215471389689 0.551671793295 -e149 0.497 42.0757738154 0.221637712856 0.551671793295 -f011 0.539 42.2266632084 0.241797816352 0.551671793295 -f041 0.561 42.8719504017 0.306521908685 0.551671793295 -f076 0.41 41.3474933283 0.256981142148 0.551671793295 -f096 0.412 41.4237121884 0.312056381676 0.551671793295 -f216 0.599 42.7434648961 0.382165112455 0.551671793295 -f231 0.619 43.056018637 0.247822481956 0.551671793295 -f235 0.422 41.7283094576 0.230713845858 0.551671793295 -f244 0.54 42.5111635186 0.282945441834 0.551671793295 -f308 0.401 42.5546203602 0.369893982362 0.551671793295 -g005 0.218 40.0754074212 0.222222737823 0.551671793295 -g050 0.633 42.2016724827 0.259877853306 0.551671793295 -g052 0.383 41.6420339038 0.251010174803 0.551671793295 -g055 0.302 41.3107982848 0.272346550274 0.551671793295 -g097 0.34 41.0800432546 0.231193988299 0.551671793295 -g120 0.51 41.8806853688 0.226762461825 0.551671793295 -g133 0.421 42.1360316298 0.322004955245 0.551671793295 -g142 0.399 41.4880817606 0.292952212036 0.551671793295 -g160 0.493 42.1469146695 0.2639575465 0.551671793295 -g240 0.687 42.9963850785 0.285188798534 0.551671793295 -h300 0.687 42.8348351217 0.272537782393 0.551671793295 -h319 0.495 42.2486633669 0.232347965963 0.551671793295 -h323 0.603 42.6462047067 0.278628902676 0.551671793295 -h342 0.421 42.1796277538 0.233773507359 0.551671793295 -h359 0.348 41.5896262003 0.217425302485 0.551671793295 -h363 0.213 40.109041203 0.22335261095 0.551671793295 -h364 0.344 41.1728711166 0.209772839036 0.551671793295 -k396 0.271 40.5323020705 0.21526700069 0.551671793295 -k411 0.564 42.3728917486 0.292022400672 0.551671793295 -k425 0.274 40.7246506476 0.208670920573 0.551671793295 -k429 0.181 39.6828885172 0.227088514354 0.551671793295 -k430 0.582 43.1639102562 0.325207917744 0.551671793295 -k441 0.68 42.9040344577 0.290951926367 0.551671793295 -k448 0.401 41.9356323131 0.332838743195 0.551671793295 -k485 0.416 41.5575369467 0.303252121943 0.551671793295 -m027 0.286 41.2111308437 0.25814462102 0.551671793295 -m034 0.562 43.0505225945 0.33321544156 0.551671793295 -m043 0.266 40.3907073912 0.250227167194 0.551671793295 -m062 0.314 41.2319680355 0.242363893683 0.551671793295 -m138 0.581 43.668679767 0.391236930244 0.551671793295 -m158 0.463 41.947753754 0.26646445376 0.551671793295 -m193 0.341 40.9915138593 0.224504319999 0.551671793295 -n256 0.631 42.8818252933 0.234369669625 0.551671793295 -n258 0.522 42.6773303466 0.24715086884 0.551671793295 -n263 0.368 41.4676952372 0.207180966902 0.551671793295 -n278 0.309 40.8607746451 0.214863256477 0.551671793295 -n285 0.528 42.3696918414 0.302654151443 0.551671793295 -n404 0.216 40.4049089433 0.21521177477 0.551671793295 -p455 0.284 40.8353825838 0.206616021304 0.551671793295 -p524 0.508 42.1942707319 0.231767202806 0.551671793295 -p528 0.781 43.4372295515 0.321348684213 0.551671793295 -p534 0.613 42.6167095942 0.260214469673 0.551671793295 -1999fw 0.278 40.5662405695 0.198452987813 0.551671793295 -1999fn 0.477 42.0543351554 0.183599933891 0.551671793295 -1999fm 0.95 43.6383176075 0.297754710705 0.551671793295 -1999fk 1.057 44.1506301001 0.238584685235 0.551671793295 -1999fj 0.816 43.6910828816 0.457863421823 0.551671793295 -1999ff 0.455 42.3239322721 0.229627540013 0.551671793295 -04Eag 1.02 44.3629247027 0.220963762065 0.551671793295 -04Gre 1.14 44.322957093 0.228701029542 0.551671793295 -04Man 0.854 43.6084642861 0.225569905838 0.551671793295 -04Mcg 1.37 45.049713349 0.262748449529 0.551671793295 -04Omb 0.975 44.3337254974 0.21278060312 0.551671793295 -04Pat 0.97 44.4626016974 0.291454456047 0.551671793295 -04Rak 0.74 43.2875467038 0.198402487067 0.551671793295 -04Sas 1.39 44.8762343403 0.250631965185 0.551671793295 -04Yow 0.46 42.1484496113 0.210095497723 0.551671793295 -05Fer 1.02 44.1640655276 0.232339527378 0.551671793295 -05Gab 1.12 44.5144054871 0.224079094547 0.551671793295 -05Lan 1.23 45.0206757228 0.235028400564 0.551671793295 -05Red 1.19 44.3628316562 0.247108113742 0.551671793295 -05Spo 0.839 43.3980657237 0.222124147444 0.551671793295 -05Str 1.01 44.9121378293 0.33374861699 0.551671793295 -05Zwi 0.521 42.3782445565 0.199332155448 0.551671793295 -2002dc 0.475 42.1048814028 0.255958316646 0.551671793295 -2002dd 0.95 43.884150294 0.235845124417 0.551671793295 -2002fw 1.3 45.0162581016 0.242719167821 0.551671793295 -2002hp 1.305 44.740169328 0.25954011501 0.551671793295 -2002kc 0.216 40.5560465985 0.244099860655 0.551671793295 -2002kd 0.735 43.0918403533 0.200575809439 0.551671793295 -2002ki 1.14 44.1969521333 0.368403293803 0.551671793295 -2003aj 1.307 45.4107441086 0.314696709568 0.551671793295 -2003az 1.265 44.944110841 0.235469959716 0.551671793295 -2003bd 0.67 43.1432095538 0.209654285426 0.551671793295 -2003be 0.64 42.9242754337 0.280114332269 0.551671793295 -2003dy 1.34 45.0675055813 0.275015093127 0.551671793295 -2003eq 0.84 43.5143043773 0.208697053243 0.551671793295 -2003XX 0.935 43.5401972359 0.227616538358 0.551671793295 -2001cw 0.953 44.2736202267 0.954717441018 0.551671793295 -2001gn 1.124 44.5675191814 0.197263047621 0.551671793295 -2001go 0.552 42.5109342761 0.103414502954 0.551671793295 -2001gq 0.671 42.9820070011 0.12054552758 0.551671793295 -2001gy 0.511 42.3736604669 0.0890617545983 0.551671793295 -2001hb 1.03 44.2400933572 0.141254202601 0.551671793295 -A-004 1.192 44.4587516 0.200258071902 0.551671793295 -C-000 1.092 44.0078477189 0.246511864633 0.551671793295 -C-001 0.974 43.8341432564 0.174238940337 0.551671793295 -F-012 1.11 44.6253358523 0.447832632795 0.551671793295 -G-004 1.35 44.8270654771 0.185705412075 0.551671793295 -H-003 0.85 43.4942573455 0.171796430655 0.551671793295 -H-005 1.241 44.5817029608 0.478371053196 0.0 -K-000 1.414 44.8037661387 0.346181483252 0.0 -N-033 1.188 44.6076425498 0.500544949291 0.551671793295 -D-000 1.017 44.2939707709 0.171046154322 1.4752251e-05 -D-006 1.315 44.9713577744 0.187508839367 0.551671793295 -P-009 0.821 43.6409387868 0.194013591972 0.551671793295 -R-012 1.215 45.2465209495 0.560317445151 0.000147652666 -Z-005 0.623 42.5145239973 0.241428134977 0.551671793295 diff --git a/evals/trace-analysis-prompt.md b/evals/trace-analysis-prompt.md deleted file mode 100644 index 120f4eb0..00000000 --- a/evals/trace-analysis-prompt.md +++ /dev/null @@ -1,22 +0,0 @@ -You are reviewing the trace of an autonomous coding agent that was asked -to build a scientific analysis with the `astra` (spec) and `lc` -(execution) CLIs. The trace digest is provided on stdin: the agent's -remarks, its tool calls, and truncated results, in order. - -Write a section titled exactly `### Confusion & pain points` containing -3-6 concise bullets. Look for and report: - -- failed commands or errored tool results, and their root cause -- moments where the agent misunderstood the spec format, the CLI - surface, the sandbox, or the execution environment -- detours: reverse-engineering source code, probing the environment, - redoing work, or fixing things the harness should have provided -- workarounds the agent invented that hint at a product gap (these are - the most valuable — call out what lightcone-cli or the eval setup - should fix) - -Each bullet: what happened, then the underlying cause, with the fix it -suggests when one is clear. Do not pad — if the run was essentially -clean, say so in one bullet and only list what little friction existed. -Do not restate the task or summarize the successful build. Output only -the markdown section, no preamble.