diff --git a/Makefile b/Makefile index ff8577aa6..7b82ba9df 100755 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ SHELL := /bin/bash -.PHONY: help docker-shell docker-check-agents docker-smoke docker-build-rdna4 docker-run docker-parallel-run docker-quality-loop docker-sikl-task-builder docker-setup-flydsl docker-setup-geak \ +.PHONY: help docker-shell docker-check-agents docker-smoke docker-build-rdna4 docker-run docker-parallel-run docker-quality-loop docker-sikl-task-builder docker-setup-flydsl docker-setup-geak docker-setup-forge \ slurm-shell slurm-smoke slurm-parallel-smoke slurm-check-agents slurm-run slurm-parallel-run slurm-submit slurm-parallel-submit \ check-docker-runner check-slurm-runner check-evaluator check-held-out check-visualization \ visualization-build visualization-serve visualization-run \ @@ -34,6 +34,7 @@ help: @echo " Use SIKL_BUILDER_ARGS='resume --run-id ...' to resume" @echo "make docker-setup-flydsl - Install FlyDSL when absent (for flydsl2flydsl, torch2flydsl, and triton2flydsl)" @echo "make docker-setup-geak - Install the Claude Agent SDK when absent (for the geak agent)" + @echo "make docker-setup-forge - Install Hyperloom[forge] from HYPERLOOM_ROOT's HEAD (for the forge agent)" @echo "make check-docker-runner - Check Docker runner syntax and runtime-specific arguments" @echo "make check-slurm-runner - Check Slurm/Spur resource and Docker handoff arguments" @echo "" @@ -170,6 +171,11 @@ docker-setup-flydsl: docker-setup-geak: @$(DOCKER_RUNNER) setup-geak +# Install Hyperloom[forge] from the committed HEAD of HYPERLOOM_ROOT into the +# mounted pip user-base. Run and preflight repeat it when that revision changes. +docker-setup-forge: + @$(DOCKER_RUNNER) setup-forge + check-docker-runner: @bash tests/test_docker_benchmark.sh diff --git a/agents/forge/README.md b/agents/forge/README.md index cd5a475a9..f424cf20a 100644 --- a/agents/forge/README.md +++ b/agents/forge/README.md @@ -318,10 +318,27 @@ checks that both commands still take the options the adapter builds. only as a library. It refuses anything else. Upstream owns its GPU DSL/compiler support and SDKs. Install the `forge` extra in -the GPU runtime environment (it also pulls the agent SDK dependencies): +the GPU runtime environment (it also pulls the agent SDK dependencies). Through +Docker, name the pinned checkout on the host: + +```bash +HYPERLOOM_ROOT=/path/to/Hyperloom make docker-setup-forge +HYPERLOOM_ROOT=/path/to/Hyperloom make docker-run CONFIG= +``` + +For `forge` runs the runner mounts `HYPERLOOM_ROOT` read-only, provisions the +agent backend's CLI and authentication (Claude Code unless the run sets +`agent.agent_backend: codex`), forwards Claude credentials by name as for GEAK, +and prepends `.aka-pyuserbase/forge-engine/` to `PYTHONPATH`, +since an image upgrade can change the Python ABI. Setup installs the +committed `HEAD` of `HYPERLOOM_ROOT` there with `pip install --target`, from an +exported copy because setuptools builds in tree. It records the revision in +`.aka-hyperloom-revision` and warns that uncommitted changes are not installed. +Run and preflight repeat the install only when that revision changes, then +require `agents/forge/engine.py --arena-probe` to pass. Outside Docker, install +into the interpreter directly: ```bash -# HYPERLOOM_ROOT names the separately mounted/pinned checkout in the container. python3 -m pip install "${HYPERLOOM_ROOT}[forge]" python3 agents/forge/engine.py --arena-probe ``` diff --git a/agents/geak/bridge.py b/agents/geak/bridge.py index b7051d15d..ff226e3d7 100644 --- a/agents/geak/bridge.py +++ b/agents/geak/bridge.py @@ -193,7 +193,8 @@ def copy_task(source: Path, destination: Path, *, remaining: Callable[[], float] def snapshot_mapping(snapshot: WorkspaceSnapshot) -> dict: return {"digests": snapshot.digests, - "initial_symbols": {k: sorted(v) for k, v in snapshot.initial_symbols.items()}} + "initial_symbols": {k: sorted(v) for k, v in snapshot.initial_symbols.items()}, + "initial_patching": snapshot.initial_patching} class Bridge: @@ -218,7 +219,8 @@ def _guard(self, root: Path, key: str = "harness") -> None: saved = self.job[key] verify_workspace_harness(WorkspaceSnapshot( root, saved["digests"], self.spec, - {k: frozenset(v) for k, v in saved["initial_symbols"].items()})) + {k: frozenset(v) for k, v in saved["initial_symbols"].items()}, + saved.get("initial_patching"))) def baseline_unchanged(self) -> None: self._guard(self.context.baseline, "baseline_harness") diff --git a/agents/geak/compatibility.py b/agents/geak/compatibility.py index 1f34e86b1..ba351c6c4 100644 --- a/agents/geak/compatibility.py +++ b/agents/geak/compatibility.py @@ -103,11 +103,11 @@ def adapt_lane(source: str) -> str: return source -def arena_contract(bridge: Bridge, python: str) -> str: +def arena_contract(bridge: Bridge, python: str, knowledge: str = "") -> str: command = shlex.join([python, str(Path(__file__).with_name("bridge.py")), "--job", str(bridge.job_path)]) spec = bridge.spec.to_mapping() - return f"""Arena task contract (takes precedence over upstream layout/harness examples) + contract = f"""Arena task contract (takes precedence over upstream layout/harness examples) Target language: {bridge.spec.candidate.language}. Initial state: {bridge.spec.candidate.initial_state}. Read the task's config.yaml, README.md if present, declared instructions and entrypoints. Keep all relative paths, multifile interfaces, and symbol-scoped restrictions exactly as declared: @@ -141,9 +141,10 @@ def arena_contract(bridge: Bridge, python: str) -> str: Upstream analysis/KB source is advisory; runtime dependencies are provisioned by Arena. GEAK metrics guide search only. Arena independently evaluates delivered candidate files. """ + return contract + (f"\n{knowledge}" if knowledge else "") -def prepare_engine(checkout: Path, bridge: Bridge, *, python: str, options: dict) -> dict: +def prepare_engine(checkout: Path, bridge: Bridge, *, python: str, options: dict, knowledge: str = "") -> dict: verify_upstream(checkout, remaining=bridge.remaining) bridge.remaining() private = bridge.root / "engine" @@ -156,7 +157,7 @@ def prepare_engine(checkout: Path, bridge: Bridge, *, python: str, options: dict roles = Path(__file__).with_name("roles") for role in roles.glob("*.md"): copy_file(role, workflow / "roles" / role.name, remaining=bridge.remaining, overwrite=True) - contract = arena_contract(bridge, python) + contract = arena_contract(bridge, python, knowledge) dispatcher = workflow / "kernel_workflow.js" for role in (workflow / "roles").glob("*.md"): bridge.remaining() diff --git a/agents/geak/launch_agent.py b/agents/geak/launch_agent.py index 6b7d3255a..67bddf726 100644 --- a/agents/geak/launch_agent.py +++ b/agents/geak/launch_agent.py @@ -17,6 +17,7 @@ from agents import register_agent from agents.geak.workflow_runner import _read_json +from src import knowledge_wiki from src.harness_guard import snapshot_workspace_harness from src.task_execution import _run_process from src.task_spec import load_task_spec @@ -175,6 +176,7 @@ def prepare_job(context: TaskContext, run_dir: Path, options: dict, *, @register_agent("geak") def launch_agent(eval_config: dict, task_config_dir: str, workspace: str) -> str: options = load_options(eval_config) + wiki = knowledge_wiki.resolve(eval_config) started = time.monotonic() deadline = started + options["timeout_seconds"] deadline_epoch = time.time() + options["timeout_seconds"] @@ -204,7 +206,13 @@ def launch_agent(eval_config: dict, task_config_dir: str, workspace: str) -> str status = {"status": "FAILED", "delivery": "NOT_ATTEMPTED", "arena_acceptance": "PENDING"} try: bridge = prepare_job(context, run_dir, options, deadline=deadline, deadline_epoch=deadline_epoch) - bridge.job["engine"] = prepare_engine(checkout, bridge, python=python, options=options) + knowledge = "" + if wiki is not None: + wiki_log = run_dir / knowledge_wiki.LOG_NAME + knowledge = wiki.instructions(python, wiki_log) + status["knowledge_wiki"] = {**wiki.identity(), "log": str(wiki_log)} + write_json(run_dir / "knowledge_wiki.json", status["knowledge_wiki"]) + bridge.job["engine"] = prepare_engine(checkout, bridge, python=python, options=options, knowledge=knowledge) write_json(bridge.job_path, bridge.job) code = _run_engine(bridge, python) result_path = run_dir / "engine_result.json" diff --git a/docs/README.md b/docs/README.md index 774e4bc4b..55276dc48 100644 --- a/docs/README.md +++ b/docs/README.md @@ -39,6 +39,7 @@ python -m sphinx -T -b html docs docs/_build/html | `how-to/task-validator.md` | How-to | Current task-validator execution, checks, and framework-finalized reports. | | `how-to/quality-loop.md` | How-to | Repository-wide validation, one-pass optimization, task hardening, and single-draft-PR publication. | | `how-to/held-out-evaluation.md` | How-to | Generate private shapes and evaluate completed runs for generalization. | +| `how-to/knowledge-wiki.md` | How-to | Offer a read-only kernel knowledge base to GEAK, with every query logged. | | `how-to/visualization.md` | How-to | Build dashboard data and serve the comparison dashboard. | | `examples/examples.md` | Examples | Step-by-step walkthroughs with expected output. | | `about/license.md` | License | Apache 2.0 notice and link to the full repository `LICENSE`. | diff --git a/docs/how-to/add-task.md b/docs/how-to/add-task.md index f85356502..3049665c6 100644 --- a/docs/how-to/add-task.md +++ b/docs/how-to/add-task.md @@ -180,7 +180,21 @@ protected function definitions. Existing editable non-entrypoint configuration helpers must pass a restricted body check: configuration data/local assignments, returns, and branches using protected predicates or supported configuration operations. Initial aliases, kernel entrypoints, and newly introduced factories -do not gain permission to run at definition time. The guard reports its +do not gain permission to run at definition time. Every editable Python file, +whatever its scope, also may not add runtime patching: calls to `globals()`, +`vars()`, `setattr`/`delattr`, `exec`/`eval`/`__import__`, `importlib` or frame +inspection (`sys._getframe`, `inspect.currentframe`), reads of code-object and +frame internals (`__code__`, `__globals__`, `f_back`, ...), assignments to an +imported module or object (`torch.Tensor.contiguous = ...`) or through an +attribute chain (`t.Tensor.contiguous = ...`), and rebinding an imported name. +Every editable C/C++/HIP file may not add library operator calls outside +comments: hipBLAS, hipBLASLt, rocBLAS, Tensile or MIOpen calls, and the +ATen/torch matrix products, softmax, losses, norms and reductions a task asks +the candidate to implement (`at::matmul`, `torch::mm`, `at::layer_norm`, +`.bmm(`, `.softmax(`, `.sum(`, ...). The host code of a Triton candidate, outside +its `@triton.jit` functions, may not add PyTorch or library operators either +(`torch.matmul`, `F.softmax`, `x @ w`, `torch.ops.*`, `aiter`, `flash_attn`, ...). +Sites the starting file already has stay allowed. The guard reports its effective policy in the validator context. It is a structural integrity check, not isolation of arbitrary Python code; reference/test manipulation remains forbidden and subject to semantic review. diff --git a/docs/how-to/knowledge-wiki.md b/docs/how-to/knowledge-wiki.md new file mode 100644 index 000000000..a3981f603 --- /dev/null +++ b/docs/how-to/knowledge-wiki.md @@ -0,0 +1,73 @@ +--- +myst: + html_meta: + "description": "Offer a read-only kernel knowledge base to the GEAK agent of an AgentKernelArena run, with every query logged." + "keywords": "AgentKernelArena, knowledge base, ROCmKernelWiki, GEAK, A/B testing" +--- + +# Offer a kernel knowledge base to agents + +A run can give its agents read-only access to a kernel knowledge base checkout +that follows the [KernelWiki](https://github.com/mit-han-lab/KernelWiki) layout, +such as [ROCmKernelWiki](https://github.com/jhinpan/ROCmKernelWiki). The feature +exists for A/B experiments that compare the same agent with and without the +knowledge base. Both arms of such an experiment must differ only in this setting. + +The GEAK agent supports it. A run that enables it for any other agent fails +before its first task. + +## Enable it + +Add the run-level key to the configuration of the knowledge-base arm only: + +```yaml +agent: + template: geak +knowledge_wiki: + enabled: true +``` + +Name the checkout on the host when starting the run: + +```bash +AKA_KNOWLEDGE_WIKI_ROOT=/path/to/ROCmKernelWiki make docker-run CONFIG="$CONFIG_PATH" +``` + +For a run that enables `knowledge_wiki`, the Docker runner mounts that checkout +read-only at the same path and forwards `AKA_KNOWLEDGE_WIKI_ROOT`. Runs that do +not enable it neither mount the checkout nor receive the variable. Keep the checkout +outside the AgentKernelArena repository, because the repository root is mounted +into every container. + +The checkout must provide `scripts/query.py`, `scripts/get_page.py` and +`scripts/grep_wiki.py`. The run stops with an error when the key is enabled but +the variable or these scripts are missing. + +## What agents see + +GEAK appends the instructions from `src/knowledge_wiki.py` to the Arena contract +on every role and Workflow call, and its agents query from the shell: + +```text + src/knowledge_wiki.py --root --log search "" [filters] + src/knowledge_wiki.py --root --log page + src/knowledge_wiki.py --root --log grep "" +``` + +The command passes the knowledge base's own output and exit status through +unchanged. The instructions name only the options these knowledge bases share +and point agents at each subcommand's `--help` for the rest. + +## Logs and identity + +Every query appends one JSON line to `knowledge_wiki_calls.jsonl` in the hidden +GEAK run directory. Each line records the timestamp, subcommand, arguments, exit +code, duration, output size, and the page IDs or files returned. + +The adapter status records the checkout's commit, whether its tree was dirty, +and the instructions version and the SHA-256 of the text it delivered. GEAK also +writes them to `knowledge_wiki.json`, and the run log repeats them at startup. +Compare these values across arms before comparing results. + +A knowledge-base arm whose log is empty received the instructions but never used +the knowledge base. Report that count rather than dropping the run. diff --git a/docs/how-to/run-evaluation.md b/docs/how-to/run-evaluation.md index a21fc858e..a6010f2e0 100644 --- a/docs/how-to/run-evaluation.md +++ b/docs/how-to/run-evaluation.md @@ -110,6 +110,13 @@ For debugging, enter the same Docker runtime used by the experiment: make docker-shell ``` +To run one command non-interactively in that runtime, without provisioning an +agent CLI or forwarding credentials (for example, a baseline-only measurement): + +```bash +AKA_VISIBLE_GPU=0 bash src/scripts/docker_benchmark.sh exec python3