From 5a0d5e368cbb7662347673114bfabcf443b05845 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 19 Oct 2026 16:56:18 +0800 Subject: [PATCH 01/14] refactor(knowledge): build the recipe journal and KB writes from named parts recipe_journal.py carried its whole job in a handful of very large functions: the session's close path, the Recipe KB amend, the per-variant fact write and the configuration arm's timeline rows were each one long body threading dozens of defaulting expressions together. Two coherent groups move to sibling modules: - `_recipe_attrs.py` owns the recipe-shaped view of SharedState: the workload tags, the KEEP'd kernel optimizations joined to their end-to-end verdict, the attempt provenance and the recipe attrs themselves. These are pure projections of settled state and never needed the collaborator. - `_config_timeline.py` owns the configuration arm's run and attempt rows on the FRAMEWORK_AGENT event. What stays is split along the jobs it already did: the close path reads as its guards, the remote publish and the local amend, and the amend builds its `put_recipe` arguments from one table of the fields a recipe carries instead of repeating "the override if the caller set it, else the live value" for each of eleven fields. Three duplicated mechanisms go with it. The framework, version and precision identity was derived twice; the store's authority row was read through three near-identical `try`/`except` blocks; and the integrate ledger's float coercion was written out three times, now `to_float`. `_collect_workload_tags`, `_build_kernel_optimizations_from_state`, `_collect_attempt_provenance` and `_build_recipe_attrs_from_state` were private helpers on the collaborator and are now module functions in `_recipe_attrs`; their tests call them there, with every assertion carried across unchanged. `test_remote_read_is_wired_into_cli_t0_and_ close_write_remains` now inspects `_publish_remote_recipe`, which is where the close path's `HyperloomRemoteKB.from_env().write` lives. No observable change: internal structure only. Co-Authored-By: Claude Opus 5 --- scripts/code_metrics_baseline.txt | 17 +- .../tests/test_coordinator_kb_writes.py | 15 +- ..._coordinator_sync_helpers_coverage_unit.py | 18 +- .../tests/test_recipe_sedimentation.py | 14 +- .../tests/test_remote_recipe_v2.py | 2 +- .../knowledge/_config_timeline.py | 269 +++ .../orchestrator/knowledge/_recipe_attrs.py | 433 +++++ .../orchestrator/knowledge/recipe_journal.py | 1490 +++++++---------- 8 files changed, 1322 insertions(+), 936 deletions(-) create mode 100644 src/hyperloom/orchestrator/knowledge/_config_timeline.py create mode 100644 src/hyperloom/orchestrator/knowledge/_recipe_attrs.py diff --git a/scripts/code_metrics_baseline.txt b/scripts/code_metrics_baseline.txt index c9ad79444c..3b65f6ee93 100644 --- a/scripts/code_metrics_baseline.txt +++ b/scripts/code_metrics_baseline.txt @@ -111,13 +111,6 @@ cognitive-complexity src/hyperloom/orchestrator/kernel/request_handlers.py::_res cognitive-complexity src/hyperloom/orchestrator/kernel/request_handlers.py::_resolve_integrate_payload 83 cognitive-complexity src/hyperloom/orchestrator/kernel/request_handlers.py::_run_forge_gemm_tuning 124 cognitive-complexity src/hyperloom/orchestrator/kernel/request_handlers.py::integrate_handler 110 -cognitive-complexity src/hyperloom/orchestrator/knowledge/recipe_journal.py::RecipeJournalCollaborator._build_kernel_optimizations_from_state 53 -cognitive-complexity src/hyperloom/orchestrator/knowledge/recipe_journal.py::RecipeJournalCollaborator._build_recipe_attrs_from_state 41 -cognitive-complexity src/hyperloom/orchestrator/knowledge/recipe_journal.py::RecipeJournalCollaborator._collect_attempt_provenance 34 -cognitive-complexity src/hyperloom/orchestrator/knowledge/recipe_journal.py::RecipeJournalCollaborator._collect_workload_tags 37 -cognitive-complexity src/hyperloom/orchestrator/knowledge/recipe_journal.py::RecipeJournalCollaborator.finalize_recipe_and_journal 63 -cognitive-complexity src/hyperloom/orchestrator/knowledge/recipe_journal.py::RecipeJournalCollaborator.kb_amend_recipe 51 -cognitive-complexity src/hyperloom/orchestrator/knowledge/recipe_journal.py::_record_config_attempts 38 cognitive-complexity src/hyperloom/orchestrator/knowledge/recipe_kb_t0.py::_build_warm_start_context 71 cognitive-complexity src/hyperloom/orchestrator/knowledge/recipe_kb_t0.py::_cascade_warm_start_search 46 cognitive-complexity src/hyperloom/orchestrator/knowledge/recipe_kb_t0.py::_find_config_donor 34 @@ -311,7 +304,6 @@ cyclomatic-complexity src/hyperloom/orchestrator/kernel/request_handlers.py::_fi cyclomatic-complexity src/hyperloom/orchestrator/kernel/request_handlers.py::_resolve_integrate_payload 24 cyclomatic-complexity src/hyperloom/orchestrator/kernel/request_handlers.py::_run_forge_gemm_tuning 56 cyclomatic-complexity src/hyperloom/orchestrator/kernel/request_handlers.py::integrate_handler 46 -cyclomatic-complexity src/hyperloom/orchestrator/knowledge/recipe_journal.py::RecipeJournalCollaborator.finalize_recipe_and_journal 21 cyclomatic-complexity src/hyperloom/orchestrator/knowledge/recipe_kb_t0.py::run_t0_anchor 33 cyclomatic-complexity src/hyperloom/orchestrator/knowledge/remote_recipe/values.py::_adopt_replayed_prior 28 cyclomatic-complexity src/hyperloom/orchestrator/loop/conversation.py::_format_inbox_event 30 @@ -367,7 +359,7 @@ duplicated-lines src/hyperloom/inference_optimizer/tests/test_column_kb.py 68 duplicated-lines src/hyperloom/inference_optimizer/tests/test_common_utils.py 913 duplicated-lines src/hyperloom/inference_optimizer/tests/test_conc_sweep.py 44 duplicated-lines src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py 18 -duplicated-lines src/hyperloom/inference_optimizer/tests/test_coordinator_sync_helpers_coverage_unit.py 18 +duplicated-lines src/hyperloom/inference_optimizer/tests/test_coordinator_sync_helpers_coverage_unit.py 15 duplicated-lines src/hyperloom/inference_optimizer/tests/test_coverage_boost_unit.py 65 duplicated-lines src/hyperloom/inference_optimizer/tests/test_coverage_gap_units.py 660 duplicated-lines src/hyperloom/inference_optimizer/tests/test_coverage_margin3_unit.py 191 @@ -772,12 +764,7 @@ function-lines src/hyperloom/orchestrator/kernel/request_handlers.py::_run_forge function-lines src/hyperloom/orchestrator/kernel/request_handlers.py::_run_forge_gemm_tuning 467 function-lines src/hyperloom/orchestrator/kernel/request_handlers.py::_run_integrate_rebaseline_with_lock_retry 111 function-lines src/hyperloom/orchestrator/kernel/request_handlers.py::integrate_handler 562 -function-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py::RecipeJournalCollaborator._build_recipe_attrs_from_state 102 function-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py::RecipeJournalCollaborator._record_fact_impl 97 -function-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py::RecipeJournalCollaborator._record_fact_per_variant 114 -function-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py::RecipeJournalCollaborator.finalize_recipe_and_journal 242 -function-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py::RecipeJournalCollaborator.kb_amend_recipe 140 -function-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py::_record_config_attempts 140 function-lines src/hyperloom/orchestrator/knowledge/recipe_kb/local_store.py::LocalRecipeStore.put_recipe 133 function-lines src/hyperloom/orchestrator/knowledge/recipe_kb/schema.py::Recipe.from_dict 102 function-lines src/hyperloom/orchestrator/knowledge/recipe_kb_t0.py::_build_warm_start_context 102 @@ -1046,7 +1033,7 @@ module-lines src/hyperloom/orchestrator/actions/executors/roofline.py 1303 module-lines src/hyperloom/orchestrator/kernel/apply_kernel_patch.py 2659 module-lines src/hyperloom/orchestrator/kernel/conc_sweep.py 1429 module-lines src/hyperloom/orchestrator/kernel/request_handlers.py 4888 -module-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py 1785 +module-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py 1473 module-lines src/hyperloom/orchestrator/knowledge/recipe_kb_t0.py 1318 module-lines src/hyperloom/orchestrator/knowledge/remote_recipe/values.py 1284 module-lines src/hyperloom/orchestrator/loop/coordinator.py 1423 diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_kb_writes.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_kb_writes.py index 176717a82c..d74a8fbd0d 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_kb_writes.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_kb_writes.py @@ -22,6 +22,9 @@ recipe_canonical_id, ) from hyperloom.orchestrator.knowledge.config import KnowledgeConfig +from hyperloom.orchestrator.knowledge._recipe_attrs import ( + recipe_attrs_from_state, +) _MODEL = "qwen3-30b-a3b" @@ -379,10 +382,10 @@ def _seed_kept_kernel(coord: Coordinator) -> None: def test_build_recipe_attrs_surfaces_kept_kernel(tmp_path: Path) -> None: - """``_build_recipe_attrs_from_state`` emits a ``kernel_optimizations`` entry carrying micro_speedup + E2E outcome.""" + """``recipe_attrs_from_state`` emits a ``kernel_optimizations`` entry carrying micro_speedup + E2E outcome.""" coord = _make_coordinator(tmp_path) _seed_kept_kernel(coord) - attrs = coord.recipe_journal._build_recipe_attrs_from_state() + attrs = recipe_attrs_from_state(coord.shared_state, coord.session_dir.name) kopts = attrs.get("kernel_optimizations") or [] assert kopts, "KEEP'd kernel k006 missing from recipe attrs" k = next((x for x in kopts if x.get("kernel_id") == "k006"), None) @@ -464,7 +467,7 @@ def test_best_config_reads_stack_args_from_canonical_server_key( } ] ss.cumulative_gain_validated = 10.0 - attrs = coord.recipe_journal._build_recipe_attrs_from_state() + attrs = recipe_attrs_from_state(coord.shared_state, coord.session_dir.name) assert attrs["best_config"]["extra_server_args"] == ("--page-size 32 --schedule-policy lpm"), ( "stack-layer launch args must be read from the canonical " "extra_server_args key; got " @@ -592,7 +595,7 @@ def test_kernel_e2e_decision_reflects_integrate_revert(tmp_path: Path) -> None: ], }, } - attrs = coord.recipe_journal._build_recipe_attrs_from_state() + attrs = recipe_attrs_from_state(coord.shared_state, coord.session_dir.name) kopts = attrs.get("kernel_optimizations") or [] k = next((x for x in kopts if x.get("kernel_id") == "k007"), None) assert k is not None, f"k007 missing from {kopts}" @@ -617,7 +620,7 @@ def test_kernel_e2e_decision_micro_only_when_not_integrated( }, } ss.kernel_integrate_attempts = {} - attrs = coord.recipe_journal._build_recipe_attrs_from_state() + attrs = recipe_attrs_from_state(coord.shared_state, coord.session_dir.name) kopts = attrs.get("kernel_optimizations") or [] k = next((x for x in kopts if x.get("kernel_id") == "k009"), None) assert k is not None @@ -644,7 +647,7 @@ def test_session_entry_carries_throughput_date_and_actions( ] ss.cumulative_gain_validated = 7.5 ss.cumulative_gain_validated_stack_len = 2 - attrs = coord.recipe_journal._build_recipe_attrs_from_state() + attrs = recipe_attrs_from_state(coord.shared_state, coord.session_dir.name) sessions = attrs.get("sessions") or [] assert sessions, "no session entry emitted" s = sessions[0] diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_sync_helpers_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_sync_helpers_coverage_unit.py index 93c5b66168..12ce7874b6 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_sync_helpers_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_sync_helpers_coverage_unit.py @@ -15,6 +15,10 @@ from hyperloom.orchestrator.loop.coordinator import Coordinator from hyperloom.orchestrator.state.objective import TargetGainObjective from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType +from hyperloom.orchestrator.knowledge._recipe_attrs import ( + kept_kernel_optimizations, + workload_kb_tags, +) def _heartbeat() -> Intent: @@ -570,7 +574,7 @@ def test_collect_workload_tags(coord: Coordinator, monkeypatch) -> None: "workload_mode": "serving", } - assert coord.recipe_journal._collect_workload_tags() == { + assert workload_kb_tags(coord.shared_state) == { "framework": "sglang", "model_class": "moe", "model_name": "Qwen3-32B", @@ -601,19 +605,19 @@ def test_workload_tags_read_ep_from_the_env_only_without_a_shared_state_value(co ss = coord.shared_state ss.ep = 8 - tags = coord.recipe_journal._collect_workload_tags() + tags = workload_kb_tags(coord.shared_state) assert tags["ep"] == 8 assert tags["pp"] == 3 ss.ep = 0 - assert coord.recipe_journal._collect_workload_tags()["ep"] == 2 + assert workload_kb_tags(coord.shared_state)["ep"] == 2 def test_an_unparseable_or_non_positive_parallelism_env_is_not_tagged(coord: Coordinator, monkeypatch) -> None: monkeypatch.setenv("EP", "two") monkeypatch.setenv("PP", "0") - tags = coord.recipe_journal._collect_workload_tags() + tags = workload_kb_tags(coord.shared_state) assert "ep" not in tags assert "pp" not in tags @@ -628,7 +632,7 @@ def test_workload_tags_survive_a_resumed_null_fingerprint_meta(coord: Coordinato ss.stack_fingerprint_meta = None ss.baseline_workload_extra = None - assert coord.recipe_journal._collect_workload_tags() == {"framework": "vllm"} + assert workload_kb_tags(coord.shared_state) == {"framework": "vllm"} def test_unknown_and_empty_fingerprint_entries_are_not_tagged(coord: Coordinator, monkeypatch) -> None: @@ -644,7 +648,7 @@ def test_unknown_and_empty_fingerprint_entries_are_not_tagged(coord: Coordinator "image_digest": "sha256:abc", } - tags = coord.recipe_journal._collect_workload_tags() + tags = workload_kb_tags(coord.shared_state) assert tags == {"framework": "vllm", "image_digest": "sha256:abc"} @@ -662,7 +666,7 @@ def test_build_kernel_optimizations_from_state(coord: Coordinator) -> None: ss.kernel_integrate_attempts = { "i1": {"kernel_id": "k1", "last_decision": "KEEP", "best_gain_pct": 5.0, "attempts": [{"new_tput": 210.0}]}, } - out = coord.recipe_journal._build_kernel_optimizations_from_state() + out = kept_kernel_optimizations(coord.shared_state) assert len(out) == 1 # only the KEEP'd k1 row = out[0] assert row["kernel_id"] == "k1" diff --git a/src/hyperloom/inference_optimizer/tests/test_recipe_sedimentation.py b/src/hyperloom/inference_optimizer/tests/test_recipe_sedimentation.py index bbdaf19554..0391d87542 100644 --- a/src/hyperloom/inference_optimizer/tests/test_recipe_sedimentation.py +++ b/src/hyperloom/inference_optimizer/tests/test_recipe_sedimentation.py @@ -17,6 +17,10 @@ ) from hyperloom.orchestrator.loop.coordinator import Coordinator from hyperloom.orchestrator.knowledge.recipe_kb import LocalRecipeStore, RecipeKB +from hyperloom.orchestrator.knowledge._recipe_attrs import ( + attempt_provenance, + recipe_attrs_from_state, +) def _make_coordinator(tmp_path: Path) -> Coordinator: @@ -67,7 +71,7 @@ def test_collect_attempt_provenance_maps_keep_and_revert(tmp_path): }, ) - kept, kept_by_gap, reverted = coord.recipe_journal._collect_attempt_provenance() + kept, kept_by_gap, reverted = attempt_provenance(coord.shared_state) assert kept == {"mtp_on": "https://pr/123"} assert kept_by_gap == {"gap.research_hint.0": "https://pr/123"} assert len(reverted) == 1 @@ -106,7 +110,7 @@ def test_provenance_resolves_by_gap_id_when_name_mismatches(tmp_path): }, ) - attrs = coord.recipe_journal._build_recipe_attrs_from_state() + attrs = recipe_attrs_from_state(coord.shared_state, coord.session_dir.name) row = next(x for x in attrs["what_worked"] if x["name"] == "k007") assert row["source"] == "https://pr/777" @@ -142,7 +146,7 @@ def test_build_recipe_attrs_sediments_source_and_revert(tmp_path): }, ) - attrs = coord.recipe_journal._build_recipe_attrs_from_state() + attrs = recipe_attrs_from_state(coord.shared_state, coord.session_dir.name) ww = attrs["what_worked"] mtp = next(x for x in ww if x["name"] == "mtp_on") assert mtp["source"] == "https://pr/123" @@ -174,7 +178,7 @@ def test_sediment_toggle_off_keeps_recipe_ephemeral(tmp_path): }, ) - attrs = coord.recipe_journal._build_recipe_attrs_from_state() + attrs = recipe_attrs_from_state(coord.shared_state, coord.session_dir.name) mtp = next(x for x in attrs["what_worked"] if x["name"] == "mtp_on") assert "source" not in mtp @@ -219,7 +223,7 @@ def test_experience_rows_survive_the_kb_round_trip(tmp_path): model="m", hardware="h", framework_name="f", - # Exactly what recipe_journal._build_recipe_attrs_from_state emits. + # Exactly what _recipe_attrs.recipe_attrs_from_state emits. what_worked=[ { "name": "mtp_on", diff --git a/src/hyperloom/inference_optimizer/tests/test_remote_recipe_v2.py b/src/hyperloom/inference_optimizer/tests/test_remote_recipe_v2.py index a0afb33a8a..f79275fdaa 100644 --- a/src/hyperloom/inference_optimizer/tests/test_remote_recipe_v2.py +++ b/src/hyperloom/inference_optimizer/tests/test_remote_recipe_v2.py @@ -3086,7 +3086,7 @@ def test_remote_read_is_wired_into_cli_t0_and_close_write_remains() -> None: fromlist=["_bootstrap_recipe_kb"], )._bootstrap_recipe_kb ) - close_source = inspect.getsource(RecipeJournalCollaborator.finalize_recipe_and_journal) + close_source = inspect.getsource(RecipeJournalCollaborator._publish_remote_recipe) assert "RemoteWarmRecipeAdapter" in bootstrap_source assert "HyperloomRemoteKB.from_env().write" in close_source assert "write_final_remote_recipe" not in close_source diff --git a/src/hyperloom/orchestrator/knowledge/_config_timeline.py b/src/hyperloom/orchestrator/knowledge/_config_timeline.py new file mode 100644 index 0000000000..0fc8d2e555 --- /dev/null +++ b/src/hyperloom/orchestrator/knowledge/_config_timeline.py @@ -0,0 +1,269 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""The configuration arm's rows on the FRAMEWORK_AGENT event. + +Timeline only. The adoption ledger and the optimization journal are written +beside these rows and do not depend on a recorder being open, so an outcome is +recorded here verbatim rather than collapsed the way the journal collapses +``KEEP_UNSTABLE`` and ``KILLED_OVERTIME`` into a plain revert. +""" + +from __future__ import annotations + +from typing import Any, Collection, Iterable, Mapping, NamedTuple, Sequence + +from hyperloom.common.timeutil import now_iso +from hyperloom.inference_optimizer.breakdown.recorder.framework_event import ( + ARM_CONFIG, + DISPOSITION_ATTEMPTED, + ROLE_CONFIG, + STEP_ATTEMPTED, + GateVerdict, +) + +from ..state.failure_evidence import UNMEASURED_OUTCOMES, classify_failure_attribution + +#: Explore outcomes that are not an attempt at anything: a variant skipped as a +#: duplicate was never measured, so no measurement would back its funnel row. +NON_ATTEMPT_OUTCOMES = frozenset({"SKIPPED_DEDUP"}) + +#: The gate whose verdict the shared accuracy block is projected from. +_ACCURACY_GATE = "accuracy" + +#: How much of a run's error text a reason field carries. +_REASON_EXCERPT_CHARS = 200 + + +class _ConfigAttemptContext(NamedTuple): + """What every attempt row produced by one grid dispatch shares.""" + + task_id: str + round_id: str + proposal_ref: str + adopted_variants: Collection[str] + + +def record_config_run(coord: Any, *, task: Any, result_dict: Mapping[str, Any]) -> None: + """Record the configuration arm's grid dispatch on the framework event. + + The arm's dispatch is one ``explore`` task, and it earns a run row for the + same reason a specialist dispatch does: the event reduces its status over + what it dispatched, so an arm whose grid came back with nothing must not + read as an arm that was never tried. Recorded at completion, where the + outcome is known, with the dispatch time taken off the task row; a grid + that measured nothing still lands, which is the case + :func:`record_config_attempts` never sees. + """ + recorder = coord._coord.phase_framework.timeline() + if recorder is None: + return + task_id = str(getattr(task, "task_id", "") or "") + if not task_id: + return + variants = result_dict.get("per_variant_outcomes") + measured = [row for row in variants if isinstance(row, dict)] if isinstance(variants, list) else [] + recorder.record_run( + task_id, + role=ROLE_CONFIG, + arm=ARM_CONFIG, + status=str(result_dict.get("status") or "") or "succeeded", + dispatched_at=str(getattr(task, "created_at", "") or ""), + completed_at=now_iso("seconds"), + reason=str(result_dict.get("error") or "")[:_REASON_EXCERPT_CHARS], + # A grid the run stopped before it measured anything is not a grid + # whose variants were measured and lost. + empty=not measured, + workspace=str(result_dict.get("workspace") or ""), + ) + + +def _attempt_measurement(metrics: Mapping[str, Any]) -> dict[str, Any]: + """What one variant's benchmark measured. + + Both ends of the pair: the anchor advances on every KEEP, so a percentage + without its denominator adds to nothing. + """ + return { + "before_tput": metrics.get("base_tput"), + "after_tput": metrics.get("tput"), + "gain_pct": metrics.get("gain_pct"), + "runtime_sec": metrics.get("runtime_sec"), + "estimated_output_throughput": metrics.get("estimated_output_throughput"), + } + + +def _attempt_config_delta(variant: Mapping[str, Any]) -> dict[str, Any]: + """The launch-configuration change one variant applied.""" + return { + "extra_server_args": variant.get("extra_server_args"), + "extra_envs": variant.get("extra_envs"), + "remove_args": variant.get("remove_args"), + "unset_envs": variant.get("unset_envs"), + "args_mode": variant.get("args_mode"), + } + + +def _attempt_accuracy(gates: Iterable[Mapping[str, Any]]) -> dict[str, Any]: + """The accuracy block, projected from the gate that ruled. + + This arm gates accuracy rather than reporting it. No gate row means nothing + gated the variant, which is not a gate that refused it. + """ + ruling = next((gate for gate in gates if str(gate.get("gate") or "") == _ACCURACY_GATE), {}) + return { + "required": True if ruling else None, + "value": ruling.get("observed"), + "reference": ruling.get("threshold"), + "passed": ruling.get("passed"), + } + + +def _attempt_failure(row: Mapping[str, Any], outcome: str) -> dict[str, Any]: + """What is known about a variant's failure, attributed only when it never measured.""" + error_class = str(row.get("error_class") or "") + error_excerpt = str(row.get("error_excerpt") or "") + attribution = "" + if outcome in UNMEASURED_OUTCOMES: + attribution = classify_failure_attribution( + error_class=error_class, + error_excerpt=error_excerpt, + reason=row.get("reason"), + explicit=row.get("failure_attribution"), + ) + return { + "error_class": error_class, + "error_excerpt": error_excerpt, + "attribution": attribution, + } + + +def _attempt_artifacts(row: Mapping[str, Any]) -> dict[str, str]: + """Where one variant's run left its workspace and its logs.""" + return { + "workspace": str(row.get("workspace") or ""), + "server_log_path": str(row.get("server_log_path") or ""), + "raw_result_path": str(row.get("raw_result_path") or ""), + } + + +def _attempt_fields( + row: Mapping[str, Any], + context: _ConfigAttemptContext, + gates: Sequence[Mapping[str, Any]], +) -> dict[str, Any]: + """The framework-event fields describing one measured configuration variant.""" + variant = row.get("variant") if isinstance(row.get("variant"), dict) else {} + metrics = row.get("metrics") if isinstance(row.get("metrics"), dict) else {} + outcome = str(row.get("outcome") or "") + fingerprint = str(row.get("fingerprint") or "") + adopted = fingerprint in context.adopted_variants + return { + "arm": ARM_CONFIG, + "round_id": context.round_id, + "task_id": context.task_id, + "proposal_ref": context.proposal_ref, + "provenance": str(row.get("provenance") or ""), + "outcome": outcome, + "reason": str(row.get("reason") or ""), + "reasoning": str(variant.get("note") or ""), + "reasoning_origin": str(variant.get("reasoning_origin") or ""), + "experience_citations": variant.get("experience_citations") or [], + "stage": str(row.get("stage") or ""), + "fingerprint": fingerprint, + # The fingerprint is the join key; the name is what a reader + # recognises the variant by. + "variant_name": str(row.get("variant_name") or ""), + "measurement": _attempt_measurement(metrics), + "config_delta": _attempt_config_delta(variant), + "accuracy": _attempt_accuracy(gates), + "failure": _attempt_failure(row, outcome), + "artifacts": _attempt_artifacts(row), + "decision": outcome, + "adopted": adopted, + # Recorded rather than referenced: every KEEP advances the stack, so + # the session's current config is not what this variant was measured + # on top of. + "measured_against": row.get("measured_against") or {}, + # What stood behind the verdict. Absent when nothing ruled. + "validation_basis": str(row.get("validation_basis") or ""), + # A pair is what makes a gain addable, so eligibility follows the pair + # being present rather than the outcome being a KEEP. + "attribution_eligible": (adopted and metrics.get("base_tput") is not None and metrics.get("tput") is not None), + } + + +def _record_attempt_gates(recorder: Any, attempt_id: str, gates: Iterable[Mapping[str, Any]]) -> None: + """Record each named gate's verdict against one attempt.""" + for gate in gates: + name = str(gate.get("gate") or "") + if not name: + continue + recorder.record_attempt_gate( + attempt_id, + GateVerdict( + gate=name, + passed=gate.get("passed"), + reason=str(gate.get("reason") or ""), + observed=gate.get("observed"), + threshold=gate.get("threshold"), + ), + ) + + +def _record_variant_attempt(recorder: Any, row: Any, context: _ConfigAttemptContext) -> bool: + """Record one measured variant and its gates; false when the row is no attempt at anything.""" + if not isinstance(row, dict): + return False + outcome = str(row.get("outcome") or "") + if outcome in NON_ATTEMPT_OUTCOMES: + return False + # The fingerprint identifies the variant within the round and the round + # within the task, so the three together identify the attempt. + fingerprint = str(row.get("fingerprint") or "") + parts = (context.task_id, context.round_id, fingerprint) + attempt_id = ":".join(part for part in parts if part) or context.task_id + if not attempt_id: + return False + gates = [gate for gate in (row.get("gates") or []) if isinstance(gate, dict)] + recorder.record_attempt(attempt_id, **_attempt_fields(row, context, gates)) + _record_attempt_gates(recorder, attempt_id, gates) + return True + + +def _settle_proposal_as_attempted(recorder: Any, proposal_ref: str, recorded: int) -> None: + """Settle the proposal behind a grid, now that its variants have been recorded. + + Settled here rather than at materialization: a grid whose task dies before + any variant runs has no attempt to stand behind an ``attempted`` reading, + and is more honestly left unsettled. + """ + recorder.record_proposal_step(proposal_ref, step=STEP_ATTEMPTED, outcome=str(recorded)) + recorder.settle_proposal(proposal_ref, disposition=DISPOSITION_ATTEMPTED) + + +def record_config_attempts( + coord: Any, + *, + task: Any, + per_variant: list[dict[str, Any]], + result_dict: Mapping[str, Any], + adopted_variants: Collection[str], +) -> None: + """Record the configuration arm's measured attempts on the framework event.""" + recorder = coord._coord.phase_framework.timeline() + if recorder is None: + return + params = getattr(task, "params", None) or {} + context = _ConfigAttemptContext( + task_id=str(getattr(task, "task_id", "") or ""), + round_id=str(result_dict.get("round_id") or ""), + proposal_ref=str(params.get("proposal_msg_id") or ""), + adopted_variants=adopted_variants, + ) + recorded = 0 + for row in per_variant: + if _record_variant_attempt(recorder, row, context): + recorded += 1 + if recorded and context.proposal_ref: + _settle_proposal_as_attempted(recorder, context.proposal_ref, recorded) diff --git a/src/hyperloom/orchestrator/knowledge/_recipe_attrs.py b/src/hyperloom/orchestrator/knowledge/_recipe_attrs.py new file mode 100644 index 0000000000..bb0e7cf0e6 --- /dev/null +++ b/src/hyperloom/orchestrator/knowledge/_recipe_attrs.py @@ -0,0 +1,433 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""The recipe-shaped view of a session's settled :class:`SharedState`. + +Every projection here is a pure read of the state the optimization loop has +already settled, shaped the way the Recipe KB stores it: the launch +configuration that won, the stack that produced it, the kernels that landed, +and the workload dimensions a warm start filters on. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Collection, Iterable, Mapping, NamedTuple, Sequence + +from hyperloom.common.coerce import to_float +from hyperloom.inference_optimizer.session.optimization_journal import OUTCOME_KEEP, OUTCOME_REVERT + +from .workload_tags import workload_shape_tags + +#: SharedState attributes naming the workload; ``model_family`` is deliberately +#: absent, being no part of the seven-dimension Recipe identity. +_MODEL_IDENTITY_ATTRS: tuple[str, ...] = ("framework", "model_class", "model_name") + +#: The serving frameworks whose version the boot-time fingerprint records. +_VERSIONED_FRAMEWORKS: frozenset[str] = frozenset({"sglang", "vllm"}) + +#: Stack-fingerprint keys that become recipe tags under their own names. +_RUNTIME_VERSION_KEYS: tuple[tuple[str, str], ...] = ( + ("rocm", "rocm_version"), + ("aiter", "aiter_version"), + ("image_digest", "image_digest"), +) + +#: The probe's own word for a version it could not read. +_UNIDENTIFIED_VERSION: str = "unknown" + +#: Baseline workload knobs carried onto the recipe, grouped by the type that +#: makes a value meaningful. A ``False`` survives: it is an explicit disable. +_POSITIVE_INT_KNOBS: tuple[str, ...] = ("max_running_requests", "max_num_seqs") +_BOOLEAN_KNOBS: tuple[str, ...] = ("chunked_prefill_enabled", "enable_torch_compile") +_TEXT_KNOBS: tuple[str, ...] = ("quant_scheme", "workload_mode") + +#: ``best_config`` fields copied from ``current_best`` as they stand. +_BEST_CONFIG_PASSTHROUGH: tuple[str, ...] = ("extra_envs", "name", "tput", "accuracy") + +#: How many recent action failures the recipe's ``what_failed`` carries. +_RECENT_FAILURE_LIMIT: int = 10 + + +class AttemptProvenance(NamedTuple): + """Where a session's proven optimizations came from, and what it reverted. + + Keepers are indexed twice because a stack layer may identify itself either + by name or by the research gap it closed. + """ + + kept_by_name: dict[str, str] + kept_by_gap: dict[str, str] + reverted: list[dict[str, Any]] + + @classmethod + def unrecorded(cls) -> "AttemptProvenance": + """The empty provenance a session that does not sediment its attempts reports.""" + return cls(kept_by_name={}, kept_by_gap={}, reverted=[]) + + +class _IntegrateIndex(NamedTuple): + """Integrate-layer entries under both keys a micro KEEP can name them by.""" + + by_kernel: dict[str, dict[str, Any]] + by_task_group: dict[str, dict[str, Any]] + + +class _E2eVerdict(NamedTuple): + """What integrating a kernel measured end to end.""" + + gain_pct: float + throughput: float + decision: str + integrated: bool + + +def _known_version(raw: Any) -> str: + """A fingerprint version, empty when the probe never identified one.""" + version = str(raw or "").strip() + return "" if version == _UNIDENTIFIED_VERSION else version + + +def _model_identity_tags(state: Any) -> dict[str, Any]: + """The workload's model-identity tags, omitting the ones this session never learned.""" + tags: dict[str, Any] = {} + for attr in _MODEL_IDENTITY_ATTRS: + text = str(getattr(state, attr, "") or "").strip() + if text: + tags[attr] = text + precision = getattr(state, "precision", None) + if precision not in (None, "", 0): + tags["precision"] = precision + return tags + + +def _runtime_version_tags(fingerprint_meta: Mapping[str, Any], framework: str) -> dict[str, str]: + """Runtime versions read off the stack fingerprint the CLI wrote at boot. + + ``framework_version`` names whichever serving framework is active, so a + recipe written under one is never matched against the other. + """ + tags: dict[str, str] = {} + framework_key = framework.lower() + if framework_key in _VERSIONED_FRAMEWORKS: + active = _known_version(fingerprint_meta.get(framework_key)) + if active: + tags["framework_version"] = active + for source_key, tag in _RUNTIME_VERSION_KEYS: + version = _known_version(fingerprint_meta.get(source_key)) + if version: + tags[tag] = version + return tags + + +def _baseline_workload_tags(workload_extra: Mapping[str, Any]) -> dict[str, Any]: + """Per-baseline workload knobs from the materialized YAML, each kept only when its type says it was set.""" + tags: dict[str, Any] = {} + for key in _POSITIVE_INT_KNOBS: + value = workload_extra.get(key) + if isinstance(value, int) and value > 0: + tags[key] = value + for key in _BOOLEAN_KNOBS: + value = workload_extra.get(key) + if isinstance(value, bool): + tags[key] = value + for key in _TEXT_KNOBS: + value = workload_extra.get(key) + if isinstance(value, str) and value.strip(): + tags[key] = value.strip() + return tags + + +def workload_kb_tags(state: Any) -> dict[str, Any]: + """The workload-shape tags describing the session this recipe came from. + + Shared by the recipe attrs and by every lesson or pitfall write, so the + warm-start reader filters on the dimensions the row was written with. + """ + tags = _model_identity_tags(state) + tags.update(workload_shape_tags(state)) + fingerprint_meta = state.stack_fingerprint_meta or {} + if isinstance(fingerprint_meta, dict): + tags.update(_runtime_version_tags(fingerprint_meta, str(tags.get("framework") or ""))) + workload_extra = state.baseline_workload_extra or {} + if isinstance(workload_extra, dict): + tags.update(_baseline_workload_tags(workload_extra)) + return tags + + +def _index_integrate_attempts(attempts: Mapping[str, Any]) -> _IntegrateIndex: + """Index the integrate ledger by kernel and by task group; the last write wins.""" + by_kernel: dict[str, dict[str, Any]] = {} + by_task_group: dict[str, dict[str, Any]] = {} + for entry in attempts.values(): + if not isinstance(entry, dict): + continue + kernel_id = str(entry.get("kernel_id") or "") + if kernel_id: + by_kernel[kernel_id] = entry + task_group_key = str(entry.get("task_group_key") or "") + if task_group_key: + by_task_group[task_group_key] = entry + return _IntegrateIndex(by_kernel=by_kernel, by_task_group=by_task_group) + + +def _last_rebenched_throughput(integration: Mapping[str, Any]) -> float: + """Throughput from the most recent integrate attempt that re-benched end to end.""" + for attempt in reversed(list(integration.get("attempts") or [])): + if isinstance(attempt, dict) and attempt.get("new_tput") is not None: + return to_float(attempt.get("new_tput"), 0.0) + return 0.0 + + +def _e2e_verdict(integration: Any) -> _E2eVerdict: + """The integrate layer's verdict on a kernel, or the never-integrated default. + + Carrying the verdict lets a warm start skip a kernel that won on the micro + bench and lost end to end. + """ + if not isinstance(integration, dict): + return _E2eVerdict(gain_pct=0.0, throughput=0.0, decision="", integrated=False) + return _E2eVerdict( + gain_pct=to_float(integration.get("best_gain_pct"), 0.0), + throughput=_last_rebenched_throughput(integration), + decision=str(integration.get("last_decision") or "").upper(), + integrated=True, + ) + + +def _kernel_optimization_row(entry: Mapping[str, Any], ledger_id: str, index: _IntegrateIndex) -> dict[str, Any]: + """One KernelOptimization row: a micro KEEP joined to its end-to-end verdict.""" + kernel_id = str(entry.get("current_kernel_id") or entry.get("kernel_id") or ledger_id) + task_group_key = str(entry.get("task_group_key") or "") + integration = index.by_task_group.get(task_group_key) if task_group_key else index.by_kernel.get(kernel_id) + e2e = _e2e_verdict(integration) + return { + "kernel_id": kernel_id, + "source_file": str(entry.get("last_source_file") or ""), + "artifact_path": str(entry.get("last_artifact_path") or ""), + "micro_speedup": to_float(entry.get("last_micro_speedup"), 0.0), + "decision": OUTCOME_KEEP, + "e2e_gain_pct": e2e.gain_pct, + "e2e_tput": e2e.throughput, + "e2e_decision": e2e.decision, + "integrated": e2e.integrated, + "ts": str(entry.get("last_ts") or ""), + } + + +def kept_kernel_optimizations(state: Any) -> list[dict[str, Any]]: + """The KEEP'd kernel optimizations, each joined to its end-to-end integrate verdict. + + A KEEP that never reached integration surfaces with ``integrated`` false + rather than being dropped, because not having been tried end to end is not + the same as having lost there. + """ + optimizations = state.kernel_opt_task_attempts or {} + if not isinstance(optimizations, dict): + return [] + integrations = state.kernel_integrate_attempts or {} + index = _index_integrate_attempts(integrations if isinstance(integrations, dict) else {}) + rows: list[dict[str, Any]] = [] + for ledger_id, entry in optimizations.items(): + if not isinstance(entry, dict): + continue + if str(entry.get("last_decision", "")).upper() != OUTCOME_KEEP: + continue + rows.append(_kernel_optimization_row(entry, ledger_id, index)) + return rows + + +def _reverted_attempt_row(name: str, attempt: Mapping[str, Any], source: str) -> dict[str, Any]: + """A ``what_failed`` row for an attempt this session measured and reverted.""" + row: dict[str, Any] = { + "name": name, + "reason": "reverted", + "gain_pct": attempt.get("gain_pct"), + } + if source: + row["source"] = source + return row + + +def _gap_attempt_provenance(gap: Mapping[str, Any]) -> AttemptProvenance: + """One research gap's attempts, split into what stuck and what was reverted.""" + source = str(gap.get("provenance") or "").strip() + canonical_id = str(gap.get("canonical_id") or "").strip() + kept_by_name: dict[str, str] = {} + kept_by_gap: dict[str, str] = {} + reverted: list[dict[str, Any]] = [] + for attempt in gap.get("attempts") or []: + if not isinstance(attempt, dict): + continue + variant = str(attempt.get("variant_name") or "").strip() + kernel = str(attempt.get("kernel_id") or "").strip() + outcome = str(attempt.get("outcome") or "").strip().upper() + if outcome == OUTCOME_KEEP and source: + for name in (variant, kernel): + if name: + kept_by_name.setdefault(name, source) + if canonical_id: + kept_by_gap.setdefault(canonical_id, source) + elif outcome == OUTCOME_REVERT and (variant or kernel): + reverted.append(_reverted_attempt_row(variant or kernel, attempt, source)) + return AttemptProvenance(kept_by_name=kept_by_name, kept_by_gap=kept_by_gap, reverted=reverted) + + +def attempt_provenance(state: Any) -> AttemptProvenance: + """Trace each proven optimization back to the research hint it came from. + + The first gap to claim a name keeps it, so a name two gaps both attempted + reads as belonging to the one that reached it first. + """ + kept_by_name: dict[str, str] = {} + kept_by_gap: dict[str, str] = {} + reverted: list[dict[str, Any]] = [] + for gap in state.gaps or []: + if not isinstance(gap, dict): + continue + from_gap = _gap_attempt_provenance(gap) + for name, source in from_gap.kept_by_name.items(): + kept_by_name.setdefault(name, source) + for canonical_id, source in from_gap.kept_by_gap.items(): + kept_by_gap.setdefault(canonical_id, source) + reverted.extend(from_gap.reverted) + return AttemptProvenance(kept_by_name=kept_by_name, kept_by_gap=kept_by_gap, reverted=reverted) + + +def _last_stack_launch_args(optimization_stack: Sequence[Any]) -> str: + """Launch args from the last validated stack layer, empty when it carries none.""" + if not optimization_stack: + return "" + last = optimization_stack[-1] + if not isinstance(last, dict): + return "" + return str(last.get("candidate_extra_server_args") or last.get("extra_server_args") or "").strip() + + +def _best_config(current_best: Mapping[str, Any], optimization_stack: Sequence[Any]) -> dict[str, Any]: + """The replayable launch configuration the recipe advertises. + + The last validated stack layer overrides ``current_best``, whose + ``extra_server_args`` may carry a string an interrupted write corrupted. + """ + config: dict[str, Any] = {} + current_args = current_best.get("extra_server_args") + if current_args: + config["extra_server_args"] = str(current_args) + for key in _BEST_CONFIG_PASSTHROUGH: + if key in current_best: + config[key] = current_best[key] + stack_args = _last_stack_launch_args(optimization_stack) + if stack_args: + config["extra_server_args"] = stack_args + return config + + +def _stack_entry_source(entry: Mapping[str, Any], name: str, provenance: AttemptProvenance) -> str: + """The research hint a stack layer came from, empty when nothing claims it. + + The layer's gap id is naming-independent and wins; the variant or kernel + name answers for layers recorded before gap ids were stamped. + """ + gap_id = str(entry.get("gap_canonical_id") or "").strip() + if gap_id: + by_gap = provenance.kept_by_gap.get(gap_id) + if by_gap: + return by_gap + kernel_id = str(entry.get("kernel_id") or "") + return provenance.kept_by_name.get(name) or provenance.kept_by_name.get(kernel_id) or "" + + +def _what_worked( + optimization_stack: Sequence[Any], + gain_per_entry: Sequence[Any], + provenance: AttemptProvenance, +) -> list[dict[str, Any]]: + """One row per stack layer: the change it made, what it gained and where it came from.""" + rows: list[dict[str, Any]] = [] + for index, entry in enumerate(optimization_stack): + if not isinstance(entry, dict): + continue + name = str(entry.get("variant_name") or entry.get("name") or entry.get("kernel_id") or "") + row: dict[str, Any] = { + "name": name, + "extra_server_args": str(entry.get("extra_server_args") or ""), + "extra_envs": dict(entry.get("extra_envs") or {}), + "gain_pct": gain_per_entry[index] if index < len(gain_per_entry) else None, + } + source = _stack_entry_source(entry, name, provenance) + if source: + row["source"] = source + rows.append(row) + return rows + + +def _what_failed(recent_failures: Sequence[Any], reverted: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + """The session's recent action failures, followed by every attempt it reverted.""" + rows: list[dict[str, Any]] = [] + for failure in recent_failures[-_RECENT_FAILURE_LIMIT:]: + if isinstance(failure, dict): + rows.append( + { + "name": str(failure.get("name") or failure.get("action") or ""), + "reason": str(failure.get("reason") or failure.get("error_class") or ""), + } + ) + rows.extend(reverted) + return rows + + +def _stack_action_names(optimization_stack: Iterable[Any]) -> list[str]: + """The named actions the optimization stack is built from, in the order they landed.""" + names: list[str] = [] + for entry in optimization_stack: + if not isinstance(entry, dict): + continue + name = str(entry.get("variant_name") or entry.get("name") or entry.get("action") or "").strip() + if name: + names.append(name) + return names + + +def _session_row( + state: Any, + session_name: str, + optimization_stack: Collection[Any], + final_throughput: float, +) -> dict[str, Any]: + """This session's row on the recipe: where it started, where it reached and by which actions. + + Self-describing in the arbor shape, so a reader needs neither the recipe's + other sessions nor this one's journal to price what it contributed. + """ + validated_stack_len = int(state.cumulative_gain_validated_stack_len or 0) + return { + "session_id": str(state.recipe_kb_session_id or session_name), + "gain_pct": float(state.cumulative_gain_validated or 0.0), + "stack_len": validated_stack_len or len(optimization_stack), + "throughput_before": float(state.baseline_tput or 0.0), + "throughput_after": final_throughput, + "date": datetime.now(timezone.utc).isoformat(), + "actions_taken": _stack_action_names(optimization_stack), + } + + +def recipe_attrs_from_state(state: Any, session_name: str) -> dict[str, Any]: + """Materialise the recipe-shaped view of one session's settled state. + + ``framework_version`` is left unset here because it is manifest-derived; + the T0 backfill writes it. + """ + optimization_stack = state.optimization_stack or [] + provenance = attempt_provenance(state) if state.recipe_sediment_enabled else AttemptProvenance.unrecorded() + final_throughput = float(state.current_best.get("tput", 0.0)) + return { + "best_config": _best_config(state.current_best, optimization_stack), + "best_throughput": final_throughput, + "what_worked": _what_worked(optimization_stack, state.gain_per_stack_entry or [], provenance), + "what_failed": _what_failed(state.last_action_failures or [], provenance.reverted), + "kernel_optimizations": kept_kernel_optimizations(state), + "last_profiled": str(state.cumulative_gain_validated_ts or ""), + "workload": workload_kb_tags(state), + "sessions": [_session_row(state, session_name, optimization_stack, final_throughput)], + } diff --git a/src/hyperloom/orchestrator/knowledge/recipe_journal.py b/src/hyperloom/orchestrator/knowledge/recipe_journal.py index 046d8bcfb7..5994a5f0b6 100644 --- a/src/hyperloom/orchestrator/knowledge/recipe_journal.py +++ b/src/hyperloom/orchestrator/knowledge/recipe_journal.py @@ -6,8 +6,10 @@ from __future__ import annotations import logging as _logging +from copy import copy from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, Collection, Mapping +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Collection, Mapping, NamedTuple from hyperloom.common.coerce import to_float, to_int from hyperloom.common.io import JsonLineFormat, append_jsonl @@ -24,12 +26,13 @@ summarize_change, ) from hyperloom.orchestrator.knowledge.recipe_kb import recipe_canonical_id -from hyperloom.orchestrator.knowledge.workload_tags import workload_shape_tags from hyperloom.orchestrator.lever import LEVER_CONFIG, lever_kind_for_task from ..collaborator import CoordinatorCollaborator from ..state.attempt_ledger import AttemptRow, record_attempt_row -from ..state.failure_evidence import UNMEASURED_OUTCOMES, classify_failure_attribution +from ._config_timeline import NON_ATTEMPT_OUTCOMES, record_config_attempts, record_config_run +from ._recipe_attrs import recipe_attrs_from_state, workload_kb_tags +from .config import KnowledgeStoreMode if TYPE_CHECKING: from ..loop.coordinator import Coordinator @@ -63,6 +66,79 @@ _REMOTE_VALIDATION_ERROR = "RemoteRecipeValidationError" +#: Recipe fields an amend carries forward when the caller does not override +#: them, each with the empty value ``put_recipe`` falls back to. +_PRESERVED_RECIPE_FIELDS: Mapping[str, Any] = MappingProxyType( + { + "best_config": {}, + "best_throughput": 0.0, + "what_worked": [], + "what_failed": [], + "remaining_gaps": [], + "last_profiled": "", + "stack_fingerprint": {}, + "sessions": [], + "authority": "EXPERIENTIAL", + "confidence": 0.85, + "evidence_refs": [], + } +) + +#: Row keys ``put_recipe`` owns; everything else on a live row is a session +#: extra and survives the amend untouched. +_RECIPE_OWNED_KEYS: frozenset[str] = frozenset(_PRESERVED_RECIPE_FIELDS) | frozenset( + { + "canonical_id", + "version", + "created_at", + "updated_at", + "model", + "hardware", + "framework", + "framework_name", + "framework_version", + "precision", + "pitfalls", + "lessons", + "provenance", + } +) + +#: Explore outcomes the journal collapses into a plain revert. +_MEASURED_LOSS_OUTCOMES: frozenset[str] = frozenset({OUTCOME_REVERT, "FAILED"}) + +#: Per-variant measurement detail the journal row carries beyond the headline +#: gain and throughput. +_VARIANT_DETAIL_METRICS: tuple[str, ...] = ( + "runtime_sec", + "wall_clock_ratio_vs_baseline", + "estimated_output_throughput", +) + + +class _FrameworkIdentity(NamedTuple): + """The framework dimensions of a Recipe identity, with the version resolved.""" + + name: str + version: str + precision: str + + +class _VariantMeasurement(NamedTuple): + """What one explore variant's benchmark reported.""" + + gain_pct: float | None + throughput_after: float | None + detail: dict[str, Any] + + +class _VariantFailure(NamedTuple): + """Why a variant was reverted; both fields absent for every other outcome.""" + + error_class: str | None + reason: str | None + + def _remote_result_type(status: str, reason: str) -> str: """The ``close_out.RESULT_*`` code for a remote KB Store write result. @@ -96,203 +172,231 @@ def _predicted_gain(*sources: dict[str, Any] | None) -> float | None: return None -#: Explore outcomes that are not an attempt at anything: a variant skipped as a -#: duplicate was never measured, so no measurement would back its funnel row. -_NON_ATTEMPT_OUTCOMES = frozenset({"SKIPPED_DEDUP"}) +def _extra_server_args(payload: Mapping[str, Any]) -> str: + """Read canonical ``extra_server_args`` from a payload.""" + value = payload.get("extra_server_args") + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, (list, tuple)): + return " ".join(str(v).strip() for v in value if str(v).strip()) + return str(value) -def _record_config_run(coord: Any, *, task: Any, result_dict: Mapping[str, Any]) -> None: - """Record the configuration arm's grid dispatch on the framework event. +def _framework_identity(state: Any) -> _FrameworkIdentity: + """The session's framework, the version it is running and its precision. - The arm's dispatch is one ``explore`` task, and it earns a run row for the - same reason a specialist dispatch does: the event reduces its status over - what it dispatched, so an arm whose grid came back with nothing must not - read as an arm that was never tried. Recorded at completion, where the - outcome is known, with the dispatch time taken off the task row; a grid - that measured nothing still lands, which is the case - :func:`_record_config_attempts` never sees. + The version is probed from the installed framework when the session never + recorded one, so a recipe is never filed under a blank version. """ - recorder = coord._coord.phase_framework.timeline() - if recorder is None: - return - from hyperloom.common.timeutil import now_iso - from hyperloom.inference_optimizer.breakdown.recorder.framework_event import ARM_CONFIG, ROLE_CONFIG - - task_id = str(getattr(task, "task_id", "") or "") - if not task_id: - return - variants = result_dict.get("per_variant_outcomes") - measured = [row for row in variants if isinstance(row, dict)] if isinstance(variants, list) else [] - status = str(result_dict.get("status") or "") or "succeeded" - recorder.record_run( - task_id, - role=ROLE_CONFIG, - arm=ARM_CONFIG, - status=status, - dispatched_at=str(getattr(task, "created_at", "") or ""), - completed_at=now_iso("seconds"), - reason=str(result_dict.get("error") or "")[:200], - # A grid the run stopped before it measured anything is not a grid - # whose variants were measured and lost. - empty=not measured, - workspace=str(result_dict.get("workspace") or ""), - ) + name = str(state.framework or "") + version = str(state.framework_version or "") + if not version and name: + version = detect_framework_version(name) + return _FrameworkIdentity(name=name, version=version, precision=str(state.precision or "")) -def _record_config_attempts( - coord: Any, +def _result_payload(result: Any) -> dict[str, Any]: + """A completed task's result dict, whatever shape the executor returned it in.""" + payload = result.result if hasattr(result, "result") else (result or {}) + return payload if isinstance(payload, dict) else {} + + +def _grid_attempt_row( + variant_outcome: Any, *, - task: Any, - per_variant: list[dict[str, Any]], - result_dict: Mapping[str, Any], - adopted_variants: Collection[str], -) -> None: - """Record the configuration arm's measured attempts on the framework event. - - Timeline only: the ledger row is written by ``fact_write_hook``, which does - not depend on a recorder being open. The outcome is recorded verbatim here, - unlike the journal beside it, which collapses ``KEEP_UNSTABLE`` and - ``KILLED_OVERTIME`` into a plain revert. + task_id: str, + round_id: str, + adopted: bool, +) -> AttemptRow | None: + """The adoption-ledger row for one measured grid variant, or ``None`` when it measured nothing.""" + outcome = str(variant_outcome.get("outcome") or "") if isinstance(variant_outcome, dict) else "" + if not outcome or outcome in NON_ATTEMPT_OUTCOMES: + return None + metrics = variant_outcome.get("metrics") if isinstance(variant_outcome.get("metrics"), dict) else {} + return AttemptRow( + lever_kind=LEVER_CONFIG, + task_id=task_id, + round_id=round_id, + fingerprint=str(variant_outcome.get("fingerprint") or ""), + variant_name=str(variant_outcome.get("variant_name") or ""), + outcome=outcome, + adopted=adopted, + gain_pct=metrics.get("gain_pct"), + before_tput=metrics.get("base_tput"), + after_tput=metrics.get("tput"), + error_class=str(variant_outcome.get("error_class") or ""), + provenance=str(variant_outcome.get("provenance") or ""), + ) + + +def _variant_verdict(outcome: str, adopted: bool) -> Verdict: + """How one explore variant's own outcome settles against the adoption rule. + + An executor KEEP the adoption rule did not lift is a refusal, not a revert: + nothing about it failed to measure. + """ + if outcome == OUTCOME_KEEP: + return Verdict.ADOPTED if adopted else Verdict.REFUSED + if outcome in _MEASURED_LOSS_OUTCOMES: + return Verdict.REVERTED + return Verdict.REFUSED + + +def _variant_measurement(metrics: Any) -> _VariantMeasurement: + """One variant's measured figures, blank throughout when it never measured.""" + if not isinstance(metrics, dict): + return _VariantMeasurement(gain_pct=None, throughput_after=None, detail={}) + return _VariantMeasurement( + gain_pct=to_float(metrics.get("gain_pct")), + throughput_after=to_float(metrics.get("output_throughput")), + detail={key: metrics[key] for key in _VARIANT_DETAIL_METRICS if metrics.get(key) is not None}, + ) + + +def _variant_failure(variant_outcome: Mapping[str, Any], outcome: str) -> _VariantFailure: + """What the journal records about a reverted variant, blank for every other outcome.""" + if outcome != OUTCOME_REVERT: + return _VariantFailure(error_class=None, reason=None) + return _VariantFailure( + error_class=str(variant_outcome.get("error_class") or "") or None, + reason=str(variant_outcome.get("reason") or "") or None, + ) + + +def _variant_pitfall_signals(variant_outcome: Mapping[str, Any], metrics: Any) -> dict[str, Any]: + """A variant's measurement plus the fields a pitfall severity is ruled from.""" + return { + **(metrics if isinstance(metrics, dict) else {}), + "error_class": variant_outcome.get("error_class"), + "status": variant_outcome.get("outcome"), + } + + +def _variant_change_attrs(variant_attrs: Any, variant_name: str) -> dict[str, Any]: + """One variant's attributes as the change summary needs them. + + A variant that names no knob falls back to its own name, else every variant + in the grid would summarise to an identical row. """ - recorder = coord._coord.phase_framework.timeline() - if recorder is None: - return - from hyperloom.inference_optimizer.breakdown.recorder.framework_event import ARM_CONFIG, GateVerdict - - task_id = str(getattr(task, "task_id", "") or "") - params = getattr(task, "params", None) or {} - round_id = str(result_dict.get("round_id") or "") - recorded = 0 - for row in per_variant: + attrs = dict(variant_attrs) if isinstance(variant_attrs, dict) else {} + named_knob = attrs.get("extra_server_args") or attrs.get("extra_envs") or attrs.get("name") + if not named_knob and variant_name: + attrs["name"] = variant_name + return attrs + + +def _finalize_outcome(*, status: str, reason: str, backend: str, result_type: str) -> dict[str, Any]: + """One terminal Recipe-finalize outcome, in the shape the close-out step records.""" + return { + "status": status, + "reason": reason, + "backend": backend, + "result_type": result_type, + } + + +def _finalize_error(exc: BaseException, *, reason: str, backend: str, result_type: str) -> dict[str, Any]: + """The outcome for a finalize attempt an exception ended.""" + return { + "status": "error", + "reason": reason, + "backend": backend, + "result_type": result_type, + "error_class": type(exc).__name__, + } + + +def _remote_publish_failure_type(exc: BaseException) -> str: + """Whether the KB Store rejected the bundle or was never reached at all.""" + if type(exc).__name__ == _REMOTE_VALIDATION_ERROR: + return _close_out.RESULT_BUNDLE_BUILD_FAILED + return _close_out.RESULT_TRANSPORT_FAILED + + +def _rows_not_superseded(live_rows: Any, superseding: Any, key: str) -> list[dict[str, Any]]: + """Copies of the live rows whose ``key`` no superseding row already claims.""" + claimed = {str(row.get(key) or "") for row in superseding or [] if isinstance(row, dict)} + kept: list[dict[str, Any]] = [] + for row in live_rows or []: if not isinstance(row, dict): continue - outcome = str(row.get("outcome") or "") - if outcome in _NON_ATTEMPT_OUTCOMES: + if str(row.get(key) or "") in claimed: continue - metrics = row.get("metrics") if isinstance(row.get("metrics"), dict) else {} - variant = row.get("variant") if isinstance(row.get("variant"), dict) else {} - fingerprint = str(row.get("fingerprint") or "") - adopted = fingerprint in adopted_variants - # The fingerprint identifies the variant within the round and the round - # within the task, so the three together identify the attempt. - attempt_id = ":".join(part for part in (task_id, round_id, fingerprint) if part) or task_id - if not attempt_id: + kept.append(dict(row)) + return kept + + +def _improved_best_config(state: Any, attrs: Mapping[str, Any], live: Mapping[str, Any]) -> dict[str, Any]: + """The ``best_config`` override, empty unless this session genuinely beat the live row. + + A session that validated nothing has no standing to overwrite a + configuration somebody else measured. + """ + my_throughput = float(attrs.get("best_throughput") or 0.0) + validated_gain = float(state.cumulative_gain_validated or 0.0) + current_args = str(state.current_best.get("extra_server_args") or "").strip() + has_validated_win = bool((state.optimization_stack or []) or validated_gain > 0.0 or current_args) + if not has_validated_win or my_throughput <= to_float(live.get("best_throughput"), 0.0): + return {} + return { + "best_config": attrs["best_config"], + "best_throughput": my_throughput, + } + + +def _appended(live_facts: Any, addition: Mapping[str, Any] | None) -> list[Any]: + """The live lesson or pitfall list with one more appended, when the caller brought one.""" + facts = list(live_facts or []) + if addition is not None: + facts.append(addition) + return facts + + +def _amend_provenance(details: Mapping[str, Any] | None) -> dict[str, Any]: + """The provenance block stamped on every amend; the central server requires one.""" + return { + "source": "hyperloom-inference-optimizer", + "generator": "coordinator", + "generated_at": datetime.now(timezone.utc).isoformat(timespec="microseconds"), + "details": dict(details or {}), + } + + +def _amended_recipe_fields(live: Mapping[str, Any], overrides: Mapping[str, Any]) -> dict[str, Any]: + """The preserved recipe fields for one amend: an override wins, an unset field keeps its live value. + + An absent or empty live value falls back to the field's own default, so + ``put_recipe`` never sees ``None`` where it declares a container. + """ + merged: dict[str, Any] = {} + for field, default in _PRESERVED_RECIPE_FIELDS.items(): + if field in overrides: + merged[field] = overrides[field] continue - gates = [gate for gate in (row.get("gates") or []) if isinstance(gate, dict)] - # This arm gates accuracy rather than reporting it, so the block both - # arms carry is projected from the gate that ruled. No gate row means - # nothing gated the variant, which is not a gate that refused it. - accuracy_gate = next((gate for gate in gates if str(gate.get("gate") or "") == "accuracy"), {}) - error_class = str(row.get("error_class") or "") - error_excerpt = str(row.get("error_excerpt") or "") - failure_attribution = "" - if outcome in UNMEASURED_OUTCOMES: - failure_attribution = classify_failure_attribution( - error_class=error_class, - error_excerpt=error_excerpt, - reason=row.get("reason"), - explicit=row.get("failure_attribution"), - ) - recorder.record_attempt( - attempt_id, - arm=ARM_CONFIG, - round_id=round_id, - task_id=task_id, - proposal_ref=str(params.get("proposal_msg_id") or ""), - provenance=str(row.get("provenance") or ""), - outcome=outcome, - reason=str(row.get("reason") or ""), - reasoning=str(variant.get("note") or ""), - reasoning_origin=str(variant.get("reasoning_origin") or ""), - experience_citations=variant.get("experience_citations") or [], - stage=str(row.get("stage") or ""), - fingerprint=fingerprint, - # The fingerprint is the join key; the name is what a reader - # recognises the variant by. - variant_name=str(row.get("variant_name") or ""), - measurement={ - # Both ends of the pair: the anchor advances on every KEEP, - # so a percentage without its denominator adds to nothing. - "before_tput": metrics.get("base_tput"), - "after_tput": metrics.get("tput"), - "gain_pct": metrics.get("gain_pct"), - "runtime_sec": metrics.get("runtime_sec"), - "estimated_output_throughput": metrics.get("estimated_output_throughput"), - }, - config_delta={ - "extra_server_args": variant.get("extra_server_args"), - "extra_envs": variant.get("extra_envs"), - "remove_args": variant.get("remove_args"), - "unset_envs": variant.get("unset_envs"), - "args_mode": variant.get("args_mode"), - }, - accuracy={ - "required": True if accuracy_gate else None, - "value": accuracy_gate.get("observed"), - "reference": accuracy_gate.get("threshold"), - "passed": accuracy_gate.get("passed"), - }, - failure={ - "error_class": error_class, - "error_excerpt": error_excerpt, - "attribution": failure_attribution, - }, - artifacts={ - "workspace": str(row.get("workspace") or ""), - "server_log_path": str(row.get("server_log_path") or ""), - "raw_result_path": str(row.get("raw_result_path") or ""), - }, - decision=outcome, - adopted=adopted, - # Recorded rather than referenced: every KEEP advances the - # stack, so the session's current config is not what this - # variant was measured on top of. - measured_against=row.get("measured_against") or {}, - # What stood behind the verdict. Absent when nothing ruled. - validation_basis=str(row.get("validation_basis") or ""), - # A pair is what makes a gain addable, so eligibility follows - # the pair being present rather than the outcome being a KEEP. - attribution_eligible=(adopted and metrics.get("base_tput") is not None and metrics.get("tput") is not None), - ) - for gate in gates: - if not str(gate.get("gate") or ""): - continue - recorder.record_attempt_gate( - attempt_id, - GateVerdict( - gate=str(gate.get("gate")), - passed=gate.get("passed"), - reason=str(gate.get("reason") or ""), - observed=gate.get("observed"), - threshold=gate.get("threshold"), - ), - ) - recorded += 1 - proposal_ref = str(params.get("proposal_msg_id") or "") - if not (recorded and proposal_ref): - return - # Settled here rather than at materialization: a grid whose task dies before - # any variant runs has no attempt to stand behind an ``attempted`` reading, - # and is more honestly left unsettled. - from hyperloom.inference_optimizer.breakdown.recorder.framework_event import ( - DISPOSITION_ATTEMPTED, - STEP_ATTEMPTED, - ) + live_value = live.get(field) + merged[field] = type(default)(live_value) if live_value else copy(default) + return merged - recorder.record_proposal_step(proposal_ref, step=STEP_ATTEMPTED, outcome=str(recorded)) - recorder.settle_proposal(proposal_ref, disposition=DISPOSITION_ATTEMPTED) +def _amended_recipe_extras(live: Mapping[str, Any], overrides: Mapping[str, Any], state: Any) -> dict[str, Any]: + """The free-form extras an amend writes back: the prior ones, the caller's, then the model's identity. -def _extra_server_args(payload: Mapping[str, Any]) -> str: - """Read canonical ``extra_server_args`` from a payload.""" - value = payload.get("extra_server_args") - if value is None: - return "" - if isinstance(value, str): - return value - if isinstance(value, (list, tuple)): - return " ".join(str(v).strip() for v in value if str(v).strip()) - return str(value) + The architecture tags are re-stamped from config.json each time because a + row written before the architecture was read carries none. + """ + extras = {key: value for key, value in live.items() if key not in _RECIPE_OWNED_KEYS} + extras.update(overrides.get("extras") or {}) + architectures = state.model_architectures or [] + if isinstance(architectures, list): + named = [str(name).strip() for name in architectures if str(name or "").strip()] + if named: + extras["architectures"] = named + model_type = str(state.model_type or "").strip() + if model_type: + extras["model_type"] = model_type + return extras class RecipeJournalCollaborator(CoordinatorCollaborator): @@ -369,45 +473,26 @@ async def fact_write_hook( adopted_variants: Fingerprints of the explore winners whose lift landed; a per-variant KEEP outside it did not adopt anything. """ - result_dict = result.result if hasattr(result, "result") else (result or {}) - if not isinstance(result_dict, dict): - result_dict = {} + result_dict = _result_payload(result) source_session_id = self._source_session_id() per_variant = result_dict.get("per_variant_outcomes") if task.kind == "explore": - _record_config_run(self, task=task, result_dict=result_dict) + record_config_run(self, task=task, result_dict=result_dict) if task.kind == "explore" and isinstance(per_variant, list) and per_variant: - _record_config_attempts( + record_config_attempts( self, task=task, per_variant=per_variant, result_dict=result_dict, adopted_variants=adopted_variants, ) - round_id = str(result_dict.get("round_id") or "") - for vo in per_variant: - adopted = str(vo.get("fingerprint") or "") in adopted_variants - outcome = str(vo.get("outcome") or "") if isinstance(vo, dict) else "" - if outcome and outcome not in _NON_ATTEMPT_OUTCOMES: - metrics = vo.get("metrics") if isinstance(vo.get("metrics"), dict) else {} - row = AttemptRow( - lever_kind=LEVER_CONFIG, - task_id=str(task.task_id or ""), - round_id=round_id, - fingerprint=str(vo.get("fingerprint") or ""), - variant_name=str(vo.get("variant_name") or ""), - outcome=outcome, - adopted=adopted, - gain_pct=metrics.get("gain_pct"), - before_tput=metrics.get("base_tput"), - after_tput=metrics.get("tput"), - error_class=str(vo.get("error_class") or ""), - provenance=str(vo.get("provenance") or ""), - ) - record_attempt_row(self.shared_state, row) - self._record_fact_per_variant( - task=task, source_session_id=source_session_id, variant_outcome=vo, adopted=adopted - ) + self._record_grid_facts( + task=task, + source_session_id=source_session_id, + per_variant=per_variant, + round_id=str(result_dict.get("round_id") or ""), + adopted_variants=adopted_variants, + ) else: self._record_fact_per_task( task=task, @@ -417,6 +502,29 @@ async def fact_write_hook( ) self.shared_state.save(self.session_dir) + def _record_grid_facts( + self, + *, + task: "Task", + source_session_id: str, + per_variant: list[dict[str, Any]], + round_id: str, + adopted_variants: Collection[str], + ) -> None: + """Record every variant of one explore grid: an adoption-ledger row and a journal fact each.""" + task_id = str(task.task_id or "") + for variant_outcome in per_variant: + adopted = str(variant_outcome.get("fingerprint") or "") in adopted_variants + row = _grid_attempt_row(variant_outcome, task_id=task_id, round_id=round_id, adopted=adopted) + if row is not None: + record_attempt_row(self.shared_state, row) + self._record_fact_per_variant( + task=task, + source_session_id=source_session_id, + variant_outcome=variant_outcome, + adopted=adopted, + ) + def ensure_journal(self) -> Journal: """Lazy-instantiate the per-session :class:`Journal` (load_or_create reads an existing file on resume). @@ -526,7 +634,7 @@ def _record_fact_impl( """ models = [str(self.shared_state.model_name or "")] if self.shared_state.model_name else [] hardware = [str(self.shared_state.gpu_type or "")] if self.shared_state.gpu_type else [] - workload_tags = self._collect_workload_tags() + workload_tags = workload_kb_tags(self.shared_state) extra = workload_tags if workload_tags else None now_iso = datetime.now(timezone.utc).isoformat(timespec="seconds") @@ -739,73 +847,42 @@ def _record_fact_per_variant( """ journal = self.ensure_journal() outcome_raw = str(variant_outcome.get("outcome") or "") - if outcome_raw == "SKIPPED_DEDUP": + if outcome_raw in NON_ATTEMPT_OUTCOMES: return - if outcome_raw == "KEEP": - verdict = Verdict.ADOPTED if adopted else Verdict.REFUSED - elif outcome_raw in ("REVERT", "FAILED"): - verdict = Verdict.REVERTED - else: - verdict = Verdict.REFUSED - outcome = derive_journal_outcome(verdict, variant_outcome) + outcome = derive_journal_outcome(_variant_verdict(outcome_raw, adopted), variant_outcome) variant_name = str(variant_outcome.get("variant_name") or "") metrics = variant_outcome.get("metrics") or {} - gain_pct = to_float(metrics.get("gain_pct") if isinstance(metrics, dict) else None) - throughput_after = to_float(metrics.get("output_throughput") if isinstance(metrics, dict) else None) + measurement = _variant_measurement(metrics) variant_attrs = variant_outcome.get("variant") or {} - kind = classify_change_kind( - task.kind, - variant_attrs if isinstance(variant_attrs, dict) else None, - ) - # Ensure the change summary is variant-specific (else every explore variant writes an identical row). - change_attrs = dict(variant_attrs) if isinstance(variant_attrs, dict) else {} - if ( - not (change_attrs.get("extra_server_args") or change_attrs.get("extra_envs") or change_attrs.get("name")) - and variant_name - ): - change_attrs["name"] = variant_name + attrs = variant_attrs if isinstance(variant_attrs, dict) else None + kind = classify_change_kind(task.kind, attrs) + change_attrs = _variant_change_attrs(variant_attrs, variant_name) change = summarize_change(task.kind, change_attrs, None) - error_class = None - reason = None - if outcome == OUTCOME_REVERT: - error_class = str(variant_outcome.get("error_class") or "") or None - reason = str(variant_outcome.get("reason") or "") or None - # Proposer attribution + per-variant measurement detail, carried from the - # explore executor's per_variant_outcomes so the decision row records who - # proposed the change and how it measured (beyond headline gain/tput). - detail_metrics = { - k: metrics[k] - for k in ( - "runtime_sec", - "wall_clock_ratio_vs_baseline", - "estimated_output_throughput", - ) - if isinstance(metrics, dict) and metrics.get(k) is not None - } + failure = _variant_failure(variant_outcome, outcome) + tick = int(self.shared_state.tick or 0) journal.append_entry( JournalEntry( phase=self.journal_entry_phase(), - lever_kind=lever_kind_for_task(kind, variant_outcome if isinstance(variant_outcome, dict) else None), - iter=int(self.shared_state.tick or 0), + lever_kind=lever_kind_for_task(kind, variant_outcome), + iter=tick, kind=kind, change=change, outcome=outcome, - gain_pct=gain_pct, - throughput_after=throughput_after, - error_class=error_class, - reason=reason, + gain_pct=measurement.gain_pct, + throughput_after=measurement.throughput_after, + error_class=failure.error_class, + reason=failure.reason, task_id=task.task_id, variant_name=variant_name, provenance=str(variant_outcome.get("provenance") or ""), scope=str(variant_outcome.get("scope") or ""), fingerprint=str(variant_outcome.get("fingerprint") or ""), - metrics=detail_metrics, - tick=int(self.shared_state.tick or 0), - predicted_gain_pct=_predicted_gain( - variant_outcome, - variant_attrs if isinstance(variant_attrs, dict) else None, - getattr(task, "params", None), - ), + # Proposer attribution and the measurement detail beyond the + # headline figures, so the row records how the change measured + # and who proposed it. + metrics=measurement.detail, + tick=tick, + predicted_gain_pct=_predicted_gain(variant_outcome, attrs, getattr(task, "params", None)), ) ) @@ -817,297 +894,18 @@ def _record_fact_per_variant( source_session_id=source_session_id, is_keep=(outcome == OUTCOME_KEEP), change=change, - gain_pct=gain_pct, - throughput_after=throughput_after, + gain_pct=measurement.gain_pct, + throughput_after=measurement.throughput_after, best_config_candidate=self.extract_kept_best_config( task=task, variant_attrs=change_attrs, ), # Workload-shape tags — see _record_fact_per_task. evidence_refs=[f"log:task-{task.task_id}", f"variant:{variant_name}"], - pitfall_severity_dict={ - **(metrics if isinstance(metrics, dict) else {}), - "error_class": variant_outcome.get("error_class"), - "status": variant_outcome.get("outcome"), - }, + pitfall_severity_dict=_variant_pitfall_signals(variant_outcome, metrics), variant_name=variant_name, ) - def _collect_workload_tags(self) -> dict[str, Any]: - """Return the workload-shape KB tag dict for the current session; shared by recipe attrs + lesson/pitfall writes so the warm-start reader filters symmetrically. - - Returns: - A dict of workload-shape KB tags (framework, model, parallelism, - runtime versions, baseline workload extras) with empty values - omitted. - """ - ss = self.shared_state - out: dict[str, Any] = {} - # model_family is not part of the seven-dimension Recipe identity. - for attr in ("framework", "model_class", "model_name"): - text = str(getattr(ss, attr, "") or "").strip() - if text: - out[attr] = text - precision = getattr(ss, "precision", None) - if precision not in (None, "", 0): - out["precision"] = precision - out.update(workload_shape_tags(ss)) - # runtime version tags from stack_fingerprint_meta (cli writes at boot, resume reads verbatim). - fp_meta = ss.stack_fingerprint_meta or {} - if isinstance(fp_meta, dict): - # framework_version is whichever of sglang/vllm is active. - fw_lc = out.get("framework", "").lower() - if fw_lc in ("sglang", "vllm"): - version = str(fp_meta.get(fw_lc) or "").strip() - if version and version != "unknown": - out["framework_version"] = version - for src_key, dst_key in ( - ("rocm", "rocm_version"), - ("aiter", "aiter_version"), - ("image_digest", "image_digest"), - ): - version = str(fp_meta.get(src_key) or "").strip() - if version and version != "unknown": - out[dst_key] = version - # per-baseline workload extras from materialized YAML; keep bool False (don't drop an "explicitly disabled" signal). - wl_extra = ss.baseline_workload_extra or {} - if isinstance(wl_extra, dict): - for k in ("max_running_requests", "max_num_seqs"): - v = wl_extra.get(k) - if isinstance(v, int) and v > 0: - out[k] = v - for k in ("chunked_prefill_enabled", "enable_torch_compile"): - v = wl_extra.get(k) - if isinstance(v, bool): - out[k] = v - for k in ("quant_scheme", "workload_mode"): - v = wl_extra.get(k) - if isinstance(v, str) and v.strip(): - out[k] = v.strip() - return out - - def _build_kernel_optimizations_from_state(self) -> list[dict[str, Any]]: - """Collect KEEP'd kernel optimizations + their E2E verdict by joining kernel_opt_task_attempts (micro) and kernel_integrate_attempts (E2E) on kernel_id; non-integrated KEEPs surface integrated=False. Returns KernelOptimization-shaped dicts. - - Returns: - A list of KernelOptimization-shaped dicts for each KEEP'd kernel, - joined with its E2E integrate verdict where available. - """ - ss = self.shared_state - opt_attempts = ss.kernel_opt_task_attempts or {} - integ_attempts = ss.kernel_integrate_attempts or {} - if not isinstance(opt_attempts, dict): - return [] - - # Index integrate results by kernel_id (last write wins; entry carries rolled-up best_gain_pct). - integ_by_kid: dict[str, dict[str, Any]] = {} - integ_by_task: dict[str, dict[str, Any]] = {} - if isinstance(integ_attempts, dict): - for entry in integ_attempts.values(): - if not isinstance(entry, dict): - continue - kid = str(entry.get("kernel_id") or "") - if kid: - integ_by_kid[kid] = entry - task_group_key = str(entry.get("task_group_key") or "") - if task_group_key: - integ_by_task[task_group_key] = entry - - out: list[dict[str, Any]] = [] - for ledger_id, e in opt_attempts.items(): - if not isinstance(e, dict): - continue - if str(e.get("last_decision", "")).upper() != "KEEP": - continue - try: - micro = float(e.get("last_micro_speedup") or 0.0) - except (TypeError, ValueError): - micro = 0.0 - kid = str(e.get("current_kernel_id") or e.get("kernel_id") or ledger_id) - task_group_key = str(e.get("task_group_key") or "") - integ = integ_by_task.get(task_group_key) if task_group_key else integ_by_kid.get(kid) - e2e_gain = 0.0 - e2e_tput = 0.0 - e2e_decision = "" - integrated = False - if isinstance(integ, dict): - integrated = True - # Integrate-layer verdict (E2E); lets warm-start skip a micro-win/E2E-loss kernel. - e2e_decision = str(integ.get("last_decision") or "").upper() - try: - e2e_gain = float(integ.get("best_gain_pct") or 0.0) - except (TypeError, ValueError): - e2e_gain = 0.0 - # Last attempt's E2E re-bench throughput. - for att in reversed(list(integ.get("attempts") or [])): - if isinstance(att, dict) and att.get("new_tput") is not None: - try: - e2e_tput = float(att.get("new_tput") or 0.0) - except (TypeError, ValueError): - e2e_tput = 0.0 - break - out.append( - { - "kernel_id": kid, - "source_file": str(e.get("last_source_file") or ""), - "artifact_path": str(e.get("last_artifact_path") or ""), - "micro_speedup": micro, - "decision": "KEEP", - "e2e_gain_pct": e2e_gain, - "e2e_tput": e2e_tput, - "e2e_decision": e2e_decision, - "integrated": integrated, - "ts": str(e.get("last_ts") or ""), - } - ) - return out - - def _collect_attempt_provenance( - self, - ) -> tuple[dict[str, str], dict[str, str], list[dict[str, Any]]]: - """Map proven optimizations to their research-hint origin from the gaps[] attempts ledger; returns (kept_sources by name/kernel, kept_by_gap by canonical_id, reverted_rows). Fail-soft. - - Returns: - A ``(kept_sources, kept_by_gap, reverted_rows)`` tuple: KEEP'd - provenance keyed by variant/kernel name, KEEP'd provenance keyed by - gap canonical_id, and reverted-attempt rows. - """ - kept_sources: dict[str, str] = {} - kept_by_gap: dict[str, str] = {} - reverted_rows: list[dict[str, Any]] = [] - gaps = self.shared_state.gaps or [] - for gap in gaps: - if not isinstance(gap, dict): - continue - provenance = str(gap.get("provenance") or "").strip() - canonical = str(gap.get("canonical_id") or "").strip() - for attempt in gap.get("attempts") or []: - if not isinstance(attempt, dict): - continue - variant = str(attempt.get("variant_name") or "").strip() - kernel = str(attempt.get("kernel_id") or "").strip() - outcome = str(attempt.get("outcome") or "").strip().upper() - if outcome == "KEEP" and provenance: - if variant: - kept_sources.setdefault(variant, provenance) - if kernel: - kept_sources.setdefault(kernel, provenance) - if canonical: - kept_by_gap.setdefault(canonical, provenance) - elif outcome == "REVERT" and (variant or kernel): - row: dict[str, Any] = { - "name": variant or kernel, - "reason": "reverted", - "gain_pct": attempt.get("gain_pct"), - } - if provenance: - row["source"] = provenance - reverted_rows.append(row) - return kept_sources, kept_by_gap, reverted_rows - - def _build_recipe_attrs_from_state(self) -> dict[str, Any]: - """Materialise the recipe-shaped view of :class:`SharedState` (defensive getattr). - - Returns: - A recipe-shaped attrs dict (best_config, what_worked, what_failed, - kernel_optimizations, workload tags, session row) for KB recipe - writes. - """ - ss = self.shared_state - current_best = ss.current_best - opt_stack = ss.optimization_stack or [] - gain_per_stack = ss.gain_per_stack_entry or [] - last_failures = ss.last_action_failures or [] - # RecipeKB best_config keys on the canonical extra_server_args field. - best_config: dict[str, Any] = {} - cb_args = current_best.get("extra_server_args") - if cb_args: - best_config["extra_server_args"] = str(cb_args) - for key in ("extra_envs", "name", "tput", "accuracy"): - if key in current_best: - best_config[key] = current_best[key] - # Prefer the last validated stack layer for launch args (current_best may carry a corrupted string). - if opt_stack: - last_entry = opt_stack[-1] - if isinstance(last_entry, dict): - stack_args = str( - last_entry.get("candidate_extra_server_args") or last_entry.get("extra_server_args") or "", - ).strip() - if stack_args: - best_config["extra_server_args"] = stack_args - sediment_on = bool(ss.recipe_sediment_enabled) - kept_sources, kept_by_gap, reverted_rows = self._collect_attempt_provenance() if sediment_on else ({}, {}, []) - what_worked: list[dict[str, Any]] = [] - for idx, entry in enumerate(opt_stack): - if not isinstance(entry, dict): - continue - gain_per: float | None = None - if idx < len(gain_per_stack): - gain_per = gain_per_stack[idx] - name = str(entry.get("variant_name") or entry.get("name") or entry.get("kernel_id") or "") - row: dict[str, Any] = { - "name": name, - "extra_server_args": str(entry.get("extra_server_args") or ""), - "extra_envs": dict(entry.get("extra_envs") or {}), - "gain_pct": gain_per, - } - # Prefer the entry's gap-id provenance (naming-independent); fall back to name/kernel_id match. - entry_gap = str(entry.get("gap_canonical_id") or "").strip() - src = ( - (kept_by_gap.get(entry_gap) if entry_gap else None) - or kept_sources.get(name) - or kept_sources.get(str(entry.get("kernel_id") or "")) - ) - if src: - row["source"] = src - what_worked.append(row) - what_failed: list[dict[str, Any]] = [] - for failure in last_failures[-10:]: - if isinstance(failure, dict): - what_failed.append( - { - "name": str(failure.get("name") or failure.get("action") or ""), - "reason": str(failure.get("reason") or failure.get("error_class") or ""), - } - ) - for rev in reverted_rows: - what_failed.append(rev) - kernel_optimizations = self._build_kernel_optimizations_from_state() - cumulative_validated = float(ss.cumulative_gain_validated or 0.0) - validated_stack_len = int(ss.cumulative_gain_validated_stack_len or 0) - # Workload-shape tags for shape-filtered warm-start queries (shared via _collect_workload_tags). - workload_tags = self._collect_workload_tags() - # framework_version left unset here (manifest-derived); the T0 backfill writes it. - return { - "best_config": best_config, - "best_throughput": float(current_best.get("tput", 0.0)), - "what_worked": what_worked, - "what_failed": what_failed, - "kernel_optimizations": kernel_optimizations, - "last_profiled": str(ss.cumulative_gain_validated_ts or ""), - "workload": workload_tags, - "sessions": [ - { - "session_id": str(ss.recipe_kb_session_id or self.session_dir.name), - "gain_pct": cumulative_validated, - "stack_len": validated_stack_len or len(opt_stack), - # arbor-shape provenance so the session row is self-describing (before/after tput + knobs). - "throughput_before": float(ss.baseline_tput or 0.0), - "throughput_after": float(current_best.get("tput", 0.0)), - "date": datetime.now(timezone.utc).isoformat(), - "actions_taken": [ - nm - for nm in ( - str(e.get("variant_name") or e.get("name") or e.get("action") or "").strip() - for e in opt_stack - if isinstance(e, dict) - ) - if nm - ], - } - ], - } - def _record_remote_recipe_audit( self, *, @@ -1280,248 +1078,225 @@ def _write_back_scope(self) -> dict[str, Any]: "osl": to_int(state.osl, None), } + def _unvalidated_stack_outcome(self) -> dict[str, Any] | None: + """The skip outcome when the working recipe outran its last validation, else ``None``. + + ``current_best`` and ``cumulative_gain_validated`` are only one Recipe's + figures when nothing was lifted since the last validation. Otherwise + every sink below -- journal, local KB, remote KB -- would pair the + newer config with the older gain, so none of them runs. + """ + state = self.shared_state + if not state.optimization_stack_has_unvalidated_keeps(): + return None + log.warning( + "Recipe finalize skipped: working recipe is not validated (stack=%s/%s generation=%s/%s)", + len(state.optimization_stack or []), + state.cumulative_gain_validated_stack_len, + state.working_recipe_generation, + state.validated_recipe_generation, + ) + return _finalize_outcome( + status="skipped", + reason="unvalidated_recipe_stack", + backend="none", + result_type=_close_out.RESULT_UNVALIDATED_RECIPE, + ) + + def _finalize_journal(self) -> None: + """Stamp the journal header with the session's settled throughput and total gain.""" + state = self.shared_state + final_tput = float(state.current_best.get("tput", 0.0)) + self.ensure_journal().finalize( + final_throughput=final_tput if final_tput > 0 else None, + total_gain_pct=float(state.cumulative_gain_validated or 0.0), + ) + def finalize_recipe_and_journal( self, *, source: str = "close", ) -> dict[str, Any]: """Finalize Recipe state and return a secret-free observable outcome.""" - # ``current_best`` and ``cumulative_gain_validated`` are only one Recipe's - # figures when nothing was lifted since the last validation. Otherwise - # every sink below -- journal, local KB, remote KB -- would pair the - # newer config with the older gain, so none of them runs. - if self.shared_state.optimization_stack_has_unvalidated_keeps(): - ss = self.shared_state - log.warning( - "Recipe finalize skipped: working recipe is not validated (stack=%s/%s generation=%s/%s)", - len(ss.optimization_stack or []), - ss.cumulative_gain_validated_stack_len, - ss.working_recipe_generation, - ss.validated_recipe_generation, - ) - return { - "status": "skipped", - "reason": "unvalidated_recipe_stack", - "backend": "none", - "result_type": _close_out.RESULT_UNVALIDATED_RECIPE, - } - journal = self.ensure_journal() - ss = self.shared_state - final_tput = float(ss.current_best.get("tput", 0.0)) - total_gain = float(ss.cumulative_gain_validated or 0.0) - journal.finalize( - final_throughput=final_tput if final_tput > 0 else None, - total_gain_pct=total_gain, - ) + unvalidated = self._unvalidated_stack_outcome() + if unvalidated is not None: + return unvalidated + self._finalize_journal() - kp = self.knowledge_plane - if bool(getattr(kp, "kb_disabled", False)): + knowledge_plane = self.knowledge_plane + if bool(getattr(knowledge_plane, "kb_disabled", False)): log.info("Recipe KB finalize skipped (--degraded-kb)") - return { - "status": "skipped", - "reason": "degraded_kb", - "backend": "disabled", - "result_type": _close_out.RESULT_KB_DISABLED, - } + return _finalize_outcome( + status="skipped", + reason="degraded_kb", + backend="disabled", + result_type=_close_out.RESULT_KB_DISABLED, + ) - from ..knowledge.config import KnowledgeConfig, KnowledgeStoreMode + from ..knowledge.config import KnowledgeConfig try: - config = getattr(kp, "config", None) or KnowledgeConfig.from_env() + config = getattr(knowledge_plane, "config", None) or KnowledgeConfig.from_env() except Exception as exc: log.exception("Recipe KB finalize configuration failed (non-fatal)") - return { - "status": "error", - "reason": f"configuration:{type(exc).__name__}", - "backend": "unknown", - "result_type": _close_out.RESULT_CONFIGURATION_FAILED, - "error_class": type(exc).__name__, - } + return _finalize_error( + exc, + reason=f"configuration:{type(exc).__name__}", + backend="unknown", + result_type=_close_out.RESULT_CONFIGURATION_FAILED, + ) from hyperloom.common.perf_metric import agentx_active - if ( - agentx_active(benchmark_mode=self.shared_state.benchmark_mode) - and config.mode is not KnowledgeStoreMode.REMOTE - ): - return { - "status": "skipped", - "reason": "agentx_local_store_unsupported", - "backend": str(getattr(config, "mode", "") or "unknown"), - "result_type": _close_out.RESULT_AGENTX_BLOCKED, - } - if config.mode is KnowledgeStoreMode.REMOTE: - # Remote mode has one Recipe sink: the KB Store final session - # writer. T0 and runtime amendment are intentionally absent. - remote_cid = "" - remote_sid = str( - self.shared_state.recipe_kb_session_id or self.shared_state.session_id or self.session_dir.name + remote = config.mode is KnowledgeStoreMode.REMOTE + if agentx_active(benchmark_mode=self.shared_state.benchmark_mode) and not remote: + return _finalize_outcome( + status="skipped", + reason="agentx_local_store_unsupported", + backend=str(getattr(config, "mode", "") or "unknown"), + result_type=_close_out.RESULT_AGENTX_BLOCKED, ) - try: - from ..knowledge.remote_recipe import HyperloomRemoteKB - - remote_cid = self.workload_canonical_id() - remote_result = HyperloomRemoteKB.from_env().write( - remote_cid, - self.shared_state, - session_id=remote_sid, - ) - log.info( - "Remote Recipe KB finalize: status=%s reason=%s cid=%s sid=%s", - remote_result.status, - remote_result.reason, - remote_cid, - remote_result.session_id, - ) - self._record_remote_recipe_audit( - source=source, - status=remote_result.status, - canonical_id=remote_cid, - session_id=remote_result.session_id, - primary_metric=remote_result.primary_metric, - primary_value=remote_result.primary_value, - reason=remote_result.reason, - ) - return { - "status": remote_result.status, - "reason": remote_result.reason, - "backend": "kb-store", - "canonical_id": str(getattr(remote_result, "canonical_id", "") or remote_cid), - "session_id": str(getattr(remote_result, "session_id", "") or remote_sid), - "result_type": _remote_result_type(remote_result.status, remote_result.reason), - } - except Exception as exc: - self._record_remote_recipe_audit( - source=source, - status="error", - canonical_id=remote_cid, - session_id=remote_sid, - error=exc, - ) - log.exception("Remote Recipe KB finalize failed (non-fatal)") - return { - "status": "error", - "reason": type(exc).__name__, - "backend": "kb-store", - "canonical_id": remote_cid, - "session_id": remote_sid, - "result_type": ( - _close_out.RESULT_BUNDLE_BUILD_FAILED - if type(exc).__name__ == _REMOTE_VALIDATION_ERROR - else _close_out.RESULT_TRANSPORT_FAILED - ), - "error_class": type(exc).__name__, - } - - # Local mode never consults ambient KB_STORE_* credentials. - if self.recipe_kb is None: + if remote: + return self._publish_remote_recipe(source) + return self._write_local_recipe() + + def _publish_remote_recipe(self, source: str) -> dict[str, Any]: + """Publish the terminal Recipe to the KB Store and audit the attempt. + + Remote mode has one Recipe sink: the KB Store final session writer. T0 + and runtime amendment are intentionally absent. + """ + state = self.shared_state + session_id = str(state.recipe_kb_session_id or state.session_id or self.session_dir.name) + canonical_id = "" + try: + from ..knowledge.remote_recipe import HyperloomRemoteKB + + canonical_id = self.workload_canonical_id() + written = HyperloomRemoteKB.from_env().write(canonical_id, state, session_id=session_id) + except Exception as exc: + self._record_remote_recipe_audit( + source=source, + status="error", + canonical_id=canonical_id, + session_id=session_id, + error=exc, + ) + log.exception("Remote Recipe KB finalize failed (non-fatal)") return { - "status": "skipped", - "reason": "no_recipe_backend", - "backend": "local", - "result_type": _close_out.RESULT_KB_DISABLED, + **_finalize_error( + exc, + reason=type(exc).__name__, + backend="kb-store", + result_type=_remote_publish_failure_type(exc), + ), + "canonical_id": canonical_id, + "session_id": session_id, } - ss = self.shared_state - model_name = ss.model_name or "" - gpu_type = ss.gpu_type or "" + log.info( + "Remote Recipe KB finalize: status=%s reason=%s cid=%s sid=%s", + written.status, + written.reason, + canonical_id, + written.session_id, + ) + self._record_remote_recipe_audit( + source=source, + status=written.status, + canonical_id=canonical_id, + session_id=written.session_id, + primary_metric=written.primary_metric, + primary_value=written.primary_value, + reason=written.reason, + ) + return { + **_finalize_outcome( + status=written.status, + reason=written.reason, + backend="kb-store", + result_type=_remote_result_type(written.status, written.reason), + ), + "canonical_id": str(getattr(written, "canonical_id", "") or canonical_id), + "session_id": str(getattr(written, "session_id", "") or session_id), + } + + def _final_recipe_overrides(self) -> dict[str, Any]: + """What this session's close writes over the live Recipe row. + + Read-modify-write under the cid flock, so two finalises of the same + session leave one row rather than tearing the lists they both append to. + """ + state = self.shared_state + attrs = recipe_attrs_from_state(state, self.session_dir.name) + live = self._authoritative_recipe_row(self.workload_canonical_id()) + my_sessions = list(attrs["sessions"] or []) + my_kernels = list(attrs.get("kernel_optimizations") or []) + # KEEP'd kernel optimizations ride the extras channel. + extras = dict(attrs.get("workload") or {}) + kernels = my_kernels + _rows_not_superseded(live.get("kernel_optimizations"), my_kernels, "kernel_id") + if kernels: + extras["kernel_optimizations"] = kernels + overrides: dict[str, Any] = { + "what_worked": attrs["what_worked"], + "what_failed": attrs["what_failed"], + "last_profiled": attrs["last_profiled"], + "sessions": _rows_not_superseded(live.get("sessions"), my_sessions, "session_id") + my_sessions, + "extras": extras, + } + overrides.update(_improved_best_config(state, attrs, live)) + return overrides + + def _write_local_recipe(self) -> dict[str, Any]: + """Amend the local Recipe row with this session's settled result. + + Local mode never consults ambient KB_STORE_* credentials. + """ + if self.recipe_kb is None: + return _finalize_outcome( + status="skipped", + reason="no_recipe_backend", + backend="local", + result_type=_close_out.RESULT_KB_DISABLED, + ) + state = self.shared_state + model_name = state.model_name or "" + gpu_type = state.gpu_type or "" if not model_name or not gpu_type: log.info( "recipe KB finalize_recipe: missing model/hardware (model=%r hardware=%r); skipping update_recipe", model_name, gpu_type, ) - return { - "status": "skipped", - "reason": "missing_model_or_hardware", - "backend": "local", - "result_type": _close_out.RESULT_INVALID_SCOPE, - } + return _finalize_outcome( + status="skipped", + reason="missing_model_or_hardware", + backend="local", + result_type=_close_out.RESULT_INVALID_SCOPE, + ) try: - attrs = self._build_recipe_attrs_from_state() - # Hoist workload tags flat into top-level recipe attrs (shallow-merged) for warm-start filters. - workload_tags = attrs.get("workload") or {} - - # sessions[] read-modify-write: read anchor, drop prior entry with our session_id (resume safety), append ours, write back. - my_sessions = list(attrs["sessions"] or []) - my_session_ids = {str((s or {}).get("session_id") or "") for s in my_sessions if isinstance(s, dict)} - # v2: read-modify-write the recipe row; sessions[] merged in-process under the cid flock so concurrent finalises don't tear. - merged_sessions: list[dict[str, Any]] = list(my_sessions) - existing_row: dict[str, Any] = {} - if self.recipe_kb is not None: - cid = self.workload_canonical_id() - # Read exactly the local store's authority row. - try: - existing_row = self.recipe_kb.get_authoritative_recipe(canonical_id=cid) or {} - except Exception as exc: # noqa: BLE001 - the recipe store may be remote - log.info("recipe read failed (%s); finalize appends the current session only", exc) - existing_sessions: list[dict[str, Any]] = [] - for row in existing_row.get("sessions") or []: - if not isinstance(row, dict): - continue - if str(row.get("session_id") or "") in my_session_ids: - # Resume/retry of the same session — our new entry supersedes the prior one. - continue - existing_sessions.append(dict(row)) - merged_sessions = existing_sessions + my_sessions - - # KEEP'd kernel optimizations ride the extras channel; merge with prior rows, dedup by kernel_id. - kopts_new = list(attrs.get("kernel_optimizations") or []) - new_kids = {str((k or {}).get("kernel_id") or "") for k in kopts_new if isinstance(k, dict)} - merged_kopts: list[dict[str, Any]] = list(kopts_new) - for prior in existing_row.get("kernel_optimizations") or []: - if not isinstance(prior, dict): - continue - if str(prior.get("kernel_id") or "") in new_kids: - continue - merged_kopts.append(dict(prior)) - - extras_payload = dict(workload_tags or {}) - if merged_kopts: - extras_payload["kernel_optimizations"] = merged_kopts - - overrides: dict[str, Any] = { - "what_worked": attrs["what_worked"], - "what_failed": attrs["what_failed"], - "last_profiled": attrs["last_profiled"], - "sessions": merged_sessions, - "extras": extras_payload, - } - # Overwrite best_config/best_throughput only on a real improvement: requires has_validated_win AND my_tput > live_tput. - my_tput = float(attrs.get("best_throughput") or 0.0) - cb_args_now = str(ss.current_best.get("extra_server_args") or "").strip() - validated_gain = float(ss.cumulative_gain_validated or 0.0) - has_validated_win = bool((ss.optimization_stack or []) or validated_gain > 0.0 or cb_args_now) - try: - live_tput = float(existing_row.get("best_throughput") or 0.0) - except (TypeError, ValueError): - live_tput = 0.0 - if has_validated_win and my_tput > live_tput: - overrides["best_config"] = attrs["best_config"] - overrides["best_throughput"] = my_tput self.kb_amend_recipe( - recipe_overrides=overrides, + recipe_overrides=self._final_recipe_overrides(), provenance_details={ "phase": "close_finalize", "evidence": [ - f"log:session-{ss.recipe_kb_session_id or self.session_dir.name}", + f"log:session-{state.recipe_kb_session_id or self.session_dir.name}", ], }, ) - return { - "status": "written", - "reason": "", - "backend": "local", - "result_type": _close_out.RESULT_WRITTEN, - } # Catch-all keeps CLOSE step 2.5 defensive against programmer bugs. except Exception as exc: log.exception("update_recipe raised unexpectedly") - return { - "status": "error", - "reason": type(exc).__name__, - "backend": "local", - "result_type": _close_out.RESULT_TRANSPORT_FAILED, - "error_class": type(exc).__name__, - } + return _finalize_error( + exc, + reason=type(exc).__name__, + backend="local", + result_type=_close_out.RESULT_TRANSPORT_FAILED, + ) + return _finalize_outcome( + status="written", + reason="", + backend="local", + result_type=_close_out.RESULT_WRITTEN, + ) def _kb_hardware_slug(self) -> str: """Topology-aware hardware dimension for the recipe ``canonical_id``.""" @@ -1533,46 +1308,42 @@ def _kb_hardware_slug(self) -> str: def workload_canonical_id(self) -> str: """Return the workload's canonical seven-dimension Recipe identity.""" - ss = self.shared_state - workload = ss.model_name or "unknown_model" - hw = self._kb_hardware_slug() - framework = str(ss.framework or "") - framework_version = str(ss.framework_version or "") - if not framework_version and framework: - framework_version = detect_framework_version(framework) - precision = str(ss.precision or "") - model_type = str(ss.model_type or "") - architectures = ss.model_architectures or [] + state = self.shared_state + identity = _framework_identity(state) from hyperloom.common.perf_metric import agentx_active return recipe_canonical_id( - model=workload, - hardware=hw, - framework_name=framework, - framework_version=framework_version, - precision=precision, - model_type=model_type, - architectures=architectures, - scheme=("agentx" if agentx_active(benchmark_mode=ss.benchmark_mode) else "inference"), + model=state.model_name or "unknown_model", + hardware=self._kb_hardware_slug(), + framework_name=identity.name, + framework_version=identity.version, + precision=identity.precision, + model_type=str(state.model_type or ""), + architectures=state.model_architectures or [], + scheme=("agentx" if agentx_active(benchmark_mode=state.benchmark_mode) else "inference"), ) + def _authoritative_recipe_row(self, canonical_id: str) -> dict[str, Any]: + """The store's exact authority row for a canonical id, empty when the read fails. + + A store that refuses the read still gets this session's write: losing + the rest of the row is better than losing what this session proved. + """ + try: + return self.recipe_kb.get_authoritative_recipe(canonical_id=canonical_id) or {} + except Exception as exc: # noqa: BLE001 - the recipe store may be remote + log.info("recipe authority read failed for cid=%s (%s); proceeding without it", canonical_id, exc) + return {} + def read_local_recipe_row(self) -> dict[str, Any]: - """Load the selected store's exact authority row for writes.""" + """Load the selected store's exact authority row for writes, cached for the current tick.""" if self.recipe_kb is None: return {} tick = int(self.shared_state.tick or 0) cache = self._local_recipe_cache if isinstance(cache, tuple) and len(cache) == 2 and cache[0] == tick: return cache[1] - try: - row = ( - self.recipe_kb.get_authoritative_recipe( - canonical_id=self.workload_canonical_id(), - ) - or {} - ) - except Exception: # noqa: BLE001 - the recipe store may be remote - row = {} + row = self._authoritative_recipe_row(self.workload_canonical_id()) self._local_recipe_cache = (tick, row) return row @@ -1642,6 +1413,20 @@ def kb_best_config_overrides_for_keep( return overrides return {} + def _amends_local_recipe(self) -> bool: + """Whether this session's Recipe row is a local one an amend may rewrite. + + Remote mode has a single Recipe sink at close, and AgentX sessions do + not write recipes at all. + """ + config = getattr(self.knowledge_plane, "config", None) + mode = getattr(getattr(config, "mode", None), "value", None) + if mode == KnowledgeStoreMode.REMOTE.value or self.recipe_kb is None: + return False + from hyperloom.common.perf_metric import agentx_active + + return not agentx_active(benchmark_mode=self.shared_state.benchmark_mode) + def kb_amend_recipe( self, *, @@ -1650,129 +1435,30 @@ def kb_amend_recipe( recipe_overrides: dict[str, Any] | None = None, provenance_details: dict[str, Any] | None = None, ) -> None: - """Read-modify-write helper for the recipe-snapshot KB: load live row, append lesson/pitfall, merge recipe_overrides (unset fields preserved), write back. Best-effort; lesson/pitfall appended without dedup.""" - config = getattr(self.knowledge_plane, "config", None) - if getattr(getattr(config, "mode", None), "value", None) == "remote" or self.recipe_kb is None: - return - from hyperloom.common.perf_metric import agentx_active + """Read-modify-write the local recipe row: append the fact, merge the overrides, write it back. - if agentx_active(benchmark_mode=self.shared_state.benchmark_mode): + Best effort. A lesson or pitfall is appended without dedup, and a field + the caller did not override keeps the value the live row carries. + """ + if not self._amends_local_recipe(): return - cid = self.workload_canonical_id() - - ss = self.shared_state - framework = str(ss.framework or "") - framework_version = str(ss.framework_version or "") - if not framework_version and framework: - framework_version = detect_framework_version(framework) - precision = str(ss.precision or "") - - # Local mode reads the exact authority row before amending it. - try: - live = self.recipe_kb.get_authoritative_recipe(canonical_id=cid) or {} - except Exception as exc: # noqa: BLE001 - log.info( - "kb_amend_recipe: authority get_recipe failed (%s); proceeding with empty live", - exc, - ) - live = {} - - lessons = list(live.get("lessons") or []) - if append_lesson is not None: - lessons.append(append_lesson) - pitfalls = list(live.get("pitfalls") or []) - if append_pitfall is not None: - pitfalls.append(append_pitfall) - - # Build put_recipe kwargs, preserving live fields the caller didn't override. + state = self.shared_state + canonical_id = self.workload_canonical_id() + identity = _framework_identity(state) + live = self._authoritative_recipe_row(canonical_id) overrides = dict(recipe_overrides or {}) - _reserved = { - "canonical_id", - "version", - "created_at", - "updated_at", - "model", - "hardware", - "framework", - "framework_name", - "framework_version", - "precision", - "best_config", - "best_throughput", - "what_worked", - "what_failed", - "remaining_gaps", - "pitfalls", - "lessons", - "last_profiled", - "stack_fingerprint", - "sessions", - "authority", - "confidence", - "evidence_refs", - "provenance", - } - prior_extras = {k: v for k, v in live.items() if k not in _reserved} - merged_extras = {**prior_extras, **(overrides.get("extras") or {})} - # Re-stamp config.json architecture-identity tags; skipped when unset. - _arch = ss.model_architectures or [] - if isinstance(_arch, list): - _arch_list = [str(a).strip() for a in _arch if str(a or "").strip()] - if _arch_list: - merged_extras["architectures"] = _arch_list - _mtype = str(ss.model_type or "").strip() - if _mtype: - merged_extras["model_type"] = _mtype put_kwargs: dict[str, Any] = { - "canonical_id": cid, - "model": ss.model_name or "unknown_model", + "canonical_id": canonical_id, + "model": state.model_name or "unknown_model", "hardware": self._kb_hardware_slug(), - "framework_name": framework, - "framework_version": framework_version, - "precision": precision, - "best_config": overrides.get("best_config") - if "best_config" in overrides - else dict(live.get("best_config") or {}), - "best_throughput": overrides.get("best_throughput") - if "best_throughput" in overrides - else float(live.get("best_throughput") or 0.0), - "what_worked": overrides.get("what_worked") - if "what_worked" in overrides - else list(live.get("what_worked") or []), - "what_failed": overrides.get("what_failed") - if "what_failed" in overrides - else list(live.get("what_failed") or []), - "remaining_gaps": overrides.get("remaining_gaps") - if "remaining_gaps" in overrides - else list(live.get("remaining_gaps") or []), - "pitfalls": pitfalls, - "lessons": lessons, - "last_profiled": overrides.get("last_profiled") - if "last_profiled" in overrides - else str(live.get("last_profiled") or ""), - "stack_fingerprint": overrides.get("stack_fingerprint") - if "stack_fingerprint" in overrides - else dict(live.get("stack_fingerprint") or {}), - "sessions": overrides.get("sessions") if "sessions" in overrides else list(live.get("sessions") or []), - "extras": merged_extras, - # Preserve audit fields across the amend (else put_recipe resets them to defaults). - "authority": overrides.get("authority") - if "authority" in overrides - else str(live.get("authority") or "EXPERIENTIAL"), - "confidence": overrides.get("confidence") - if "confidence" in overrides - else float(live.get("confidence") or 0.85), - "evidence_refs": overrides.get("evidence_refs") - if "evidence_refs" in overrides - else list(live.get("evidence_refs") or []), - "provenance": { - "source": "hyperloom-inference-optimizer", - "generator": "coordinator", - "generated_at": datetime.now(timezone.utc).isoformat( - timespec="microseconds", - ), - "details": dict(provenance_details or {}), - }, + "framework_name": identity.name, + "framework_version": identity.version, + "precision": identity.precision, + "pitfalls": _appended(live.get("pitfalls"), append_pitfall), + "lessons": _appended(live.get("lessons"), append_lesson), + "extras": _amended_recipe_extras(live, overrides, state), + "provenance": _amend_provenance(provenance_details), + **_amended_recipe_fields(live, overrides), } try: self.recipe_kb.put_recipe(**put_kwargs) @@ -1780,7 +1466,7 @@ def kb_amend_recipe( except Exception: log.exception( "kb_amend_recipe: put_recipe failed for cid=%s", - cid, + canonical_id, ) From 20d08235fd6788501247f81743ad601753efb68f Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 19 Oct 2026 18:49:40 +0800 Subject: [PATCH 02/14] refactor(knowledge): merge a session's result into the live Recipe row in its own module The read-modify-write an amend performs is one job: the caller's overrides win, each field it left alone keeps the value the live row carries, and the extras another session wrote survive untouched. That rule, its table of the fields a recipe carries and the row-supersession it shares with the close path now live in `_recipe_amend.py` rather than among the journal's own helpers. No observable change: internal structure only. Co-Authored-By: Claude Opus 5 --- scripts/code_metrics_baseline.txt | 2 +- .../orchestrator/knowledge/_recipe_amend.py | 141 +++++++++++++++++ .../orchestrator/knowledge/recipe_journal.py | 148 ++---------------- 3 files changed, 158 insertions(+), 133 deletions(-) create mode 100644 src/hyperloom/orchestrator/knowledge/_recipe_amend.py diff --git a/scripts/code_metrics_baseline.txt b/scripts/code_metrics_baseline.txt index 7e2b63224a..de57e0ad94 100644 --- a/scripts/code_metrics_baseline.txt +++ b/scripts/code_metrics_baseline.txt @@ -960,7 +960,7 @@ module-lines src/hyperloom/orchestrator/actions/executors/roofline.py 1303 module-lines src/hyperloom/orchestrator/kernel/apply_kernel_patch.py 2657 module-lines src/hyperloom/orchestrator/kernel/conc_sweep.py 1429 module-lines src/hyperloom/orchestrator/kernel/request_handlers.py 4888 -module-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py 1473 +module-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py 1376 module-lines src/hyperloom/orchestrator/knowledge/recipe_kb_t0.py 1318 module-lines src/hyperloom/orchestrator/knowledge/remote_recipe/values.py 1284 module-lines src/hyperloom/orchestrator/loop/coordinator.py 1423 diff --git a/src/hyperloom/orchestrator/knowledge/_recipe_amend.py b/src/hyperloom/orchestrator/knowledge/_recipe_amend.py new file mode 100644 index 0000000000..90c8a8aca1 --- /dev/null +++ b/src/hyperloom/orchestrator/knowledge/_recipe_amend.py @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Merging one session's result into the live Recipe row. + +A recipe row outlives the session amending it, so every write here is a +read-modify-write: the caller's overrides win, each field it left alone keeps +the value the live row carries, and the free-form extras some other session +wrote survive untouched. +""" + +from __future__ import annotations + +from copy import copy +from datetime import datetime, timezone +from types import MappingProxyType +from typing import Any, Mapping + +from hyperloom.common.coerce import to_float + +#: Recipe fields an amend carries forward when the caller does not override +#: them, each with the empty value ``put_recipe`` falls back to. +PRESERVED_RECIPE_FIELDS: Mapping[str, Any] = MappingProxyType( + { + "best_config": {}, + "best_throughput": 0.0, + "what_worked": [], + "what_failed": [], + "remaining_gaps": [], + "last_profiled": "", + "stack_fingerprint": {}, + "sessions": [], + "authority": "EXPERIENTIAL", + "confidence": 0.85, + "evidence_refs": [], + } +) + +#: Row keys ``put_recipe`` owns; everything else on a live row is a session +#: extra and survives the amend untouched. +_RECIPE_OWNED_KEYS: frozenset[str] = frozenset(PRESERVED_RECIPE_FIELDS) | frozenset( + { + "canonical_id", + "version", + "created_at", + "updated_at", + "model", + "hardware", + "framework", + "framework_name", + "framework_version", + "precision", + "pitfalls", + "lessons", + "provenance", + } +) + + +def rows_not_superseded(live_rows: Any, superseding: Any, key: str) -> list[dict[str, Any]]: + """Copies of the live rows whose ``key`` no superseding row already claims.""" + claimed = {str(row.get(key) or "") for row in superseding or [] if isinstance(row, dict)} + kept: list[dict[str, Any]] = [] + for row in live_rows or []: + if not isinstance(row, dict): + continue + if str(row.get(key) or "") in claimed: + continue + kept.append(dict(row)) + return kept + + +def improved_best_config(state: Any, attrs: Mapping[str, Any], live: Mapping[str, Any]) -> dict[str, Any]: + """The ``best_config`` override, empty unless this session genuinely beat the live row. + + A session that validated nothing has no standing to overwrite a + configuration somebody else measured. + """ + my_throughput = float(attrs.get("best_throughput") or 0.0) + validated_gain = float(state.cumulative_gain_validated or 0.0) + current_args = str(state.current_best.get("extra_server_args") or "").strip() + has_validated_win = bool((state.optimization_stack or []) or validated_gain > 0.0 or current_args) + if not has_validated_win or my_throughput <= to_float(live.get("best_throughput"), 0.0): + return {} + return { + "best_config": attrs["best_config"], + "best_throughput": my_throughput, + } + + +def appended(live_facts: Any, addition: Mapping[str, Any] | None) -> list[Any]: + """The live lesson or pitfall list with one more appended, when the caller brought one.""" + facts = list(live_facts or []) + if addition is not None: + facts.append(addition) + return facts + + +def amend_provenance(details: Mapping[str, Any] | None) -> dict[str, Any]: + """The provenance block stamped on every amend; the central server requires one.""" + return { + "source": "hyperloom-inference-optimizer", + "generator": "coordinator", + "generated_at": datetime.now(timezone.utc).isoformat(timespec="microseconds"), + "details": dict(details or {}), + } + + +def amended_recipe_fields(live: Mapping[str, Any], overrides: Mapping[str, Any]) -> dict[str, Any]: + """The preserved recipe fields for one amend: an override wins, an unset field keeps its live value. + + An absent or empty live value falls back to the field's own default, so + ``put_recipe`` never sees ``None`` where it declares a container. + """ + merged: dict[str, Any] = {} + for field, default in PRESERVED_RECIPE_FIELDS.items(): + if field in overrides: + merged[field] = overrides[field] + continue + live_value = live.get(field) + merged[field] = type(default)(live_value) if live_value else copy(default) + return merged + + +def amended_recipe_extras(live: Mapping[str, Any], overrides: Mapping[str, Any], state: Any) -> dict[str, Any]: + """The free-form extras an amend writes back: the prior ones, the caller's, then the model's identity. + + The architecture tags are re-stamped from config.json each time because a + row written before the architecture was read carries none. + """ + extras = {key: value for key, value in live.items() if key not in _RECIPE_OWNED_KEYS} + extras.update(overrides.get("extras") or {}) + architectures = state.model_architectures or [] + if isinstance(architectures, list): + named = [str(name).strip() for name in architectures if str(name or "").strip()] + if named: + extras["architectures"] = named + model_type = str(state.model_type or "").strip() + if model_type: + extras["model_type"] = model_type + return extras diff --git a/src/hyperloom/orchestrator/knowledge/recipe_journal.py b/src/hyperloom/orchestrator/knowledge/recipe_journal.py index ab760d49c5..c94ccbfdb7 100644 --- a/src/hyperloom/orchestrator/knowledge/recipe_journal.py +++ b/src/hyperloom/orchestrator/knowledge/recipe_journal.py @@ -6,10 +6,8 @@ from __future__ import annotations import logging as _logging -from copy import copy from dataclasses import dataclass from datetime import datetime, timezone -from types import MappingProxyType from typing import TYPE_CHECKING, Any, Collection, Mapping, NamedTuple from hyperloom.common.coerce import to_float, to_int @@ -32,6 +30,14 @@ from ..collaborator import CoordinatorCollaborator from ..state.attempt_ledger import AttemptRow, record_attempt_row from ._config_timeline import NON_ATTEMPT_OUTCOMES, record_config_attempts, record_config_run +from ._recipe_amend import ( + amend_provenance, + amended_recipe_extras, + amended_recipe_fields, + appended, + improved_best_config, + rows_not_superseded, +) from ._recipe_attrs import recipe_attrs_from_state, workload_kb_tags from .config import KnowledgeStoreMode @@ -67,44 +73,6 @@ _REMOTE_VALIDATION_ERROR = "RemoteRecipeValidationError" -#: Recipe fields an amend carries forward when the caller does not override -#: them, each with the empty value ``put_recipe`` falls back to. -_PRESERVED_RECIPE_FIELDS: Mapping[str, Any] = MappingProxyType( - { - "best_config": {}, - "best_throughput": 0.0, - "what_worked": [], - "what_failed": [], - "remaining_gaps": [], - "last_profiled": "", - "stack_fingerprint": {}, - "sessions": [], - "authority": "EXPERIENTIAL", - "confidence": 0.85, - "evidence_refs": [], - } -) - -#: Row keys ``put_recipe`` owns; everything else on a live row is a session -#: extra and survives the amend untouched. -_RECIPE_OWNED_KEYS: frozenset[str] = frozenset(_PRESERVED_RECIPE_FIELDS) | frozenset( - { - "canonical_id", - "version", - "created_at", - "updated_at", - "model", - "hardware", - "framework", - "framework_name", - "framework_version", - "precision", - "pitfalls", - "lessons", - "provenance", - } -) - #: Explore outcomes the journal collapses into a plain revert. _MEASURED_LOSS_OUTCOMES: frozenset[str] = frozenset({OUTCOME_REVERT, "FAILED"}) @@ -363,90 +331,6 @@ def _remote_publish_failure_type(exc: BaseException) -> str: return _close_out.RESULT_TRANSPORT_FAILED -def _rows_not_superseded(live_rows: Any, superseding: Any, key: str) -> list[dict[str, Any]]: - """Copies of the live rows whose ``key`` no superseding row already claims.""" - claimed = {str(row.get(key) or "") for row in superseding or [] if isinstance(row, dict)} - kept: list[dict[str, Any]] = [] - for row in live_rows or []: - if not isinstance(row, dict): - continue - if str(row.get(key) or "") in claimed: - continue - kept.append(dict(row)) - return kept - - -def _improved_best_config(state: Any, attrs: Mapping[str, Any], live: Mapping[str, Any]) -> dict[str, Any]: - """The ``best_config`` override, empty unless this session genuinely beat the live row. - - A session that validated nothing has no standing to overwrite a - configuration somebody else measured. - """ - my_throughput = float(attrs.get("best_throughput") or 0.0) - validated_gain = float(state.cumulative_gain_validated or 0.0) - current_args = str(state.current_best.get("extra_server_args") or "").strip() - has_validated_win = bool((state.optimization_stack or []) or validated_gain > 0.0 or current_args) - if not has_validated_win or my_throughput <= to_float(live.get("best_throughput"), 0.0): - return {} - return { - "best_config": attrs["best_config"], - "best_throughput": my_throughput, - } - - -def _appended(live_facts: Any, addition: Mapping[str, Any] | None) -> list[Any]: - """The live lesson or pitfall list with one more appended, when the caller brought one.""" - facts = list(live_facts or []) - if addition is not None: - facts.append(addition) - return facts - - -def _amend_provenance(details: Mapping[str, Any] | None) -> dict[str, Any]: - """The provenance block stamped on every amend; the central server requires one.""" - return { - "source": "hyperloom-inference-optimizer", - "generator": "coordinator", - "generated_at": datetime.now(timezone.utc).isoformat(timespec="microseconds"), - "details": dict(details or {}), - } - - -def _amended_recipe_fields(live: Mapping[str, Any], overrides: Mapping[str, Any]) -> dict[str, Any]: - """The preserved recipe fields for one amend: an override wins, an unset field keeps its live value. - - An absent or empty live value falls back to the field's own default, so - ``put_recipe`` never sees ``None`` where it declares a container. - """ - merged: dict[str, Any] = {} - for field, default in _PRESERVED_RECIPE_FIELDS.items(): - if field in overrides: - merged[field] = overrides[field] - continue - live_value = live.get(field) - merged[field] = type(default)(live_value) if live_value else copy(default) - return merged - - -def _amended_recipe_extras(live: Mapping[str, Any], overrides: Mapping[str, Any], state: Any) -> dict[str, Any]: - """The free-form extras an amend writes back: the prior ones, the caller's, then the model's identity. - - The architecture tags are re-stamped from config.json each time because a - row written before the architecture was read carries none. - """ - extras = {key: value for key, value in live.items() if key not in _RECIPE_OWNED_KEYS} - extras.update(overrides.get("extras") or {}) - architectures = state.model_architectures or [] - if isinstance(architectures, list): - named = [str(name).strip() for name in architectures if str(name or "").strip()] - if named: - extras["architectures"] = named - model_type = str(state.model_type or "").strip() - if model_type: - extras["model_type"] = model_type - return extras - - class RecipeJournalCollaborator(CoordinatorCollaborator): """Records settled results into the optimization journal, the Recipe KB facts and the final Recipe.""" @@ -1251,17 +1135,17 @@ def _final_recipe_overrides(self) -> dict[str, Any]: my_kernels = list(attrs.get("kernel_optimizations") or []) # KEEP'd kernel optimizations ride the extras channel. extras = dict(attrs.get("workload") or {}) - kernels = my_kernels + _rows_not_superseded(live.get("kernel_optimizations"), my_kernels, "kernel_id") + kernels = my_kernels + rows_not_superseded(live.get("kernel_optimizations"), my_kernels, "kernel_id") if kernels: extras["kernel_optimizations"] = kernels overrides: dict[str, Any] = { "what_worked": attrs["what_worked"], "what_failed": attrs["what_failed"], "last_profiled": attrs["last_profiled"], - "sessions": _rows_not_superseded(live.get("sessions"), my_sessions, "session_id") + my_sessions, + "sessions": rows_not_superseded(live.get("sessions"), my_sessions, "session_id") + my_sessions, "extras": extras, } - overrides.update(_improved_best_config(state, attrs, live)) + overrides.update(improved_best_config(state, attrs, live)) return overrides def _write_local_recipe(self) -> dict[str, Any]: @@ -1473,11 +1357,11 @@ def kb_amend_recipe( "framework_name": identity.name, "framework_version": identity.version, "precision": identity.precision, - "pitfalls": _appended(live.get("pitfalls"), append_pitfall), - "lessons": _appended(live.get("lessons"), append_lesson), - "extras": _amended_recipe_extras(live, overrides, state), - "provenance": _amend_provenance(provenance_details), - **_amended_recipe_fields(live, overrides), + "pitfalls": appended(live.get("pitfalls"), append_pitfall), + "lessons": appended(live.get("lessons"), append_lesson), + "extras": amended_recipe_extras(live, overrides, state), + "provenance": amend_provenance(provenance_details), + **amended_recipe_fields(live, overrides), } try: self.recipe_kb.put_recipe(**put_kwargs) From e420134aec61007cc6c1fe4b32df99ed37cc14c1 Mon Sep 17 00:00:00 2001 From: Test Date: Tue, 20 Oct 2026 10:46:45 +0800 Subject: [PATCH 03/14] chore(metrics): lower the recipe journal's module-lines baseline after the merge Co-Authored-By: Claude Opus 5 --- scripts/code_metrics_baseline.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/code_metrics_baseline.txt b/scripts/code_metrics_baseline.txt index 9ed2fa979c..19182bb3ea 100644 --- a/scripts/code_metrics_baseline.txt +++ b/scripts/code_metrics_baseline.txt @@ -927,7 +927,7 @@ module-lines src/hyperloom/orchestrator/actions/executors/roofline.py 1303 module-lines src/hyperloom/orchestrator/kernel/apply_kernel_patch.py 2657 module-lines src/hyperloom/orchestrator/kernel/conc_sweep.py 1429 module-lines src/hyperloom/orchestrator/kernel/request_handlers.py 4887 -module-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py 1376 +module-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py 1362 module-lines src/hyperloom/orchestrator/knowledge/recipe_kb_t0.py 1318 module-lines src/hyperloom/orchestrator/knowledge/remote_recipe/values.py 1284 module-lines src/hyperloom/orchestrator/loop/coordinator.py 1423 From 47ff719bb46c36a9f4abdfb624630e3c804f1676 Mon Sep 17 00:00:00 2001 From: Test Date: Tue, 20 Oct 2026 10:56:02 +0800 Subject: [PATCH 04/14] test(common): pin that a recorded measurement keeps a non-finite value Co-Authored-By: Claude Opus 5 --- .../inference_optimizer/tests/test_coerce.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/hyperloom/inference_optimizer/tests/test_coerce.py b/src/hyperloom/inference_optimizer/tests/test_coerce.py index 09358f6acb..2c2808e578 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coerce.py +++ b/src/hyperloom/inference_optimizer/tests/test_coerce.py @@ -12,6 +12,7 @@ first_float, first_int, optional_positive_int, + recorded_float, to_float, to_int, to_str_list, @@ -66,6 +67,30 @@ def test_custom_default(self): assert to_float(None, default=-1.0) == -1.0 +class TestRecordedFloat: + @pytest.mark.parametrize( + "value,expected", + [ + (1, 1.0), + (1.5, 1.5), + ("2.5", 2.5), + (-4, -4.0), + ], + ) + def test_parses(self, value, expected): + assert recorded_float(value) == expected + + @pytest.mark.parametrize("value", [None, "", "abc", object(), [1]]) + def test_absent_or_unparseable_reads_as_the_default(self, value): + assert recorded_float(value) == 0.0 + assert recorded_float(value, -1.0) == -1.0 + + def test_infinity_survives_so_a_broken_benchmark_stays_visible(self): + assert recorded_float(float("inf")) == math.inf + assert recorded_float("Infinity") == math.inf + assert math.isnan(recorded_float(float("nan"))) + + class TestToInt: @pytest.mark.parametrize( "value,expected", From 6658097fe9341774b8c95a100f532ee50cc2db9f Mon Sep 17 00:00:00 2001 From: Test Date: Tue, 20 Oct 2026 11:16:27 +0800 Subject: [PATCH 05/14] refactor(knowledge): delete five single-use units instead of moving them Two wrappers whose whole body was a dict literal, one whose body was two recorder calls, a NamedTuple built in one place and unpacked in the next, and a classmethod constructor with one caller. Each is inlined at its only call site; no function crosses the complexity ceiling as a result. Co-Authored-By: Claude Opus 5 --- src/hyperloom/common/coerce.py | 8 +-- .../knowledge/_config_timeline.py | 61 +++++++------------ .../orchestrator/knowledge/_recipe_attrs.py | 45 +++----------- 3 files changed, 35 insertions(+), 79 deletions(-) diff --git a/src/hyperloom/common/coerce.py b/src/hyperloom/common/coerce.py index ee036f2d42..de54c0c3c3 100644 --- a/src/hyperloom/common/coerce.py +++ b/src/hyperloom/common/coerce.py @@ -24,11 +24,11 @@ def to_float(value: Any, default: _T | None = None) -> float | _T | None: def recorded_float(value: Any, default: float = 0.0) -> float: - """Coerce a recorded measurement to ``float``, keeping ``inf`` / ``nan`` as they were recorded. + """Coerce a recorded value to ``float``, keeping ``inf`` / ``nan`` as they were recorded. - Unlike :func:`to_float`, a non-finite figure survives the coercion. It says - the benchmark behind it is broken, and *default* would render that as a - measurement somebody took. + Unlike :func:`to_float`, a non-finite figure survives the coercion. It + records that whatever produced it was degenerate, and *default* would + render that as an ordinary value. """ try: return float(value or default) diff --git a/src/hyperloom/orchestrator/knowledge/_config_timeline.py b/src/hyperloom/orchestrator/knowledge/_config_timeline.py index 3416903111..d1962814dd 100644 --- a/src/hyperloom/orchestrator/knowledge/_config_timeline.py +++ b/src/hyperloom/orchestrator/knowledge/_config_timeline.py @@ -78,32 +78,6 @@ def record_config_run(coord: Any, *, task: Any, result_dict: Mapping[str, Any]) ) -def _attempt_measurement(metrics: Mapping[str, Any]) -> dict[str, Any]: - """What one variant's benchmark measured. - - Both ends of the pair: the anchor advances on every KEEP, so a percentage - without its denominator adds to nothing. - """ - return { - "before_tput": metrics.get("base_tput"), - "after_tput": metrics.get("tput"), - "gain_pct": metrics.get("gain_pct"), - "runtime_sec": metrics.get("runtime_sec"), - "estimated_output_throughput": metrics.get("estimated_output_throughput"), - } - - -def _attempt_config_delta(variant: Mapping[str, Any]) -> dict[str, Any]: - """The launch-configuration change one variant applied.""" - return { - "extra_server_args": variant.get("extra_server_args"), - "extra_envs": variant.get("extra_envs"), - "remove_args": variant.get("remove_args"), - "unset_envs": variant.get("unset_envs"), - "args_mode": variant.get("args_mode"), - } - - def _attempt_accuracy(gates: Iterable[Mapping[str, Any]]) -> dict[str, Any]: """The accuracy block, projected from the gate that ruled. @@ -174,8 +148,22 @@ def _attempt_fields( # The fingerprint is the join key; the name is what a reader # recognises the variant by. "variant_name": str(row.get("variant_name") or ""), - "measurement": _attempt_measurement(metrics), - "config_delta": _attempt_config_delta(variant), + "measurement": { + # Both ends of the pair: the anchor advances on every KEEP, so a + # percentage without its denominator adds to nothing. + "before_tput": metrics.get("base_tput"), + "after_tput": metrics.get("tput"), + "gain_pct": metrics.get("gain_pct"), + "runtime_sec": metrics.get("runtime_sec"), + "estimated_output_throughput": metrics.get("estimated_output_throughput"), + }, + "config_delta": { + "extra_server_args": variant.get("extra_server_args"), + "extra_envs": variant.get("extra_envs"), + "remove_args": variant.get("remove_args"), + "unset_envs": variant.get("unset_envs"), + "args_mode": variant.get("args_mode"), + }, "accuracy": _attempt_accuracy(gates), "failure": _attempt_failure(row, outcome), "artifacts": _attempt_artifacts(row), @@ -233,17 +221,6 @@ def _record_variant_attempt(recorder: Any, row: Any, context: _ConfigAttemptCont return True -def _settle_proposal_as_attempted(recorder: Any, proposal_ref: str, recorded: int) -> None: - """Settle the proposal behind a grid, now that its variants have been recorded. - - Settled here rather than at materialization: a grid whose task dies before - any variant runs has no attempt to stand behind an ``attempted`` reading, - and is more honestly left unsettled. - """ - recorder.record_proposal_step(proposal_ref, step=STEP_ATTEMPTED, outcome=str(recorded)) - recorder.settle_proposal(proposal_ref, disposition=DISPOSITION_ATTEMPTED) - - def record_config_attempts( coord: Any, *, @@ -268,4 +245,8 @@ def record_config_attempts( if _record_variant_attempt(recorder, row, context): recorded += 1 if recorded and context.proposal_ref: - _settle_proposal_as_attempted(recorder, context.proposal_ref, recorded) + # Settled here rather than at materialization: a grid whose task dies + # before any variant runs has no attempt to stand behind an + # ``attempted`` reading, and is more honestly left unsettled. + recorder.record_proposal_step(context.proposal_ref, step=STEP_ATTEMPTED, outcome=str(recorded)) + recorder.settle_proposal(context.proposal_ref, disposition=DISPOSITION_ATTEMPTED) diff --git a/src/hyperloom/orchestrator/knowledge/_recipe_attrs.py b/src/hyperloom/orchestrator/knowledge/_recipe_attrs.py index f073c9b016..2962f439ee 100644 --- a/src/hyperloom/orchestrator/knowledge/_recipe_attrs.py +++ b/src/hyperloom/orchestrator/knowledge/_recipe_attrs.py @@ -60,11 +60,6 @@ class AttemptProvenance(NamedTuple): kept_by_gap: dict[str, str] reverted: list[dict[str, Any]] - @classmethod - def unrecorded(cls) -> "AttemptProvenance": - """The empty provenance a session that does not sediment its attempts reports.""" - return cls(kept_by_name={}, kept_by_gap={}, reverted=[]) - class _IntegrateIndex(NamedTuple): """Integrate-layer entries under both keys a micro KEEP can name them by.""" @@ -73,15 +68,6 @@ class _IntegrateIndex(NamedTuple): by_task_group: dict[str, dict[str, Any]] -class _E2eVerdict(NamedTuple): - """What integrating a kernel measured end to end.""" - - gain_pct: float - throughput: float - decision: str - integrated: bool - - def _known_version(raw: Any) -> str: """A fingerprint version, empty when the probe never identified one.""" version = str(raw or "").strip() @@ -179,38 +165,26 @@ def _last_rebenched_throughput(integration: Mapping[str, Any]) -> float: return 0.0 -def _e2e_verdict(integration: Any) -> _E2eVerdict: - """The integrate layer's verdict on a kernel, or the never-integrated default. +def _kernel_optimization_row(entry: Mapping[str, Any], ledger_id: str, index: _IntegrateIndex) -> dict[str, Any]: + """One KernelOptimization row: a micro KEEP joined to its end-to-end verdict. - Carrying the verdict lets a warm start skip a kernel that won on the micro + Carrying that verdict lets a warm start skip a kernel that won on the micro bench and lost end to end. """ - if not isinstance(integration, dict): - return _E2eVerdict(gain_pct=0.0, throughput=0.0, decision="", integrated=False) - return _E2eVerdict( - gain_pct=recorded_float(integration.get("best_gain_pct")), - throughput=_last_rebenched_throughput(integration), - decision=str(integration.get("last_decision") or "").upper(), - integrated=True, - ) - - -def _kernel_optimization_row(entry: Mapping[str, Any], ledger_id: str, index: _IntegrateIndex) -> dict[str, Any]: - """One KernelOptimization row: a micro KEEP joined to its end-to-end verdict.""" kernel_id = str(entry.get("current_kernel_id") or entry.get("kernel_id") or ledger_id) task_group_key = str(entry.get("task_group_key") or "") integration = index.by_task_group.get(task_group_key) if task_group_key else index.by_kernel.get(kernel_id) - e2e = _e2e_verdict(integration) + integrated = isinstance(integration, dict) return { "kernel_id": kernel_id, "source_file": str(entry.get("last_source_file") or ""), "artifact_path": str(entry.get("last_artifact_path") or ""), "micro_speedup": recorded_float(entry.get("last_micro_speedup")), "decision": OUTCOME_KEEP, - "e2e_gain_pct": e2e.gain_pct, - "e2e_tput": e2e.throughput, - "e2e_decision": e2e.decision, - "integrated": e2e.integrated, + "e2e_gain_pct": recorded_float(integration.get("best_gain_pct")) if integrated else 0.0, + "e2e_tput": _last_rebenched_throughput(integration) if integrated else 0.0, + "e2e_decision": str(integration.get("last_decision") or "").upper() if integrated else "", + "integrated": integrated, "ts": str(entry.get("last_ts") or ""), } @@ -419,7 +393,8 @@ def recipe_attrs_from_state(state: Any, session_name: str) -> dict[str, Any]: the T0 backfill writes it. """ optimization_stack = state.optimization_stack or [] - provenance = attempt_provenance(state) if state.recipe_sediment_enabled else AttemptProvenance.unrecorded() + # A session that does not sediment its attempts reports no provenance. + provenance = attempt_provenance(state) if state.recipe_sediment_enabled else AttemptProvenance({}, {}, []) final_throughput = float(state.current_best.get("tput", 0.0)) return { "best_config": _best_config(state.current_best, optimization_stack), From db77fa14a068a380a5968b977043d9228fad1c42 Mon Sep 17 00:00:00 2001 From: Test Date: Tue, 20 Oct 2026 11:28:03 +0800 Subject: [PATCH 06/14] test(common): pin the three inputs recorded_float and to_float disagree on to_float is not a drop-in replacement for a guarded float(x or 0.0): it returns its default for inf, nan and bool. Pinning all three keeps the next swap from being silent, and says so in recorded_float's own docstring. Co-Authored-By: Claude Opus 5 --- src/hyperloom/common/coerce.py | 10 ++++++---- src/hyperloom/inference_optimizer/tests/test_coerce.py | 8 +++++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/hyperloom/common/coerce.py b/src/hyperloom/common/coerce.py index de54c0c3c3..88adf87986 100644 --- a/src/hyperloom/common/coerce.py +++ b/src/hyperloom/common/coerce.py @@ -24,11 +24,13 @@ def to_float(value: Any, default: _T | None = None) -> float | _T | None: def recorded_float(value: Any, default: float = 0.0) -> float: - """Coerce a recorded value to ``float``, keeping ``inf`` / ``nan`` as they were recorded. + """Coerce a recorded value to ``float``, keeping ``inf``, ``nan`` and ``bool`` as they were recorded. - Unlike :func:`to_float`, a non-finite figure survives the coercion. It - records that whatever produced it was degenerate, and *default* would - render that as an ordinary value. + :func:`to_float` is not a drop-in replacement and the two disagree on + exactly those three: it returns *default* for each. A non-finite figure + survives here because it records that whatever produced it was degenerate, + and *default* would render that as an ordinary value nobody can tell from a + real one afterwards. """ try: return float(value or default) diff --git a/src/hyperloom/inference_optimizer/tests/test_coerce.py b/src/hyperloom/inference_optimizer/tests/test_coerce.py index 2c2808e578..4dff4f96ee 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coerce.py +++ b/src/hyperloom/inference_optimizer/tests/test_coerce.py @@ -85,11 +85,17 @@ def test_absent_or_unparseable_reads_as_the_default(self, value): assert recorded_float(value) == 0.0 assert recorded_float(value, -1.0) == -1.0 - def test_infinity_survives_so_a_broken_benchmark_stays_visible(self): + def test_infinity_survives_so_a_degenerate_measurement_stays_visible(self): assert recorded_float(float("inf")) == math.inf assert recorded_float("Infinity") == math.inf assert math.isnan(recorded_float(float("nan"))) + @pytest.mark.parametrize("value", [float("inf"), float("nan"), True]) + def test_keeps_what_to_float_rejects(self, value): + """The three inputs on which the two coercions disagree, so a swap is never silent.""" + assert to_float(value, 0.0) == 0.0 + assert recorded_float(value) != 0.0 or math.isnan(recorded_float(value)) + class TestToInt: @pytest.mark.parametrize( From f9fa9c1d349dec1b2c2a4651f6282f43c0a6b8a9 Mon Sep 17 00:00:00 2001 From: Test Date: Tue, 20 Oct 2026 19:08:55 +0800 Subject: [PATCH 07/14] docs(knowledge): collapse the added helpers' docstrings to their summary line The restructuring added helpers, and each arrived with a multi-paragraph block. Reduce every added helper to its summary line, keeping only the sentences that state an invariant or a constraint from outside the file. Co-Authored-By: Claude Opus 5 --- src/hyperloom/common/coerce.py | 7 +-- .../knowledge/_config_timeline.py | 14 +----- .../orchestrator/knowledge/_recipe_amend.py | 10 ++--- .../orchestrator/knowledge/_recipe_attrs.py | 43 +++++-------------- .../orchestrator/knowledge/recipe_journal.py | 31 ++++--------- 5 files changed, 27 insertions(+), 78 deletions(-) diff --git a/src/hyperloom/common/coerce.py b/src/hyperloom/common/coerce.py index 88adf87986..a0e58c32d1 100644 --- a/src/hyperloom/common/coerce.py +++ b/src/hyperloom/common/coerce.py @@ -26,11 +26,8 @@ def to_float(value: Any, default: _T | None = None) -> float | _T | None: def recorded_float(value: Any, default: float = 0.0) -> float: """Coerce a recorded value to ``float``, keeping ``inf``, ``nan`` and ``bool`` as they were recorded. - :func:`to_float` is not a drop-in replacement and the two disagree on - exactly those three: it returns *default* for each. A non-finite figure - survives here because it records that whatever produced it was degenerate, - and *default* would render that as an ordinary value nobody can tell from a - real one afterwards. + :func:`to_float` returns *default* for those three instead, which would + render a degenerate figure as one nobody can tell from a real measurement. """ try: return float(value or default) diff --git a/src/hyperloom/orchestrator/knowledge/_config_timeline.py b/src/hyperloom/orchestrator/knowledge/_config_timeline.py index d1962814dd..e8ce09b734 100644 --- a/src/hyperloom/orchestrator/knowledge/_config_timeline.py +++ b/src/hyperloom/orchestrator/knowledge/_config_timeline.py @@ -45,16 +45,7 @@ class _ConfigAttemptContext(NamedTuple): def record_config_run(coord: Any, *, task: Any, result_dict: Mapping[str, Any]) -> None: - """Record the configuration arm's grid dispatch on the framework event. - - The arm's dispatch is one ``explore`` task, and it earns a run row for the - same reason a specialist dispatch does: the event reduces its status over - what it dispatched, so an arm whose grid came back with nothing must not - read as an arm that was never tried. Recorded at completion, where the - outcome is known, with the dispatch time taken off the task row; a grid - that measured nothing still lands, which is the case - :func:`record_config_attempts` never sees. - """ + """Record the configuration arm's grid dispatch on the framework event.""" recorder = coord._coord.phase_framework.timeline() if recorder is None: return @@ -81,8 +72,7 @@ def record_config_run(coord: Any, *, task: Any, result_dict: Mapping[str, Any]) def _attempt_accuracy(gates: Iterable[Mapping[str, Any]]) -> dict[str, Any]: """The accuracy block, projected from the gate that ruled. - This arm gates accuracy rather than reporting it. No gate row means nothing - gated the variant, which is not a gate that refused it. + No gate row means nothing gated the variant, which is not a gate that refused it. """ ruling = next((gate for gate in gates if str(gate.get("gate") or "") == _ACCURACY_GATE), {}) return { diff --git a/src/hyperloom/orchestrator/knowledge/_recipe_amend.py b/src/hyperloom/orchestrator/knowledge/_recipe_amend.py index 82d0a3198b..ee8fbc06b0 100644 --- a/src/hyperloom/orchestrator/knowledge/_recipe_amend.py +++ b/src/hyperloom/orchestrator/knowledge/_recipe_amend.py @@ -71,11 +71,7 @@ def rows_not_superseded(live_rows: Any, superseding: Any, key: str) -> list[dict def improved_best_config(state: Any, attrs: Mapping[str, Any], live: Mapping[str, Any]) -> dict[str, Any]: - """The ``best_config`` override, empty unless this session genuinely beat the live row. - - A session that validated nothing has no standing to overwrite a - configuration somebody else measured. - """ + """The ``best_config`` override, empty unless this session genuinely beat the live row.""" my_throughput = float(attrs.get("best_throughput") or 0.0) validated_gain = float(state.cumulative_gain_validated or 0.0) current_args = str(state.current_best.get("extra_server_args") or "").strip() @@ -125,8 +121,8 @@ def amended_recipe_fields(live: Mapping[str, Any], overrides: Mapping[str, Any]) def amended_recipe_extras(live: Mapping[str, Any], overrides: Mapping[str, Any], state: Any) -> dict[str, Any]: """The free-form extras an amend writes back: the prior ones, the caller's, then the model's identity. - The architecture tags are re-stamped from config.json each time because a - row written before the architecture was read carries none. + The architecture tags are re-stamped from config.json each time because a row + written before the architecture was read carries none. """ extras = {key: value for key, value in live.items() if key not in _RECIPE_OWNED_KEYS} extras.update(overrides.get("extras") or {}) diff --git a/src/hyperloom/orchestrator/knowledge/_recipe_attrs.py b/src/hyperloom/orchestrator/knowledge/_recipe_attrs.py index 2962f439ee..00f7fc2631 100644 --- a/src/hyperloom/orchestrator/knowledge/_recipe_attrs.py +++ b/src/hyperloom/orchestrator/knowledge/_recipe_attrs.py @@ -90,8 +90,8 @@ def _model_identity_tags(state: Any) -> dict[str, Any]: def _runtime_version_tags(fingerprint_meta: Mapping[str, Any], framework: str) -> dict[str, str]: """Runtime versions read off the stack fingerprint the CLI wrote at boot. - ``framework_version`` names whichever serving framework is active, so a - recipe written under one is never matched against the other. + ``framework_version`` names whichever serving framework is active, so a recipe + written under one is never matched against the other. """ tags: dict[str, str] = {} framework_key = framework.lower() @@ -125,11 +125,7 @@ def _baseline_workload_tags(workload_extra: Mapping[str, Any]) -> dict[str, Any] def workload_kb_tags(state: Any) -> dict[str, Any]: - """The workload-shape tags describing the session this recipe came from. - - Shared by the recipe attrs and by every lesson or pitfall write, so the - warm-start reader filters on the dimensions the row was written with. - """ + """The workload-shape tags describing the session this recipe came from.""" tags = _model_identity_tags(state) tags.update(workload_shape_tags(state)) fingerprint_meta = state.stack_fingerprint_meta or {} @@ -166,11 +162,7 @@ def _last_rebenched_throughput(integration: Mapping[str, Any]) -> float: def _kernel_optimization_row(entry: Mapping[str, Any], ledger_id: str, index: _IntegrateIndex) -> dict[str, Any]: - """One KernelOptimization row: a micro KEEP joined to its end-to-end verdict. - - Carrying that verdict lets a warm start skip a kernel that won on the micro - bench and lost end to end. - """ + """One KernelOptimization row: a micro KEEP joined to its end-to-end verdict.""" kernel_id = str(entry.get("current_kernel_id") or entry.get("kernel_id") or ledger_id) task_group_key = str(entry.get("task_group_key") or "") integration = index.by_task_group.get(task_group_key) if task_group_key else index.by_kernel.get(kernel_id) @@ -190,12 +182,7 @@ def _kernel_optimization_row(entry: Mapping[str, Any], ledger_id: str, index: _I def kept_kernel_optimizations(state: Any) -> list[dict[str, Any]]: - """The KEEP'd kernel optimizations, each joined to its end-to-end integrate verdict. - - A KEEP that never reached integration surfaces with ``integrated`` false - rather than being dropped, because not having been tried end to end is not - the same as having lost there. - """ + """The KEEP'd kernel optimizations, each joined to its end-to-end integrate verdict.""" optimizations = state.kernel_opt_task_attempts or {} if not isinstance(optimizations, dict): return [] @@ -248,11 +235,7 @@ def _gap_attempt_provenance(gap: Mapping[str, Any]) -> AttemptProvenance: def attempt_provenance(state: Any) -> AttemptProvenance: - """Trace each proven optimization back to the research hint it came from. - - The first gap to claim a name keeps it, so a name two gaps both attempted - reads as belonging to the one that reached it first. - """ + """Trace each proven optimization back to the research hint it came from.""" kept_by_name: dict[str, str] = {} kept_by_gap: dict[str, str] = {} reverted: list[dict[str, Any]] = [] @@ -300,8 +283,8 @@ def _best_config(current_best: Mapping[str, Any], optimization_stack: Sequence[A def _stack_entry_source(entry: Mapping[str, Any], name: str, provenance: AttemptProvenance) -> str: """The research hint a stack layer came from, empty when nothing claims it. - The layer's gap id is naming-independent and wins; the variant or kernel - name answers for layers recorded before gap ids were stamped. + The layer's gap id wins; the variant or kernel name answers for layers + recorded before gap ids were stamped. """ gap_id = str(entry.get("gap_canonical_id") or "").strip() if gap_id: @@ -369,11 +352,7 @@ def _session_row( optimization_stack: Collection[Any], final_throughput: float, ) -> dict[str, Any]: - """This session's row on the recipe: where it started, where it reached and by which actions. - - Self-describing in the arbor shape, so a reader needs neither the recipe's - other sessions nor this one's journal to price what it contributed. - """ + """This session's row on the recipe: where it started, where it reached and by which actions.""" validated_stack_len = int(state.cumulative_gain_validated_stack_len or 0) return { "session_id": str(state.recipe_kb_session_id or session_name), @@ -389,8 +368,8 @@ def _session_row( def recipe_attrs_from_state(state: Any, session_name: str) -> dict[str, Any]: """Materialise the recipe-shaped view of one session's settled state. - ``framework_version`` is left unset here because it is manifest-derived; - the T0 backfill writes it. + ``framework_version`` is left unset here because it is manifest-derived; the + T0 backfill writes it. """ optimization_stack = state.optimization_stack or [] # A session that does not sediment its attempts reports no provenance. diff --git a/src/hyperloom/orchestrator/knowledge/recipe_journal.py b/src/hyperloom/orchestrator/knowledge/recipe_journal.py index dfd362290b..d8326e80e4 100644 --- a/src/hyperloom/orchestrator/knowledge/recipe_journal.py +++ b/src/hyperloom/orchestrator/knowledge/recipe_journal.py @@ -248,11 +248,7 @@ def _grid_attempt_row( def _variant_verdict(outcome: str, adopted: bool) -> Verdict: - """How one explore variant's own outcome settles against the adoption rule. - - An executor KEEP the adoption rule did not lift is a refusal, not a revert: - nothing about it failed to measure. - """ + """How one explore variant's own outcome settles against the adoption rule.""" if outcome == OUTCOME_KEEP: return Verdict.ADOPTED if adopted else Verdict.REFUSED if outcome in _MEASURED_LOSS_OUTCOMES: @@ -291,11 +287,7 @@ def _variant_pitfall_signals(variant_outcome: Mapping[str, Any], metrics: Any) - def _variant_change_attrs(variant_attrs: Any, variant_name: str) -> dict[str, Any]: - """One variant's attributes as the change summary needs them. - - A variant that names no knob falls back to its own name, else every variant - in the grid would summarise to an identical row. - """ + """One variant's attributes as the change summary needs them.""" attrs = dict(variant_attrs) if isinstance(variant_attrs, dict) else {} named_knob = attrs.get("extra_server_args") or attrs.get("extra_envs") or attrs.get("name") if not named_knob and variant_name: @@ -985,9 +977,8 @@ def _unvalidated_stack_outcome(self) -> dict[str, Any] | None: """The skip outcome when the working recipe outran its last validation, else ``None``. ``current_best`` and ``cumulative_gain_validated`` are only one Recipe's - figures when nothing was lifted since the last validation. Otherwise - every sink below -- journal, local KB, remote KB -- would pair the - newer config with the older gain, so none of them runs. + figures when nothing was lifted since the last validation; otherwise every + sink below would pair the newer config with the older gain. """ state = self.shared_state if not state.optimization_stack_has_unvalidated_keeps(): @@ -1065,8 +1056,8 @@ def finalize_recipe_and_journal( def _publish_remote_recipe(self, source: str) -> dict[str, Any]: """Publish the terminal Recipe to the KB Store and audit the attempt. - Remote mode has one Recipe sink: the KB Store final session writer. T0 - and runtime amendment are intentionally absent. + Remote mode has one Recipe sink: the KB Store final session writer. T0 and + runtime amendment are intentionally absent. """ state = self.shared_state session_id = str(state.recipe_kb_session_id or state.session_id or self.session_dir.name) @@ -1129,8 +1120,8 @@ def _publish_remote_recipe(self, source: str) -> dict[str, Any]: def _final_recipe_overrides(self) -> dict[str, Any]: """What this session's close writes over the live Recipe row. - Read-modify-write under the cid flock, so two finalises of the same - session leave one row rather than tearing the lists they both append to. + Read-modify-write under the cid flock, so two finalises of the same session + leave one row rather than tearing the lists they both append to. """ state = self.shared_state attrs = recipe_attrs_from_state(state, self.session_dir.name) @@ -1299,11 +1290,7 @@ def kb_best_config_overrides_for_keep( return {} def _amends_local_recipe(self) -> bool: - """Whether this session's Recipe row is a local one an amend may rewrite. - - Remote mode has a single Recipe sink at close, and AgentX sessions do - not write recipes at all. - """ + """Whether this session's Recipe row is a local one an amend may rewrite.""" config = getattr(self.knowledge_plane, "config", None) mode = getattr(getattr(config, "mode", None), "value", None) if mode == KnowledgeStoreMode.REMOTE.value or self.recipe_kb is None: From dd913a0b92613dfaed49c1d6d882a349767e59bf Mon Sep 17 00:00:00 2001 From: Test Date: Tue, 20 Oct 2026 19:17:11 +0800 Subject: [PATCH 08/14] chore(metrics): lower the module-lines baseline the docstring collapse made stale Co-Authored-By: Claude Opus 5 --- scripts/code_metrics_baseline.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/code_metrics_baseline.txt b/scripts/code_metrics_baseline.txt index d634e26264..b6169dd860 100644 --- a/scripts/code_metrics_baseline.txt +++ b/scripts/code_metrics_baseline.txt @@ -898,7 +898,7 @@ module-lines src/hyperloom/orchestrator/actions/executors/roofline.py 1303 module-lines src/hyperloom/orchestrator/kernel/apply_kernel_patch.py 2657 module-lines src/hyperloom/orchestrator/kernel/conc_sweep.py 1429 module-lines src/hyperloom/orchestrator/kernel/request_handlers.py 4887 -module-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py 1362 +module-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py 1340 module-lines src/hyperloom/orchestrator/knowledge/recipe_kb_t0.py 1318 module-lines src/hyperloom/orchestrator/knowledge/remote_recipe/values.py 1284 module-lines src/hyperloom/orchestrator/loop/coordinator.py 1423 From c151c7119b8530dc9541c15e51e1b58b86dd0014 Mon Sep 17 00:00:00 2001 From: Test Date: Wed, 21 Oct 2026 15:01:44 +0800 Subject: [PATCH 09/14] refactor(knowledge): restore the units the ceiling never asked to change fact_write_hook, _record_fact_impl and _record_config_run were inside every repo threshold before this branch, so they read as they did: the grid loop and the provenance dict are back in their callers and the configuration arm's run row is back beside them. Four more single-use units -- a dict-literal wrapper, a two-call wrapper and two predicates -- go the same way. Co-Authored-By: Claude Opus 5 --- scripts/code_metrics_baseline.txt | 2 +- .../knowledge/_config_timeline.py | 30 --- .../orchestrator/knowledge/recipe_journal.py | 176 +++++++++--------- 3 files changed, 87 insertions(+), 121 deletions(-) diff --git a/scripts/code_metrics_baseline.txt b/scripts/code_metrics_baseline.txt index 3b9f842520..3f4d623e4b 100644 --- a/scripts/code_metrics_baseline.txt +++ b/scripts/code_metrics_baseline.txt @@ -806,7 +806,7 @@ module-lines src/hyperloom/orchestrator/actions/executors/roofline.py 1303 module-lines src/hyperloom/orchestrator/kernel/apply_kernel_patch.py 2657 module-lines src/hyperloom/orchestrator/kernel/conc_sweep.py 1429 module-lines src/hyperloom/orchestrator/kernel/request_handlers.py 4887 -module-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py 1340 +module-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py 1336 module-lines src/hyperloom/orchestrator/knowledge/recipe_kb_t0.py 1318 module-lines src/hyperloom/orchestrator/knowledge/remote_recipe/values.py 1224 module-lines src/hyperloom/orchestrator/loop/coordinator.py 1423 diff --git a/src/hyperloom/orchestrator/knowledge/_config_timeline.py b/src/hyperloom/orchestrator/knowledge/_config_timeline.py index e8ce09b734..b96616a63c 100644 --- a/src/hyperloom/orchestrator/knowledge/_config_timeline.py +++ b/src/hyperloom/orchestrator/knowledge/_config_timeline.py @@ -13,11 +13,9 @@ from typing import Any, Collection, Iterable, Mapping, NamedTuple, Sequence -from hyperloom.common.timeutil import now_iso from hyperloom.inference_optimizer.breakdown.recorder.framework_event import ( ARM_CONFIG, DISPOSITION_ATTEMPTED, - ROLE_CONFIG, STEP_ATTEMPTED, GateVerdict, ) @@ -31,9 +29,6 @@ #: The gate whose verdict the shared accuracy block is projected from. _ACCURACY_GATE = "accuracy" -#: How much of a run's error text a reason field carries. -_REASON_EXCERPT_CHARS = 200 - class _ConfigAttemptContext(NamedTuple): """What every attempt row produced by one grid dispatch shares.""" @@ -44,31 +39,6 @@ class _ConfigAttemptContext(NamedTuple): adopted_variants: Collection[str] -def record_config_run(coord: Any, *, task: Any, result_dict: Mapping[str, Any]) -> None: - """Record the configuration arm's grid dispatch on the framework event.""" - recorder = coord._coord.phase_framework.timeline() - if recorder is None: - return - task_id = str(getattr(task, "task_id", "") or "") - if not task_id: - return - variants = result_dict.get("per_variant_outcomes") - measured = [row for row in variants if isinstance(row, dict)] if isinstance(variants, list) else [] - recorder.record_run( - task_id, - role=ROLE_CONFIG, - arm=ARM_CONFIG, - status=str(result_dict.get("status") or "") or "succeeded", - dispatched_at=str(getattr(task, "created_at", "") or ""), - completed_at=now_iso("seconds"), - reason=str(result_dict.get("error") or "")[:_REASON_EXCERPT_CHARS], - # A grid the run stopped before it measured anything is not a grid - # whose variants were measured and lost. - empty=not measured, - workspace=str(result_dict.get("workspace") or ""), - ) - - def _attempt_accuracy(gates: Iterable[Mapping[str, Any]]) -> dict[str, Any]: """The accuracy block, projected from the gate that ruled. diff --git a/src/hyperloom/orchestrator/knowledge/recipe_journal.py b/src/hyperloom/orchestrator/knowledge/recipe_journal.py index d8326e80e4..27901a84c7 100644 --- a/src/hyperloom/orchestrator/knowledge/recipe_journal.py +++ b/src/hyperloom/orchestrator/knowledge/recipe_journal.py @@ -29,7 +29,7 @@ from ..collaborator import CoordinatorCollaborator from ..state.attempt_ledger import AttemptRow, record_attempt_row -from ._config_timeline import NON_ATTEMPT_OUTCOMES, record_config_attempts, record_config_run +from ._config_timeline import NON_ATTEMPT_OUTCOMES, record_config_attempts from ._recipe_amend import ( amend_provenance, amended_recipe_extras, @@ -200,6 +200,44 @@ def _extra_server_args(payload: Mapping[str, Any]) -> str: return str(value) +def _record_config_run(coord: Any, *, task: Any, result_dict: Mapping[str, Any]) -> None: + """Record the configuration arm's grid dispatch on the framework event. + + The arm's dispatch is one ``explore`` task, and it earns a run row for the + same reason a specialist dispatch does: the event reduces its status over + what it dispatched, so an arm whose grid came back with nothing must not + read as an arm that was never tried. Recorded at completion, where the + outcome is known, with the dispatch time taken off the task row; a grid + that measured nothing still lands, which is the case + :func:`record_config_attempts` never sees. + """ + recorder = coord._coord.phase_framework.timeline() + if recorder is None: + return + from hyperloom.common.timeutil import now_iso + from hyperloom.inference_optimizer.breakdown.recorder.framework_event import ARM_CONFIG, ROLE_CONFIG + + task_id = str(getattr(task, "task_id", "") or "") + if not task_id: + return + variants = result_dict.get("per_variant_outcomes") + measured = [row for row in variants if isinstance(row, dict)] if isinstance(variants, list) else [] + status = str(result_dict.get("status") or "") or "succeeded" + recorder.record_run( + task_id, + role=ROLE_CONFIG, + arm=ARM_CONFIG, + status=status, + dispatched_at=str(getattr(task, "created_at", "") or ""), + completed_at=now_iso("seconds"), + reason=str(result_dict.get("error") or "")[:200], + # A grid the run stopped before it measured anything is not a grid + # whose variants were measured and lost. + empty=not measured, + workspace=str(result_dict.get("workspace") or ""), + ) + + def _framework_identity(state: Any) -> _FrameworkIdentity: """The session's framework, the version it is running and its precision. @@ -277,15 +315,6 @@ def _variant_failure(variant_outcome: Mapping[str, Any], outcome: str) -> _Varia ) -def _variant_pitfall_signals(variant_outcome: Mapping[str, Any], metrics: Any) -> dict[str, Any]: - """A variant's measurement plus the fields a pitfall severity is ruled from.""" - return { - **(metrics if isinstance(metrics, dict) else {}), - "error_class": variant_outcome.get("error_class"), - "status": variant_outcome.get("outcome"), - } - - def _variant_change_attrs(variant_attrs: Any, variant_name: str) -> dict[str, Any]: """One variant's attributes as the change summary needs them.""" attrs = dict(variant_attrs) if isinstance(variant_attrs, dict) else {} @@ -316,13 +345,6 @@ def _finalize_error(exc: BaseException, *, reason: str, backend: str, result_typ } -def _remote_publish_failure_type(exc: BaseException) -> str: - """Whether the KB Store rejected the bundle or was never reached at all.""" - if type(exc).__name__ == _REMOTE_VALIDATION_ERROR: - return _close_out.RESULT_BUNDLE_BUILD_FAILED - return _close_out.RESULT_TRANSPORT_FAILED - - class RecipeJournalCollaborator(CoordinatorCollaborator): """Records settled results into the optimization journal, the Recipe KB facts and the final Recipe.""" @@ -401,7 +423,7 @@ async def fact_write_hook( source_session_id = self._source_session_id() per_variant = result_dict.get("per_variant_outcomes") if task.kind == "explore": - record_config_run(self, task=task, result_dict=result_dict) + _record_config_run(self, task=task, result_dict=result_dict) if task.kind == "explore" and isinstance(per_variant, list) and per_variant: record_config_attempts( self, @@ -410,13 +432,16 @@ async def fact_write_hook( result_dict=result_dict, adopted_variants=adopted_variants, ) - self._record_grid_facts( - task=task, - source_session_id=source_session_id, - per_variant=per_variant, - round_id=str(result_dict.get("round_id") or ""), - adopted_variants=adopted_variants, - ) + round_id = str(result_dict.get("round_id") or "") + task_id = str(task.task_id or "") + for vo in per_variant: + adopted = str(vo.get("fingerprint") or "") in adopted_variants + row = _grid_attempt_row(vo, task_id=task_id, round_id=round_id, adopted=adopted) + if row is not None: + record_attempt_row(self.shared_state, row) + self._record_fact_per_variant( + task=task, source_session_id=source_session_id, variant_outcome=vo, adopted=adopted + ) else: self._record_fact_per_task( task=task, @@ -426,29 +451,6 @@ async def fact_write_hook( ) self.shared_state.save(self.session_dir) - def _record_grid_facts( - self, - *, - task: "Task", - source_session_id: str, - per_variant: list[dict[str, Any]], - round_id: str, - adopted_variants: Collection[str], - ) -> None: - """Record every variant of one explore grid: an adoption-ledger row and a journal fact each.""" - task_id = str(task.task_id or "") - for variant_outcome in per_variant: - adopted = str(variant_outcome.get("fingerprint") or "") in adopted_variants - row = _grid_attempt_row(variant_outcome, task_id=task_id, round_id=round_id, adopted=adopted) - if row is not None: - record_attempt_row(self.shared_state, row) - self._record_fact_per_variant( - task=task, - source_session_id=source_session_id, - variant_outcome=variant_outcome, - adopted=adopted, - ) - def ensure_journal(self) -> Journal: """Lazy-instantiate the per-session :class:`Journal` (load_or_create reads an existing file on resume). @@ -518,23 +520,6 @@ def journal_entry_phase(self) -> str: """ return str(self.shared_state.phase or "").strip().upper() or "UNKNOWN" - def _fact_provenance(self, fact: FactRecord, now_iso: str) -> dict[str, Any]: - """The provenance details stamped on the lesson or pitfall this fact writes.""" - state = self.shared_state - workload_tags = workload_kb_tags(state) - provenance: dict[str, Any] = { - "source_session_id": fact.source_session_id, - "source_task_id": fact.task.task_id, - "evidence": list(fact.evidence_refs or []), - "applicable_models": [str(state.model_name)] if state.model_name else [], - "applicable_hardware": [str(state.gpu_type)] if state.gpu_type else [], - "extra": dict(workload_tags), - "now": now_iso, - } - if fact.variant_name is not None: - provenance["source_variant_name"] = fact.variant_name - return provenance - def _record_fact_impl(self, fact: FactRecord) -> None: """Shared KB write for _record_fact_per_task and _record_fact_per_variant. @@ -547,8 +532,19 @@ def _record_fact_impl(self, fact: FactRecord) -> None: fact: The completed task, its outcome and the evidence to stamp on the provenance. """ + state = self.shared_state now_iso = datetime.now(timezone.utc).isoformat(timespec="seconds") - provenance_base = self._fact_provenance(fact, now_iso) + provenance_base: dict[str, Any] = { + "source_session_id": fact.source_session_id, + "source_task_id": fact.task.task_id, + "evidence": list(fact.evidence_refs or []), + "applicable_models": [str(state.model_name)] if state.model_name else [], + "applicable_hardware": [str(state.gpu_type)] if state.gpu_type else [], + "extra": dict(workload_kb_tags(state)), + "now": now_iso, + } + if fact.variant_name is not None: + provenance_base["source_variant_name"] = fact.variant_name lesson = None pitfall = None @@ -807,7 +803,11 @@ def _record_fact_per_variant( ), # Workload-shape tags — see _record_fact_per_task. evidence_refs=[f"log:task-{task.task_id}", f"variant:{variant_name}"], - pitfall_severity_dict=_variant_pitfall_signals(variant_outcome, metrics), + pitfall_severity_dict={ + **(metrics if isinstance(metrics, dict) else {}), + "error_class": variant_outcome.get("error_class"), + "status": variant_outcome.get("outcome"), + }, variant_name=variant_name, ) ) @@ -997,15 +997,6 @@ def _unvalidated_stack_outcome(self) -> dict[str, Any] | None: result_type=_close_out.RESULT_UNVALIDATED_RECIPE, ) - def _finalize_journal(self) -> None: - """Stamp the journal header with the session's settled throughput and total gain.""" - state = self.shared_state - final_tput = float(state.current_best.get("tput", 0.0)) - self.ensure_journal().finalize( - final_throughput=final_tput if final_tput > 0 else None, - total_gain_pct=float(state.cumulative_gain_validated or 0.0), - ) - def finalize_recipe_and_journal( self, *, @@ -1015,7 +1006,12 @@ def finalize_recipe_and_journal( unvalidated = self._unvalidated_stack_outcome() if unvalidated is not None: return unvalidated - self._finalize_journal() + state = self.shared_state + final_tput = float(state.current_best.get("tput", 0.0)) + self.ensure_journal().finalize( + final_throughput=final_tput if final_tput > 0 else None, + total_gain_pct=float(state.cumulative_gain_validated or 0.0), + ) knowledge_plane = self.knowledge_plane if bool(getattr(knowledge_plane, "kb_disabled", False)): @@ -1083,7 +1079,11 @@ def _publish_remote_recipe(self, source: str) -> dict[str, Any]: exc, reason=type(exc).__name__, backend="kb-store", - result_type=_remote_publish_failure_type(exc), + result_type=( + _close_out.RESULT_BUNDLE_BUILD_FAILED + if type(exc).__name__ == _REMOTE_VALIDATION_ERROR + else _close_out.RESULT_TRANSPORT_FAILED + ), ), "canonical_id": canonical_id, "session_id": session_id, @@ -1212,7 +1212,7 @@ def workload_canonical_id(self) -> str: ) def read_local_recipe_row(self) -> dict[str, Any]: - """Load the selected store's exact authority row for writes, cached for the current tick.""" + """Load the selected store's exact authority row for writes.""" if self.recipe_kb is None: return {} tick = int(self.shared_state.tick or 0) @@ -1289,16 +1289,6 @@ def kb_best_config_overrides_for_keep( return overrides return {} - def _amends_local_recipe(self) -> bool: - """Whether this session's Recipe row is a local one an amend may rewrite.""" - config = getattr(self.knowledge_plane, "config", None) - mode = getattr(getattr(config, "mode", None), "value", None) - if mode == KnowledgeStoreMode.REMOTE.value or self.recipe_kb is None: - return False - from hyperloom.common.perf_metric import agentx_active - - return not agentx_active(benchmark_mode=self.shared_state.benchmark_mode) - def kb_amend_recipe( self, *, @@ -1313,9 +1303,15 @@ def kb_amend_recipe( without dedup, and a field the caller did not override keeps the value the live row carries. """ - if not self._amends_local_recipe(): + config = getattr(self.knowledge_plane, "config", None) + mode = getattr(getattr(config, "mode", None), "value", None) + if mode == KnowledgeStoreMode.REMOTE.value or self.recipe_kb is None: return + from hyperloom.common.perf_metric import agentx_active + state = self.shared_state + if agentx_active(benchmark_mode=state.benchmark_mode): + return canonical_id = self.workload_canonical_id() identity = _framework_identity(state) live = self.recipe_kb.get_authoritative_recipe(canonical_id=canonical_id) or {} From 54a552193c6b87222d5c360429aafcd5717a53ef Mon Sep 17 00:00:00 2001 From: Test Date: Wed, 21 Oct 2026 17:11:39 +0800 Subject: [PATCH 10/14] refactor(recipe-journal): delete the three sibling modules and their scaffolding The split answered the complexity ceiling by relocation: `_config_timeline.py`, `_recipe_amend.py` and `_recipe_attrs.py` held 38 helpers, 29 of them with one caller, each carrying a docstring restating its name and two blank lines. All three are inlined back into `recipe_journal.py`, and what only existed to carry arguments across the module boundary is deleted with them: - `_ConfigAttemptContext`, `_attempt_accuracy` and `_attempt_artifacts`; - the caller-side `str(x or "")` on every field `record_attempt` already projects through `event_fields.text`, and the empty-gate skip `record_attempt_gate` already makes; - `_IntegrateIndex`, nine single-use constants, `_model_identity_tags`, `_last_stack_launch_args`, `_stack_action_names`, `_stack_entry_source`, `amend_provenance` and `improved_best_config`; - every docstring that restated a private helper's name; the invariants the longer ones carried are kept as comments. Co-Authored-By: Claude Opus 5 --- .../tests/test_coordinator_kb_writes.py | 2 +- ..._coordinator_sync_helpers_coverage_unit.py | 2 +- .../tests/test_recipe_sedimentation.py | 4 +- .../knowledge/_config_timeline.py | 212 ------ .../orchestrator/knowledge/_recipe_amend.py | 137 ---- .../orchestrator/knowledge/_recipe_attrs.py | 387 ----------- .../orchestrator/knowledge/recipe_journal.py | 635 ++++++++++++++++-- 7 files changed, 583 insertions(+), 796 deletions(-) delete mode 100644 src/hyperloom/orchestrator/knowledge/_config_timeline.py delete mode 100644 src/hyperloom/orchestrator/knowledge/_recipe_amend.py delete mode 100644 src/hyperloom/orchestrator/knowledge/_recipe_attrs.py diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_kb_writes.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_kb_writes.py index 490339cf3a..6d83744e12 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_kb_writes.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_kb_writes.py @@ -25,7 +25,7 @@ recipe_canonical_id, ) from hyperloom.orchestrator.knowledge.config import KnowledgeConfig -from hyperloom.orchestrator.knowledge._recipe_attrs import ( +from hyperloom.orchestrator.knowledge.recipe_journal import ( recipe_attrs_from_state, ) from hyperloom.inference_optimizer.breakdown.recorder import close_out as _close_out diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_sync_helpers_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_sync_helpers_coverage_unit.py index 8237bf196f..33afce5397 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_sync_helpers_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_sync_helpers_coverage_unit.py @@ -15,7 +15,7 @@ from hyperloom.orchestrator.loop.coordinator import Coordinator from hyperloom.orchestrator.state.objective import TargetGainObjective from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType -from hyperloom.orchestrator.knowledge._recipe_attrs import ( +from hyperloom.orchestrator.knowledge.recipe_journal import ( kept_kernel_optimizations, workload_kb_tags, ) diff --git a/src/hyperloom/inference_optimizer/tests/test_recipe_sedimentation.py b/src/hyperloom/inference_optimizer/tests/test_recipe_sedimentation.py index 47c1a3f7c9..b27df6da9d 100644 --- a/src/hyperloom/inference_optimizer/tests/test_recipe_sedimentation.py +++ b/src/hyperloom/inference_optimizer/tests/test_recipe_sedimentation.py @@ -17,7 +17,7 @@ ) from hyperloom.orchestrator.loop.coordinator import AdvisoryServices, Coordinator from hyperloom.orchestrator.knowledge.recipe_kb import LocalRecipeStore, RecipeKB -from hyperloom.orchestrator.knowledge._recipe_attrs import ( +from hyperloom.orchestrator.knowledge.recipe_journal import ( attempt_provenance, recipe_attrs_from_state, ) @@ -223,7 +223,7 @@ def test_experience_rows_survive_the_kb_round_trip(tmp_path): model="m", hardware="h", framework_name="f", - # Exactly what _recipe_attrs.recipe_attrs_from_state emits. + # Exactly what recipe_journal.recipe_attrs_from_state emits. what_worked=[ { "name": "mtp_on", diff --git a/src/hyperloom/orchestrator/knowledge/_config_timeline.py b/src/hyperloom/orchestrator/knowledge/_config_timeline.py deleted file mode 100644 index b96616a63c..0000000000 --- a/src/hyperloom/orchestrator/knowledge/_config_timeline.py +++ /dev/null @@ -1,212 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -"""The configuration arm's rows on the FRAMEWORK_AGENT event. - -Timeline only. The adoption ledger and the optimization journal are written -beside these rows and do not depend on a recorder being open, so an outcome is -recorded here verbatim rather than collapsed the way the journal collapses -``KEEP_UNSTABLE`` and ``KILLED_OVERTIME`` into a plain revert. -""" - -from __future__ import annotations - -from typing import Any, Collection, Iterable, Mapping, NamedTuple, Sequence - -from hyperloom.inference_optimizer.breakdown.recorder.framework_event import ( - ARM_CONFIG, - DISPOSITION_ATTEMPTED, - STEP_ATTEMPTED, - GateVerdict, -) - -from ..state.failure_evidence import UNMEASURED_OUTCOMES, classify_failure_attribution - -#: Explore outcomes that are not an attempt at anything: a variant skipped as a -#: duplicate was never measured, so no measurement would back its funnel row. -NON_ATTEMPT_OUTCOMES = frozenset({"SKIPPED_DEDUP"}) - -#: The gate whose verdict the shared accuracy block is projected from. -_ACCURACY_GATE = "accuracy" - - -class _ConfigAttemptContext(NamedTuple): - """What every attempt row produced by one grid dispatch shares.""" - - task_id: str - round_id: str - proposal_ref: str - adopted_variants: Collection[str] - - -def _attempt_accuracy(gates: Iterable[Mapping[str, Any]]) -> dict[str, Any]: - """The accuracy block, projected from the gate that ruled. - - No gate row means nothing gated the variant, which is not a gate that refused it. - """ - ruling = next((gate for gate in gates if str(gate.get("gate") or "") == _ACCURACY_GATE), {}) - return { - "required": True if ruling else None, - "value": ruling.get("observed"), - "reference": ruling.get("threshold"), - "passed": ruling.get("passed"), - } - - -def _attempt_failure(row: Mapping[str, Any], outcome: str) -> dict[str, Any]: - """What is known about a variant's failure, attributed only when it never measured.""" - error_class = str(row.get("error_class") or "") - error_excerpt = str(row.get("error_excerpt") or "") - attribution = "" - if outcome in UNMEASURED_OUTCOMES: - attribution = classify_failure_attribution( - error_class=error_class, - error_excerpt=error_excerpt, - reason=row.get("reason"), - explicit=row.get("failure_attribution"), - ) - return { - "error_class": error_class, - "error_excerpt": error_excerpt, - "attribution": attribution, - } - - -def _attempt_artifacts(row: Mapping[str, Any]) -> dict[str, str]: - """Where one variant's run left its workspace and its logs.""" - return { - "workspace": str(row.get("workspace") or ""), - "server_log_path": str(row.get("server_log_path") or ""), - "raw_result_path": str(row.get("raw_result_path") or ""), - } - - -def _attempt_fields( - row: Mapping[str, Any], - context: _ConfigAttemptContext, - gates: Sequence[Mapping[str, Any]], -) -> dict[str, Any]: - """The framework-event fields describing one measured configuration variant.""" - variant = row.get("variant") if isinstance(row.get("variant"), dict) else {} - metrics = row.get("metrics") if isinstance(row.get("metrics"), dict) else {} - outcome = str(row.get("outcome") or "") - fingerprint = str(row.get("fingerprint") or "") - adopted = fingerprint in context.adopted_variants - return { - "arm": ARM_CONFIG, - "round_id": context.round_id, - "task_id": context.task_id, - "proposal_ref": context.proposal_ref, - "provenance": str(row.get("provenance") or ""), - "outcome": outcome, - "reason": str(row.get("reason") or ""), - "reasoning": str(variant.get("note") or ""), - "reasoning_origin": str(variant.get("reasoning_origin") or ""), - "experience_citations": variant.get("experience_citations") or [], - "stage": str(row.get("stage") or ""), - "fingerprint": fingerprint, - # The fingerprint is the join key; the name is what a reader - # recognises the variant by. - "variant_name": str(row.get("variant_name") or ""), - "measurement": { - # Both ends of the pair: the anchor advances on every KEEP, so a - # percentage without its denominator adds to nothing. - "before_tput": metrics.get("base_tput"), - "after_tput": metrics.get("tput"), - "gain_pct": metrics.get("gain_pct"), - "runtime_sec": metrics.get("runtime_sec"), - "estimated_output_throughput": metrics.get("estimated_output_throughput"), - }, - "config_delta": { - "extra_server_args": variant.get("extra_server_args"), - "extra_envs": variant.get("extra_envs"), - "remove_args": variant.get("remove_args"), - "unset_envs": variant.get("unset_envs"), - "args_mode": variant.get("args_mode"), - }, - "accuracy": _attempt_accuracy(gates), - "failure": _attempt_failure(row, outcome), - "artifacts": _attempt_artifacts(row), - "decision": outcome, - "adopted": adopted, - # Every variant recorded here ran: the deduped one is skipped above. - "benched": True, - # Recorded rather than referenced: every KEEP advances the stack, so - # the session's current config is not what this variant was measured - # on top of. - "measured_against": row.get("measured_against") or {}, - # What stood behind the verdict. Absent when nothing ruled. - "validation_basis": str(row.get("validation_basis") or ""), - # A pair is what makes a gain addable, so eligibility follows the pair - # being present rather than the outcome being a KEEP. - "attribution_eligible": (adopted and metrics.get("base_tput") is not None and metrics.get("tput") is not None), - } - - -def _record_attempt_gates(recorder: Any, attempt_id: str, gates: Iterable[Mapping[str, Any]]) -> None: - """Record each named gate's verdict against one attempt.""" - for gate in gates: - name = str(gate.get("gate") or "") - if not name: - continue - recorder.record_attempt_gate( - attempt_id, - GateVerdict( - gate=name, - passed=gate.get("passed"), - reason=str(gate.get("reason") or ""), - observed=gate.get("observed"), - threshold=gate.get("threshold"), - ), - ) - - -def _record_variant_attempt(recorder: Any, row: Any, context: _ConfigAttemptContext) -> bool: - """Record one measured variant and its gates; false when the row is no attempt at anything.""" - if not isinstance(row, dict): - return False - outcome = str(row.get("outcome") or "") - if outcome in NON_ATTEMPT_OUTCOMES: - return False - # The fingerprint identifies the variant within the round and the round - # within the task, so the three together identify the attempt. - fingerprint = str(row.get("fingerprint") or "") - parts = (context.task_id, context.round_id, fingerprint) - attempt_id = ":".join(part for part in parts if part) or context.task_id - if not attempt_id: - return False - gates = [gate for gate in (row.get("gates") or []) if isinstance(gate, dict)] - recorder.record_attempt(attempt_id, **_attempt_fields(row, context, gates)) - _record_attempt_gates(recorder, attempt_id, gates) - return True - - -def record_config_attempts( - coord: Any, - *, - task: Any, - per_variant: list[dict[str, Any]], - result_dict: Mapping[str, Any], - adopted_variants: Collection[str], -) -> None: - """Record the configuration arm's measured attempts on the framework event.""" - recorder = coord._coord.phase_framework.timeline() - if recorder is None: - return - params = getattr(task, "params", None) or {} - context = _ConfigAttemptContext( - task_id=str(getattr(task, "task_id", "") or ""), - round_id=str(result_dict.get("round_id") or ""), - proposal_ref=str(params.get("proposal_msg_id") or ""), - adopted_variants=adopted_variants, - ) - recorded = 0 - for row in per_variant: - if _record_variant_attempt(recorder, row, context): - recorded += 1 - if recorded and context.proposal_ref: - # Settled here rather than at materialization: a grid whose task dies - # before any variant runs has no attempt to stand behind an - # ``attempted`` reading, and is more honestly left unsettled. - recorder.record_proposal_step(context.proposal_ref, step=STEP_ATTEMPTED, outcome=str(recorded)) - recorder.settle_proposal(context.proposal_ref, disposition=DISPOSITION_ATTEMPTED) diff --git a/src/hyperloom/orchestrator/knowledge/_recipe_amend.py b/src/hyperloom/orchestrator/knowledge/_recipe_amend.py deleted file mode 100644 index ee8fbc06b0..0000000000 --- a/src/hyperloom/orchestrator/knowledge/_recipe_amend.py +++ /dev/null @@ -1,137 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -"""Merging one session's result into the live Recipe row. - -A recipe row outlives the session amending it, so every write here is a -read-modify-write: the caller's overrides win, each field it left alone keeps -the value the live row carries, and the free-form extras some other session -wrote survive untouched. -""" - -from __future__ import annotations - -from copy import copy -from datetime import datetime, timezone -from types import MappingProxyType -from typing import Any, Mapping - -from hyperloom.common.coerce import recorded_float - -#: Recipe fields an amend carries forward when the caller does not override -#: them, each with the empty value ``put_recipe`` falls back to. -PRESERVED_RECIPE_FIELDS: Mapping[str, Any] = MappingProxyType( - { - "best_config": {}, - "best_throughput": 0.0, - "what_worked": [], - "what_failed": [], - "remaining_gaps": [], - "last_profiled": "", - "stack_fingerprint": {}, - "sessions": [], - "authority": "EXPERIENTIAL", - "confidence": 0.85, - "evidence_refs": [], - } -) - -#: Row keys ``put_recipe`` owns; everything else on a live row is a session -#: extra and survives the amend untouched. -_RECIPE_OWNED_KEYS: frozenset[str] = frozenset(PRESERVED_RECIPE_FIELDS) | frozenset( - { - "canonical_id", - "version", - "created_at", - "updated_at", - "model", - "hardware", - "framework", - "framework_name", - "framework_version", - "precision", - "pitfalls", - "lessons", - "provenance", - } -) - - -def rows_not_superseded(live_rows: Any, superseding: Any, key: str) -> list[dict[str, Any]]: - """Copies of the live rows whose ``key`` no superseding row already claims.""" - claimed = {str(row.get(key) or "") for row in superseding or [] if isinstance(row, dict)} - kept: list[dict[str, Any]] = [] - for row in live_rows or []: - if not isinstance(row, dict): - continue - if str(row.get(key) or "") in claimed: - continue - kept.append(dict(row)) - return kept - - -def improved_best_config(state: Any, attrs: Mapping[str, Any], live: Mapping[str, Any]) -> dict[str, Any]: - """The ``best_config`` override, empty unless this session genuinely beat the live row.""" - my_throughput = float(attrs.get("best_throughput") or 0.0) - validated_gain = float(state.cumulative_gain_validated or 0.0) - current_args = str(state.current_best.get("extra_server_args") or "").strip() - has_validated_win = bool((state.optimization_stack or []) or validated_gain > 0.0 or current_args) - if not has_validated_win or my_throughput <= recorded_float(live.get("best_throughput")): - return {} - return { - "best_config": attrs["best_config"], - "best_throughput": my_throughput, - } - - -def appended(live_facts: Any, addition: Mapping[str, Any] | None) -> list[Any]: - """The live lesson or pitfall list with one more appended, when the caller brought one.""" - facts = list(live_facts or []) - if addition is not None: - facts.append(addition) - return facts - - -def amend_provenance(details: Mapping[str, Any] | None) -> dict[str, Any]: - """The provenance block stamped on every amend; the central server requires one.""" - return { - "source": "hyperloom-inference-optimizer", - "generator": "coordinator", - "generated_at": datetime.now(timezone.utc).isoformat(timespec="microseconds"), - "details": dict(details or {}), - } - - -def amended_recipe_fields(live: Mapping[str, Any], overrides: Mapping[str, Any]) -> dict[str, Any]: - """The preserved recipe fields for one amend: an override wins, an unset field keeps its live value. - - An absent or empty live value falls back to the field's own default, so - ``put_recipe`` never sees ``None`` where it declares a container. - """ - merged: dict[str, Any] = {} - for field, default in PRESERVED_RECIPE_FIELDS.items(): - if field in overrides: - merged[field] = overrides[field] - continue - live_value = live.get(field) - merged[field] = type(default)(live_value) if live_value else copy(default) - return merged - - -def amended_recipe_extras(live: Mapping[str, Any], overrides: Mapping[str, Any], state: Any) -> dict[str, Any]: - """The free-form extras an amend writes back: the prior ones, the caller's, then the model's identity. - - The architecture tags are re-stamped from config.json each time because a row - written before the architecture was read carries none. - """ - extras = {key: value for key, value in live.items() if key not in _RECIPE_OWNED_KEYS} - extras.update(overrides.get("extras") or {}) - architectures = state.model_architectures or [] - if isinstance(architectures, list): - named = [str(name).strip() for name in architectures if str(name or "").strip()] - if named: - extras["architectures"] = named - model_type = str(state.model_type or "").strip() - if model_type: - extras["model_type"] = model_type - return extras diff --git a/src/hyperloom/orchestrator/knowledge/_recipe_attrs.py b/src/hyperloom/orchestrator/knowledge/_recipe_attrs.py deleted file mode 100644 index 00f7fc2631..0000000000 --- a/src/hyperloom/orchestrator/knowledge/_recipe_attrs.py +++ /dev/null @@ -1,387 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -"""The recipe-shaped view of a session's settled :class:`SharedState`. - -Every projection here is a pure read of the state the optimization loop has -already settled, shaped the way the Recipe KB stores it: the launch -configuration that won, the stack that produced it, the kernels that landed, -and the workload dimensions a warm start filters on. -""" - -from __future__ import annotations - -from datetime import datetime, timezone -from typing import Any, Collection, Iterable, Mapping, NamedTuple, Sequence - -from hyperloom.common.coerce import recorded_float -from hyperloom.inference_optimizer.session.optimization_journal import OUTCOME_KEEP, OUTCOME_REVERT - -from .workload_tags import workload_shape_tags - -#: SharedState attributes naming the workload; ``model_family`` is deliberately -#: absent, being no part of the seven-dimension Recipe identity. -_MODEL_IDENTITY_ATTRS: tuple[str, ...] = ("framework", "model_class", "model_name") - -#: The serving frameworks whose version the boot-time fingerprint records. -_VERSIONED_FRAMEWORKS: frozenset[str] = frozenset({"sglang", "vllm"}) - -#: Stack-fingerprint keys that become recipe tags under their own names. -_RUNTIME_VERSION_KEYS: tuple[tuple[str, str], ...] = ( - ("rocm", "rocm_version"), - ("aiter", "aiter_version"), - ("image_digest", "image_digest"), -) - -#: The probe's own word for a version it could not read. -_UNIDENTIFIED_VERSION: str = "unknown" - -#: Baseline workload knobs carried onto the recipe, grouped by the type that -#: makes a value meaningful. A ``False`` survives: it is an explicit disable. -_POSITIVE_INT_KNOBS: tuple[str, ...] = ("max_running_requests", "max_num_seqs") -_BOOLEAN_KNOBS: tuple[str, ...] = ("chunked_prefill_enabled", "enable_torch_compile") -_TEXT_KNOBS: tuple[str, ...] = ("quant_scheme", "workload_mode") - -#: ``best_config`` fields copied from ``current_best`` as they stand. -_BEST_CONFIG_PASSTHROUGH: tuple[str, ...] = ("extra_envs", "name", "tput", "accuracy") - -#: How many recent action failures the recipe's ``what_failed`` carries. -_RECENT_FAILURE_LIMIT: int = 10 - - -class AttemptProvenance(NamedTuple): - """Where a session's proven optimizations came from, and what it reverted. - - Keepers are indexed twice because a stack layer may identify itself either - by name or by the research gap it closed. - """ - - kept_by_name: dict[str, str] - kept_by_gap: dict[str, str] - reverted: list[dict[str, Any]] - - -class _IntegrateIndex(NamedTuple): - """Integrate-layer entries under both keys a micro KEEP can name them by.""" - - by_kernel: dict[str, dict[str, Any]] - by_task_group: dict[str, dict[str, Any]] - - -def _known_version(raw: Any) -> str: - """A fingerprint version, empty when the probe never identified one.""" - version = str(raw or "").strip() - return "" if version == _UNIDENTIFIED_VERSION else version - - -def _model_identity_tags(state: Any) -> dict[str, Any]: - """The workload's model-identity tags, omitting the ones this session never learned.""" - tags: dict[str, Any] = {} - for attr in _MODEL_IDENTITY_ATTRS: - text = str(getattr(state, attr, "") or "").strip() - if text: - tags[attr] = text - precision = getattr(state, "precision", None) - if precision not in (None, "", 0): - tags["precision"] = precision - return tags - - -def _runtime_version_tags(fingerprint_meta: Mapping[str, Any], framework: str) -> dict[str, str]: - """Runtime versions read off the stack fingerprint the CLI wrote at boot. - - ``framework_version`` names whichever serving framework is active, so a recipe - written under one is never matched against the other. - """ - tags: dict[str, str] = {} - framework_key = framework.lower() - if framework_key in _VERSIONED_FRAMEWORKS: - active = _known_version(fingerprint_meta.get(framework_key)) - if active: - tags["framework_version"] = active - for source_key, tag in _RUNTIME_VERSION_KEYS: - version = _known_version(fingerprint_meta.get(source_key)) - if version: - tags[tag] = version - return tags - - -def _baseline_workload_tags(workload_extra: Mapping[str, Any]) -> dict[str, Any]: - """Per-baseline workload knobs from the materialized YAML, each kept only when its type says it was set.""" - tags: dict[str, Any] = {} - for key in _POSITIVE_INT_KNOBS: - value = workload_extra.get(key) - if isinstance(value, int) and value > 0: - tags[key] = value - for key in _BOOLEAN_KNOBS: - value = workload_extra.get(key) - if isinstance(value, bool): - tags[key] = value - for key in _TEXT_KNOBS: - value = workload_extra.get(key) - if isinstance(value, str) and value.strip(): - tags[key] = value.strip() - return tags - - -def workload_kb_tags(state: Any) -> dict[str, Any]: - """The workload-shape tags describing the session this recipe came from.""" - tags = _model_identity_tags(state) - tags.update(workload_shape_tags(state)) - fingerprint_meta = state.stack_fingerprint_meta or {} - if isinstance(fingerprint_meta, dict): - tags.update(_runtime_version_tags(fingerprint_meta, str(tags.get("framework") or ""))) - workload_extra = state.baseline_workload_extra or {} - if isinstance(workload_extra, dict): - tags.update(_baseline_workload_tags(workload_extra)) - return tags - - -def _index_integrate_attempts(attempts: Mapping[str, Any]) -> _IntegrateIndex: - """Index the integrate ledger by kernel and by task group; the last write wins.""" - by_kernel: dict[str, dict[str, Any]] = {} - by_task_group: dict[str, dict[str, Any]] = {} - for entry in attempts.values(): - if not isinstance(entry, dict): - continue - kernel_id = str(entry.get("kernel_id") or "") - if kernel_id: - by_kernel[kernel_id] = entry - task_group_key = str(entry.get("task_group_key") or "") - if task_group_key: - by_task_group[task_group_key] = entry - return _IntegrateIndex(by_kernel=by_kernel, by_task_group=by_task_group) - - -def _last_rebenched_throughput(integration: Mapping[str, Any]) -> float: - """Throughput from the most recent integrate attempt that re-benched end to end.""" - for attempt in reversed(list(integration.get("attempts") or [])): - if isinstance(attempt, dict) and attempt.get("new_tput") is not None: - return recorded_float(attempt.get("new_tput")) - return 0.0 - - -def _kernel_optimization_row(entry: Mapping[str, Any], ledger_id: str, index: _IntegrateIndex) -> dict[str, Any]: - """One KernelOptimization row: a micro KEEP joined to its end-to-end verdict.""" - kernel_id = str(entry.get("current_kernel_id") or entry.get("kernel_id") or ledger_id) - task_group_key = str(entry.get("task_group_key") or "") - integration = index.by_task_group.get(task_group_key) if task_group_key else index.by_kernel.get(kernel_id) - integrated = isinstance(integration, dict) - return { - "kernel_id": kernel_id, - "source_file": str(entry.get("last_source_file") or ""), - "artifact_path": str(entry.get("last_artifact_path") or ""), - "micro_speedup": recorded_float(entry.get("last_micro_speedup")), - "decision": OUTCOME_KEEP, - "e2e_gain_pct": recorded_float(integration.get("best_gain_pct")) if integrated else 0.0, - "e2e_tput": _last_rebenched_throughput(integration) if integrated else 0.0, - "e2e_decision": str(integration.get("last_decision") or "").upper() if integrated else "", - "integrated": integrated, - "ts": str(entry.get("last_ts") or ""), - } - - -def kept_kernel_optimizations(state: Any) -> list[dict[str, Any]]: - """The KEEP'd kernel optimizations, each joined to its end-to-end integrate verdict.""" - optimizations = state.kernel_opt_task_attempts or {} - if not isinstance(optimizations, dict): - return [] - integrations = state.kernel_integrate_attempts or {} - index = _index_integrate_attempts(integrations if isinstance(integrations, dict) else {}) - rows: list[dict[str, Any]] = [] - for ledger_id, entry in optimizations.items(): - if not isinstance(entry, dict): - continue - if str(entry.get("last_decision", "")).upper() != OUTCOME_KEEP: - continue - rows.append(_kernel_optimization_row(entry, ledger_id, index)) - return rows - - -def _reverted_attempt_row(name: str, attempt: Mapping[str, Any], source: str) -> dict[str, Any]: - """A ``what_failed`` row for an attempt this session measured and reverted.""" - row: dict[str, Any] = { - "name": name, - "reason": "reverted", - "gain_pct": attempt.get("gain_pct"), - } - if source: - row["source"] = source - return row - - -def _gap_attempt_provenance(gap: Mapping[str, Any]) -> AttemptProvenance: - """One research gap's attempts, split into what stuck and what was reverted.""" - source = str(gap.get("provenance") or "").strip() - canonical_id = str(gap.get("canonical_id") or "").strip() - kept_by_name: dict[str, str] = {} - kept_by_gap: dict[str, str] = {} - reverted: list[dict[str, Any]] = [] - for attempt in gap.get("attempts") or []: - if not isinstance(attempt, dict): - continue - variant = str(attempt.get("variant_name") or "").strip() - kernel = str(attempt.get("kernel_id") or "").strip() - outcome = str(attempt.get("outcome") or "").strip().upper() - if outcome == OUTCOME_KEEP and source: - for name in (variant, kernel): - if name: - kept_by_name.setdefault(name, source) - if canonical_id: - kept_by_gap.setdefault(canonical_id, source) - elif outcome == OUTCOME_REVERT and (variant or kernel): - reverted.append(_reverted_attempt_row(variant or kernel, attempt, source)) - return AttemptProvenance(kept_by_name=kept_by_name, kept_by_gap=kept_by_gap, reverted=reverted) - - -def attempt_provenance(state: Any) -> AttemptProvenance: - """Trace each proven optimization back to the research hint it came from.""" - kept_by_name: dict[str, str] = {} - kept_by_gap: dict[str, str] = {} - reverted: list[dict[str, Any]] = [] - for gap in state.gaps or []: - if not isinstance(gap, dict): - continue - from_gap = _gap_attempt_provenance(gap) - for name, source in from_gap.kept_by_name.items(): - kept_by_name.setdefault(name, source) - for canonical_id, source in from_gap.kept_by_gap.items(): - kept_by_gap.setdefault(canonical_id, source) - reverted.extend(from_gap.reverted) - return AttemptProvenance(kept_by_name=kept_by_name, kept_by_gap=kept_by_gap, reverted=reverted) - - -def _last_stack_launch_args(optimization_stack: Sequence[Any]) -> str: - """Launch args from the last validated stack layer, empty when it carries none.""" - if not optimization_stack: - return "" - last = optimization_stack[-1] - if not isinstance(last, dict): - return "" - return str(last.get("candidate_extra_server_args") or last.get("extra_server_args") or "").strip() - - -def _best_config(current_best: Mapping[str, Any], optimization_stack: Sequence[Any]) -> dict[str, Any]: - """The replayable launch configuration the recipe advertises. - - The last validated stack layer overrides ``current_best``, whose - ``extra_server_args`` may carry a string an interrupted write corrupted. - """ - config: dict[str, Any] = {} - current_args = current_best.get("extra_server_args") - if current_args: - config["extra_server_args"] = str(current_args) - for key in _BEST_CONFIG_PASSTHROUGH: - if key in current_best: - config[key] = current_best[key] - stack_args = _last_stack_launch_args(optimization_stack) - if stack_args: - config["extra_server_args"] = stack_args - return config - - -def _stack_entry_source(entry: Mapping[str, Any], name: str, provenance: AttemptProvenance) -> str: - """The research hint a stack layer came from, empty when nothing claims it. - - The layer's gap id wins; the variant or kernel name answers for layers - recorded before gap ids were stamped. - """ - gap_id = str(entry.get("gap_canonical_id") or "").strip() - if gap_id: - by_gap = provenance.kept_by_gap.get(gap_id) - if by_gap: - return by_gap - kernel_id = str(entry.get("kernel_id") or "") - return provenance.kept_by_name.get(name) or provenance.kept_by_name.get(kernel_id) or "" - - -def _what_worked( - optimization_stack: Sequence[Any], - gain_per_entry: Sequence[Any], - provenance: AttemptProvenance, -) -> list[dict[str, Any]]: - """One row per stack layer: the change it made, what it gained and where it came from.""" - rows: list[dict[str, Any]] = [] - for index, entry in enumerate(optimization_stack): - if not isinstance(entry, dict): - continue - name = str(entry.get("variant_name") or entry.get("name") or entry.get("kernel_id") or "") - row: dict[str, Any] = { - "name": name, - "extra_server_args": str(entry.get("extra_server_args") or ""), - "extra_envs": dict(entry.get("extra_envs") or {}), - "gain_pct": gain_per_entry[index] if index < len(gain_per_entry) else None, - } - source = _stack_entry_source(entry, name, provenance) - if source: - row["source"] = source - rows.append(row) - return rows - - -def _what_failed(recent_failures: Sequence[Any], reverted: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: - """The session's recent action failures, followed by every attempt it reverted.""" - rows: list[dict[str, Any]] = [] - for failure in recent_failures[-_RECENT_FAILURE_LIMIT:]: - if isinstance(failure, dict): - rows.append( - { - "name": str(failure.get("name") or failure.get("action") or ""), - "reason": str(failure.get("reason") or failure.get("error_class") or ""), - } - ) - rows.extend(reverted) - return rows - - -def _stack_action_names(optimization_stack: Iterable[Any]) -> list[str]: - """The named actions the optimization stack is built from, in the order they landed.""" - names: list[str] = [] - for entry in optimization_stack: - if not isinstance(entry, dict): - continue - name = str(entry.get("variant_name") or entry.get("name") or entry.get("action") or "").strip() - if name: - names.append(name) - return names - - -def _session_row( - state: Any, - session_name: str, - optimization_stack: Collection[Any], - final_throughput: float, -) -> dict[str, Any]: - """This session's row on the recipe: where it started, where it reached and by which actions.""" - validated_stack_len = int(state.cumulative_gain_validated_stack_len or 0) - return { - "session_id": str(state.recipe_kb_session_id or session_name), - "gain_pct": float(state.cumulative_gain_validated or 0.0), - "stack_len": validated_stack_len or len(optimization_stack), - "throughput_before": float(state.baseline_tput or 0.0), - "throughput_after": final_throughput, - "date": datetime.now(timezone.utc).isoformat(), - "actions_taken": _stack_action_names(optimization_stack), - } - - -def recipe_attrs_from_state(state: Any, session_name: str) -> dict[str, Any]: - """Materialise the recipe-shaped view of one session's settled state. - - ``framework_version`` is left unset here because it is manifest-derived; the - T0 backfill writes it. - """ - optimization_stack = state.optimization_stack or [] - # A session that does not sediment its attempts reports no provenance. - provenance = attempt_provenance(state) if state.recipe_sediment_enabled else AttemptProvenance({}, {}, []) - final_throughput = float(state.current_best.get("tput", 0.0)) - return { - "best_config": _best_config(state.current_best, optimization_stack), - "best_throughput": final_throughput, - "what_worked": _what_worked(optimization_stack, state.gain_per_stack_entry or [], provenance), - "what_failed": _what_failed(state.last_action_failures or [], provenance.reverted), - "kernel_optimizations": kept_kernel_optimizations(state), - "last_profiled": str(state.cumulative_gain_validated_ts or ""), - "workload": workload_kb_tags(state), - "sessions": [_session_row(state, session_name, optimization_stack, final_throughput)], - } diff --git a/src/hyperloom/orchestrator/knowledge/recipe_journal.py b/src/hyperloom/orchestrator/knowledge/recipe_journal.py index 27901a84c7..47b07ba141 100644 --- a/src/hyperloom/orchestrator/knowledge/recipe_journal.py +++ b/src/hyperloom/orchestrator/knowledge/recipe_journal.py @@ -6,13 +6,21 @@ from __future__ import annotations import logging as _logging +from copy import copy from dataclasses import dataclass from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, Collection, Mapping, NamedTuple +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Collection, Mapping, NamedTuple, Sequence -from hyperloom.common.coerce import to_float, to_int +from hyperloom.common.coerce import recorded_float, to_float, to_int from hyperloom.common.io import JsonLineFormat, append_jsonl from hyperloom.inference_optimizer.breakdown.recorder import close_out as _close_out +from hyperloom.inference_optimizer.breakdown.recorder.framework_event import ( + ARM_CONFIG, + DISPOSITION_ATTEMPTED, + STEP_ATTEMPTED, + GateVerdict, +) from hyperloom.inference_optimizer.recipe_snapshot_constants import detect_framework_version from hyperloom.inference_optimizer.session.optimization_journal import ( OUTCOME_KEEP, @@ -29,17 +37,9 @@ from ..collaborator import CoordinatorCollaborator from ..state.attempt_ledger import AttemptRow, record_attempt_row -from ._config_timeline import NON_ATTEMPT_OUTCOMES, record_config_attempts -from ._recipe_amend import ( - amend_provenance, - amended_recipe_extras, - amended_recipe_fields, - appended, - improved_best_config, - rows_not_superseded, -) -from ._recipe_attrs import recipe_attrs_from_state, workload_kb_tags +from ..state.failure_evidence import UNMEASURED_OUTCOMES, classify_failure_attribution from .config import KnowledgeStoreMode +from .workload_tags import workload_shape_tags if TYPE_CHECKING: from ..loop.coordinator import Coordinator @@ -76,6 +76,48 @@ #: Explore outcomes the journal collapses into a plain revert. _MEASURED_LOSS_OUTCOMES: frozenset[str] = frozenset({OUTCOME_REVERT, "FAILED"}) +#: Explore outcomes that are not an attempt at anything: a variant skipped as a +#: duplicate was never measured, so no measurement would back its funnel row. +_NON_ATTEMPT_OUTCOMES: frozenset[str] = frozenset({"SKIPPED_DEDUP"}) + +#: Recipe fields an amend carries forward when the caller does not override +#: them, each with the empty value ``put_recipe`` falls back to. +PRESERVED_RECIPE_FIELDS: Mapping[str, Any] = MappingProxyType( + { + "best_config": {}, + "best_throughput": 0.0, + "what_worked": [], + "what_failed": [], + "remaining_gaps": [], + "last_profiled": "", + "stack_fingerprint": {}, + "sessions": [], + "authority": "EXPERIENTIAL", + "confidence": 0.85, + "evidence_refs": [], + } +) + +#: Row keys ``put_recipe`` owns; everything else on a live row is a session +#: extra and survives the amend untouched. +_RECIPE_OWNED_KEYS: frozenset[str] = frozenset(PRESERVED_RECIPE_FIELDS) | frozenset( + { + "canonical_id", + "version", + "created_at", + "updated_at", + "model", + "hardware", + "framework", + "framework_name", + "framework_version", + "precision", + "pitfalls", + "lessons", + "provenance", + } +) + #: Per-variant measurement detail the journal row carries beyond the headline #: gain and throughput. _VARIANT_DETAIL_METRICS: tuple[str, ...] = ( @@ -133,24 +175,18 @@ class RemoteRecipeAudit: class _FrameworkIdentity(NamedTuple): - """The framework dimensions of a Recipe identity, with the version resolved.""" - name: str version: str precision: str class _VariantMeasurement(NamedTuple): - """What one explore variant's benchmark reported.""" - gain_pct: float | None throughput_after: float | None detail: dict[str, Any] class _VariantFailure(NamedTuple): - """Why a variant was reverted; both fields absent for every other outcome.""" - error_class: str | None reason: str | None @@ -215,7 +251,7 @@ def _record_config_run(coord: Any, *, task: Any, result_dict: Mapping[str, Any]) if recorder is None: return from hyperloom.common.timeutil import now_iso - from hyperloom.inference_optimizer.breakdown.recorder.framework_event import ARM_CONFIG, ROLE_CONFIG + from hyperloom.inference_optimizer.breakdown.recorder.framework_event import ROLE_CONFIG task_id = str(getattr(task, "task_id", "") or "") if not task_id: @@ -238,12 +274,212 @@ def _record_config_run(coord: Any, *, task: Any, result_dict: Mapping[str, Any]) ) -def _framework_identity(state: Any) -> _FrameworkIdentity: - """The session's framework, the version it is running and its precision. +def _attempt_failure(row: Mapping[str, Any], outcome: str) -> dict[str, Any]: + error_class = str(row.get("error_class") or "") + error_excerpt = str(row.get("error_excerpt") or "") + attribution = "" + if outcome in UNMEASURED_OUTCOMES: + attribution = classify_failure_attribution( + error_class=error_class, + error_excerpt=error_excerpt, + reason=row.get("reason"), + explicit=row.get("failure_attribution"), + ) + return {"error_class": error_class, "error_excerpt": error_excerpt, "attribution": attribution} + + +def _attempt_fields( + row: Mapping[str, Any], + gates: Sequence[Mapping[str, Any]], + shared: Mapping[str, str], + adopted: bool, +) -> dict[str, Any]: + variant = row.get("variant") if isinstance(row.get("variant"), dict) else {} + metrics = row.get("metrics") if isinstance(row.get("metrics"), dict) else {} + outcome = str(row.get("outcome") or "") + # This arm gates accuracy rather than reporting it, so the block both arms + # carry is projected from the gate that ruled. No gate row means nothing + # gated the variant, which is not a gate that refused it. + ruling = next((gate for gate in gates if str(gate.get("gate") or "") == "accuracy"), {}) + return { + "arm": ARM_CONFIG, + **shared, + "provenance": row.get("provenance"), + "outcome": outcome, + "reason": row.get("reason"), + "reasoning": variant.get("note"), + "reasoning_origin": variant.get("reasoning_origin"), + "experience_citations": variant.get("experience_citations"), + "stage": row.get("stage"), + "fingerprint": row.get("fingerprint"), + # The fingerprint is the join key; the name is what a reader + # recognises the variant by. + "variant_name": row.get("variant_name"), + "measurement": { + # Both ends of the pair: the anchor advances on every KEEP, so a + # percentage without its denominator adds to nothing. + "before_tput": metrics.get("base_tput"), + "after_tput": metrics.get("tput"), + "gain_pct": metrics.get("gain_pct"), + "runtime_sec": metrics.get("runtime_sec"), + "estimated_output_throughput": metrics.get("estimated_output_throughput"), + }, + "config_delta": { + "extra_server_args": variant.get("extra_server_args"), + "extra_envs": variant.get("extra_envs"), + "remove_args": variant.get("remove_args"), + "unset_envs": variant.get("unset_envs"), + "args_mode": variant.get("args_mode"), + }, + "accuracy": { + "required": True if ruling else None, + "value": ruling.get("observed"), + "reference": ruling.get("threshold"), + "passed": ruling.get("passed"), + }, + "failure": _attempt_failure(row, outcome), + "artifacts": { + "workspace": row.get("workspace"), + "server_log_path": row.get("server_log_path"), + "raw_result_path": row.get("raw_result_path"), + }, + "decision": outcome, + "adopted": adopted, + # Every variant recorded here ran: the deduped one is skipped above. + "benched": True, + # Recorded rather than referenced: every KEEP advances the stack, so + # the session's current config is not what this variant was measured + # on top of. + "measured_against": row.get("measured_against"), + # What stood behind the verdict. Absent when nothing ruled. + "validation_basis": row.get("validation_basis"), + # A pair is what makes a gain addable, so eligibility follows the pair + # being present rather than the outcome being a KEEP. + "attribution_eligible": (adopted and metrics.get("base_tput") is not None and metrics.get("tput") is not None), + } + + +def _record_variant_attempt( + recorder: Any, + row: Any, + shared: Mapping[str, str], + adopted_variants: Collection[str], +) -> bool: + if not isinstance(row, dict): + return False + if str(row.get("outcome") or "") in _NON_ATTEMPT_OUTCOMES: + return False + # The fingerprint identifies the variant within the round and the round + # within the task, so the three together identify the attempt. + fingerprint = str(row.get("fingerprint") or "") + parts = (shared["task_id"], shared["round_id"], fingerprint) + attempt_id = ":".join(part for part in parts if part) or shared["task_id"] + if not attempt_id: + return False + gates = [gate for gate in (row.get("gates") or []) if isinstance(gate, dict)] + recorder.record_attempt(attempt_id, **_attempt_fields(row, gates, shared, fingerprint in adopted_variants)) + for gate in gates: + recorder.record_attempt_gate( + attempt_id, + GateVerdict( + gate=str(gate.get("gate") or ""), + passed=gate.get("passed"), + reason=str(gate.get("reason") or ""), + observed=gate.get("observed"), + threshold=gate.get("threshold"), + ), + ) + return True + - The version is probed from the installed framework when the session never - recorded one, so a recipe is never filed under a blank version. +def _record_config_attempts( + coord: Any, + *, + task: Any, + per_variant: list[dict[str, Any]], + result_dict: Mapping[str, Any], + adopted_variants: Collection[str], +) -> None: + """Record the configuration arm's measured attempts on the framework event. + + Timeline only: the ledger row is written by ``fact_write_hook``, which does + not depend on a recorder being open. The outcome is recorded verbatim here, + unlike the journal beside it, which collapses ``KEEP_UNSTABLE`` and + ``KILLED_OVERTIME`` into a plain revert. """ + recorder = coord._coord.phase_framework.timeline() + if recorder is None: + return + params = getattr(task, "params", None) or {} + shared = { + "task_id": str(getattr(task, "task_id", "") or ""), + "round_id": str(result_dict.get("round_id") or ""), + "proposal_ref": str(params.get("proposal_msg_id") or ""), + } + recorded = 0 + for row in per_variant: + if _record_variant_attempt(recorder, row, shared, adopted_variants): + recorded += 1 + if not (recorded and shared["proposal_ref"]): + return + # Settled here rather than at materialization: a grid whose task dies before + # any variant runs has no attempt to stand behind an ``attempted`` reading, + # and is more honestly left unsettled. + recorder.record_proposal_step(shared["proposal_ref"], step=STEP_ATTEMPTED, outcome=str(recorded)) + recorder.settle_proposal(shared["proposal_ref"], disposition=DISPOSITION_ATTEMPTED) + + +def rows_not_superseded(live_rows: Any, superseding: Any, key: str) -> list[dict[str, Any]]: + claimed = {str(row.get(key) or "") for row in superseding or [] if isinstance(row, dict)} + kept: list[dict[str, Any]] = [] + for row in live_rows or []: + if not isinstance(row, dict): + continue + if str(row.get(key) or "") in claimed: + continue + kept.append(dict(row)) + return kept + + +def appended(live_facts: Any, addition: Mapping[str, Any] | None) -> list[Any]: + facts = list(live_facts or []) + if addition is not None: + facts.append(addition) + return facts + + +def amended_recipe_fields(live: Mapping[str, Any], overrides: Mapping[str, Any]) -> dict[str, Any]: + # An absent or empty live value falls back to the field's own default, so + # ``put_recipe`` never sees ``None`` where it declares a container. + merged: dict[str, Any] = {} + for field, default in PRESERVED_RECIPE_FIELDS.items(): + if field in overrides: + merged[field] = overrides[field] + continue + live_value = live.get(field) + merged[field] = type(default)(live_value) if live_value else copy(default) + return merged + + +def amended_recipe_extras(live: Mapping[str, Any], overrides: Mapping[str, Any], state: Any) -> dict[str, Any]: + # The architecture tags are re-stamped from config.json each time because a + # row written before the architecture was read carries none. + extras = {key: value for key, value in live.items() if key not in _RECIPE_OWNED_KEYS} + extras.update(overrides.get("extras") or {}) + architectures = state.model_architectures or [] + if isinstance(architectures, list): + named = [str(name).strip() for name in architectures if str(name or "").strip()] + if named: + extras["architectures"] = named + model_type = str(state.model_type or "").strip() + if model_type: + extras["model_type"] = model_type + return extras + + +def _framework_identity(state: Any) -> _FrameworkIdentity: + # Probed from the installed framework when the session never recorded a + # version, so a recipe is never filed under a blank one. name = str(state.framework or "") version = str(state.framework_version or "") if not version and name: @@ -252,7 +488,6 @@ def _framework_identity(state: Any) -> _FrameworkIdentity: def _result_payload(result: Any) -> dict[str, Any]: - """A completed task's result dict, whatever shape the executor returned it in.""" payload = result.result if hasattr(result, "result") else (result or {}) return payload if isinstance(payload, dict) else {} @@ -264,9 +499,8 @@ def _grid_attempt_row( round_id: str, adopted: bool, ) -> AttemptRow | None: - """The adoption-ledger row for one measured grid variant, or ``None`` when it measured nothing.""" outcome = str(variant_outcome.get("outcome") or "") if isinstance(variant_outcome, dict) else "" - if not outcome or outcome in NON_ATTEMPT_OUTCOMES: + if not outcome or outcome in _NON_ATTEMPT_OUTCOMES: return None metrics = variant_outcome.get("metrics") if isinstance(variant_outcome.get("metrics"), dict) else {} return AttemptRow( @@ -286,7 +520,6 @@ def _grid_attempt_row( def _variant_verdict(outcome: str, adopted: bool) -> Verdict: - """How one explore variant's own outcome settles against the adoption rule.""" if outcome == OUTCOME_KEEP: return Verdict.ADOPTED if adopted else Verdict.REFUSED if outcome in _MEASURED_LOSS_OUTCOMES: @@ -295,7 +528,6 @@ def _variant_verdict(outcome: str, adopted: bool) -> Verdict: def _variant_measurement(metrics: Any) -> _VariantMeasurement: - """One variant's measured figures, blank throughout when it never measured.""" if not isinstance(metrics, dict): return _VariantMeasurement(gain_pct=None, throughput_after=None, detail={}) return _VariantMeasurement( @@ -306,7 +538,6 @@ def _variant_measurement(metrics: Any) -> _VariantMeasurement: def _variant_failure(variant_outcome: Mapping[str, Any], outcome: str) -> _VariantFailure: - """What the journal records about a reverted variant, blank for every other outcome.""" if outcome != OUTCOME_REVERT: return _VariantFailure(error_class=None, reason=None) return _VariantFailure( @@ -316,7 +547,6 @@ def _variant_failure(variant_outcome: Mapping[str, Any], outcome: str) -> _Varia def _variant_change_attrs(variant_attrs: Any, variant_name: str) -> dict[str, Any]: - """One variant's attributes as the change summary needs them.""" attrs = dict(variant_attrs) if isinstance(variant_attrs, dict) else {} named_knob = attrs.get("extra_server_args") or attrs.get("extra_envs") or attrs.get("name") if not named_knob and variant_name: @@ -325,7 +555,6 @@ def _variant_change_attrs(variant_attrs: Any, variant_name: str) -> dict[str, An def _finalize_outcome(*, status: str, reason: str, backend: str, result_type: str) -> dict[str, Any]: - """One terminal Recipe-finalize outcome, in the shape the close-out step records.""" return { "status": status, "reason": reason, @@ -335,7 +564,6 @@ def _finalize_outcome(*, status: str, reason: str, backend: str, result_type: st def _finalize_error(exc: BaseException, *, reason: str, backend: str, result_type: str) -> dict[str, Any]: - """The outcome for a finalize attempt an exception ended.""" return { "status": "error", "reason": reason, @@ -425,7 +653,7 @@ async def fact_write_hook( if task.kind == "explore": _record_config_run(self, task=task, result_dict=result_dict) if task.kind == "explore" and isinstance(per_variant, list) and per_variant: - record_config_attempts( + _record_config_attempts( self, task=task, per_variant=per_variant, @@ -747,7 +975,7 @@ def _record_fact_per_variant( """ journal = self.ensure_journal() outcome_raw = str(variant_outcome.get("outcome") or "") - if outcome_raw in NON_ATTEMPT_OUTCOMES: + if outcome_raw in _NON_ATTEMPT_OUTCOMES: return outcome = derive_journal_outcome(_variant_verdict(outcome_raw, adopted), variant_outcome) variant_name = str(variant_outcome.get("variant_name") or "") @@ -974,12 +1202,10 @@ def _write_back_scope(self) -> dict[str, Any]: } def _unvalidated_stack_outcome(self) -> dict[str, Any] | None: - """The skip outcome when the working recipe outran its last validation, else ``None``. - - ``current_best`` and ``cumulative_gain_validated`` are only one Recipe's - figures when nothing was lifted since the last validation; otherwise every - sink below would pair the newer config with the older gain. - """ + # ``current_best`` and ``cumulative_gain_validated`` are only one + # Recipe's figures when nothing was lifted since the last validation; + # otherwise every sink below would pair the newer config with the + # older gain. state = self.shared_state if not state.optimization_stack_has_unvalidated_keeps(): return None @@ -1050,11 +1276,8 @@ def finalize_recipe_and_journal( return self._write_local_recipe() def _publish_remote_recipe(self, source: str) -> dict[str, Any]: - """Publish the terminal Recipe to the KB Store and audit the attempt. - - Remote mode has one Recipe sink: the KB Store final session writer. T0 and - runtime amendment are intentionally absent. - """ + # Remote mode has one Recipe sink: the KB Store final session writer. + # T0 and runtime amendment are intentionally absent. state = self.shared_state session_id = str(state.recipe_kb_session_id or state.session_id or self.session_dir.name) canonical_id = "" @@ -1118,11 +1341,8 @@ def _publish_remote_recipe(self, source: str) -> dict[str, Any]: } def _final_recipe_overrides(self) -> dict[str, Any]: - """What this session's close writes over the live Recipe row. - - Read-modify-write under the cid flock, so two finalises of the same session - leave one row rather than tearing the lists they both append to. - """ + # Read-modify-write under the cid flock, so two finalises of the same + # session leave one row rather than tearing the lists they both append to. state = self.shared_state attrs = recipe_attrs_from_state(state, self.session_dir.name) live = self.recipe_kb.get_authoritative_recipe(canonical_id=self.workload_canonical_id()) or {} @@ -1140,14 +1360,18 @@ def _final_recipe_overrides(self) -> dict[str, Any]: "sessions": rows_not_superseded(live.get("sessions"), my_sessions, "session_id") + my_sessions, "extras": extras, } - overrides.update(improved_best_config(state, attrs, live)) + # The row's own figures survive a session that never validated a win, + # or whose win does not beat what the row already advertises. + my_throughput = float(attrs.get("best_throughput") or 0.0) + won = bool(state.optimization_stack or float(state.cumulative_gain_validated or 0.0) > 0.0) + won = won or bool(str(state.current_best.get("extra_server_args") or "").strip()) + if won and my_throughput > recorded_float(live.get("best_throughput")): + overrides["best_config"] = attrs["best_config"] + overrides["best_throughput"] = my_throughput return overrides def _write_local_recipe(self) -> dict[str, Any]: - """Amend the local Recipe row with this session's settled result. - - Local mode never consults ambient KB_STORE_* credentials. - """ + # Local mode never consults ambient KB_STORE_* credentials. if self.recipe_kb is None: return _finalize_outcome( status="skipped", @@ -1326,7 +1550,12 @@ def kb_amend_recipe( "pitfalls": appended(live.get("pitfalls"), append_pitfall), "lessons": appended(live.get("lessons"), append_lesson), "extras": amended_recipe_extras(live, overrides, state), - "provenance": amend_provenance(provenance_details), + "provenance": { + "source": "hyperloom-inference-optimizer", + "generator": "coordinator", + "generated_at": datetime.now(timezone.utc).isoformat(timespec="microseconds"), + "details": dict(provenance_details or {}), + }, **amended_recipe_fields(live, overrides), } self.recipe_kb.put_recipe(**put_kwargs) @@ -1334,3 +1563,297 @@ def kb_amend_recipe( __all__ = ["RecipeJournalCollaborator"] + + +#: The probe's own word for a version it could not read. +_UNIDENTIFIED_VERSION: str = "unknown" + + +class AttemptProvenance(NamedTuple): + """Where a session's proven optimizations came from, and what it reverted. + + Keepers are indexed twice because a stack layer may identify itself either + by name or by the research gap it closed. + """ + + kept_by_name: dict[str, str] + kept_by_gap: dict[str, str] + reverted: list[dict[str, Any]] + + +def _known_version(raw: Any) -> str: + version = str(raw or "").strip() + return "" if version == _UNIDENTIFIED_VERSION else version + + +def _runtime_version_tags(fingerprint_meta: Mapping[str, Any], framework: str) -> dict[str, str]: + tags: dict[str, str] = {} + framework_key = framework.lower() + if framework_key in {"sglang", "vllm"}: + active = _known_version(fingerprint_meta.get(framework_key)) + if active: + tags["framework_version"] = active + for source_key, tag in (("rocm", "rocm_version"), ("aiter", "aiter_version"), ("image_digest", "image_digest")): + version = _known_version(fingerprint_meta.get(source_key)) + if version: + tags[tag] = version + return tags + + +def _baseline_workload_tags(workload_extra: Mapping[str, Any]) -> dict[str, Any]: + tags: dict[str, Any] = {} + for key in ("max_running_requests", "max_num_seqs"): + value = workload_extra.get(key) + if isinstance(value, int) and value > 0: + tags[key] = value + for key in ("chunked_prefill_enabled", "enable_torch_compile"): + value = workload_extra.get(key) + if isinstance(value, bool): + tags[key] = value + for key in ("quant_scheme", "workload_mode"): + value = workload_extra.get(key) + if isinstance(value, str) and value.strip(): + tags[key] = value.strip() + return tags + + +def workload_kb_tags(state: Any) -> dict[str, Any]: + """The workload-shape tags describing the session this recipe came from.""" + tags: dict[str, Any] = {} + # ``model_family`` is deliberately absent, being no part of the + # seven-dimension Recipe identity. + for attr in ("framework", "model_class", "model_name"): + text = str(getattr(state, attr, "") or "").strip() + if text: + tags[attr] = text + precision = getattr(state, "precision", None) + if precision not in (None, "", 0): + tags["precision"] = precision + tags.update(workload_shape_tags(state)) + fingerprint_meta = state.stack_fingerprint_meta or {} + if isinstance(fingerprint_meta, dict): + tags.update(_runtime_version_tags(fingerprint_meta, str(tags.get("framework") or ""))) + workload_extra = state.baseline_workload_extra or {} + if isinstance(workload_extra, dict): + tags.update(_baseline_workload_tags(workload_extra)) + return tags + + +def _index_integrate_attempts(attempts: Mapping[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: + by_kernel: dict[str, dict[str, Any]] = {} + by_task_group: dict[str, dict[str, Any]] = {} + for entry in attempts.values(): + if not isinstance(entry, dict): + continue + kernel_id = str(entry.get("kernel_id") or "") + if kernel_id: + by_kernel[kernel_id] = entry + task_group_key = str(entry.get("task_group_key") or "") + if task_group_key: + by_task_group[task_group_key] = entry + return by_kernel, by_task_group + + +def _last_rebenched_throughput(integration: Mapping[str, Any]) -> float: + for attempt in reversed(list(integration.get("attempts") or [])): + if isinstance(attempt, dict) and attempt.get("new_tput") is not None: + return recorded_float(attempt.get("new_tput")) + return 0.0 + + +def _kernel_optimization_row( + entry: Mapping[str, Any], + ledger_id: str, + by_kernel: Mapping[str, Any], + by_task_group: Mapping[str, Any], +) -> dict[str, Any]: + kernel_id = str(entry.get("current_kernel_id") or entry.get("kernel_id") or ledger_id) + task_group_key = str(entry.get("task_group_key") or "") + integration = by_task_group.get(task_group_key) if task_group_key else by_kernel.get(kernel_id) + integrated = isinstance(integration, dict) + return { + "kernel_id": kernel_id, + "source_file": str(entry.get("last_source_file") or ""), + "artifact_path": str(entry.get("last_artifact_path") or ""), + "micro_speedup": recorded_float(entry.get("last_micro_speedup")), + "decision": OUTCOME_KEEP, + "e2e_gain_pct": recorded_float(integration.get("best_gain_pct")) if integrated else 0.0, + "e2e_tput": _last_rebenched_throughput(integration) if integrated else 0.0, + "e2e_decision": str(integration.get("last_decision") or "").upper() if integrated else "", + "integrated": integrated, + "ts": str(entry.get("last_ts") or ""), + } + + +def kept_kernel_optimizations(state: Any) -> list[dict[str, Any]]: + """The KEEP'd kernel optimizations, each joined to its end-to-end integrate verdict.""" + optimizations = state.kernel_opt_task_attempts or {} + if not isinstance(optimizations, dict): + return [] + integrations = state.kernel_integrate_attempts or {} + by_kernel, by_task_group = _index_integrate_attempts(integrations if isinstance(integrations, dict) else {}) + rows: list[dict[str, Any]] = [] + for ledger_id, entry in optimizations.items(): + if not isinstance(entry, dict): + continue + if str(entry.get("last_decision", "")).upper() != OUTCOME_KEEP: + continue + rows.append(_kernel_optimization_row(entry, ledger_id, by_kernel, by_task_group)) + return rows + + +def _reverted_attempt_row(name: str, attempt: Mapping[str, Any], source: str) -> dict[str, Any]: + row: dict[str, Any] = { + "name": name, + "reason": "reverted", + "gain_pct": attempt.get("gain_pct"), + } + if source: + row["source"] = source + return row + + +def _gap_attempt_provenance(gap: Mapping[str, Any]) -> AttemptProvenance: + source = str(gap.get("provenance") or "").strip() + canonical_id = str(gap.get("canonical_id") or "").strip() + kept_by_name: dict[str, str] = {} + kept_by_gap: dict[str, str] = {} + reverted: list[dict[str, Any]] = [] + for attempt in gap.get("attempts") or []: + if not isinstance(attempt, dict): + continue + variant = str(attempt.get("variant_name") or "").strip() + kernel = str(attempt.get("kernel_id") or "").strip() + outcome = str(attempt.get("outcome") or "").strip().upper() + if outcome == OUTCOME_KEEP and source: + for name in (variant, kernel): + if name: + kept_by_name.setdefault(name, source) + if canonical_id: + kept_by_gap.setdefault(canonical_id, source) + elif outcome == OUTCOME_REVERT and (variant or kernel): + reverted.append(_reverted_attempt_row(variant or kernel, attempt, source)) + return AttemptProvenance(kept_by_name=kept_by_name, kept_by_gap=kept_by_gap, reverted=reverted) + + +def attempt_provenance(state: Any) -> AttemptProvenance: + """Trace each proven optimization back to the research hint it came from.""" + kept_by_name: dict[str, str] = {} + kept_by_gap: dict[str, str] = {} + reverted: list[dict[str, Any]] = [] + for gap in state.gaps or []: + if not isinstance(gap, dict): + continue + from_gap = _gap_attempt_provenance(gap) + for name, source in from_gap.kept_by_name.items(): + kept_by_name.setdefault(name, source) + for canonical_id, source in from_gap.kept_by_gap.items(): + kept_by_gap.setdefault(canonical_id, source) + reverted.extend(from_gap.reverted) + return AttemptProvenance(kept_by_name=kept_by_name, kept_by_gap=kept_by_gap, reverted=reverted) + + +def _best_config(current_best: Mapping[str, Any], optimization_stack: Sequence[Any]) -> dict[str, Any]: + config: dict[str, Any] = {} + current_args = current_best.get("extra_server_args") + if current_args: + config["extra_server_args"] = str(current_args) + for key in ("extra_envs", "name", "tput", "accuracy"): + if key in current_best: + config[key] = current_best[key] + last = optimization_stack[-1] if optimization_stack else None + stack_args = ( + str(last.get("candidate_extra_server_args") or last.get("extra_server_args") or "").strip() + if isinstance(last, dict) + else "" + ) + if stack_args: + config["extra_server_args"] = stack_args + return config + + +def _what_worked( + optimization_stack: Sequence[Any], + gain_per_entry: Sequence[Any], + provenance: AttemptProvenance, +) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for index, entry in enumerate(optimization_stack): + if not isinstance(entry, dict): + continue + name = str(entry.get("variant_name") or entry.get("name") or entry.get("kernel_id") or "") + row: dict[str, Any] = { + "name": name, + "extra_server_args": str(entry.get("extra_server_args") or ""), + "extra_envs": dict(entry.get("extra_envs") or {}), + "gain_pct": gain_per_entry[index] if index < len(gain_per_entry) else None, + } + # The layer's gap id wins; the variant or kernel name answers for + # layers recorded before gap ids were stamped. + gap_id = str(entry.get("gap_canonical_id") or "").strip() + by_name = provenance.kept_by_name + source = (provenance.kept_by_gap.get(gap_id) if gap_id else "") or by_name.get(name) + source = source or by_name.get(str(entry.get("kernel_id") or "")) or "" + if source: + row["source"] = source + rows.append(row) + return rows + + +def _what_failed(recent_failures: Sequence[Any], reverted: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for failure in recent_failures[-10:]: + if isinstance(failure, dict): + rows.append( + { + "name": str(failure.get("name") or failure.get("action") or ""), + "reason": str(failure.get("reason") or failure.get("error_class") or ""), + } + ) + rows.extend(reverted) + return rows + + +def _session_row( + state: Any, + session_name: str, + optimization_stack: Collection[Any], + final_throughput: float, +) -> dict[str, Any]: + validated_stack_len = int(state.cumulative_gain_validated_stack_len or 0) + actions = [ + str(entry.get("variant_name") or entry.get("name") or entry.get("action") or "").strip() + for entry in optimization_stack + if isinstance(entry, dict) + ] + return { + "session_id": str(state.recipe_kb_session_id or session_name), + "gain_pct": float(state.cumulative_gain_validated or 0.0), + "stack_len": validated_stack_len or len(optimization_stack), + "throughput_before": float(state.baseline_tput or 0.0), + "throughput_after": final_throughput, + "date": datetime.now(timezone.utc).isoformat(), + "actions_taken": [name for name in actions if name], + } + + +def recipe_attrs_from_state(state: Any, session_name: str) -> dict[str, Any]: + """Materialise the recipe-shaped view of one session's settled state. + + ``framework_version`` is left unset here because it is manifest-derived; the + T0 backfill writes it. + """ + optimization_stack = state.optimization_stack or [] + # A session that does not sediment its attempts reports no provenance. + provenance = attempt_provenance(state) if state.recipe_sediment_enabled else AttemptProvenance({}, {}, []) + final_throughput = float(state.current_best.get("tput", 0.0)) + return { + "best_config": _best_config(state.current_best, optimization_stack), + "best_throughput": final_throughput, + "what_worked": _what_worked(optimization_stack, state.gain_per_stack_entry or [], provenance), + "what_failed": _what_failed(state.last_action_failures or [], provenance.reverted), + "kernel_optimizations": kept_kernel_optimizations(state), + "last_profiled": str(state.cumulative_gain_validated_ts or ""), + "workload": workload_kb_tags(state), + "sessions": [_session_row(state, session_name, optimization_stack, final_throughput)], + } From e37c3998de79477e6e609b76324f1122f62117f7 Mon Sep 17 00:00:00 2001 From: Test Date: Wed, 21 Oct 2026 17:18:00 +0800 Subject: [PATCH 11/14] refactor(recipe-journal): bring _record_fact_per_variant back under the length gate Its docstring restated each parameter name; only the no_promote rule said something the signature could not. Co-Authored-By: Claude Opus 5 --- .../orchestrator/knowledge/recipe_journal.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/hyperloom/orchestrator/knowledge/recipe_journal.py b/src/hyperloom/orchestrator/knowledge/recipe_journal.py index 47b07ba141..555f8f9b1e 100644 --- a/src/hyperloom/orchestrator/knowledge/recipe_journal.py +++ b/src/hyperloom/orchestrator/knowledge/recipe_journal.py @@ -876,10 +876,8 @@ def _record_fact_per_task( ), ) ) - if self.recipe_kb is None: return - self._record_fact_impl( FactRecord( task=task, @@ -964,14 +962,7 @@ def _record_fact_per_variant( ) -> None: """Per-variant fact write — mirror of _record_fact_per_task for explore per-variant decisions. - Args: - task: The completed explore task. - source_session_id: The hyperloom-local session id stamped on the - fact provenance. - variant_outcome: One per-variant outcome row (name, outcome, - metrics). - adopted: Whether this variant's lift landed; an executor KEEP that - was not adopted journals as ``no_promote``. + An executor KEEP that was not adopted journals as ``no_promote``. """ journal = self.ensure_journal() outcome_raw = str(variant_outcome.get("outcome") or "") From 81ca18e006e0714aa400052d3fd5dda6e713f9d1 Mon Sep 17 00:00:00 2001 From: Test Date: Thu, 22 Oct 2026 17:22:00 +0800 Subject: [PATCH 12/14] refactor(knowledge): delete what recipe_journal derived, coerced and documented twice The module carried a second copy of several things it already had. The framework / version / precision identity was derived at two sites with the same fallback chain around each; `_extra_server_args` re-implemented `common.coerce.to_str_list`'s whole None / str / list / scalar ladder; four figures were parsed through their own `try: float(x or 0.0)` blocks, and the `gain_pct` / `delta_pct` fallback was an open-coded `first_float`. Deleted with them: the `isinstance` guards whose producers are all typed (`lever_kind_for_task`'s two dict arguments, the variant metrics checked twice, and the tuple shape of a cache this class is the sole writer of), and eleven docstring `Args:` / `Returns:` blocks that restated the parameter names and the body back to the reader. What those docstrings said that the code cannot is kept. The module goes from 1671 lines to 1654 and its baseline entry comes down with it; production lines across both changed files are -4 against main. No observable field, value or log line changes. Co-Authored-By: Claude Opus 5 --- scripts/code_metrics_baseline.txt | 2 +- .../orchestrator/knowledge/recipe_journal.py | 211 ++++-------------- 2 files changed, 46 insertions(+), 167 deletions(-) diff --git a/scripts/code_metrics_baseline.txt b/scripts/code_metrics_baseline.txt index 236c32639d..0de74d0b8d 100644 --- a/scripts/code_metrics_baseline.txt +++ b/scripts/code_metrics_baseline.txt @@ -501,7 +501,7 @@ module-lines src/hyperloom/orchestrator/actions/executors/roofline.py 1303 module-lines src/hyperloom/orchestrator/kernel/apply_kernel_patch.py 2656 module-lines src/hyperloom/orchestrator/kernel/conc_sweep.py 1429 module-lines src/hyperloom/orchestrator/kernel/request_handlers.py 4692 -module-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py 1336 +module-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py 1654 module-lines src/hyperloom/orchestrator/loop/coordinator.py 1337 module-lines src/hyperloom/orchestrator/loop/dispatcher.py 1454 module-lines src/hyperloom/orchestrator/loop/intent_router.py 1409 diff --git a/src/hyperloom/orchestrator/knowledge/recipe_journal.py b/src/hyperloom/orchestrator/knowledge/recipe_journal.py index 70ccbabc17..01e385c6d1 100644 --- a/src/hyperloom/orchestrator/knowledge/recipe_journal.py +++ b/src/hyperloom/orchestrator/knowledge/recipe_journal.py @@ -12,7 +12,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Collection, Mapping, NamedTuple, Sequence -from hyperloom.common.coerce import recorded_float, to_float, to_int +from hyperloom.common.coerce import first_float, recorded_float, to_float, to_int, to_mapping, to_str_list from hyperloom.common.io import JsonLineFormat, append_jsonl from hyperloom.inference_optimizer.breakdown.recorder import close_out as _close_out from hyperloom.inference_optimizer.breakdown.recorder.framework_event import ( @@ -130,22 +130,9 @@ class FactRecord: """One settled outcome to write to the Recipe KB as a lesson or a pitfall. - Attributes: - task: The completed task (provides task_id). - source_session_id: Hyperloom-local session id stamped on provenance. - is_keep: True when the outcome is a validated KEEP. - change: Summarized change string (used in the statement). - gain_pct: Measured gain percentage, or ``None``. - throughput_after: Measured throughput after the change, or ``None``. - best_config_candidate: Pre-extracted best-config dict (differs between - per-task and per-variant callers). - evidence_refs: Evidence reference strings stamped on the provenance - (task-only or task+variant refs). - pitfall_severity_dict: The dict passed to ``_pitfall_severity_for`` - (per-task passes the result dict; per-variant passes a merged - metrics + outcome dict). - variant_name: Variant name, present only for per-variant calls; added to - ``provenance_details`` when non-None. + ``pitfall_severity_dict`` is the result dict per-task, and the variant's merged + metrics + outcome per-variant. ``variant_name`` is set only by the per-variant + caller, and reaching ``provenance_details`` is what it is for. """ task: "Task" @@ -185,11 +172,6 @@ class _VariantMeasurement(NamedTuple): detail: dict[str, Any] -class _VariantFailure(NamedTuple): - error_class: str | None - reason: str | None - - def _remote_result_type(status: str, reason: str) -> str: """The ``close_out.RESULT_*`` code for a remote KB Store write result. @@ -208,11 +190,10 @@ def _remote_result_type(status: str, reason: str) -> str: def _predicted_gain(*sources: dict[str, Any] | None) -> float | None: - """First non-zero ``predicted_gain_pct`` (``to_float``-parsed) across ordered sources. + """First non-zero ``predicted_gain_pct`` across ordered sources. - Sources are checked in order; a non-zero prediction wins. Returns ``None`` - when none carry a usable value so the journal row stays ``predicted``-free - for unpredicted (default-grid) changes rather than recording a fake 0. + ``None`` when none carry one, so the journal row for an unpredicted + (default-grid) change stays ``predicted``-free rather than recording a fake 0. """ for src in sources: if not isinstance(src, dict): @@ -224,15 +205,7 @@ def _predicted_gain(*sources: dict[str, Any] | None) -> float | None: def _extra_server_args(payload: Mapping[str, Any]) -> str: - """Read canonical ``extra_server_args`` from a payload.""" - value = payload.get("extra_server_args") - if value is None: - return "" - if isinstance(value, str): - return value - if isinstance(value, (list, tuple)): - return " ".join(str(v).strip() for v in value if str(v).strip()) - return str(value) + return " ".join(to_str_list(payload.get("extra_server_args"))) def _record_config_run(coord: Any, *, task: Any, result_dict: Mapping[str, Any]) -> None: @@ -521,9 +494,7 @@ def _variant_verdict(outcome: str, adopted: bool) -> Verdict: return Verdict.REFUSED -def _variant_measurement(metrics: Any) -> _VariantMeasurement: - if not isinstance(metrics, dict): - return _VariantMeasurement(gain_pct=None, throughput_after=None, detail={}) +def _variant_measurement(metrics: Mapping[str, Any]) -> _VariantMeasurement: return _VariantMeasurement( gain_pct=to_float(metrics.get("gain_pct")), throughput_after=to_float(metrics.get("output_throughput")), @@ -531,15 +502,6 @@ def _variant_measurement(metrics: Any) -> _VariantMeasurement: ) -def _variant_failure(variant_outcome: Mapping[str, Any], outcome: str) -> _VariantFailure: - if outcome != OUTCOME_REVERT: - return _VariantFailure(error_class=None, reason=None) - return _VariantFailure( - error_class=str(variant_outcome.get("error_class") or "") or None, - reason=str(variant_outcome.get("reason") or "") or None, - ) - - def _variant_change_attrs(variant_attrs: Any, variant_name: str) -> dict[str, Any]: attrs = dict(variant_attrs) if isinstance(variant_attrs, dict) else {} named_knob = attrs.get("extra_server_args") or attrs.get("extra_envs") or attrs.get("name") @@ -549,22 +511,12 @@ def _variant_change_attrs(variant_attrs: Any, variant_name: str) -> dict[str, An def _finalize_outcome(*, status: str, reason: str, backend: str, result_type: str) -> dict[str, Any]: - return { - "status": status, - "reason": reason, - "backend": backend, - "result_type": result_type, - } + return {"status": status, "reason": reason, "backend": backend, "result_type": result_type} def _finalize_error(exc: BaseException, *, reason: str, backend: str, result_type: str) -> dict[str, Any]: - return { - "status": "error", - "reason": reason, - "backend": backend, - "result_type": result_type, - "error_class": type(exc).__name__, - } + failed = _finalize_outcome(status="error", reason=reason, backend=backend, result_type=result_type) + return {**failed, "error_class": type(exc).__name__} class RecipeJournalCollaborator(CoordinatorCollaborator): @@ -597,7 +549,7 @@ def journal_integrate_keep(self, result: dict[str, Any], *, lift_kind: str, new_ journal.append_entry( JournalEntry( phase=self.journal_entry_phase(), - lever_kind=lever_kind_for_task(lift_kind, result if isinstance(result, dict) else None), + lever_kind=lever_kind_for_task(lift_kind, result), iter=int(self.shared_state.tick or 0), kind=kind, change=change, @@ -614,14 +566,7 @@ def journal_integrate_keep(self, result: dict[str, Any], *, lift_kind: str, new_ log.exception("integrate journal append failed") def _source_session_id(self) -> str: - """Return the hyperloom-local session id used as source_session_id on KB fact writes. - - NOT a KB-side session id; prefers recipe_kb_session_id, falls back to session_dir.name. - - Returns: - The hyperloom-local session id (recipe_kb_session_id when set, else - ``session_dir.name``). - """ + """The hyperloom-local session id stamped on KB fact writes, never a KB-side one.""" return str(self.shared_state.recipe_kb_session_id or "") or self.session_dir.name async def fact_write_hook( @@ -634,12 +579,8 @@ async def fact_write_hook( ) -> None: """Per-task fact-write entry point (per_variant for explore grids, else per-task). - Args: - task: The completed task being recorded. - result: The task's :class:`SubAgentResult` (or result dict). - verdict: The settlement verdict. - adopted_variants: Fingerprints of the explore winners whose lift - landed; a per-variant KEEP outside it did not adopt anything. + ``adopted_variants`` holds the fingerprints of the explore winners whose lift + landed; a per-variant KEEP outside it did not adopt anything. """ result_dict = _result_payload(result) source_session_id = self._source_session_id() @@ -674,12 +615,7 @@ async def fact_write_hook( self.shared_state.save(self.session_dir) def ensure_journal(self) -> Journal: - """Lazy-instantiate the per-session :class:`Journal` (load_or_create reads an existing file on resume). - - Returns: - The per-session :class:`Journal` instance (created on first call, - with the baseline backfilled on subsequent calls). - """ + """Lazy-instantiate the per-session :class:`Journal` (load_or_create reads an existing file on resume).""" existing = self._journal if existing is None: ss = self.shared_state @@ -700,22 +636,8 @@ def _pitfall_severity_for( self, result_dict: dict[str, Any] | None, ) -> str | None: - """Decide whether a failed result warrants a pitfall row. - - ``crash`` / ``oom`` / ``hang`` / ``detokenizer_stall`` on ``error_class``, - or ``crash`` / ``oom`` / ``hang`` on ``status``, yield - ``SEVERITY_CRASH``; a ``gain_pct`` at or below - ``PITFALL_REGRESS_THRESHOLD_PCT`` (-5.0) yields ``SEVERITY_REGRESS``; - otherwise ``None``. - - Args: - result_dict: The failed task's result dict; non-dict yields ``None``. - - Returns: - The pitfall severity (``SEVERITY_CRASH`` / ``SEVERITY_REGRESS``), or - ``None`` when no pitfall is warranted. - """ - if not isinstance(result_dict, dict): + """The severity of the pitfall a failed result warrants, or ``None`` for no pitfall.""" + if not result_dict: return None error_class = str(result_dict.get("error_class") or "").lower() # ``detokenizer_stall`` is a hang in all but name; record it as a @@ -725,34 +647,20 @@ def _pitfall_severity_for( status = str(result_dict.get("status") or "").lower() if status in ("crash", "oom", "hang"): return _SEVERITY_CRASH - gain = result_dict.get("gain_pct") - try: - gain_pct = float(gain) if gain is not None else None - except (TypeError, ValueError): - gain_pct = None + gain_pct = to_float(result_dict.get("gain_pct")) if gain_pct is not None and gain_pct <= self.PITFALL_REGRESS_THRESHOLD_PCT: return _SEVERITY_REGRESS return None def journal_entry_phase(self) -> str: - """Return the current phase label for journal entries. - - Returns: - str: The uppercased phase name, or ``"UNKNOWN"`` when unset. - """ + """The uppercased phase name journal entries are stamped with, ``"UNKNOWN"`` when unset.""" return str(self.shared_state.phase or "").strip().upper() or "UNKNOWN" def _record_fact_impl(self, fact: FactRecord) -> None: - """Shared KB write for _record_fact_per_task and _record_fact_per_variant. + """Write one KB lesson (KEEP with positive gain) or one KB pitfall (REVERT/failure). - Writes one KB lesson (on KEEP with positive gain) or one KB pitfall - (on REVERT/failure) to the recipe row, then returns. Call only after - the journal entry has been appended and ``recipe_kb`` is confirmed - non-None by the caller. - - Args: - fact: The completed task, its outcome and the evidence to stamp on - the provenance. + Call only after the journal entry has been appended and ``recipe_kb`` is + confirmed non-None by the caller. """ state = self.shared_state now_iso = datetime.now(timezone.utc).isoformat(timespec="seconds") @@ -820,23 +728,16 @@ def _record_fact_per_task( result_dict: dict[str, Any], verdict: Verdict, ) -> None: - """Per-task fact write — one journal row + maybe one KB fact (source_session_id is hyperloom-local). - - Args: - task: The completed task being recorded. - source_session_id: The hyperloom-local session id stamped on the - fact provenance. - result_dict: The task result dict. - verdict: The settlement verdict (ADOPTED → KEEP and a lesson, else - a non-KEEP row and maybe a pitfall). + """Per-task fact write — one journal row + maybe one KB fact. + + ``ADOPTED`` journals a KEEP and may write a lesson; any other verdict + journals a non-KEEP row and may write a pitfall. """ journal = self.ensure_journal() # integrate_patch reports its delta under ``delta_pct``; # fall back to it so a reverted/kept patch shows its REAL measured delta # in the journal instead of a null gain. - gain_pct = to_float(result_dict.get("gain_pct")) - if gain_pct is None: - gain_pct = to_float(result_dict.get("delta_pct")) + gain_pct = first_float(result_dict.get("gain_pct"), result_dict.get("delta_pct")) throughput_after = to_float(result_dict.get("output_throughput")) kind = classify_change_kind(task.kind, None) change = summarize_change(task.kind, None, result_dict) @@ -853,7 +754,7 @@ def _record_fact_per_task( journal.append_entry( JournalEntry( phase=self.journal_entry_phase(), - lever_kind=lever_kind_for_task(kind, result_dict if isinstance(result_dict, dict) else None), + lever_kind=lever_kind_for_task(kind, result_dict), iter=int(self.shared_state.tick or 0), kind=kind, change=change, @@ -897,16 +798,10 @@ def _build_statement( kind: str, severity: str | None = None, ) -> str: - """Build the lesson statement / pitfall description hashed into the KB canonical_id; MUST exclude volatile fields (e.g. gain_pct) so N sessions merge instead of producing N rows. Identity = framework + change + model/hw. - - Args: - change: The summarized change description. - kind: ``"lesson"`` or ``"pitfall"`` — selects the rendered form. - severity: The pitfall severity, rendered only when ``kind`` is - ``"pitfall"``. + """Build the lesson statement / pitfall description hashed into the KB canonical_id. - Returns: - The identity-stable statement / description string. + Identity is framework + change + model/hw. It MUST exclude volatile fields + (``gain_pct`` and the like) so N sessions merge instead of producing N rows. """ framework = str(self.shared_state.framework or "").strip() fw_tag = f"[{framework or '?'}] " @@ -925,16 +820,9 @@ def _build_measured_impact( stack_depth: int, measured_at: str, ) -> dict[str, Any]: - """Structured ``measured_impact`` payload (dict not legacy string so consumers parse without regex); stack_depth = stack length before this lesson lands. + """Structured ``measured_impact`` payload, a dict rather than a legacy string so consumers parse without regex. - Args: - gain_pct: The measured gain percent, or ``None``. - throughput_after: Throughput after the change, or ``None``. - stack_depth: Optimization-stack length before this lesson lands. - measured_at: ISO timestamp of the measurement. - - Returns: - A compact ``measured_impact`` dict with ``None`` fields stripped. + ``stack_depth`` is the optimization-stack length before this lesson lands. """ out: dict[str, Any] = { "gain_pct": float(gain_pct) if gain_pct is not None else None, @@ -964,14 +852,15 @@ def _record_fact_per_variant( return outcome = derive_journal_outcome(_variant_verdict(outcome_raw, adopted), variant_outcome) variant_name = str(variant_outcome.get("variant_name") or "") - metrics = variant_outcome.get("metrics") or {} + metrics = to_mapping(variant_outcome.get("metrics")) measurement = _variant_measurement(metrics) variant_attrs = variant_outcome.get("variant") or {} attrs = variant_attrs if isinstance(variant_attrs, dict) else None kind = classify_change_kind(task.kind, attrs) change_attrs = _variant_change_attrs(variant_attrs, variant_name) change = summarize_change(task.kind, change_attrs, None) - failure = _variant_failure(variant_outcome, outcome) + # Only a measured loss carries a failure; a refusal has nothing to report. + reverted = outcome == OUTCOME_REVERT tick = int(self.shared_state.tick or 0) journal.append_entry( JournalEntry( @@ -983,8 +872,8 @@ def _record_fact_per_variant( outcome=outcome, gain_pct=measurement.gain_pct, throughput_after=measurement.throughput_after, - error_class=failure.error_class, - reason=failure.reason, + error_class=(str(variant_outcome.get("error_class") or "") or None) if reverted else None, + reason=(str(variant_outcome.get("reason") or "") or None) if reverted else None, task_id=task.task_id, variant_name=variant_name, provenance=str(variant_outcome.get("provenance") or ""), @@ -1017,7 +906,7 @@ def _record_fact_per_variant( # Workload-shape tags — see _record_fact_per_task. evidence_refs=[f"log:task-{task.task_id}", f"variant:{variant_name}"], pitfall_severity_dict={ - **(metrics if isinstance(metrics, dict) else {}), + **metrics, "error_class": variant_outcome.get("error_class"), "status": variant_outcome.get("outcome"), }, @@ -1173,10 +1062,7 @@ def _record_write_back_settled( ) def _write_back_scope(self) -> dict[str, Any]: - """The workload dimensions the published recipe is partitioned by. - - Matches the ``RecipeScope`` shape the section already published. - """ + """The workload dimensions the published recipe is partitioned by, in ``RecipeScope`` shape.""" state = self.shared_state return { "kernel_optimizer": str(state.kernel_optimizer or ""), @@ -1421,7 +1307,7 @@ def read_local_recipe_row(self) -> dict[str, Any]: return {} tick = int(self.shared_state.tick or 0) cache = self._local_recipe_cache - if isinstance(cache, tuple) and len(cache) == 2 and cache[0] == tick: + if cache is not None and cache[0] == tick: return cache[1] row = self.recipe_kb.get_authoritative_recipe(canonical_id=self.workload_canonical_id()) or {} self._local_recipe_cache = (tick, row) @@ -1475,15 +1361,8 @@ def kb_best_config_overrides_for_keep( _extra_server_args(live_bc).strip() or (isinstance(live_bc.get("extra_envs"), Mapping) and live_bc.get("extra_envs")) ) - try: - live_tput = float(live.get("best_throughput") or 0.0) - except (TypeError, ValueError): - live_tput = 0.0 - try: - new_tput = float(throughput_after or 0.0) - except (TypeError, ValueError): - new_tput = 0.0 - + live_tput = recorded_float(live.get("best_throughput")) + new_tput = recorded_float(throughput_after) if not live_has_config or (new_tput > 0.0 and new_tput >= live_tput): overrides: dict[str, Any] = { "best_config": dict(best_config_candidate), From cd6db858d3b6f7be4299b7ca9962c636f0fe619b Mon Sep 17 00:00:00 2001 From: Test Date: Thu, 22 Oct 2026 17:48:06 +0800 Subject: [PATCH 13/14] refactor(knowledge): give the config timeline and the recipe attributes their own modules recipe_journal.py carried the FRAMEWORK_AGENT configuration-arm attempt rows and the session's recipe attributes alongside the journal itself. Each is a complete job with one entry point, so each becomes its own module and the journal keeps only what it owns. _NON_ATTEMPT_OUTCOMES travels with the attempt rows that read it, so the dependency points one way. Co-Authored-By: Claude Opus 5 --- .../knowledge/_config_timeline.py | 169 ++++++++ .../orchestrator/knowledge/_recipe_attrs.py | 242 +++++++++++ .../orchestrator/knowledge/recipe_journal.py | 393 +----------------- 3 files changed, 415 insertions(+), 389 deletions(-) create mode 100644 src/hyperloom/orchestrator/knowledge/_config_timeline.py create mode 100644 src/hyperloom/orchestrator/knowledge/_recipe_attrs.py diff --git a/src/hyperloom/orchestrator/knowledge/_config_timeline.py b/src/hyperloom/orchestrator/knowledge/_config_timeline.py new file mode 100644 index 0000000000..6fb6f6564c --- /dev/null +++ b/src/hyperloom/orchestrator/knowledge/_config_timeline.py @@ -0,0 +1,169 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""The configuration arm's attempt rows on the FRAMEWORK_AGENT event.""" + +from __future__ import annotations + +from typing import Any, Collection, Mapping, Sequence + +from hyperloom.inference_optimizer.breakdown.recorder.framework_event import ( + ARM_CONFIG, + DISPOSITION_ATTEMPTED, + STEP_ATTEMPTED, + GateVerdict, +) + +#: Explore outcomes that are not an attempt at anything: a variant skipped as a +#: duplicate was never measured, so no measurement would back its funnel row. +_NON_ATTEMPT_OUTCOMES: frozenset[str] = frozenset({"SKIPPED_DEDUP"}) + + +def _attempt_failure(row: Mapping[str, Any]) -> dict[str, Any]: + return { + "error_class": str(row.get("error_class") or ""), + "error_excerpt": str(row.get("error_excerpt") or ""), + # Explore classified this variant's failure once; the verdict it stored is the one verdict. + "attribution": str(row.get("failure_attribution") or ""), + } + + +def _attempt_fields( + row: Mapping[str, Any], + gates: Sequence[Mapping[str, Any]], + shared: Mapping[str, str], + adopted: bool, +) -> dict[str, Any]: + variant = row.get("variant") if isinstance(row.get("variant"), dict) else {} + metrics = row.get("metrics") if isinstance(row.get("metrics"), dict) else {} + outcome = str(row.get("outcome") or "") + # This arm gates accuracy rather than reporting it, so the block both arms + # carry is projected from the gate that ruled. No gate row means nothing + # gated the variant, which is not a gate that refused it. + ruling = next((gate for gate in gates if str(gate.get("gate") or "") == "accuracy"), {}) + return { + "arm": ARM_CONFIG, + **shared, + "provenance": row.get("provenance"), + "outcome": outcome, + "reason": row.get("reason"), + "reasoning": variant.get("note"), + "reasoning_origin": variant.get("reasoning_origin"), + "experience_citations": variant.get("experience_citations"), + "stage": row.get("stage"), + "fingerprint": row.get("fingerprint"), + # The fingerprint is the join key; the name is what a reader + # recognises the variant by. + "variant_name": row.get("variant_name"), + "measurement": { + # Both ends of the pair: the anchor advances on every KEEP, so a + # percentage without its denominator adds to nothing. + "before_tput": metrics.get("base_tput"), + "after_tput": metrics.get("tput"), + "gain_pct": metrics.get("gain_pct"), + "runtime_sec": metrics.get("runtime_sec"), + "estimated_output_throughput": metrics.get("estimated_output_throughput"), + }, + "config_delta": { + "extra_server_args": variant.get("extra_server_args"), + "extra_envs": variant.get("extra_envs"), + "remove_args": variant.get("remove_args"), + "unset_envs": variant.get("unset_envs"), + "args_mode": variant.get("args_mode"), + }, + "accuracy": { + "required": True if ruling else None, + "value": ruling.get("observed"), + "reference": ruling.get("threshold"), + "passed": ruling.get("passed"), + }, + "failure": _attempt_failure(row), + "artifacts": { + "workspace": row.get("workspace"), + "server_log_path": row.get("server_log_path"), + "raw_result_path": row.get("raw_result_path"), + }, + "decision": outcome, + "adopted": adopted, + # Every variant recorded here ran: the deduped one is skipped above. + "benched": True, + # Recorded rather than referenced: every KEEP advances the stack, so + # the session's current config is not what this variant was measured + # on top of. + "measured_against": row.get("measured_against"), + # What stood behind the verdict. Absent when nothing ruled. + "validation_basis": row.get("validation_basis"), + # A pair is what makes a gain addable, so eligibility follows the pair + # being present rather than the outcome being a KEEP. + "attribution_eligible": (adopted and metrics.get("base_tput") is not None and metrics.get("tput") is not None), + } + + +def _record_variant_attempt( + recorder: Any, + row: Any, + shared: Mapping[str, str], + adopted_variants: Collection[str], +) -> bool: + if not isinstance(row, dict): + return False + if str(row.get("outcome") or "") in _NON_ATTEMPT_OUTCOMES: + return False + # The fingerprint identifies the variant within the round and the round + # within the task, so the three together identify the attempt. + fingerprint = str(row.get("fingerprint") or "") + parts = (shared["task_id"], shared["round_id"], fingerprint) + attempt_id = ":".join(part for part in parts if part) or shared["task_id"] + if not attempt_id: + return False + gates = [gate for gate in (row.get("gates") or []) if isinstance(gate, dict)] + recorder.record_attempt(attempt_id, **_attempt_fields(row, gates, shared, fingerprint in adopted_variants)) + for gate in gates: + recorder.record_attempt_gate( + attempt_id, + GateVerdict( + gate=str(gate.get("gate") or ""), + passed=gate.get("passed"), + reason=str(gate.get("reason") or ""), + observed=gate.get("observed"), + threshold=gate.get("threshold"), + ), + ) + return True + + +def _record_config_attempts( + coord: Any, + *, + task: Any, + per_variant: list[dict[str, Any]], + result_dict: Mapping[str, Any], + adopted_variants: Collection[str], +) -> None: + """Record the configuration arm's measured attempts on the framework event. + + Timeline only: the ledger row is written by ``fact_write_hook``, which does + not depend on a recorder being open. The outcome is recorded verbatim here, + unlike the journal beside it, which collapses ``KEEP_UNSTABLE`` and + ``KILLED_OVERTIME`` into a plain revert. + """ + recorder = coord._coord.phase_framework.timeline() + if recorder is None: + return + params = getattr(task, "params", None) or {} + shared = { + "task_id": str(getattr(task, "task_id", "") or ""), + "round_id": str(result_dict.get("round_id") or ""), + "proposal_ref": str(params.get("proposal_msg_id") or ""), + } + recorded = 0 + for row in per_variant: + if _record_variant_attempt(recorder, row, shared, adopted_variants): + recorded += 1 + if not (recorded and shared["proposal_ref"]): + return + # Settled here rather than at materialization: a grid whose task dies before + # any variant runs has no attempt to stand behind an ``attempted`` reading, + # and is more honestly left unsettled. + recorder.record_proposal_step(shared["proposal_ref"], step=STEP_ATTEMPTED, outcome=str(recorded)) + recorder.settle_proposal(shared["proposal_ref"], disposition=DISPOSITION_ATTEMPTED) diff --git a/src/hyperloom/orchestrator/knowledge/_recipe_attrs.py b/src/hyperloom/orchestrator/knowledge/_recipe_attrs.py new file mode 100644 index 0000000000..18171e3e84 --- /dev/null +++ b/src/hyperloom/orchestrator/knowledge/_recipe_attrs.py @@ -0,0 +1,242 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""What a session's recipe attributes say about where its optimizations came from.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Collection, Mapping, NamedTuple, Sequence + +from hyperloom.inference_optimizer.session.optimization_journal import OUTCOME_KEEP, OUTCOME_REVERT + +from .workload_tags import workload_shape_tags + +#: The probe's own word for a version it could not read. +_UNIDENTIFIED_VERSION: str = "unknown" + + +class AttemptProvenance(NamedTuple): + """Where a session's proven optimizations came from, and what it reverted. + + Keepers are indexed twice because a stack layer may identify itself either + by name or by the research gap it closed. + """ + + kept_by_name: dict[str, str] + kept_by_gap: dict[str, str] + reverted: list[dict[str, Any]] + + +def _known_version(raw: Any) -> str: + version = str(raw or "").strip() + return "" if version == _UNIDENTIFIED_VERSION else version + + +def _runtime_version_tags(fingerprint_meta: Mapping[str, Any], framework: str) -> dict[str, str]: + tags: dict[str, str] = {} + framework_key = framework.lower() + if framework_key in {"sglang", "vllm"}: + active = _known_version(fingerprint_meta.get(framework_key)) + if active: + tags["framework_version"] = active + for source_key, tag in (("rocm", "rocm_version"), ("aiter", "aiter_version"), ("image_digest", "image_digest")): + version = _known_version(fingerprint_meta.get(source_key)) + if version: + tags[tag] = version + return tags + + +def _baseline_workload_tags(workload_extra: Mapping[str, Any]) -> dict[str, Any]: + tags: dict[str, Any] = {} + for key in ("max_running_requests", "max_num_seqs"): + value = workload_extra.get(key) + if isinstance(value, int) and value > 0: + tags[key] = value + for key in ("chunked_prefill_enabled", "enable_torch_compile"): + value = workload_extra.get(key) + if isinstance(value, bool): + tags[key] = value + for key in ("quant_scheme", "workload_mode"): + value = workload_extra.get(key) + if isinstance(value, str) and value.strip(): + tags[key] = value.strip() + return tags + + +def workload_kb_tags(state: Any) -> dict[str, Any]: + """The workload-shape tags describing the session this recipe came from.""" + tags: dict[str, Any] = {} + # ``model_family`` is deliberately absent, being no part of the + # seven-dimension Recipe identity. + for attr in ("framework", "model_class", "model_name"): + text = str(getattr(state, attr, "") or "").strip() + if text: + tags[attr] = text + precision = getattr(state, "precision", None) + if precision not in (None, "", 0): + tags["precision"] = precision + tags.update(workload_shape_tags(state)) + fingerprint_meta = state.stack_fingerprint_meta or {} + if isinstance(fingerprint_meta, dict): + tags.update(_runtime_version_tags(fingerprint_meta, str(tags.get("framework") or ""))) + workload_extra = state.baseline_workload_extra or {} + if isinstance(workload_extra, dict): + tags.update(_baseline_workload_tags(workload_extra)) + return tags + + +def _reverted_attempt_row(name: str, attempt: Mapping[str, Any], source: str) -> dict[str, Any]: + row: dict[str, Any] = { + "name": name, + "reason": "reverted", + "gain_pct": attempt.get("gain_pct"), + } + if source: + row["source"] = source + return row + + +def _gap_attempt_provenance(gap: Mapping[str, Any]) -> AttemptProvenance: + source = str(gap.get("provenance") or "").strip() + canonical_id = str(gap.get("canonical_id") or "").strip() + kept_by_name: dict[str, str] = {} + kept_by_gap: dict[str, str] = {} + reverted: list[dict[str, Any]] = [] + for attempt in gap.get("attempts") or []: + if not isinstance(attempt, dict): + continue + variant = str(attempt.get("variant_name") or "").strip() + kernel = str(attempt.get("kernel_id") or "").strip() + outcome = str(attempt.get("outcome") or "").strip().upper() + if outcome == OUTCOME_KEEP and source: + for name in (variant, kernel): + if name: + kept_by_name.setdefault(name, source) + if canonical_id: + kept_by_gap.setdefault(canonical_id, source) + elif outcome == OUTCOME_REVERT and (variant or kernel): + reverted.append(_reverted_attempt_row(variant or kernel, attempt, source)) + return AttemptProvenance(kept_by_name=kept_by_name, kept_by_gap=kept_by_gap, reverted=reverted) + + +def attempt_provenance(state: Any) -> AttemptProvenance: + """Trace each proven optimization back to the research hint it came from.""" + kept_by_name: dict[str, str] = {} + kept_by_gap: dict[str, str] = {} + reverted: list[dict[str, Any]] = [] + for gap in state.gaps or []: + if not isinstance(gap, dict): + continue + from_gap = _gap_attempt_provenance(gap) + for name, source in from_gap.kept_by_name.items(): + kept_by_name.setdefault(name, source) + for canonical_id, source in from_gap.kept_by_gap.items(): + kept_by_gap.setdefault(canonical_id, source) + reverted.extend(from_gap.reverted) + return AttemptProvenance(kept_by_name=kept_by_name, kept_by_gap=kept_by_gap, reverted=reverted) + + +def _best_config(current_best: Mapping[str, Any], optimization_stack: Sequence[Any]) -> dict[str, Any]: + config: dict[str, Any] = {} + current_args = current_best.get("extra_server_args") + if current_args: + config["extra_server_args"] = str(current_args) + for key in ("extra_envs", "name", "tput", "accuracy"): + if key in current_best: + config[key] = current_best[key] + last = optimization_stack[-1] if optimization_stack else None + stack_args = ( + str(last.get("candidate_extra_server_args") or last.get("extra_server_args") or "").strip() + if isinstance(last, dict) + else "" + ) + if stack_args: + config["extra_server_args"] = stack_args + return config + + +def _what_worked( + optimization_stack: Sequence[Any], + gain_per_entry: Sequence[Any], + provenance: AttemptProvenance, +) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for index, entry in enumerate(optimization_stack): + if not isinstance(entry, dict): + continue + name = str(entry.get("variant_name") or entry.get("name") or entry.get("kernel_id") or "") + row: dict[str, Any] = { + "name": name, + "extra_server_args": str(entry.get("extra_server_args") or ""), + "extra_envs": dict(entry.get("extra_envs") or {}), + "gain_pct": gain_per_entry[index] if index < len(gain_per_entry) else None, + } + # The layer's gap id wins; the variant or kernel name answers for + # layers recorded before gap ids were stamped. + gap_id = str(entry.get("gap_canonical_id") or "").strip() + by_name = provenance.kept_by_name + source = (provenance.kept_by_gap.get(gap_id) if gap_id else "") or by_name.get(name) + source = source or by_name.get(str(entry.get("kernel_id") or "")) or "" + if source: + row["source"] = source + rows.append(row) + return rows + + +def _what_failed(recent_failures: Sequence[Any], reverted: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for failure in recent_failures[-10:]: + if isinstance(failure, dict): + rows.append( + { + "name": str(failure.get("name") or failure.get("action") or ""), + "reason": str(failure.get("reason") or failure.get("error_class") or ""), + } + ) + rows.extend(reverted) + return rows + + +def _session_row( + state: Any, + session_name: str, + optimization_stack: Collection[Any], + final_throughput: float, +) -> dict[str, Any]: + validated_stack_len = int(state.cumulative_gain_validated_stack_len or 0) + actions = [ + str(entry.get("variant_name") or entry.get("name") or entry.get("action") or "").strip() + for entry in optimization_stack + if isinstance(entry, dict) + ] + return { + "session_id": str(state.recipe_kb_session_id or session_name), + "gain_pct": float(state.cumulative_gain_validated or 0.0), + "stack_len": validated_stack_len or len(optimization_stack), + "throughput_before": float(state.baseline_tput or 0.0), + "throughput_after": final_throughput, + "date": datetime.now(timezone.utc).isoformat(), + "actions_taken": [name for name in actions if name], + } + + +def recipe_attrs_from_state(state: Any, session_name: str) -> dict[str, Any]: + """Materialise the recipe-shaped view of one session's settled state. + + ``framework_version`` is left unset here because it is manifest-derived; the + T0 backfill writes it. + """ + optimization_stack = state.optimization_stack or [] + # A session that does not sediment its attempts reports no provenance. + provenance = attempt_provenance(state) if state.recipe_sediment_enabled else AttemptProvenance({}, {}, []) + final_throughput = float(state.current_best.get("tput", 0.0)) + return { + "best_config": _best_config(state.current_best, optimization_stack), + "best_throughput": final_throughput, + "what_worked": _what_worked(optimization_stack, state.gain_per_stack_entry or [], provenance), + "what_failed": _what_failed(state.last_action_failures or [], provenance.reverted), + "last_profiled": str(state.cumulative_gain_validated_ts or ""), + "workload": workload_kb_tags(state), + "sessions": [_session_row(state, session_name, optimization_stack, final_throughput)], + } diff --git a/src/hyperloom/orchestrator/knowledge/recipe_journal.py b/src/hyperloom/orchestrator/knowledge/recipe_journal.py index 01e385c6d1..0d433a6689 100644 --- a/src/hyperloom/orchestrator/knowledge/recipe_journal.py +++ b/src/hyperloom/orchestrator/knowledge/recipe_journal.py @@ -10,16 +10,13 @@ from dataclasses import dataclass from datetime import datetime, timezone from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Collection, Mapping, NamedTuple, Sequence +from typing import TYPE_CHECKING, Any, Collection, Mapping, NamedTuple from hyperloom.common.coerce import first_float, recorded_float, to_float, to_int, to_mapping, to_str_list from hyperloom.common.io import JsonLineFormat, append_jsonl from hyperloom.inference_optimizer.breakdown.recorder import close_out as _close_out from hyperloom.inference_optimizer.breakdown.recorder.framework_event import ( ARM_CONFIG, - DISPOSITION_ATTEMPTED, - STEP_ATTEMPTED, - GateVerdict, ) from hyperloom.inference_optimizer.recipe_snapshot_constants import detect_framework_version from hyperloom.inference_optimizer.session.optimization_journal import ( @@ -38,7 +35,8 @@ from ..collaborator import CoordinatorCollaborator from ..state.attempt_ledger import AttemptRow, record_attempt_row from .config import KnowledgeStoreMode -from .workload_tags import workload_shape_tags +from ._config_timeline import _NON_ATTEMPT_OUTCOMES, _record_config_attempts +from ._recipe_attrs import attempt_provenance, recipe_attrs_from_state, workload_kb_tags if TYPE_CHECKING: from ..loop.coordinator import Coordinator @@ -75,9 +73,6 @@ #: Explore outcomes the journal collapses into a plain revert. _MEASURED_LOSS_OUTCOMES: frozenset[str] = frozenset({OUTCOME_REVERT, "FAILED"}) -#: Explore outcomes that are not an attempt at anything: a variant skipped as a -#: duplicate was never measured, so no measurement would back its funnel row. -_NON_ATTEMPT_OUTCOMES: frozenset[str] = frozenset({"SKIPPED_DEDUP"}) #: Recipe fields an amend carries forward when the caller does not override #: them, each with the empty value ``put_recipe`` falls back to. @@ -246,156 +241,6 @@ def _record_config_run(coord: Any, *, task: Any, result_dict: Mapping[str, Any]) ) -def _attempt_failure(row: Mapping[str, Any]) -> dict[str, Any]: - return { - "error_class": str(row.get("error_class") or ""), - "error_excerpt": str(row.get("error_excerpt") or ""), - # Explore classified this variant's failure once; the verdict it stored is the one verdict. - "attribution": str(row.get("failure_attribution") or ""), - } - - -def _attempt_fields( - row: Mapping[str, Any], - gates: Sequence[Mapping[str, Any]], - shared: Mapping[str, str], - adopted: bool, -) -> dict[str, Any]: - variant = row.get("variant") if isinstance(row.get("variant"), dict) else {} - metrics = row.get("metrics") if isinstance(row.get("metrics"), dict) else {} - outcome = str(row.get("outcome") or "") - # This arm gates accuracy rather than reporting it, so the block both arms - # carry is projected from the gate that ruled. No gate row means nothing - # gated the variant, which is not a gate that refused it. - ruling = next((gate for gate in gates if str(gate.get("gate") or "") == "accuracy"), {}) - return { - "arm": ARM_CONFIG, - **shared, - "provenance": row.get("provenance"), - "outcome": outcome, - "reason": row.get("reason"), - "reasoning": variant.get("note"), - "reasoning_origin": variant.get("reasoning_origin"), - "experience_citations": variant.get("experience_citations"), - "stage": row.get("stage"), - "fingerprint": row.get("fingerprint"), - # The fingerprint is the join key; the name is what a reader - # recognises the variant by. - "variant_name": row.get("variant_name"), - "measurement": { - # Both ends of the pair: the anchor advances on every KEEP, so a - # percentage without its denominator adds to nothing. - "before_tput": metrics.get("base_tput"), - "after_tput": metrics.get("tput"), - "gain_pct": metrics.get("gain_pct"), - "runtime_sec": metrics.get("runtime_sec"), - "estimated_output_throughput": metrics.get("estimated_output_throughput"), - }, - "config_delta": { - "extra_server_args": variant.get("extra_server_args"), - "extra_envs": variant.get("extra_envs"), - "remove_args": variant.get("remove_args"), - "unset_envs": variant.get("unset_envs"), - "args_mode": variant.get("args_mode"), - }, - "accuracy": { - "required": True if ruling else None, - "value": ruling.get("observed"), - "reference": ruling.get("threshold"), - "passed": ruling.get("passed"), - }, - "failure": _attempt_failure(row), - "artifacts": { - "workspace": row.get("workspace"), - "server_log_path": row.get("server_log_path"), - "raw_result_path": row.get("raw_result_path"), - }, - "decision": outcome, - "adopted": adopted, - # Every variant recorded here ran: the deduped one is skipped above. - "benched": True, - # Recorded rather than referenced: every KEEP advances the stack, so - # the session's current config is not what this variant was measured - # on top of. - "measured_against": row.get("measured_against"), - # What stood behind the verdict. Absent when nothing ruled. - "validation_basis": row.get("validation_basis"), - # A pair is what makes a gain addable, so eligibility follows the pair - # being present rather than the outcome being a KEEP. - "attribution_eligible": (adopted and metrics.get("base_tput") is not None and metrics.get("tput") is not None), - } - - -def _record_variant_attempt( - recorder: Any, - row: Any, - shared: Mapping[str, str], - adopted_variants: Collection[str], -) -> bool: - if not isinstance(row, dict): - return False - if str(row.get("outcome") or "") in _NON_ATTEMPT_OUTCOMES: - return False - # The fingerprint identifies the variant within the round and the round - # within the task, so the three together identify the attempt. - fingerprint = str(row.get("fingerprint") or "") - parts = (shared["task_id"], shared["round_id"], fingerprint) - attempt_id = ":".join(part for part in parts if part) or shared["task_id"] - if not attempt_id: - return False - gates = [gate for gate in (row.get("gates") or []) if isinstance(gate, dict)] - recorder.record_attempt(attempt_id, **_attempt_fields(row, gates, shared, fingerprint in adopted_variants)) - for gate in gates: - recorder.record_attempt_gate( - attempt_id, - GateVerdict( - gate=str(gate.get("gate") or ""), - passed=gate.get("passed"), - reason=str(gate.get("reason") or ""), - observed=gate.get("observed"), - threshold=gate.get("threshold"), - ), - ) - return True - - -def _record_config_attempts( - coord: Any, - *, - task: Any, - per_variant: list[dict[str, Any]], - result_dict: Mapping[str, Any], - adopted_variants: Collection[str], -) -> None: - """Record the configuration arm's measured attempts on the framework event. - - Timeline only: the ledger row is written by ``fact_write_hook``, which does - not depend on a recorder being open. The outcome is recorded verbatim here, - unlike the journal beside it, which collapses ``KEEP_UNSTABLE`` and - ``KILLED_OVERTIME`` into a plain revert. - """ - recorder = coord._coord.phase_framework.timeline() - if recorder is None: - return - params = getattr(task, "params", None) or {} - shared = { - "task_id": str(getattr(task, "task_id", "") or ""), - "round_id": str(result_dict.get("round_id") or ""), - "proposal_ref": str(params.get("proposal_msg_id") or ""), - } - recorded = 0 - for row in per_variant: - if _record_variant_attempt(recorder, row, shared, adopted_variants): - recorded += 1 - if not (recorded and shared["proposal_ref"]): - return - # Settled here rather than at materialization: a grid whose task dies before - # any variant runs has no attempt to stand behind an ``attempted`` reading, - # and is more honestly left unsettled. - recorder.record_proposal_step(shared["proposal_ref"], step=STEP_ATTEMPTED, outcome=str(recorded)) - recorder.settle_proposal(shared["proposal_ref"], disposition=DISPOSITION_ATTEMPTED) - - def rows_not_superseded(live_rows: Any, superseding: Any, key: str) -> list[dict[str, Any]]: claimed = {str(row.get(key) or "") for row in superseding or [] if isinstance(row, dict)} kept: list[dict[str, Any]] = [] @@ -1421,234 +1266,4 @@ def kb_amend_recipe( self._local_recipe_cache = None -__all__ = ["RecipeJournalCollaborator"] - - -#: The probe's own word for a version it could not read. -_UNIDENTIFIED_VERSION: str = "unknown" - - -class AttemptProvenance(NamedTuple): - """Where a session's proven optimizations came from, and what it reverted. - - Keepers are indexed twice because a stack layer may identify itself either - by name or by the research gap it closed. - """ - - kept_by_name: dict[str, str] - kept_by_gap: dict[str, str] - reverted: list[dict[str, Any]] - - -def _known_version(raw: Any) -> str: - version = str(raw or "").strip() - return "" if version == _UNIDENTIFIED_VERSION else version - - -def _runtime_version_tags(fingerprint_meta: Mapping[str, Any], framework: str) -> dict[str, str]: - tags: dict[str, str] = {} - framework_key = framework.lower() - if framework_key in {"sglang", "vllm"}: - active = _known_version(fingerprint_meta.get(framework_key)) - if active: - tags["framework_version"] = active - for source_key, tag in (("rocm", "rocm_version"), ("aiter", "aiter_version"), ("image_digest", "image_digest")): - version = _known_version(fingerprint_meta.get(source_key)) - if version: - tags[tag] = version - return tags - - -def _baseline_workload_tags(workload_extra: Mapping[str, Any]) -> dict[str, Any]: - tags: dict[str, Any] = {} - for key in ("max_running_requests", "max_num_seqs"): - value = workload_extra.get(key) - if isinstance(value, int) and value > 0: - tags[key] = value - for key in ("chunked_prefill_enabled", "enable_torch_compile"): - value = workload_extra.get(key) - if isinstance(value, bool): - tags[key] = value - for key in ("quant_scheme", "workload_mode"): - value = workload_extra.get(key) - if isinstance(value, str) and value.strip(): - tags[key] = value.strip() - return tags - - -def workload_kb_tags(state: Any) -> dict[str, Any]: - """The workload-shape tags describing the session this recipe came from.""" - tags: dict[str, Any] = {} - # ``model_family`` is deliberately absent, being no part of the - # seven-dimension Recipe identity. - for attr in ("framework", "model_class", "model_name"): - text = str(getattr(state, attr, "") or "").strip() - if text: - tags[attr] = text - precision = getattr(state, "precision", None) - if precision not in (None, "", 0): - tags["precision"] = precision - tags.update(workload_shape_tags(state)) - fingerprint_meta = state.stack_fingerprint_meta or {} - if isinstance(fingerprint_meta, dict): - tags.update(_runtime_version_tags(fingerprint_meta, str(tags.get("framework") or ""))) - workload_extra = state.baseline_workload_extra or {} - if isinstance(workload_extra, dict): - tags.update(_baseline_workload_tags(workload_extra)) - return tags - - -def _reverted_attempt_row(name: str, attempt: Mapping[str, Any], source: str) -> dict[str, Any]: - row: dict[str, Any] = { - "name": name, - "reason": "reverted", - "gain_pct": attempt.get("gain_pct"), - } - if source: - row["source"] = source - return row - - -def _gap_attempt_provenance(gap: Mapping[str, Any]) -> AttemptProvenance: - source = str(gap.get("provenance") or "").strip() - canonical_id = str(gap.get("canonical_id") or "").strip() - kept_by_name: dict[str, str] = {} - kept_by_gap: dict[str, str] = {} - reverted: list[dict[str, Any]] = [] - for attempt in gap.get("attempts") or []: - if not isinstance(attempt, dict): - continue - variant = str(attempt.get("variant_name") or "").strip() - kernel = str(attempt.get("kernel_id") or "").strip() - outcome = str(attempt.get("outcome") or "").strip().upper() - if outcome == OUTCOME_KEEP and source: - for name in (variant, kernel): - if name: - kept_by_name.setdefault(name, source) - if canonical_id: - kept_by_gap.setdefault(canonical_id, source) - elif outcome == OUTCOME_REVERT and (variant or kernel): - reverted.append(_reverted_attempt_row(variant or kernel, attempt, source)) - return AttemptProvenance(kept_by_name=kept_by_name, kept_by_gap=kept_by_gap, reverted=reverted) - - -def attempt_provenance(state: Any) -> AttemptProvenance: - """Trace each proven optimization back to the research hint it came from.""" - kept_by_name: dict[str, str] = {} - kept_by_gap: dict[str, str] = {} - reverted: list[dict[str, Any]] = [] - for gap in state.gaps or []: - if not isinstance(gap, dict): - continue - from_gap = _gap_attempt_provenance(gap) - for name, source in from_gap.kept_by_name.items(): - kept_by_name.setdefault(name, source) - for canonical_id, source in from_gap.kept_by_gap.items(): - kept_by_gap.setdefault(canonical_id, source) - reverted.extend(from_gap.reverted) - return AttemptProvenance(kept_by_name=kept_by_name, kept_by_gap=kept_by_gap, reverted=reverted) - - -def _best_config(current_best: Mapping[str, Any], optimization_stack: Sequence[Any]) -> dict[str, Any]: - config: dict[str, Any] = {} - current_args = current_best.get("extra_server_args") - if current_args: - config["extra_server_args"] = str(current_args) - for key in ("extra_envs", "name", "tput", "accuracy"): - if key in current_best: - config[key] = current_best[key] - last = optimization_stack[-1] if optimization_stack else None - stack_args = ( - str(last.get("candidate_extra_server_args") or last.get("extra_server_args") or "").strip() - if isinstance(last, dict) - else "" - ) - if stack_args: - config["extra_server_args"] = stack_args - return config - - -def _what_worked( - optimization_stack: Sequence[Any], - gain_per_entry: Sequence[Any], - provenance: AttemptProvenance, -) -> list[dict[str, Any]]: - rows: list[dict[str, Any]] = [] - for index, entry in enumerate(optimization_stack): - if not isinstance(entry, dict): - continue - name = str(entry.get("variant_name") or entry.get("name") or entry.get("kernel_id") or "") - row: dict[str, Any] = { - "name": name, - "extra_server_args": str(entry.get("extra_server_args") or ""), - "extra_envs": dict(entry.get("extra_envs") or {}), - "gain_pct": gain_per_entry[index] if index < len(gain_per_entry) else None, - } - # The layer's gap id wins; the variant or kernel name answers for - # layers recorded before gap ids were stamped. - gap_id = str(entry.get("gap_canonical_id") or "").strip() - by_name = provenance.kept_by_name - source = (provenance.kept_by_gap.get(gap_id) if gap_id else "") or by_name.get(name) - source = source or by_name.get(str(entry.get("kernel_id") or "")) or "" - if source: - row["source"] = source - rows.append(row) - return rows - - -def _what_failed(recent_failures: Sequence[Any], reverted: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: - rows: list[dict[str, Any]] = [] - for failure in recent_failures[-10:]: - if isinstance(failure, dict): - rows.append( - { - "name": str(failure.get("name") or failure.get("action") or ""), - "reason": str(failure.get("reason") or failure.get("error_class") or ""), - } - ) - rows.extend(reverted) - return rows - - -def _session_row( - state: Any, - session_name: str, - optimization_stack: Collection[Any], - final_throughput: float, -) -> dict[str, Any]: - validated_stack_len = int(state.cumulative_gain_validated_stack_len or 0) - actions = [ - str(entry.get("variant_name") or entry.get("name") or entry.get("action") or "").strip() - for entry in optimization_stack - if isinstance(entry, dict) - ] - return { - "session_id": str(state.recipe_kb_session_id or session_name), - "gain_pct": float(state.cumulative_gain_validated or 0.0), - "stack_len": validated_stack_len or len(optimization_stack), - "throughput_before": float(state.baseline_tput or 0.0), - "throughput_after": final_throughput, - "date": datetime.now(timezone.utc).isoformat(), - "actions_taken": [name for name in actions if name], - } - - -def recipe_attrs_from_state(state: Any, session_name: str) -> dict[str, Any]: - """Materialise the recipe-shaped view of one session's settled state. - - ``framework_version`` is left unset here because it is manifest-derived; the - T0 backfill writes it. - """ - optimization_stack = state.optimization_stack or [] - # A session that does not sediment its attempts reports no provenance. - provenance = attempt_provenance(state) if state.recipe_sediment_enabled else AttemptProvenance({}, {}, []) - final_throughput = float(state.current_best.get("tput", 0.0)) - return { - "best_config": _best_config(state.current_best, optimization_stack), - "best_throughput": final_throughput, - "what_worked": _what_worked(optimization_stack, state.gain_per_stack_entry or [], provenance), - "what_failed": _what_failed(state.last_action_failures or [], provenance.reverted), - "last_profiled": str(state.cumulative_gain_validated_ts or ""), - "workload": workload_kb_tags(state), - "sessions": [_session_row(state, session_name, optimization_stack, final_throughput)], - } +__all__ = ["RecipeJournalCollaborator", "attempt_provenance", "recipe_attrs_from_state", "workload_kb_tags"] From feda1ad28f96166a80cf700e72243052839a1bae Mon Sep 17 00:00:00 2001 From: Test Date: Thu, 22 Oct 2026 17:48:40 +0800 Subject: [PATCH 14/14] chore(metrics): follow recipe_journal down to its split length Co-Authored-By: Claude Opus 5 --- scripts/code_metrics_baseline.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/code_metrics_baseline.txt b/scripts/code_metrics_baseline.txt index 21a19a92f7..b3030fa464 100644 --- a/scripts/code_metrics_baseline.txt +++ b/scripts/code_metrics_baseline.txt @@ -480,7 +480,7 @@ module-lines src/hyperloom/orchestrator/actions/executors/roofline.py 1303 module-lines src/hyperloom/orchestrator/kernel/apply_kernel_patch.py 2656 module-lines src/hyperloom/orchestrator/kernel/conc_sweep.py 1429 module-lines src/hyperloom/orchestrator/kernel/request_handlers.py 4690 -module-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py 1654 +module-lines src/hyperloom/orchestrator/knowledge/recipe_journal.py 1269 module-lines src/hyperloom/orchestrator/loop/coordinator.py 1337 module-lines src/hyperloom/orchestrator/loop/dispatcher.py 1453 module-lines src/hyperloom/orchestrator/loop/intent_router.py 1409