diff --git a/CHANGELOG.md b/CHANGELOG.md index fb9d2fb..11380a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ onward the project follows SemVer normally: patch releases are bug-fix-only, minor releases are backward-compatible additions, and breaking changes require a major bump. +## v2.5.3: Close the benchmark learning loop + +A focused follow-up to v2.5.2. Adds the host-side path that lets a graded +task feed its outcome back into the brain, so the consolidation machinery +(usefulness, query-routes, staples) receives a real signal on every solve. + +### Added + +- `kimetsu brain benchmark-credit --task --passed`: on a pass, groups a + citation for the memories most relevant to the task and routes it through + the same grouped, query-linked path consolidation consumes. Driven by a + benchmark harness after grading, so the learning signal never depends on + the agent inside the sandbox calling a tool. This is what makes an + iterated-benchmark learning loop actually reach the brain; before it, the + benchmark memory path recorded outcomes but never exercised consolidation. + ## v2.5.2: The brain that learns from its own outcomes Consolidation v1: the first release where citation outcomes reshape the brain diff --git a/Cargo.lock b/Cargo.lock index 4c9f26d..7a0fcec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -800,9 +800,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] @@ -1989,7 +1989,7 @@ dependencies = [ [[package]] name = "kimetsu-agent" -version = "2.5.2" +version = "2.5.3" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -2009,7 +2009,7 @@ dependencies = [ [[package]] name = "kimetsu-brain" -version = "2.5.2" +version = "2.5.3" dependencies = [ "blake3", "fastembed", @@ -2030,7 +2030,7 @@ dependencies = [ [[package]] name = "kimetsu-chat" -version = "2.5.2" +version = "2.5.3" dependencies = [ "base64 0.22.1", "crossterm", @@ -2046,7 +2046,7 @@ dependencies = [ [[package]] name = "kimetsu-cli" -version = "2.5.2" +version = "2.5.3" dependencies = [ "clap", "flate2", @@ -2071,7 +2071,7 @@ dependencies = [ [[package]] name = "kimetsu-core" -version = "2.5.2" +version = "2.5.3" dependencies = [ "serde", "serde_json", @@ -2082,7 +2082,7 @@ dependencies = [ [[package]] name = "kimetsu-e2e" -version = "2.5.2" +version = "2.5.3" dependencies = [ "kimetsu-agent", "kimetsu-brain", @@ -2095,7 +2095,7 @@ dependencies = [ [[package]] name = "kimetsu-remote" -version = "2.5.2" +version = "2.5.3" dependencies = [ "axum", "axum-server", diff --git a/Cargo.toml b/Cargo.toml index 6480c74..fe4291c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ resolver = "3" # everything below except the per-crate `description`, `keywords`, # and `categories` which live in each `[package]` block since # crates.io enforces them per crate. -version = "2.5.2" +version = "2.5.3" edition = "2024" # Rust ecosystem dual-license (matches tokio, serde, fastembed-rs, # etc.). UNLICENSED would block crates.io entirely. diff --git a/crates/kimetsu-agent/Cargo.toml b/crates/kimetsu-agent/Cargo.toml index f91b2ce..d0516af 100644 --- a/crates/kimetsu-agent/Cargo.toml +++ b/crates/kimetsu-agent/Cargo.toml @@ -27,8 +27,8 @@ aws-sigv4.workspace = true aws-smithy-runtime-api.workspace = true blake3.workspace = true http.workspace = true -kimetsu-brain = { path = "../kimetsu-brain", version = "2.5.2" } -kimetsu-core = { path = "../kimetsu-core", version = "2.5.2" } +kimetsu-brain = { path = "../kimetsu-brain", version = "2.5.3" } +kimetsu-core = { path = "../kimetsu-core", version = "2.5.3" } regex.workspace = true reqwest.workspace = true rusqlite.workspace = true diff --git a/crates/kimetsu-brain/Cargo.toml b/crates/kimetsu-brain/Cargo.toml index 12a4aad..a3519ba 100644 --- a/crates/kimetsu-brain/Cargo.toml +++ b/crates/kimetsu-brain/Cargo.toml @@ -50,7 +50,7 @@ hf-hub = { version = "0.5", optional = true, default-features = false, features # already native (ort). The lean build never links it. usearch = { version = "2", optional = true } ignore.workspace = true -kimetsu-core = { path = "../kimetsu-core", version = "2.5.2" } +kimetsu-core = { path = "../kimetsu-core", version = "2.5.3" } # v0.4.5: regex backs the secret-redaction patterns in # `kimetsu_brain::redact`. The crate is already a workspace pin # elsewhere; we just opt this crate into it now. diff --git a/crates/kimetsu-brain/src/reinforce.rs b/crates/kimetsu-brain/src/reinforce.rs index f016ff6..d97836d 100644 --- a/crates/kimetsu-brain/src/reinforce.rs +++ b/crates/kimetsu-brain/src/reinforce.rs @@ -57,6 +57,45 @@ pub struct ReinforceSummary { pub routes_embedded: usize, } +/// Close the benchmark learning loop for one graded task (v2.5.2): when a +/// task PASSES, the memories most relevant to it get a grouped, query-linked +/// citation — the exact signal consolidation consumes (usefulness +1.0 each, +/// query-routes from task -> those memories, and staples from their +/// co-citation). Driven host-side by the benchmark harness after grading, so +/// the learning signal never depends on the in-container agent calling any +/// tool. Failures produce no citation (the retrieved memories are not +/// necessarily to blame). Returns how many memories were credited. +/// +/// Why this exists: the MCP `kimetsu_brain_cite` tool routes through the +/// SINGLETON `record_mcp_citation` path (one id, no query, fresh run) which +/// feeds none of the three consumers. This routes through the grouped +/// `record_citations` path, which feeds all three. +pub fn credit_benchmark_outcome( + start: &Path, + task: &str, + passed: bool, + top_k: usize, +) -> KimetsuResult { + if !passed { + return Ok(0); + } + // Retrieve the memories most relevant to this task, then credit the top + // few as "in context when the task was solved". search_memories ranks by + // BM25 over the task text across project + user brains. + let hits = crate::project::search_memories(start, task, top_k.max(1) as u32, 0, None, None)?; + let ids: Vec = hits.into_iter().take(top_k).map(|h| h.memory_id).collect(); + if ids.is_empty() { + return Ok(0); + } + crate::project::record_citations( + start, + &ids, + Some("benchmark: in context when task passed"), + Some(task), + )?; + Ok(ids.len()) +} + /// Run the offline consolidation pass: staple qualifying co-citations and/or /// rebuild the query-routes table. Both idempotent; safe to run every /// session end or between benchmark iterations. diff --git a/crates/kimetsu-chat/Cargo.toml b/crates/kimetsu-chat/Cargo.toml index efcaaf7..5c91f93 100644 --- a/crates/kimetsu-chat/Cargo.toml +++ b/crates/kimetsu-chat/Cargo.toml @@ -38,9 +38,9 @@ openclaw = ["dep:json5"] # surface, not a benchmark harness. [dependencies] -kimetsu-agent = { path = "../kimetsu-agent", version = "2.5.2" } -kimetsu-brain = { path = "../kimetsu-brain", version = "2.5.2" } -kimetsu-core = { path = "../kimetsu-core", version = "2.5.2" } +kimetsu-agent = { path = "../kimetsu-agent", version = "2.5.3" } +kimetsu-brain = { path = "../kimetsu-brain", version = "2.5.3" } +kimetsu-core = { path = "../kimetsu-core", version = "2.5.3" } base64.workspace = true crossterm.workspace = true json5 = { workspace = true, optional = true } diff --git a/crates/kimetsu-cli/Cargo.toml b/crates/kimetsu-cli/Cargo.toml index 1a16deb..39b1f85 100644 --- a/crates/kimetsu-cli/Cargo.toml +++ b/crates/kimetsu-cli/Cargo.toml @@ -43,10 +43,10 @@ path = "src/main.rs" [dependencies] clap.workspace = true -kimetsu-agent = { path = "../kimetsu-agent", version = "2.5.2" } -kimetsu-brain = { path = "../kimetsu-brain", version = "2.5.2" } -kimetsu-chat = { path = "../kimetsu-chat", version = "2.5.2" } -kimetsu-core = { path = "../kimetsu-core", version = "2.5.2" } +kimetsu-agent = { path = "../kimetsu-agent", version = "2.5.3" } +kimetsu-brain = { path = "../kimetsu-brain", version = "2.5.3" } +kimetsu-chat = { path = "../kimetsu-chat", version = "2.5.3" } +kimetsu-core = { path = "../kimetsu-core", version = "2.5.3" } # v0.4.6: `kimetsu doctor` serializes its report struct so --json # output can be piped into CI / hooks. flate2 = "1" diff --git a/crates/kimetsu-cli/src/commands/brain.rs b/crates/kimetsu-cli/src/commands/brain.rs index 02d8cbb..6ebc08b 100644 --- a/crates/kimetsu-cli/src/commands/brain.rs +++ b/crates/kimetsu-cli/src/commands/brain.rs @@ -200,6 +200,7 @@ pub(crate) fn brain(command: BrainCommand) -> KimetsuResult<()> { BrainCommand::Forget(args) => brain_forget(args), BrainCommand::Cite(args) => brain_cite(args), BrainCommand::Reinforce(args) => brain_reinforce(args), + BrainCommand::BenchmarkCredit(args) => brain_benchmark_credit(args), BrainCommand::Regret(args) => brain_regret(args), BrainCommand::Distill(args) => brain_distill(args), BrainCommand::Graph { command } => brain_graph(command), @@ -2756,7 +2757,7 @@ pub(crate) fn brain_forget(args: ForgetArgs) -> KimetsuResult<()> { c.use_count, c.usefulness_score, c.age_days, - &c.text_preview + c.text_preview ); } } @@ -2896,6 +2897,30 @@ pub(crate) fn brain_reinforce(args: ReinforceArgs) -> KimetsuResult<()> { Ok(()) } +pub(crate) fn brain_benchmark_credit(args: BenchmarkCreditArgs) -> KimetsuResult<()> { + let workspace = args + .workspace + .unwrap_or_else(|| env::current_dir().unwrap_or_default()); + let credited = kimetsu_brain::reinforce::credit_benchmark_outcome( + &workspace, + &args.task, + args.passed, + args.top_k, + )?; + println!( + "benchmark-credit: {} memor{} cited for task \"{}\" ({})", + credited, + if credited == 1 { "y" } else { "ies" }, + args.task, + if args.passed { + "passed" + } else { + "not passed — no citation" + } + ); + Ok(()) +} + pub(crate) fn brain_regret(args: RegretArgs) -> KimetsuResult<()> { let workspace = args .workspace diff --git a/crates/kimetsu-cli/src/main.rs b/crates/kimetsu-cli/src/main.rs index 14387d1..1fbaf91 100644 --- a/crates/kimetsu-cli/src/main.rs +++ b/crates/kimetsu-cli/src/main.rs @@ -847,6 +847,11 @@ enum BrainCommand { /// before get a bounded retrieval boost. Model-free; run offline /// (session end / between benchmark iterations). Reinforce(ReinforceArgs), + /// Close the benchmark learning loop for one graded task: on a PASS, + /// grouped-cite the memories most relevant to the task (feeds usefulness, + /// query-routes, and staples). Driven host-side by the benchmark harness + /// so the learning signal never depends on the in-container agent. + BenchmarkCredit(BenchmarkCreditArgs), /// Record a regret: mark that a surfaced memory was unhelpful/misleading. /// /// Writes a `retrieval.regret` telemetry event for the memory — the negative @@ -1359,6 +1364,23 @@ struct ReinforceArgs { workspace: Option, } +/// Args for `kimetsu brain benchmark-credit`. +#[derive(Debug, Args)] +struct BenchmarkCreditArgs { + /// The task description / query the graded task represents. + #[arg(long)] + task: String, + /// Mark the task as PASSED — only passes produce a citation. + #[arg(long)] + passed: bool, + /// How many top-ranked memories to credit on a pass. + #[arg(long, default_value_t = 3)] + top_k: usize, + /// Override the brain workspace path (defaults to current directory). + #[arg(long)] + workspace: Option, +} + /// Args for `kimetsu brain regret`. #[derive(Debug, Args)] struct RegretArgs { diff --git a/crates/kimetsu-e2e/Cargo.toml b/crates/kimetsu-e2e/Cargo.toml index 0f99b4d..183e26b 100644 --- a/crates/kimetsu-e2e/Cargo.toml +++ b/crates/kimetsu-e2e/Cargo.toml @@ -20,9 +20,9 @@ publish = false # Real (non-dev) deps so the test fixtures + scripted provider can be # re-exported from `kimetsu_e2e::prelude` to the integration tests in # `tests/`. Integration tests treat this crate as a normal library. -kimetsu-agent = { path = "../kimetsu-agent", version = "2.5.2" } -kimetsu-brain = { path = "../kimetsu-brain", version = "2.5.2" } -kimetsu-core = { path = "../kimetsu-core", version = "2.5.2" } +kimetsu-agent = { path = "../kimetsu-agent", version = "2.5.3" } +kimetsu-brain = { path = "../kimetsu-brain", version = "2.5.3" } +kimetsu-core = { path = "../kimetsu-core", version = "2.5.3" } rusqlite.workspace = true serde_json.workspace = true time.workspace = true