From 0daaeb5b38b9b96033f7659ee11476af9515d776 Mon Sep 17 00:00:00 2001 From: shinyobjectz Date: Fri, 20 Nov 2026 15:47:58 -0800 Subject: [PATCH] AK6-AK7: raea ask learns from the person: verdicts, notes and approvals label it, calibrate and try A label belongs to a question and the clips a call sent, so new words asked of the same clips share its labels. raea ask verdict ANSWER WORD is the person's answer in the question's own terms; a note of theirs on the posted render, inside the window and holding one of the question's label words, is a yes, and a post they approved without one is a no. Each call now keeps its file's identity, so the note is matched to the render it was left on, never to a later render at the same path. An agent's own look (--eye agent) is kept and never counted. raea ask calibrate holds each labelled version to the person's labels through the fingerprint's own AUC, quantile and gates (16 of each kind, 3 projects, 0.8); a version that passes flags where 10% of their clean work would, and goes out with the question from then on, in place of raea's shipped calibration. One under 0.6 after 32 labels lists what it missed. raea ask try sends new words with the kept clips through the same runner and prints both AUCs. The record now writes each row in one write, since several runs append to the same files. --- Cargo.lock | 2 + apps/ask/Cargo.toml | 4 + apps/ask/src/calibrate.rs | 214 ++++++++++++++++++++++++++++ apps/ask/src/labels.rs | 236 +++++++++++++++++++++++++++++++ apps/ask/src/labels_tests.rs | 132 +++++++++++++++++ apps/ask/src/lib.rs | 9 +- apps/ask/src/record.rs | 63 ++++++++- apps/ask/src/runner.rs | 31 +++- apps/ask/src/tests.rs | 2 +- apps/ask/src/verbs.rs | 93 +++++++++--- docs/ask.feature | 32 ++++- docs/manual/reference/cli/ask.md | 26 +++- 12 files changed, 806 insertions(+), 38 deletions(-) create mode 100644 apps/ask/src/calibrate.rs create mode 100644 apps/ask/src/labels.rs create mode 100644 apps/ask/src/labels_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 61149fb4..8076b221 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13761,6 +13761,8 @@ dependencies = [ "raea-io-assets", "raea-io-clef", "raea-io-home", + "raea-io-trace", + "raea-sys-fingerprint", "serde", "serde_json", "sha2 0.10.9", diff --git a/apps/ask/Cargo.toml b/apps/ask/Cargo.toml index e88f68a2..55a4a746 100644 --- a/apps/ask/Cargo.toml +++ b/apps/ask/Cargo.toml @@ -11,6 +11,10 @@ description = "raea ask: questions about a piece's picture and sound, prepared f raea-core-ask = { path = "../../core/ask" } raea-io-clef = { path = "../../io/clef" } raea-io-home = { path = "../../io/home" } +# the fingerprint's AUC, quantile and gates, so a question is calibrated as an axis is (AK7) +raea-sys-fingerprint = { path = "../../systems/fingerprint" } +# today's date for a calibration +raea-io-trace = { path = "../../io/trace" } dunce = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/apps/ask/src/calibrate.rs b/apps/ask/src/calibrate.rs new file mode 100644 index 00000000..c19cc6d4 --- /dev/null +++ b/apps/ask/src/calibrate.rs @@ -0,0 +1,214 @@ +//! Calibration (docs/ask.feature AK7): a question's answers held to the person's labels (labels.rs), per version of its +//! words. Clef isn't trained on the labels, so its AUC on them is held out already; the one thing fitted is the +//! threshold, the problem score that flags 10% of the clean labels. The gates are the fingerprint's (16 labels of each +//! kind, from 3 projects, an AUC of 0.8). A question that passes is calibrated at its threshold: its answers are +//! findings, and the person's calibration replaces raea's shipped one. One under 0.6 after 32 labels doesn't separate, +//! and says which labels it missed, so its words can be fixed. `raea ask try` asks new words of the labelled clips and +//! sets both AUCs side by side. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::path::PathBuf; +use std::sync::Arc; + +use raea_core_ask::{Calibration, Question}; +use raea_io_clef::Media; +use raea_sys_fingerprint::head::{auc, quantile}; +use raea_sys_fingerprint::train::{GATE_AUC, GATE_LABELS, GATE_PROJECTS}; +use serde::Serialize; +use serde_json::Value; + +use crate::labels::{self, Label}; +use crate::prepare::Piece; +use crate::record; +use crate::runner::{self, Event, Group, Kept, Opts, Summary}; + +pub const FILE: &str = "calibration.json"; +/// An AUC under this after twice the gate's labels: the words don't separate the problem. +const SEPARATES: f64 = 0.6; + +/// The person's calibrations that passed the gates, by question version. +pub fn local() -> BTreeMap { + std::fs::read_to_string(record::dir().join(FILE)).ok().and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default() +} + +/// A question with the person's calibration in place of raea's, when it has one. +pub fn apply(q: &mut Question, local: &BTreeMap) { + if let Some(c) = local.get(&q.version()) { + q.calibration = Some(c.clone()); + } +} + +/// A label the answers missed: a problem it scored low, or clean work it scored high. +#[derive(Debug, Clone, Serialize)] +pub struct Miss { + pub file: String, + pub span: String, + pub problem: f64, + pub bad: bool, + pub note: Option, +} + +/// One version's standing. +#[derive(Debug, Clone, Serialize)] +pub struct Standing { + pub id: String, + pub version: String, + pub ask: String, + pub problems: usize, + pub clean: usize, + pub projects: usize, + pub auc: Option, + pub threshold: Option, + /// calibrated, advice or "doesn't separate" + pub state: String, + pub why: String, + pub misses: Vec, + /// other eyes' labels on its clips, kept and not counted + pub others: usize, +} + +/// Each labelled clip's latest problem score for one version, beside its label. +fn scored<'a>(version: &str, standing: &HashMap<(String, String), &'a Label>, calls: &HashMap) -> Vec<(f64, &'a Label)> { + let mut latest: HashMap = HashMap::new(); + for a in record::rows("answers.jsonl").iter().filter(|a| a["version"] == version) { + let (Some(p), Some(call)) = (a["problem"].as_f64(), calls.get(a["call"].as_str().unwrap_or_default())) else { continue }; + let key = (a["question"].as_str().unwrap_or_default().to_string(), labels::media_key(call)); + if let Some(l) = standing.get(&key) { + latest.insert(key.1, (p, l)); + } + } + latest.into_values().collect() +} + +/// One version held to its labels. +pub fn standing_of(q: &Question, all: &[Label], calls: &HashMap) -> Standing { + let standing = labels::standing(all); + let s = scored(&q.version(), &standing, calls); + let score = |bad: bool| s.iter().filter(|(_, l)| l.bad == bad).map(|(p, _)| *p as f32).collect::>(); + let (pos, neg) = (score(true), score(false)); + let projects = s.iter().map(|(_, l)| l.project.as_str()).collect::>().len(); + let a = Some(auc(&pos, &neg)).filter(|a| a.is_finite()); + let threshold = Some(quantile(&neg, 0.9) as f64).filter(|t| t.is_finite()); + let mut why = Vec::new(); + if pos.len() < GATE_LABELS || neg.len() < GATE_LABELS { + why.push(format!("{} problem and {} clean labels (needs {GATE_LABELS} of each)", pos.len(), neg.len())); + } + if projects < GATE_PROJECTS { + why.push(format!("labels from {projects} projects (needs {GATE_PROJECTS})")); + } + match a { + Some(v) if v < GATE_AUC => why.push(format!("AUC {v:.2} (needs {GATE_AUC})")), + None => why.push("no AUC yet: it needs labels of both kinds".into()), + _ => {} + } + let stuck = s.len() >= 2 * GATE_LABELS && a.is_some_and(|v| v < SEPARATES); + let mut misses: Vec = Vec::new(); + if stuck { + let mut by: Vec<&(f64, &Label)> = s.iter().collect(); + by.sort_by(|x, y| x.0.total_cmp(&y.0)); + let low = by.iter().filter(|(_, l)| l.bad).take(5); + let high = by.iter().rev().filter(|(_, l)| !l.bad).take(5); + misses = low.chain(high).map(|(p, l)| Miss { file: l.file.clone(), span: labels::span(l.from, l.to), problem: *p, bad: l.bad, note: l.note.clone() }).collect(); + } + let others = all.iter().filter(|l| !l.counted() && l.question == q.id).count(); + let state = match (why.is_empty(), stuck) { + (true, _) => "calibrated", + (false, true) => "doesn't separate", + (false, false) => "advice", + }; + let why = if why.is_empty() { format!("earned: {} labels from {projects} projects, flags past {:.2}", s.len(), threshold.unwrap_or(0.0)) } else { why.join("; ") }; + Standing { id: q.id.clone(), version: q.version(), ask: q.ask.clone(), problems: pos.len(), clean: neg.len(), projects, auc: a, threshold, state: state.into(), why, misses, others } +} + +/// Every labelled question's versions as the record kept them (or only `id`'s), newest wording last. +fn versions(only: Option<&str>, all: &[Label]) -> Vec { + let labelled: BTreeSet<&str> = all.iter().map(|l| l.question.as_str()).collect(); + let mut seen = BTreeSet::new(); + let mut out = Vec::new(); + for a in record::rows("answers.jsonl") { + let id = a["question"].as_str().unwrap_or_default(); + if only.is_some_and(|o| o != id) || !labelled.contains(id) || !seen.insert(a["version"].as_str().unwrap_or_default().to_string()) { + continue; + } + out.push(labels::question_of(&a)); + } + out +} + +/// `raea ask calibrate [ID]`: the feed's labels read again, then every labelled version held to them. The ones that +/// pass are kept as the person's calibration; one that no longer passes loses it. +pub fn calibrate(only: Option<&str>) -> Result<(Vec, (usize, usize)), String> { + let feed = labels::from_feed()?; + let (all, calls) = (labels::all(), labels::calls()); + let out: Vec = versions(only, &all).iter().map(|q| standing_of(q, &all, &calls)).collect(); + let mut keep = local(); + for s in &out { + keep.remove(&s.version); + if s.state == "calibrated" { + let n = (s.problems + s.clean) as u32; + let c = Calibration { auc: s.auc.unwrap_or(0.0), n, positives: s.problems as u32, threshold: s.threshold, date: today(), by: "you".into() }; + keep.insert(s.version.clone(), c); + } + } + record::write_whole(FILE, &serde_json::to_string_pretty(&keep).map_err(|e| e.to_string())?)?; + Ok((out, feed)) +} + +/// YYYY-MM-DD, UTC. +fn today() -> String { + raea_io_trace::utc(std::time::SystemTime::now()).0.chars().take(10).collect() +} + +/// The groups `raea ask try` sends: the new question on each labelled clip set, with the state it was asked with. +pub fn try_groups(like: &str, words: &str) -> Result<(Question, Question, Vec), String> { + let all = labels::all(); + let standing = labels::standing(&all); + let rows = record::rows("answers.jsonl"); + let old = rows.iter().rev().find(|a| a["question"] == like).map(labels::question_of).ok_or_else(|| format!("ask try: no answers to {like} in the record"))?; + let new = Question { ask: words.to_string(), calibration: None, ..old.clone() }; + let calls = labels::calls(); + let mut groups: Vec = Vec::new(); + let mut sent = BTreeSet::new(); + for call in calls.values().filter(|c| standing.contains_key(&(like.to_string(), labels::media_key(c)))) { + let names: Vec<&str> = call["media"].as_array().into_iter().flatten().filter_map(Value::as_str).collect(); + let Some(kept) = media(&names) else { continue }; + if !sent.insert(labels::media_key(call)) { + continue; + } + let piece = Arc::new(Piece { file: PathBuf::from(call["file"].as_str().unwrap_or_default()), project: call["project"].as_str().map(PathBuf::from), ..Default::default() }); + let (from, to) = (call["from"].as_f64().unwrap_or(0.0), call["to"].as_f64()); + let seconds = to.map_or(60.0, |t| t - from); + let kept = Kept { video_seconds: if kept.videos.is_empty() { 0.0 } else { seconds }, sound_seconds: if kept.sounds.is_empty() { 0.0 } else { seconds }, media: kept, state: call["state"].as_str().unwrap_or_default().into() }; + groups.push(Group { piece, from, to, sees: new.sees, hears: new.hears, with: vec![], questions: vec![new.clone()], kept: Some(Arc::new(kept)) }); + } + if groups.is_empty() { + return Err(format!("ask try: none of {like}'s labelled clips are still kept (raea ask calibrate lists its labels)")); + } + Ok((old, new, groups)) +} + +/// A call's clips from the kept media, in the kinds they were sent as; None when one has been pruned. +fn media(names: &[&str]) -> Option { + let mut m = Media::default(); + for n in names { + let bytes = record::media(n)?; + match n.rsplit('.').next() { + Some("mp4") => m.videos.push(bytes), + Some("mp3") => m.sounds.push(bytes), + _ => m.stills.push(bytes), + } + } + Some(m) +} + +/// `raea ask try "WORDS" --like ID`: the new words asked of the old question's labelled clips, and both versions held to +/// the same labels. +pub fn try_words(like: &str, words: &str, o: &Opts, out: &mut dyn FnMut(Event)) -> Result<(Summary, Option<(Standing, Standing)>), String> { + let (old, new, groups) = try_groups(like, words)?; + let sum = runner::run_groups(groups, Vec::new(), o, out); + if o.dry { + return Ok((sum, None)); + } + let (all, calls) = (labels::all(), labels::calls()); + Ok((sum, Some((standing_of(&old, &all, &calls), standing_of(&new, &all, &calls))))) +} diff --git a/apps/ask/src/labels.rs b/apps/ask/src/labels.rs new file mode 100644 index 00000000..179a5f6c --- /dev/null +++ b/apps/ask/src/labels.rs @@ -0,0 +1,236 @@ +//! Labels (docs/ask.feature AK6): whether a question's problem was really there in what one call sent, so the question +//! can be calibrated on the person's own eye (calibrate.rs). A label belongs to a question and to the clips a call sent +//! (by their hashes), never to one answer: the same question reworded, asked again of the same clips, has the same +//! labels, which is what lets `raea ask try` measure new words. +//! +//! Where labels come from: +//! - `raea ask verdict ANSWER WORD`: the person answers the question themselves, in its own terms (yes or no, an option, +//! a level). Theirs counts first. +//! - The feed: a note of the person's on the render (its copy in the project's feed, known by `record::ident`), pinned +//! inside the window or not pinned at all, holding one of the question's label words, is a yes; an approved post (done) +//! with no such note is a no. Read again by every `raea ask calibrate`. +//! - Another eye (`verdict --eye agent`: an agent's own look) is kept as `eye:NAME` and never counted, as the +//! fingerprint keeps another eye's readings (FP6d). + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use raea_core_ask::{Kind, Question}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::record; + +pub const FILE: &str = "labels.jsonl"; + +/// A note pinned at t covers 1.5 s before it to 0.5 s after (the fingerprint's span). +const BEFORE: f64 = 1.5; +const AFTER: f64 = 0.5; + +/// Whether a question's problem is there in one call's clips, and who said so. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Label { + pub question: String, + /// the clips the call sent, by hash + pub media: String, + pub bad: bool, + /// "verdict" (the person's), "note" or "approval" (read from the feed), or "eye:NAME" (kept, never counted) + pub source: String, + pub project: String, + pub file: String, + pub from: f64, + pub to: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub note: Option, + pub at: String, +} + +impl Label { + /// The person's own: a verdict, a note or an approval. + pub fn counted(&self) -> bool { + self.rank() > 0 + } + fn rank(&self) -> u8 { + match self.source.as_str() { + "verdict" => 3, + "note" => 2, + "approval" => 1, + _ => 0, + } + } +} + +/// The clips a call sent, as one key. +pub fn media_key(call: &Value) -> String { + let mut m: Vec<&str> = call["media"].as_array().into_iter().flatten().filter_map(Value::as_str).collect(); + m.sort_unstable(); + m.join("+") +} + +/// The feed a call's render was posted to: its project's, else the nearest one above its file. +fn feed_of(call: &Value) -> Option { + let file = Path::new(call["file"].as_str().unwrap_or_default()); + call["project"].as_str().map(Path::new).into_iter().chain(file.ancestors()).map(|d| d.join(".raea/feed")).find(|d| d.is_dir()) +} + +/// A call's project: the folder holding its brief, else the folder holding the feed it was posted to, else its file's. +pub fn project_of(call: &Value) -> String { + if let Some(p) = call["project"].as_str() { + return p.to_string(); + } + let home = feed_of(call).and_then(|f| f.parent()?.parent().map(Path::to_path_buf)); + let file = Path::new(call["file"].as_str().unwrap_or_default()); + home.as_deref().or(file.parent()).map(|p| p.display().to_string()).unwrap_or_default() +} + +pub fn all() -> Vec