diff --git a/Cargo.lock b/Cargo.lock index 61149fb4..8076b221 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13761,6 +13761,8 @@ dependencies = [ "raea-io-assets", "raea-io-clef", "raea-io-home", + "raea-io-trace", + "raea-sys-fingerprint", "serde", "serde_json", "sha2 0.10.9", diff --git a/apps/ask/Cargo.toml b/apps/ask/Cargo.toml index e88f68a2..55a4a746 100644 --- a/apps/ask/Cargo.toml +++ b/apps/ask/Cargo.toml @@ -11,6 +11,10 @@ description = "raea ask: questions about a piece's picture and sound, prepared f raea-core-ask = { path = "../../core/ask" } raea-io-clef = { path = "../../io/clef" } raea-io-home = { path = "../../io/home" } +# the fingerprint's AUC, quantile and gates, so a question is calibrated as an axis is (AK7) +raea-sys-fingerprint = { path = "../../systems/fingerprint" } +# today's date for a calibration +raea-io-trace = { path = "../../io/trace" } dunce = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/apps/ask/src/calibrate.rs b/apps/ask/src/calibrate.rs new file mode 100644 index 00000000..c19cc6d4 --- /dev/null +++ b/apps/ask/src/calibrate.rs @@ -0,0 +1,214 @@ +//! Calibration (docs/ask.feature AK7): a question's answers held to the person's labels (labels.rs), per version of its +//! words. Clef isn't trained on the labels, so its AUC on them is held out already; the one thing fitted is the +//! threshold, the problem score that flags 10% of the clean labels. The gates are the fingerprint's (16 labels of each +//! kind, from 3 projects, an AUC of 0.8). A question that passes is calibrated at its threshold: its answers are +//! findings, and the person's calibration replaces raea's shipped one. One under 0.6 after 32 labels doesn't separate, +//! and says which labels it missed, so its words can be fixed. `raea ask try` asks new words of the labelled clips and +//! sets both AUCs side by side. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::path::PathBuf; +use std::sync::Arc; + +use raea_core_ask::{Calibration, Question}; +use raea_io_clef::Media; +use raea_sys_fingerprint::head::{auc, quantile}; +use raea_sys_fingerprint::train::{GATE_AUC, GATE_LABELS, GATE_PROJECTS}; +use serde::Serialize; +use serde_json::Value; + +use crate::labels::{self, Label}; +use crate::prepare::Piece; +use crate::record; +use crate::runner::{self, Event, Group, Kept, Opts, Summary}; + +pub const FILE: &str = "calibration.json"; +/// An AUC under this after twice the gate's labels: the words don't separate the problem. +const SEPARATES: f64 = 0.6; + +/// The person's calibrations that passed the gates, by question version. +pub fn local() -> BTreeMap { + std::fs::read_to_string(record::dir().join(FILE)).ok().and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default() +} + +/// A question with the person's calibration in place of raea's, when it has one. +pub fn apply(q: &mut Question, local: &BTreeMap) { + if let Some(c) = local.get(&q.version()) { + q.calibration = Some(c.clone()); + } +} + +/// A label the answers missed: a problem it scored low, or clean work it scored high. +#[derive(Debug, Clone, Serialize)] +pub struct Miss { + pub file: String, + pub span: String, + pub problem: f64, + pub bad: bool, + pub note: Option, +} + +/// One version's standing. +#[derive(Debug, Clone, Serialize)] +pub struct Standing { + pub id: String, + pub version: String, + pub ask: String, + pub problems: usize, + pub clean: usize, + pub projects: usize, + pub auc: Option, + pub threshold: Option, + /// calibrated, advice or "doesn't separate" + pub state: String, + pub why: String, + pub misses: Vec, + /// other eyes' labels on its clips, kept and not counted + pub others: usize, +} + +/// Each labelled clip's latest problem score for one version, beside its label. +fn scored<'a>(version: &str, standing: &HashMap<(String, String), &'a Label>, calls: &HashMap) -> Vec<(f64, &'a Label)> { + let mut latest: HashMap = HashMap::new(); + for a in record::rows("answers.jsonl").iter().filter(|a| a["version"] == version) { + let (Some(p), Some(call)) = (a["problem"].as_f64(), calls.get(a["call"].as_str().unwrap_or_default())) else { continue }; + let key = (a["question"].as_str().unwrap_or_default().to_string(), labels::media_key(call)); + if let Some(l) = standing.get(&key) { + latest.insert(key.1, (p, l)); + } + } + latest.into_values().collect() +} + +/// One version held to its labels. +pub fn standing_of(q: &Question, all: &[Label], calls: &HashMap) -> Standing { + let standing = labels::standing(all); + let s = scored(&q.version(), &standing, calls); + let score = |bad: bool| s.iter().filter(|(_, l)| l.bad == bad).map(|(p, _)| *p as f32).collect::>(); + let (pos, neg) = (score(true), score(false)); + let projects = s.iter().map(|(_, l)| l.project.as_str()).collect::>().len(); + let a = Some(auc(&pos, &neg)).filter(|a| a.is_finite()); + let threshold = Some(quantile(&neg, 0.9) as f64).filter(|t| t.is_finite()); + let mut why = Vec::new(); + if pos.len() < GATE_LABELS || neg.len() < GATE_LABELS { + why.push(format!("{} problem and {} clean labels (needs {GATE_LABELS} of each)", pos.len(), neg.len())); + } + if projects < GATE_PROJECTS { + why.push(format!("labels from {projects} projects (needs {GATE_PROJECTS})")); + } + match a { + Some(v) if v < GATE_AUC => why.push(format!("AUC {v:.2} (needs {GATE_AUC})")), + None => why.push("no AUC yet: it needs labels of both kinds".into()), + _ => {} + } + let stuck = s.len() >= 2 * GATE_LABELS && a.is_some_and(|v| v < SEPARATES); + let mut misses: Vec = Vec::new(); + if stuck { + let mut by: Vec<&(f64, &Label)> = s.iter().collect(); + by.sort_by(|x, y| x.0.total_cmp(&y.0)); + let low = by.iter().filter(|(_, l)| l.bad).take(5); + let high = by.iter().rev().filter(|(_, l)| !l.bad).take(5); + misses = low.chain(high).map(|(p, l)| Miss { file: l.file.clone(), span: labels::span(l.from, l.to), problem: *p, bad: l.bad, note: l.note.clone() }).collect(); + } + let others = all.iter().filter(|l| !l.counted() && l.question == q.id).count(); + let state = match (why.is_empty(), stuck) { + (true, _) => "calibrated", + (false, true) => "doesn't separate", + (false, false) => "advice", + }; + let why = if why.is_empty() { format!("earned: {} labels from {projects} projects, flags past {:.2}", s.len(), threshold.unwrap_or(0.0)) } else { why.join("; ") }; + Standing { id: q.id.clone(), version: q.version(), ask: q.ask.clone(), problems: pos.len(), clean: neg.len(), projects, auc: a, threshold, state: state.into(), why, misses, others } +} + +/// Every labelled question's versions as the record kept them (or only `id`'s), newest wording last. +fn versions(only: Option<&str>, all: &[Label]) -> Vec { + let labelled: BTreeSet<&str> = all.iter().map(|l| l.question.as_str()).collect(); + let mut seen = BTreeSet::new(); + let mut out = Vec::new(); + for a in record::rows("answers.jsonl") { + let id = a["question"].as_str().unwrap_or_default(); + if only.is_some_and(|o| o != id) || !labelled.contains(id) || !seen.insert(a["version"].as_str().unwrap_or_default().to_string()) { + continue; + } + out.push(labels::question_of(&a)); + } + out +} + +/// `raea ask calibrate [ID]`: the feed's labels read again, then every labelled version held to them. The ones that +/// pass are kept as the person's calibration; one that no longer passes loses it. +pub fn calibrate(only: Option<&str>) -> Result<(Vec, (usize, usize)), String> { + let feed = labels::from_feed()?; + let (all, calls) = (labels::all(), labels::calls()); + let out: Vec = versions(only, &all).iter().map(|q| standing_of(q, &all, &calls)).collect(); + let mut keep = local(); + for s in &out { + keep.remove(&s.version); + if s.state == "calibrated" { + let n = (s.problems + s.clean) as u32; + let c = Calibration { auc: s.auc.unwrap_or(0.0), n, positives: s.problems as u32, threshold: s.threshold, date: today(), by: "you".into() }; + keep.insert(s.version.clone(), c); + } + } + record::write_whole(FILE, &serde_json::to_string_pretty(&keep).map_err(|e| e.to_string())?)?; + Ok((out, feed)) +} + +/// YYYY-MM-DD, UTC. +fn today() -> String { + raea_io_trace::utc(std::time::SystemTime::now()).0.chars().take(10).collect() +} + +/// The groups `raea ask try` sends: the new question on each labelled clip set, with the state it was asked with. +pub fn try_groups(like: &str, words: &str) -> Result<(Question, Question, Vec), String> { + let all = labels::all(); + let standing = labels::standing(&all); + let rows = record::rows("answers.jsonl"); + let old = rows.iter().rev().find(|a| a["question"] == like).map(labels::question_of).ok_or_else(|| format!("ask try: no answers to {like} in the record"))?; + let new = Question { ask: words.to_string(), calibration: None, ..old.clone() }; + let calls = labels::calls(); + let mut groups: Vec = Vec::new(); + let mut sent = BTreeSet::new(); + for call in calls.values().filter(|c| standing.contains_key(&(like.to_string(), labels::media_key(c)))) { + let names: Vec<&str> = call["media"].as_array().into_iter().flatten().filter_map(Value::as_str).collect(); + let Some(kept) = media(&names) else { continue }; + if !sent.insert(labels::media_key(call)) { + continue; + } + let piece = Arc::new(Piece { file: PathBuf::from(call["file"].as_str().unwrap_or_default()), project: call["project"].as_str().map(PathBuf::from), ..Default::default() }); + let (from, to) = (call["from"].as_f64().unwrap_or(0.0), call["to"].as_f64()); + let seconds = to.map_or(60.0, |t| t - from); + let kept = Kept { video_seconds: if kept.videos.is_empty() { 0.0 } else { seconds }, sound_seconds: if kept.sounds.is_empty() { 0.0 } else { seconds }, media: kept, state: call["state"].as_str().unwrap_or_default().into() }; + groups.push(Group { piece, from, to, sees: new.sees, hears: new.hears, with: vec![], questions: vec![new.clone()], kept: Some(Arc::new(kept)) }); + } + if groups.is_empty() { + return Err(format!("ask try: none of {like}'s labelled clips are still kept (raea ask calibrate lists its labels)")); + } + Ok((old, new, groups)) +} + +/// A call's clips from the kept media, in the kinds they were sent as; None when one has been pruned. +fn media(names: &[&str]) -> Option { + let mut m = Media::default(); + for n in names { + let bytes = record::media(n)?; + match n.rsplit('.').next() { + Some("mp4") => m.videos.push(bytes), + Some("mp3") => m.sounds.push(bytes), + _ => m.stills.push(bytes), + } + } + Some(m) +} + +/// `raea ask try "WORDS" --like ID`: the new words asked of the old question's labelled clips, and both versions held to +/// the same labels. +pub fn try_words(like: &str, words: &str, o: &Opts, out: &mut dyn FnMut(Event)) -> Result<(Summary, Option<(Standing, Standing)>), String> { + let (old, new, groups) = try_groups(like, words)?; + let sum = runner::run_groups(groups, Vec::new(), o, out); + if o.dry { + return Ok((sum, None)); + } + let (all, calls) = (labels::all(), labels::calls()); + Ok((sum, Some((standing_of(&old, &all, &calls), standing_of(&new, &all, &calls))))) +} diff --git a/apps/ask/src/labels.rs b/apps/ask/src/labels.rs new file mode 100644 index 00000000..179a5f6c --- /dev/null +++ b/apps/ask/src/labels.rs @@ -0,0 +1,236 @@ +//! Labels (docs/ask.feature AK6): whether a question's problem was really there in what one call sent, so the question +//! can be calibrated on the person's own eye (calibrate.rs). A label belongs to a question and to the clips a call sent +//! (by their hashes), never to one answer: the same question reworded, asked again of the same clips, has the same +//! labels, which is what lets `raea ask try` measure new words. +//! +//! Where labels come from: +//! - `raea ask verdict ANSWER WORD`: the person answers the question themselves, in its own terms (yes or no, an option, +//! a level). Theirs counts first. +//! - The feed: a note of the person's on the render (its copy in the project's feed, known by `record::ident`), pinned +//! inside the window or not pinned at all, holding one of the question's label words, is a yes; an approved post (done) +//! with no such note is a no. Read again by every `raea ask calibrate`. +//! - Another eye (`verdict --eye agent`: an agent's own look) is kept as `eye:NAME` and never counted, as the +//! fingerprint keeps another eye's readings (FP6d). + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use raea_core_ask::{Kind, Question}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::record; + +pub const FILE: &str = "labels.jsonl"; + +/// A note pinned at t covers 1.5 s before it to 0.5 s after (the fingerprint's span). +const BEFORE: f64 = 1.5; +const AFTER: f64 = 0.5; + +/// Whether a question's problem is there in one call's clips, and who said so. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Label { + pub question: String, + /// the clips the call sent, by hash + pub media: String, + pub bad: bool, + /// "verdict" (the person's), "note" or "approval" (read from the feed), or "eye:NAME" (kept, never counted) + pub source: String, + pub project: String, + pub file: String, + pub from: f64, + pub to: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub note: Option, + pub at: String, +} + +impl Label { + /// The person's own: a verdict, a note or an approval. + pub fn counted(&self) -> bool { + self.rank() > 0 + } + fn rank(&self) -> u8 { + match self.source.as_str() { + "verdict" => 3, + "note" => 2, + "approval" => 1, + _ => 0, + } + } +} + +/// The clips a call sent, as one key. +pub fn media_key(call: &Value) -> String { + let mut m: Vec<&str> = call["media"].as_array().into_iter().flatten().filter_map(Value::as_str).collect(); + m.sort_unstable(); + m.join("+") +} + +/// The feed a call's render was posted to: its project's, else the nearest one above its file. +fn feed_of(call: &Value) -> Option { + let file = Path::new(call["file"].as_str().unwrap_or_default()); + call["project"].as_str().map(Path::new).into_iter().chain(file.ancestors()).map(|d| d.join(".raea/feed")).find(|d| d.is_dir()) +} + +/// A call's project: the folder holding its brief, else the folder holding the feed it was posted to, else its file's. +pub fn project_of(call: &Value) -> String { + if let Some(p) = call["project"].as_str() { + return p.to_string(); + } + let home = feed_of(call).and_then(|f| f.parent()?.parent().map(Path::to_path_buf)); + let file = Path::new(call["file"].as_str().unwrap_or_default()); + home.as_deref().or(file.parent()).map(|p| p.display().to_string()).unwrap_or_default() +} + +pub fn all() -> Vec