From af2b35f4396f76f33238ddc055edff86e2325165 Mon Sep 17 00:00:00 2001 From: shinyobjectz Date: Fri, 20 Nov 2026 16:24:11 -0800 Subject: [PATCH] AK8: the fingerprint's flags asked again of clef raea fingerprint score FILE --ask flags (or ask: "flags" in fingerprint.ts) puts each flagged moment to clef as a yes/no question in its axis's own words, looking or listening as the axis does, on that moment's window alone, so it costs by the flags and never by the piece; --ask unsure also asks the windows an axis puts in its plausible band without flagging them. It goes through raea ask's runner (routes, cache, record, the person's calibration), and the axis's words are the question's label words, so the person's notes calibrate it (AK6, AK7). Each flag gets a line under the score ("dark: clef doesn't see it (0.00)", and a clef array with --json), and each answer replaces clef's last eye:clef evidence row for that moment, which trains the head and never counts as the person's. A moment it couldn't ask says why, and the score goes on. The double check is its own crate, raea-app-double, since raea-app-ask holds 1841 of its 2000 lines. The manual page covers score --ask beside readings' and postmortem's --ask N; docs/ask.feature marks AK8 done. --- Cargo.lock | 16 +++ Cargo.toml | 1 + apps/double/Cargo.toml | 23 ++++ apps/double/src/lib.rs | 167 +++++++++++++++++++++++ apps/double/src/tests.rs | 88 ++++++++++++ apps/fingerprint/Cargo.toml | 2 + apps/fingerprint/src/lib.rs | 2 +- apps/fingerprint/src/verbs.rs | 15 +- docs/ask.feature | 18 ++- docs/manual/reference/cli/fingerprint.md | 13 +- sdk/raea/src/fingerprint.ts | 3 + systems/fingerprint/src/taxonomy.rs | 6 +- tools/layers.toml | 1 + 13 files changed, 347 insertions(+), 8 deletions(-) create mode 100644 apps/double/Cargo.toml create mode 100644 apps/double/src/lib.rs create mode 100644 apps/double/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 8076b221c..7b8bec51d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13849,6 +13849,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "raea-app-double" +version = "0.1.0" +dependencies = [ + "raea-app-ask", + "raea-core-ask", + "raea-io-assets", + "raea-io-clef", + "raea-sys-columns", + "raea-sys-fingerprint", + "raea-sys-fingerprint-use", + "serde", + "serde_json", +] + [[package]] name = "raea-app-engines" version = "0.1.0" @@ -13872,6 +13887,7 @@ dependencies = [ "dunce", "fastrand", "raea-app-check", + "raea-app-double", "raea-io-home", "raea-sys-columns", "raea-sys-fingerprint", diff --git a/Cargo.toml b/Cargo.toml index 18a78ac38..df0aabc55 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -189,6 +189,7 @@ members = [ "runtime/scene2d", "runtime/shared", "apps/ask", + "apps/double", "apps/check", "apps/check/flash", "apps/check/inbox", diff --git a/apps/double/Cargo.toml b/apps/double/Cargo.toml new file mode 100644 index 000000000..f5095fa84 --- /dev/null +++ b/apps/double/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "raea-app-double" +version = "0.1.0" +edition = "2024" +license = "FSL-1.1-ALv2" +publish = false +description = "The fingerprint's flags asked again of the decision model (docs/ask.feature AK8): each flagged moment in its axis's own words on its own window, the answers beside the flags and kept as another eye's evidence" + +[dependencies] +# the runner every ask goes through, and the question format +raea-app-ask = { path = "../ask" } +raea-core-ask = { path = "../../core/ask" } +# the score's moments, the axes' words and the evidence rows clef's answers become +raea-sys-fingerprint = { path = "../../systems/fingerprint" } +raea-sys-fingerprint-use = { path = "../../systems/fingerprint-use" } +raea-sys-columns = { path = "../../systems/columns" } +serde = { version = "1", features = ["derive"] } + +[dev-dependencies] +# a clip made with FFmpeg, and clef's answers as the tests give them +raea-io-assets = { path = "../../io/assets" } +raea-io-clef = { path = "../../io/clef" } +serde_json = "1" diff --git a/apps/double/src/lib.rs b/apps/double/src/lib.rs new file mode 100644 index 000000000..69d7b8ae7 --- /dev/null +++ b/apps/double/src/lib.rs @@ -0,0 +1,167 @@ +//! The fingerprint's flags asked again (docs/ask.feature AK8). `raea fingerprint score FILE --ask flags` (or `ask: +//! "flags"` in fingerprint.ts) puts each flagged moment to the decision model in its axis's own words, on that moment's +//! window alone, so it costs by the flags and never by the piece; `--ask unsure` also asks each window an axis puts in +//! its plausible band without flagging it. Moments on the same window share a call. Each flag then says "clef agrees +//! (0.71)" or "clef doesn't see it (0.08)", and each answer becomes an `eye:clef` evidence row, which trains the axis's +//! head and never counts as the person's (FP6d). The question carries the axis's words as its label words, so the +//! person's notes calibrate it as they do any question (AK6, AK7). + +use std::path::Path; +use std::sync::Arc; + +use raea_app_ask::prepare::Piece; +use raea_app_ask::runner::{self, Entry, Event}; +pub use raea_app_ask::runner::Opts; +use raea_core_ask::{Question, Sees, Window}; +use raea_sys_columns::Columns; +use raea_sys_columns::window::Modality; +use raea_sys_fingerprint::evidence::{Row, features}; +use raea_sys_fingerprint::folder::Folder; +use raea_sys_fingerprint::taxonomy::{Axis, Fingerprint}; +use raea_sys_fingerprint_use::score::{Scored, clock}; +use serde::Serialize; + +/// One moment asked again. +#[derive(Debug, Clone, Serialize)] +pub struct Double { + pub a: f64, + pub b: f64, + pub axis: String, + /// the axis in the person's words + pub words: String, + /// whether the fingerprint flagged it (else it was unsure of it) + pub flagged: bool, + /// clef's problem score, when it answered + pub problem: Option, + /// clef agrees: past even odds, or past the question's threshold once the person's labels calibrated it + pub agrees: Option, + /// its answer's id in raea ask's record, for `raea ask verdict` + pub answer: Option, + /// why it wasn't asked or still waits + pub not_asked: Option, +} + +impl Double { + /// What clef made of it, as a flag's line says it. + pub fn said(&self) -> String { + match (self.agrees, self.problem, &self.not_asked) { + (Some(true), Some(p), _) => format!("clef agrees ({p:.2})"), + (Some(false), Some(p), _) => format!("clef doesn't see it ({p:.2})"), + (_, _, Some(why)) => format!("clef wasn't asked: {why}"), + _ => "clef gave no answer".into(), + } + } +} + +/// The moments asked again, a line each under the score's summary, as its flags are written (the app reads them). +pub fn lines(ds: &[Double]) -> String { + let each = ds.iter().map(|d| format!(" {}-{} {}: {}{}\n", clock(d.a), clock(d.b), d.axis, d.said(), if d.flagged { "" } else { " (the fingerprint was unsure)" })); + format!("asked again of clef ({} moment{}):\n{}", ds.len(), if ds.len() == 1 { "" } else { "s" }, each.collect::()) +} + +/// The moments to ask: every flag, and with `unsure` each run of windows an axis puts in its plausible band unflagged. +pub fn moments(s: &Scored, fp: &Fingerprint, which: &str) -> Vec { + let words = |axis: &str| fp.axis(axis).and_then(|a| a.words.first().cloned()).unwrap_or_else(|| axis.to_string()); + let one = |a: f64, b: f64, axis: &str, flagged: bool| Double { a, b, axis: axis.into(), words: words(axis), flagged, problem: None, agrees: None, answer: None, not_asked: None }; + let mut out: Vec = s.moments.iter().map(|m| one(m.a, m.b, &m.axis, true)).collect(); + if which == "unsure" { + for w in &s.windows { + for x in w.axes.iter().filter(|x| !x.flagged && x.band == Some("plausible")) { + match out.iter_mut().rev().find(|d| d.axis == x.axis && !d.flagged) { + Some(d) if w.a <= d.b => d.b = w.b, + _ => out.push(one(w.a, w.b, &x.axis, false)), + } + } + } + } + out +} + +/// An axis as a question: is its defect there (or, for a quality, is it there), looking or listening as it does. +pub fn question(axis: &Axis) -> Question { + let words = axis.words.first().cloned().unwrap_or_else(|| axis.name.replace(['-', '_'], " ")); + let good = axis.polarity == "good"; + let ask = if good { format!("Is this {words}?") } else { format!("Is there {words} here?") }; + let mut q = Question::plain(&format!("fp.{}", axis.name), &ask); + if good { + q.flags = vec!["no".into()]; + } + let listens = matches!(axis.modality, Modality::Sound | Modality::Events); + q.sees = if listens { Sees::Nothing } else { Sees::Video }; + q.hears = listens; + q.label_words = axis.words.clone(); + q +} + +/// Each moment asked of `file`, a call a window, at most `o.jobs` at once; the moments with clef's answers. A moment +/// that couldn't be asked (no route, the pool spent, the file unreadable) says why, and the score goes on without it. +pub fn ask(file: &Path, fp: &Fingerprint, mut ds: Vec, o: &Opts) -> Vec { + let piece = match Piece::read(file, None, None, None, None) { + Ok(p) => Arc::new(p), + Err(why) => { + for d in &mut ds { + d.not_asked = Some(format!("the file couldn't be read: {why}")); + } + return ds; + } + }; + let entries = ds + .iter() + .filter_map(|d| { + let q = question(fp.axis(&d.axis)?); + Some(Entry { piece: piece.clone(), window: Some(Window { from: Some(d.a), to: Some(d.b) }), questions: vec![q], dropped: vec![], with: vec![] }) + }) + .collect(); + runner::run(entries, o, &mut |e| { + let Event::Answer(r) = e else { return }; + let Some(d) = ds.iter_mut().find(|d| format!("fp.{}", d.axis) == r.id && d.a == r.from && r.to == Some(d.b)) else { return }; + d.problem = r.problem; + d.agrees = r.says.as_ref().map(|_| r.flagged); + d.answer = r.answer.clone(); + d.not_asked = r.not_asked.clone().or_else(|| r.waits.clone()); + }); + ds +} + +/// The answered moments kept in the fingerprint's ledger as clef's evidence, over the file's columns; a moment asked +/// again replaces clef's last row for it. How many it kept. +pub fn keep(dir: &Folder, c: &Columns, project: &str, ds: &[Double]) -> Result { + let rows: Vec = ds.iter().filter_map(|d| row(d, &c.media, project)).collect(); + if rows.is_empty() { + return Ok(0); + } + let mut l = dir.ledger().map_err(|e| e.to_string())?; + let mut n = 0; + for r in rows { + l.retain(|x| !(x.source == r.source && x.media == r.media && x.axis == r.axis && x.a == r.a && x.b == r.b)); + let f = features(c, r.a, r.b); + l.push(r, &f); + n += 1; + } + dir.write_ledger(&l).map_err(|e| e.to_string())?; + Ok(n) +} + +/// An answered moment as another eye's evidence row. +pub fn row(d: &Double, media: &str, project: &str) -> Option { + Some(Row { + media: media.into(), + a: d.a, + b: d.b, + axis: d.axis.clone(), + bad: d.agrees?, + weight: 1.0, + who: "clef".into(), + source: "eye:clef".into(), + project: project.into(), + kind: None, + region: None, + words: None, + at: 0, + pin: None, + band: None, + }) +} + +#[cfg(test)] +mod tests; diff --git a/apps/double/src/tests.rs b/apps/double/src/tests.rs new file mode 100644 index 000000000..087b0b570 --- /dev/null +++ b/apps/double/src/tests.rs @@ -0,0 +1,88 @@ +use std::path::PathBuf; + +use raea_app_ask::runner::Opts; +use raea_core_ask::Sees; +use raea_io_clef::{Answered, Call, Fail}; +use raea_sys_fingerprint::taxonomy::base; +use raea_sys_fingerprint_use::score::{AxisScore, Moment, Scored, Window}; +use serde_json::json; + +use crate::{ask, lines, moments, question, row}; + +fn moment(a: f64, b: f64, axis: &str) -> Moment { + Moment { a, b, axis: axis.into(), words: String::new(), by: "head", state: "gating".into(), band: Some("strong"), gating: true, peak: 1.0, at: None } +} + +fn window(a: f64, axis: &str, band: &'static str, flagged: bool) -> Window { + let s = AxisScore { axis: axis.into(), value: 0.5, by: "head", state: "advice".into(), band: Some(band), flagged, gating: false }; + Window { a, b: a + 2.0, axes: vec![s], seen: None, heard: None, level: -20.0, overall: None } +} + +fn scored(moments: Vec, windows: Vec) -> Scored { + Scored { file: "a.mp4".into(), media: "m".into(), dur: 10.0, fingerprint: "test".into(), moments, windows } +} + +#[test] +fn flags_are_asked_and_unsure_adds_the_plausible_windows_run_together() { + let fp = base("test"); + let s = scored(vec![moment(0.0, 3.0, "blur")], vec![window(4.0, "glitch", "plausible", false), window(5.0, "glitch", "plausible", false), window(8.0, "glitch", "weak", false)]); + assert_eq!(moments(&s, &fp, "flags").len(), 1); + let unsure = moments(&s, &fp, "unsure"); + assert_eq!(unsure.iter().map(|d| (d.a, d.b, d.axis.as_str(), d.flagged)).collect::>(), [(0.0, 3.0, "blur", true), (4.0, 7.0, "glitch", false)]); + assert!(unsure[0].words.contains("blurry"), "an axis is asked in its own words: {}", unsure[0].words); +} + +#[test] +fn an_axis_asks_as_it_senses_and_a_quality_is_flagged_on_no() { + let fp = base("test"); + let blur = question(fp.axis("blur").unwrap()); + assert!(blur.ask.starts_with("Is there a blurry") && blur.sees == Sees::Video && !blur.hears && blur.flags.is_empty()); + assert_eq!(blur.label_words, fp.axis("blur").unwrap().words, "the person's notes in the axis's words label it"); + let muffled = question(fp.axis("muffled voice").unwrap()); + assert!(muffled.sees == Sees::Nothing && muffled.hears); + let mut good = fp.axis("blur").unwrap().clone(); + good.polarity = "good".into(); + let q = question(&good); + assert!(q.ask.starts_with("Is this ") && q.flags == ["no"]); +} + +/// clef as a fake: yes to the blur question, no to anything else. +fn blur_only(call: &Call) -> Result { + let answers = call.questions.iter().map(|q| (q.id.clone(), json!({"noul": if q.id == "fp.blur" { 0.8 } else { 0.1 }}))).collect(); + Ok(Answered { answers, cost: 0.001, cached: false, route: "a test".into(), pool_left: None }) +} + +/// A 6 s clip with a picture and a tone. +fn clip() -> PathBuf { + let out = std::env::temp_dir().join(format!("raea-double-{}.mp4", std::process::id())); + let ff = raea_io_assets::ffmpeg_tool("ffmpeg", false).expect("FFmpeg"); + let ok = std::process::Command::new(ff) + .args(["-y", "-v", "error", "-f", "lavfi", "-i", "testsrc=size=320x240:rate=10:duration=6", "-f", "lavfi", "-i", "sine=frequency=440:duration=6"]) + .args(["-c:v", "mpeg4", "-c:a", "aac", "-shortest"]) + .arg(&out) + .status() + .expect("FFmpeg ran"); + assert!(ok.success()); + out +} + +#[test] +fn each_flag_is_asked_on_its_own_window_and_said_beside_it_and_kept_as_clefs() { + // SAFETY: this crate's only test that reads it, set before the runner starts + unsafe { std::env::set_var("RAEA_ASK_DIR", std::env::temp_dir().join(format!("raea-double-ask-{}", std::process::id()))) }; + let fp = base("test"); + let file = clip(); + let s = scored(vec![moment(0.0, 2.0, "blur"), moment(0.0, 2.0, "glitch"), moment(3.0, 5.0, "muffled voice")], vec![]); + let ds = ask(&file, &fp, moments(&s, &fp, "flags"), &Opts { jobs: 2, dry: false, ask: blur_only }); + let said: Vec = ds.iter().map(|d| format!("{}: {}", d.axis, d.said())).collect(); + assert_eq!(said, ["blur: clef agrees (0.80)", "glitch: clef doesn't see it (0.10)", "muffled voice: clef doesn't see it (0.10)"]); + assert!(lines(&ds).starts_with("asked again of clef (3 moments):\n 0:00-0:02 blur: clef agrees (0.80)\n"), "{}", lines(&ds)); + let gone = ask(&std::env::temp_dir().join("raea-double-none.mp4"), &fp, moments(&s, &fp, "flags"), &Opts { jobs: 2, dry: false, ask: blur_only }); + assert!(gone.iter().all(|d| d.agrees.is_none() && d.said().starts_with("clef wasn't asked")), "a file it can't read asks nothing: {gone:?}"); + let r = row(&ds[0], "m", "p").unwrap(); + assert!(r.bad && r.source == "eye:clef" && r.who == "clef" && (r.a, r.b) == (0.0, 2.0)); + assert!(!row(&ds[1], "m", "p").unwrap().bad); + let mut unanswered = ds[0].clone(); + unanswered.agrees = None; + assert!(row(&unanswered, "m", "p").is_none(), "a moment clef didn't answer isn't evidence"); +} diff --git a/apps/fingerprint/Cargo.toml b/apps/fingerprint/Cargo.toml index 50a7e4c4d..5b7953e8f 100644 --- a/apps/fingerprint/Cargo.toml +++ b/apps/fingerprint/Cargo.toml @@ -13,6 +13,8 @@ raea-io-home = { path = "../../io/home" } raea-sys-fingerprint = { path = "../../systems/fingerprint" } # scoring and the sheet, which one judges a piece, diff and profiles raea-sys-fingerprint-use = { path = "../../systems/fingerprint-use" } +# score --ask: each flag asked again of the decision model (docs/ask.feature AK8) +raea-app-double = { path = "../double" } raea-sys-fingerprint-things = { path = "../../systems/fingerprint-things" } # the encoders and the columns they make raea-sys-columns = { path = "../../systems/columns" } diff --git a/apps/fingerprint/src/lib.rs b/apps/fingerprint/src/lib.rs index f58d318dd..f5f95648e 100644 --- a/apps/fingerprint/src/lib.rs +++ b/apps/fingerprint/src/lib.rs @@ -21,7 +21,7 @@ use std::path::Path; pub use serve::serve; -pub const USAGE: &str = "raea fingerprint new NAME [--who W] [--about TEXT] [--extends FP] [--kinds K,K] [--tags T,T] | raea fingerprint bootstrap FILE... [--from LIST] [--per N] | raea fingerprint teach FILE --axis A --at T [--bad|--good] [--span S] [--box x0,y0,x1,y1] [--words W] [--project P] | raea fingerprint approve FILE... [--project P] | raea fingerprint pair WORSE BETTER --axis A [--at T] [--then T2] | raea fingerprint versions V1 V2... | raea fingerprint profile REFS... --as NAME [--against LIST] | raea fingerprint score FILE [--was OLD] [--json] [--sheet out.png] [--timeline] [--profile NAME] | raea fingerprint report [--json] | raea fingerprint which [PATH] [--json] | raea fingerprint use NAME [--in project|workspace] [PATH] | raea fingerprint diff A B [--json] | raea fingerprint share NAME [--in project|workspace] [PATH] | raea fingerprint reembed PLACE... | raea fingerprint read PROJECT [--by kimi|claude] | raea fingerprint readings [PROJECT] [--ask N] [--tally] | raea fingerprint learn-notes [PROJECT] | raea fingerprint move [--fp CHILD] [--to PARENT] [--axis A] [--source S] [--project P] | raea fingerprint postmortem [PROJECT] [--ask N] [--tally] | raea fingerprint things FILE [--project P] [--json] | raea fingerprint stop [--fp NAME] (a person's sense of good work as axes with small heads over on-device encoders, docs/fingerprint.md: new makes ~/.raea/fingerprints/NAME with raea's base taxonomy in fingerprint.ts (or FP's axes with --extends), what it's for, the kinds of piece and project tags it judges, and every axis scoring from its words; bootstrap puts known defects into windows of real work and trains the mechanical axes, held out by source, and patch heads that say where in the frame a picture defect is; teach adds a label at a moment; approve makes approved work clean on every axis, away from its notes; pair says one moment is better than another on an axis (a pick, a hand edit); versions makes weak pairs of noted moments the next version changed; score prints the moments it expects a note on first (where in the frame, and the layers there, when its patch head points held out), then every axis per window with --json, a sheet PNG with --sheet, or the piece second by second in words with --timeline, and with --was OLD whether each of OLD's gating flags is fixed, still there, or looks disguised (nearer the axis's disguised defects than clean work); profile takes the ranges of work you admire (loudness, tilt, cut rate, motion, brightness, contrast, saturation, sharpness), checked leave-one-out, and score --profile says what falls outside them, where and which way; report says each head's held-out score, labels, cheats, gate and what it can't judge; which says the one that judges a piece (its project's, its workspace's, then yours) and why, use pins one there, diff lists the axes two fingerprints rank the same evidence most differently on, and share copies one's taxonomy and evidence (no heads, no media) into a project or workspace for its team; reembed moves a fingerprint's evidence to this machine's encoders, re-embedding the rows whose media it finds; read has a fresh kimi or claude session read a project's pinned notes into kinds and axes, readings puts them to you to check and tallies your answers, and learn-notes makes the defects and praise evidence once you've agreed with 45 kinds and 40 axes in 50, each project's in the fingerprint that judges it; move takes a child's own labels (by axis, source or project) up into a fingerprint it extends, which never reads them otherwise; postmortem scores every version in a project's feed and asks you at most ten yes/no questions in the app, each with its moment playing, where the fingerprint disagrees with your labels or is least sure, and --tally makes your answers evidence and says how each head's held-out score moved; things finds a piece's shots, its things (a render's layers, and the marks drawn on its notes) with their depth, and the camera's path, and says where a thing floats, two pass through each other, the camera lurches mid-move or nothing is on screen, as JSON for `raea check --things` with --json)"; +pub const USAGE: &str = "raea fingerprint new NAME [--who W] [--about TEXT] [--extends FP] [--kinds K,K] [--tags T,T] | raea fingerprint bootstrap FILE... [--from LIST] [--per N] | raea fingerprint teach FILE --axis A --at T [--bad|--good] [--span S] [--box x0,y0,x1,y1] [--words W] [--project P] | raea fingerprint approve FILE... [--project P] | raea fingerprint pair WORSE BETTER --axis A [--at T] [--then T2] | raea fingerprint versions V1 V2... | raea fingerprint profile REFS... --as NAME [--against LIST] | raea fingerprint score FILE [--was OLD] [--ask flags|unsure] [--json] [--sheet out.png] [--timeline] [--profile NAME] | raea fingerprint report [--json] | raea fingerprint which [PATH] [--json] | raea fingerprint use NAME [--in project|workspace] [PATH] | raea fingerprint diff A B [--json] | raea fingerprint share NAME [--in project|workspace] [PATH] | raea fingerprint reembed PLACE... | raea fingerprint read PROJECT [--by kimi|claude] | raea fingerprint readings [PROJECT] [--ask N] [--tally] | raea fingerprint learn-notes [PROJECT] | raea fingerprint move [--fp CHILD] [--to PARENT] [--axis A] [--source S] [--project P] | raea fingerprint postmortem [PROJECT] [--ask N] [--tally] | raea fingerprint things FILE [--project P] [--json] | raea fingerprint stop [--fp NAME] (a person's sense of good work as axes with small heads over on-device encoders, docs/fingerprint.md: new makes ~/.raea/fingerprints/NAME with raea's base taxonomy in fingerprint.ts (or FP's axes with --extends), what it's for, the kinds of piece and project tags it judges, and every axis scoring from its words; bootstrap puts known defects into windows of real work and trains the mechanical axes, held out by source, and patch heads that say where in the frame a picture defect is; teach adds a label at a moment; approve makes approved work clean on every axis, away from its notes; pair says one moment is better than another on an axis (a pick, a hand edit); versions makes weak pairs of noted moments the next version changed; score prints the moments it expects a note on first (where in the frame, and the layers there, when its patch head points held out), then every axis per window with --json, a sheet PNG with --sheet, or the piece second by second in words with --timeline, and with --was OLD whether each of OLD's gating flags is fixed, still there, or looks disguised (nearer the axis's disguised defects than clean work), and with --ask flags each flag put to clef again on its own window, whether clef agrees said beside it and kept as clef's evidence (unsure: the moments it's unsure of too); profile takes the ranges of work you admire (loudness, tilt, cut rate, motion, brightness, contrast, saturation, sharpness), checked leave-one-out, and score --profile says what falls outside them, where and which way; report says each head's held-out score, labels, cheats, gate and what it can't judge; which says the one that judges a piece (its project's, its workspace's, then yours) and why, use pins one there, diff lists the axes two fingerprints rank the same evidence most differently on, and share copies one's taxonomy and evidence (no heads, no media) into a project or workspace for its team; reembed moves a fingerprint's evidence to this machine's encoders, re-embedding the rows whose media it finds; read has a fresh kimi or claude session read a project's pinned notes into kinds and axes, readings puts them to you to check and tallies your answers, and learn-notes makes the defects and praise evidence once you've agreed with 45 kinds and 40 axes in 50, each project's in the fingerprint that judges it; move takes a child's own labels (by axis, source or project) up into a fingerprint it extends, which never reads them otherwise; postmortem scores every version in a project's feed and asks you at most ten yes/no questions in the app, each with its moment playing, where the fingerprint disagrees with your labels or is least sure, and --tally makes your answers evidence and says how each head's held-out score moved; things finds a piece's shots, its things (a render's layers, and the marks drawn on its notes) with their depth, and the camera's path, and says where a thing floats, two pass through each other, the camera lurches mid-move or nothing is on screen, as JSON for `raea check --things` with --json)"; /// Evaluates a .ts file's default export to its JSON (the CLI's spec host) pub type Eval = fn(&Path) -> Result; diff --git a/apps/fingerprint/src/verbs.rs b/apps/fingerprint/src/verbs.rs index c80ba25bc..af770d2d9 100644 --- a/apps/fingerprint/src/verbs.rs +++ b/apps/fingerprint/src/verbs.rs @@ -303,6 +303,16 @@ fn scored(enc: &mut Encoders, args: &[String], cwd: &Path, eval: Eval) -> Result let found = checks(&file); arbitrate(&mut s, &fp, &found); raea_sys_fingerprint_things::located::locate(enc, &file, &raea_sys_fingerprint_use::patch::load(&dir.dir), &mut s)?; + // AK8: each flag asked again of the decision model on its own window, the answers kept as clef's evidence + let doubles = match flag(args, "--ask").map(str::to_string).or_else(|| fp.ask.clone()) { + Some(which) => { + anyhow::ensure!(which == "flags" || which == "unsure", "--ask takes flags or unsure, not {which}"); + let d = raea_app_double::ask(&file, &fp, raea_app_double::moments(&s, &fp, &which), &raea_app_double::Opts::new(4, false)); + raea_app_double::keep(&dir, &c, &project_of(&file), &d).map_err(|e| anyhow!(e))?; + d + } + None => Vec::new(), + }; // FP9a: each gating flag on the cut before, judged on this one, and its axis's disguises rerun on the fix let fixes = match flag(args, "--was") { Some(old) => { @@ -314,7 +324,7 @@ fn scored(enc: &mut Encoders, args: &[String], cwd: &Path, eval: Eval) -> Result None => None, }; let mut out = if has(args, "--json") { - serde_json::to_string_pretty(&json!({"fingerprint": {"name": fp.name, "about": fp.about, "why": why}, "scored": s, "checks": found.iter().map(|(r, a, b, says)| json!({"check": r, "a": a, "b": b, "says": says})).collect::>(), "fixes": fixes}))? + serde_json::to_string_pretty(&json!({"fingerprint": {"name": fp.name, "about": fp.about, "why": why}, "scored": s, "clef": doubles, "checks": found.iter().map(|(r, a, b, says)| json!({"check": r, "a": a, "b": b, "says": says})).collect::>(), "fixes": fixes}))? } else if let Some(f) = &fixes { let mut t = format!("judged by {}: {why}\n", fp.name); t += &raea_sys_fingerprint_use::fixed::said(f); @@ -323,6 +333,9 @@ fn scored(enc: &mut Encoders, args: &[String], cwd: &Path, eval: Eval) -> Result } else { let mut t = format!("judged by {}: {why}\n", fp.name); t += &summary(&s); + if !doubles.is_empty() { + t += &raea_app_double::lines(&doubles); + } t += &format!("checks: {}\n", if found.is_empty() { "no findings".to_string() } else { format!("{} findings", found.len()) }); for (r, a, _, says) in &found { t += &format!(" {} {r}: {says}\n", clock(*a)); diff --git a/docs/ask.feature b/docs/ask.feature index a5fd263dc..d6c712628 100644 --- a/docs/ask.feature +++ b/docs/ask.feature @@ -207,7 +207,7 @@ Feature: Ask: questions about a piece's picture and sound, answered by the decis Rule: AK8. The fingerprint asks too - @needs:AK3 + @needs:AK3 @done Scenario: AK8. A double check on the fingerprint's flags Given the plan | crate | file | lines | job | @@ -222,6 +222,22 @@ Feature: Ask: questions about a piece's picture and sound, answered by the decis And it costs by the flags, never the whole piece And the app's rule_check asks the rules nothing measures, and the fingerprint with `--ask unsure`, before it asks the agent for its own verdicts, so the agent opens only the frames clef flags + Status 11-20: done. raea-app-double (apps/double/src/lib.rs, 165 lines) asks each flag, and with `unsure` each run + of windows an axis puts in its plausible band unflagged, as a yes/no question in the axis's first words (a + quality axis is flagged on no), seeing the picture or hearing the sound as the axis does, on that moment's + window, through raea-app-ask's runner: moments on the same window share a call, and the question carries the + axis's words as its label words, so the person's notes calibrate it (AK6, AK7). A moment it couldn't ask (no + route, the pool spent) says why, and the score goes on. Each answer replaces clef's last `eye:clef` row for that + moment in the ledger. In the app (raea-feed), rule_check scores with `--ask unsure`: a gating flag clef doesn't + see becomes one to look at, not to fix; advice flags clef sees none of hold; a moment the fingerprint was only + unsure of is flagged where clef sees it; the rules nothing measures go to clef in one call, and the agent's + verdict settles any of them. Tests: flags_are_asked_and_unsure_adds_the_plausible_windows_run_together, + an_axis_asks_as_it_senses_and_a_quality_is_flagged_on_no, + each_flag_is_asked_on_its_own_window_and_said_beside_it_and_kept_as_clefs; the feed's rules-tools tests. Real + run: a 10 s render's five flags; clef saw no `dark` on a frame that is plainly lit (the fingerprint had it + gating), agreed it's a computer-generated scene (0.96), $0.0007, and a second run cost nothing (cache). The + same run showed the fingerprint judging sound axes on a render with no sound track (an empty clip's CLAP + vector); fixed apart from AK8, in raea-sys-columns: sound is in a file only when it has levels Rule: AK9. Handing a question to a local head diff --git a/docs/manual/reference/cli/fingerprint.md b/docs/manual/reference/cli/fingerprint.md index d3482b853..dfd6a47d6 100644 --- a/docs/manual/reference/cli/fingerprint.md +++ b/docs/manual/reference/cli/fingerprint.md @@ -2,7 +2,7 @@ title: "raea fingerprint" summary: "A person's sense of good work, learned on this Mac: axes with small heads over on-device encoders that score a render moment by moment and say what they can't judge yet." order: 44 -sources: ["apps/fingerprint/src/lib.rs", "systems/fingerprint/src/taxonomy.rs", "systems/fingerprint-things/src/things.rs"] +sources: ["apps/fingerprint/src/lib.rs", "systems/fingerprint/src/taxonomy.rs", "systems/fingerprint-things/src/things.rs", "apps/double/src/lib.rs"] --- A fingerprint is one person's sense of good work, kept in `~/.raea/fingerprints/NAME`. It is a list of axes (blur, @@ -13,7 +13,11 @@ only what a file has. A new axis judges from its words until it has been trained you point at. A person can keep several, each saying what it's for, and one can build on another with `--extends`. The app's Fingerprint page shows the same thing: what each one knows, what it's still learning, and what it can't judge yet. `postmortem` looks back on a finished project with at most ten questions, and `things` says where a -thing floats or passes through another, the camera lurches or the frame is empty, for `raea check --things`. See +thing floats or passes through another, the camera lurches or the frame is empty, for `raea check --things`. +`score --ask flags` puts each flag to clef again, through `raea ask`, on that moment's window alone. Beside the flag it +says "clef agrees (0.96)" or "clef doesn't see it (0.00)". It costs by the flags, never by the piece, and a render +asked again costs nothing. Clef's answers are kept as its evidence (`eye:clef`), which trains the axis's head but never +counts as your own. See [docs/fingerprint.md](https://github.com/shinyobjectz/raea/blob/main/docs/fingerprint.md) for the plan. ```sh @@ -21,6 +25,7 @@ raea fingerprint new me raea fingerprint new client-ads --about "Short ads for one brand" --extends me raea fingerprint bootstrap renders/*.mp4 --fp client-ads raea fingerprint score cut.mp4 --timeline +raea fingerprint score cut.mp4 --ask flags raea fingerprint teach cut.mp4 --axis blur --at 4.2 --bad raea fingerprint report raea fingerprint postmortem projects/film @@ -32,7 +37,7 @@ raea fingerprint things shot.mp4 --json > things.json ## Usage ```sh -raea fingerprint new NAME [--who W] [--about TEXT] [--extends FP] [--kinds K,K] [--tags T,T] | raea fingerprint bootstrap FILE... [--from LIST] [--per N] | raea fingerprint teach FILE --axis A --at T [--bad|--good] [--span S] [--box x0,y0,x1,y1] [--words W] [--project P] | raea fingerprint approve FILE... [--project P] | raea fingerprint pair WORSE BETTER --axis A [--at T] [--then T2] | raea fingerprint versions V1 V2... | raea fingerprint profile REFS... --as NAME [--against LIST] | raea fingerprint score FILE [--was OLD] [--json] [--sheet out.png] [--timeline] [--profile NAME] | raea fingerprint report [--json] | raea fingerprint which [PATH] [--json] | raea fingerprint use NAME [--in project|workspace] [PATH] | raea fingerprint diff A B [--json] | raea fingerprint share NAME [--in project|workspace] [PATH] | raea fingerprint reembed PLACE... | raea fingerprint read PROJECT [--by kimi|claude] | raea fingerprint readings [PROJECT] [--ask N] [--tally] | raea fingerprint learn-notes [PROJECT] | raea fingerprint move [--fp CHILD] [--to PARENT] [--axis A] [--source S] [--project P] | raea fingerprint postmortem [PROJECT] [--ask N] [--tally] | raea fingerprint things FILE [--project P] [--json] | raea fingerprint stop [--fp NAME] (a person's sense of good work as axes with small heads over on-device encoders, docs/fingerprint.md: new makes ~/.raea/fingerprints/NAME with raea's base taxonomy in fingerprint.ts (or FP's axes with --extends), what it's for, the kinds of piece and project tags it judges, and every axis scoring from its words; bootstrap puts known defects into windows of real work and trains the mechanical axes, held out by source, and patch heads that say where in the frame a picture defect is; teach adds a label at a moment; approve makes approved work clean on every axis, away from its notes; pair says one moment is better than another on an axis (a pick, a hand edit); versions makes weak pairs of noted moments the next version changed; score prints the moments it expects a note on first (where in the frame, and the layers there, when its patch head points held out), then every axis per window with --json, a sheet PNG with --sheet, or the piece second by second in words with --timeline, and with --was OLD whether each of OLD's gating flags is fixed, still there, or looks disguised (nearer the axis's disguised defects than clean work); profile takes the ranges of work you admire (loudness, tilt, cut rate, motion, brightness, contrast, saturation, sharpness), checked leave-one-out, and score --profile says what falls outside them, where and which way; report says each head's held-out score, labels, cheats, gate and what it can't judge; which says the one that judges a piece (its project's, its workspace's, then yours) and why, use pins one there, diff lists the axes two fingerprints rank the same evidence most differently on, and share copies one's taxonomy and evidence (no heads, no media) into a project or workspace for its team; reembed moves a fingerprint's evidence to this machine's encoders, re-embedding the rows whose media it finds; read has a fresh kimi or claude session read a project's pinned notes into kinds and axes, readings puts them to you to check and tallies your answers, and learn-notes makes the defects and praise evidence once you've agreed with 45 kinds and 40 axes in 50, each project's in the fingerprint that judges it; move takes a child's own labels (by axis, source or project) up into a fingerprint it extends, which never reads them otherwise; postmortem scores every version in a project's feed and asks you at most ten yes/no questions in the app, each with its moment playing, where the fingerprint disagrees with your labels or is least sure, and --tally makes your answers evidence and says how each head's held-out score moved; things finds a piece's shots, its things (a render's layers, and the marks drawn on its notes) with their depth, and the camera's path, and says where a thing floats, two pass through each other, the camera lurches mid-move or nothing is on screen, as JSON for `raea check --things` with --json) +raea fingerprint new NAME [--who W] [--about TEXT] [--extends FP] [--kinds K,K] [--tags T,T] | raea fingerprint bootstrap FILE... [--from LIST] [--per N] | raea fingerprint teach FILE --axis A --at T [--bad|--good] [--span S] [--box x0,y0,x1,y1] [--words W] [--project P] | raea fingerprint approve FILE... [--project P] | raea fingerprint pair WORSE BETTER --axis A [--at T] [--then T2] | raea fingerprint versions V1 V2... | raea fingerprint profile REFS... --as NAME [--against LIST] | raea fingerprint score FILE [--was OLD] [--ask flags|unsure] [--json] [--sheet out.png] [--timeline] [--profile NAME] | raea fingerprint report [--json] | raea fingerprint which [PATH] [--json] | raea fingerprint use NAME [--in project|workspace] [PATH] | raea fingerprint diff A B [--json] | raea fingerprint share NAME [--in project|workspace] [PATH] | raea fingerprint reembed PLACE... | raea fingerprint read PROJECT [--by kimi|claude] | raea fingerprint readings [PROJECT] [--ask N] [--tally] | raea fingerprint learn-notes [PROJECT] | raea fingerprint move [--fp CHILD] [--to PARENT] [--axis A] [--source S] [--project P] | raea fingerprint postmortem [PROJECT] [--ask N] [--tally] | raea fingerprint things FILE [--project P] [--json] | raea fingerprint stop [--fp NAME] (a person's sense of good work as axes with small heads over on-device encoders, docs/fingerprint.md: new makes ~/.raea/fingerprints/NAME with raea's base taxonomy in fingerprint.ts (or FP's axes with --extends), what it's for, the kinds of piece and project tags it judges, and every axis scoring from its words; bootstrap puts known defects into windows of real work and trains the mechanical axes, held out by source, and patch heads that say where in the frame a picture defect is; teach adds a label at a moment; approve makes approved work clean on every axis, away from its notes; pair says one moment is better than another on an axis (a pick, a hand edit); versions makes weak pairs of noted moments the next version changed; score prints the moments it expects a note on first (where in the frame, and the layers there, when its patch head points held out), then every axis per window with --json, a sheet PNG with --sheet, or the piece second by second in words with --timeline, and with --was OLD whether each of OLD's gating flags is fixed, still there, or looks disguised (nearer the axis's disguised defects than clean work), and with --ask flags each flag put to clef again on its own window, whether clef agrees said beside it and kept as clef's evidence (unsure: the moments it's unsure of too); profile takes the ranges of work you admire (loudness, tilt, cut rate, motion, brightness, contrast, saturation, sharpness), checked leave-one-out, and score --profile says what falls outside them, where and which way; report says each head's held-out score, labels, cheats, gate and what it can't judge; which says the one that judges a piece (its project's, its workspace's, then yours) and why, use pins one there, diff lists the axes two fingerprints rank the same evidence most differently on, and share copies one's taxonomy and evidence (no heads, no media) into a project or workspace for its team; reembed moves a fingerprint's evidence to this machine's encoders, re-embedding the rows whose media it finds; read has a fresh kimi or claude session read a project's pinned notes into kinds and axes, readings puts them to you to check and tallies your answers, and learn-notes makes the defects and praise evidence once you've agreed with 45 kinds and 40 axes in 50, each project's in the fingerprint that judges it; move takes a child's own labels (by axis, source or project) up into a fingerprint it extends, which never reads them otherwise; postmortem scores every version in a project's feed and asks you at most ten yes/no questions in the app, each with its moment playing, where the fingerprint disagrees with your labels or is least sure, and --tally makes your answers evidence and says how each head's held-out score moved; things finds a piece's shots, its things (a render's layers, and the marks drawn on its notes) with their depth, and the camera's path, and says where a thing floats, two pass through each other, the camera lurches mid-move or nothing is on screen, as JSON for `raea check --things` with --json) ``` Headless: it needs nothing running and opens no window. Takes any number of arguments besides its flags. @@ -44,7 +49,7 @@ Headless: it needs nothing running and opens no window. Takes any number of argu | `--about` | `TEXT` | What this fingerprint is for, in a sentence, shown beside its name in the app | | `--against` | `LIST` | With profile: a file listing unrelated work, one path a line, that the references must fit their own ranges better than | | `--as` | `NAME` | With profile: the name the profile is kept under, for score --profile | -| `--ask` | `N` | With readings: put N of the readings to you in the app, one question each, to check before notes train; with postmortem: ask at most N (10 at most) | +| `--ask` | `flags\|unsure` | With score: put each flag to clef again, on that moment's window alone, and say beside it whether clef agrees (`unsure`: also the moments an axis puts in its plausible band without flagging); the answers are kept as clef's evidence, never yours, and `ask: "flags"` in fingerprint.ts does it on every score. With readings (`--ask N`): put N of the readings to you in the app, one question each, to check before notes train; with postmortem (`--ask N`): ask at most N (10 at most) | | `--at` | `T` | The moment a label is for, in seconds | | `--axis` | `A` | The axis a label is for, by name, such as blur or clip | | `--bad` | – | The moment has the problem the axis names (the default) | diff --git a/sdk/raea/src/fingerprint.ts b/sdk/raea/src/fingerprint.ts index 4b2b00118..6a3cbe8c2 100644 --- a/sdk/raea/src/fingerprint.ts +++ b/sdk/raea/src/fingerprint.ts @@ -44,6 +44,9 @@ export interface Fingerprint { extends?: string; /** Which pieces it judges when its project or workspace holds more than one. */ applies?: Applies; + /** Its flags asked again of the decision model as it scores: "flags", or "unsure" for the moments it's unsure of too + * (`raea fingerprint score --ask`). Costs by the flags; each answer is clef's evidence, never yours. */ + ask?: "flags" | "unsure"; /** What it notices, each axis with its words and the sense its head reads. */ axes: Axis[]; /** Each encoder's id and its file's sha256: heads trained on other encoders' columns are refused. */ diff --git a/systems/fingerprint/src/taxonomy.rs b/systems/fingerprint/src/taxonomy.rs index 95bf27e2d..d4d608c1e 100644 --- a/systems/fingerprint/src/taxonomy.rs +++ b/systems/fingerprint/src/taxonomy.rs @@ -54,6 +54,10 @@ pub struct Fingerprint { /// which pieces it judges when a project or workspace holds more than one (FP14c) #[serde(default, skip_serializing_if = "Option::is_none")] pub applies: Option, + /// its flags asked again of the decision model as it scores (docs/ask.feature AK8): "flags", or "unsure" for the + /// moments it's unsure of too + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ask: Option, pub axes: Vec, /// each encoder's id and its file's sha256 (FP2a): heads trained on other encoders' columns are refused #[serde(default)] @@ -120,7 +124,7 @@ pub fn base(name: &str) -> Fingerprint { ax("design and type", "unreadable text", Picture, &["small, cluttered text that is hard to read"], None, true), ax("camera", "shaky camera", Change, &["a shaky handheld camera"], None, false), ]; - Fingerprint { name: name.into(), who: None, about: None, extends: None, applies: None, axes, encoders: Default::default() } + Fingerprint { name: name.into(), who: None, about: None, extends: None, applies: None, ask: None, axes, encoders: Default::default() } } /// The background vocabularies a word head is calibrated against (earshot's method: an axis's similarity as a z diff --git a/tools/layers.toml b/tools/layers.toml index f33ae8d8b..3d868dfe1 100644 --- a/tools/layers.toml +++ b/tools/layers.toml @@ -195,6 +195,7 @@ raea-app-engines = "apps" raea-app-interchange = "apps" raea-app-judge = "apps" raea-app-ask = "apps" +raea-app-double = "apps" raea-app-solid = "apps" raea-app-media = "apps" raea-app-targets = "apps"