diff --git a/Cargo.lock b/Cargo.lock index bb598681a..61149fb45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13752,6 +13752,20 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "019b4b213425016d7d84a153c4c73afb0946fbb4840e4eece7ba8848b9d6da22" +[[package]] +name = "raea-app-ask" +version = "0.1.0" +dependencies = [ + "dunce", + "raea-core-ask", + "raea-io-assets", + "raea-io-clef", + "raea-io-home", + "serde", + "serde_json", + "sha2 0.10.9", +] + [[package]] name = "raea-app-capcut" version = "0.1.0" @@ -13902,14 +13916,13 @@ dependencies = [ name = "raea-app-judge" version = "0.1.0" dependencies = [ - "base64 0.22.1", "dunce", - "image", + "raea-core-ask", "raea-core-facts", + "raea-io-clef", "raea-io-home", "raea-runtime-bundle", "serde_json", - "ureq 2.12.1", ] [[package]] @@ -13996,6 +14009,7 @@ name = "raea-app-solid" version = "0.1.0" dependencies = [ "dunce", + "raea-app-ask", "raea-app-judge", "raea-core-facts", "raea-core-meshcheck", @@ -14269,6 +14283,7 @@ version = "0.1.0" dependencies = [ "bevy", "dunce", + "raea-app-ask", "raea-app-capcut", "raea-app-check", "raea-app-ci", @@ -14327,6 +14342,15 @@ dependencies = [ "serde_json", ] +[[package]] +name = "raea-core-ask" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "ts-rs", +] + [[package]] name = "raea-core-brief" version = "0.1.0" @@ -14557,6 +14581,20 @@ dependencies = [ "serde_json", ] +[[package]] +name = "raea-io-clef" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "raea-core-ask", + "raea-io-assets", + "raea-io-ffmpeg", + "raea-io-home", + "serde_json", + "sha2 0.10.9", + "ureq 2.12.1", +] + [[package]] name = "raea-io-codec" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index a9c85e6e7..18a78ac38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ resolver = "2" members = [ "core/animate", + "core/ask", "core/brief", "core/facts", "core/ease", @@ -61,6 +62,7 @@ members = [ "io/codec", "io/media", "io/assets", + "io/clef", "io/gen", "io/gen-comfy", "io/gen-core", @@ -186,6 +188,7 @@ members = [ "runtime/scene", "runtime/scene2d", "runtime/shared", + "apps/ask", "apps/check", "apps/check/flash", "apps/check/inbox", diff --git a/apps/ask/Cargo.toml b/apps/ask/Cargo.toml new file mode 100644 index 000000000..e88f68a29 --- /dev/null +++ b/apps/ask/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "raea-app-ask" +version = "0.1.0" +edition = "2024" +license = "FSL-1.1-ALv2" +publish = false +description = "raea ask: questions about a piece's picture and sound, prepared from what raea knows of it, asked of clef in parallel from the person's plan, and every answer kept" + +[dependencies] +# the question format and the standard sets; the one clef client, routed to the person's plan first (docs/ask.feature) +raea-core-ask = { path = "../../core/ask" } +raea-io-clef = { path = "../../io/clef" } +raea-io-home = { path = "../../io/home" } +dunce = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" + +[dev-dependencies] +raea-io-assets = { path = "../../io/assets" } diff --git a/apps/ask/src/lib.rs b/apps/ask/src/lib.rs new file mode 100644 index 000000000..3cd85e820 --- /dev/null +++ b/apps/ask/src/lib.rs @@ -0,0 +1,14 @@ +//! `raea ask` (docs/ask.feature AK2-AK3): questions about a piece's picture and sound. `prepare` reads what raea knows +//! of the piece and keeps the questions that apply; `runner` plans the calls, cuts each clip once and asks in parallel; +//! `record` keeps every call and answer for labels and calibration; `verbs` is the command. `raea check`, the app's +//! critique and the fingerprint's double-check go through the same runner. + +pub mod prepare; +pub mod record; +pub mod runner; +pub mod verbs; + +pub use verbs::{USAGE, cli}; + +#[cfg(test)] +mod tests; diff --git a/apps/ask/src/prepare.rs b/apps/ask/src/prepare.rs new file mode 100644 index 000000000..02c04b74d --- /dev/null +++ b/apps/ask/src/prepare.rs @@ -0,0 +1,164 @@ +//! Ask understands a question before it spends (docs/ask.feature AK2): what the file holds (ffprobe), what the project's +//! brief says it's for, whether anyone speaks in it, and from those which questions can be asked, which are swapped for +//! their variant (sound for sound-silent) and which are dropped, each with why. The state paragraph tells clef what the +//! piece is, as the app's critique writes it. When it isn't known whether anyone speaks, both variants are kept, the +//! voice question goes in the same call (runner.rs), free, and the variant that doesn't apply is set aside. + +use std::path::{Path, PathBuf}; + +use raea_core_ask::{Facts, Question, library}; +use raea_io_clef::media::{self, Probe}; +use serde_json::{Value, json}; + +/// The question asked beside the rest when no one knows yet whether anyone speaks in the piece. +pub const VOICE: &str = "voice-heard"; + +/// What raea knows of the piece. +#[derive(Debug, Clone, Default)] +pub struct Piece { + pub file: PathBuf, + pub probe: Probe, + /// the brief's fields, as `raea spec brief.ts --as brief` reads them ({} without a brief) + pub brief: Value, + /// the project's folder (the one holding the brief), when there is one + pub project: Option, + /// the words heard, when a words file was given + pub words: Option, + /// someone speaks: from --voice, or from the words file (none heard is no) + pub voice: Option, + /// what the caller adds to the state (what raea measured, the platform) + pub extra: Option, +} + +/// The brief nearest the file: brief.ts in its folder or any folder above it. +pub fn brief_near(file: &Path) -> Option { + let abs = dunce::canonicalize(file).ok()?; + abs.ancestors().skip(1).map(|d| d.join("brief.ts")).find(|b| b.is_file()) +} + +/// A brief's fields, through `raea spec FILE --as brief` (this raea), or {} when it can't be read. +fn brief_of(path: &Path) -> Value { + let Ok(exe) = std::env::current_exe() else { return json!({}) }; + let Ok(out) = std::process::Command::new(exe).args(["spec"]).arg(path).args(["--as", "brief"]).output() else { return json!({}) }; + let text = String::from_utf8_lossy(&out.stdout); + let v: Value = text.find('{').and_then(|i| serde_json::from_str(&text[i..]).ok()).unwrap_or_else(|| json!({})); + v.get("brief").cloned().unwrap_or(v) +} + +/// The words heard, from a words file (a JSON list of words, each a string or {word} or {text}): their text, and +/// whether anyone speaks (an empty list is no one). +pub fn words_of(path: &Path) -> Result<(String, bool), String> { + let v: Value = serde_json::from_str(&std::fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?).map_err(|e| format!("{}: {e}", path.display()))?; + let list = v.as_array().or(v["words"].as_array()).ok_or_else(|| format!("{}: a list of words", path.display()))?; + let words: Vec<&str> = list.iter().filter_map(|w| w.as_str().or(w["word"].as_str()).or(w["text"].as_str())).collect(); + Ok((words.join(" "), !words.is_empty())) +} + +impl Piece { + /// The piece as raea finds it: the file probed, its brief (given or the nearest), its words. + pub fn read(file: &Path, brief: Option<&Path>, words: Option<&Path>, voice: Option, extra: Option) -> Result { + let probe = media::probe(file)?; + // "none": no brief, even with one near (a caller that writes the whole state itself, as the app's critique does) + let brief_file = match brief { + Some(p) if p.as_os_str() == "none" => None, + Some(p) => Some(p.to_path_buf()), + None => brief_near(file), + }; + let (words, heard) = match words { + Some(w) => words_of(w).map(|(t, h)| (Some(t).filter(|t| !t.is_empty()), Some(h)))?, + None => (None, None), + }; + Ok(Piece { + file: file.to_path_buf(), + probe, + brief: brief_file.as_deref().map(brief_of).unwrap_or_else(|| json!({})), + project: brief_file.and_then(|b| b.parent().map(Path::to_path_buf)), + words, + voice: voice.or(heard).or((!probe.sound).then_some(false)), + extra, + }) + } + + pub fn modes(&self) -> Vec { + let m = &self.brief["mode"]; + let list: Vec<&str> = m.as_array().map(|a| a.iter().filter_map(Value::as_str).collect()).unwrap_or_else(|| m.as_str().into_iter().collect()); + list.iter().flat_map(|s| s.split(',')).map(|s| s.replace("(assumed)", "").trim().to_string()).filter(|s| !s.is_empty()).collect() + } + + pub fn facts(&self) -> Facts { + let modes = self.modes(); + Facts { game: modes.iter().any(|m| m == "play"), modes, voice: self.voice, sound: self.probe.sound, video: self.probe.video } + } + + /// What clef reads before the media: what the piece is and what's known of it. + pub fn state(&self) -> String { + let b = &self.brief; + let modes = self.modes(); + let what = if modes.is_empty() { "video".to_string() } else { modes.join(" and ") }; + // with no brief and the caller's own state, the caller says what the piece is + let briefless = b.as_object().is_none_or(|o| o.is_empty()); + let mut s = match (b["title"].as_str(), briefless && self.extra.is_some()) { + (_, true) => vec![], + (Some(t), _) => vec![format!("A {what} piece: \"{t}\".")], + (None, _) => vec![format!("A {what} piece.")], + }; + for (k, say) in [("audience", "It's for"), ("believe", "It should leave them believing"), ("logline", "Its storyboard"), ("voice", "Its voice")] { + if let Some(v) = b[k].as_str().filter(|v| !v.trim().is_empty()) { + s.push(format!("{say}: {}.", v.trim().trim_end_matches('.'))); + } + } + if self.voice == Some(false) { + s.push("No one speaks in it: its sound is the music and the effects, with no voice.".into()); + } + if let Some(w) = &self.words { + let cut: String = w.chars().take(1200).collect(); + s.push(format!("The words heard: \"{cut}{}\"", if cut.len() < w.len() { "..." } else { "" })); + } + if self.probe.seconds > 0.0 { + s.push(format!("It runs {:.0} s.", self.probe.seconds)); + } + if let Some(x) = &self.extra { + s.push(x.trim().to_string()); + } + s.join(" ") + } +} + +/// Is anyone heard speaking: asked beside the questions that depend on it, at no cost (clef costs by media, not by +/// questions), when no one knows yet. +pub fn voice_question() -> Question { + let mut v = Question::plain(VOICE, "Is anyone heard speaking in it (a voice saying words, not singing)?"); + v.sees = raea_core_ask::Sees::Nothing; + v +} + +/// The questions kept for this piece, and those dropped with why. A question that doesn't apply is swapped for its +/// variant when that one does; not knowing whether anyone speaks, both are kept. +pub fn keep(questions: Vec, f: &Facts, rubric_md: Option<&str>) -> (Vec, Vec<(String, String)>) { + let mut asked: Vec = Vec::new(); + let mut dropped = Vec::new(); + let add = |q: Question, asked: &mut Vec| { + if !asked.iter().any(|a| a.id == q.id) { + asked.push(q); + } + }; + for q in questions { + let Some(why) = q.not_for(f) else { + // not knowing whether anyone speaks, its variant goes in too: one call answers either way + let twin = (f.voice.is_none() && q.when.voice.is_some()).then(|| q.swap.as_deref().and_then(|s| library::find(s, rubric_md))).flatten(); + add(q, &mut asked); + if let Some(t) = twin { + add(t, &mut asked); + } + continue; + }; + match q.swap.as_deref().and_then(|s| library::find(s, rubric_md)).filter(|s| s.not_for(f).is_none()) { + Some(s) => { + dropped.push((q.id.clone(), format!("{why}: asked {} instead", s.id))); + add(s, &mut asked); + } + None => dropped.push((q.id.clone(), why)), + } + } + (asked, dropped) +} diff --git a/apps/ask/src/record.rs b/apps/ask/src/record.rs new file mode 100644 index 000000000..be274d41a --- /dev/null +++ b/apps/ask/src/record.rs @@ -0,0 +1,103 @@ +//! Every answer kept (docs/ask.feature AK3), so a question can be labelled, calibrated and reworded later (AK6, AK7): +//! each call in calls.jsonl (the file, the window, the state, the clips it sent by hash, the route and the cost) and each +//! answer in answers.jsonl (the call, the question and its version, the reading). The clips themselves are kept by hash +//! in media/, at most 2 GB, the oldest out first, so new words can be asked of the same media. + +use std::io::Write as _; +use std::path::PathBuf; + +use raea_core_ask::{Question, Reading}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; + +/// How much cut media is kept. +pub const MEDIA_CAP: u64 = 2 << 30; + +pub fn dir() -> PathBuf { + raea_io_clef::cache::dir() +} + +/// A new id: a prefix and twelve hex digits, from the time, the process and a counter. +pub fn new_id(prefix: &str) -> String { + static N: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let t = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_nanos()).unwrap_or(0); + let seed = format!("{t}-{}-{}", std::process::id(), N.fetch_add(1, std::sync::atomic::Ordering::Relaxed)); + let h = seed.bytes().fold(0xcbf2_9ce4_8422_2325u64, |h, b| (h ^ b as u64).wrapping_mul(0x0100_0000_01b3)); + format!("{prefix}-{:012x}", h & 0xffff_ffff_ffff) +} + +fn now() -> String { + let s = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0); + s.to_string() +} + +fn append(file: &str, row: &Value) { + let d = dir(); + if std::fs::create_dir_all(&d).is_err() { + return; + } + if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(d.join(file)) { + let _ = writeln!(f, "{row}"); + } +} + +/// A clip kept by its hash, as `HASH.EXT` (already kept is left alone). +pub fn keep_media(bytes: &[u8], ext: &str) -> String { + let hash: String = Sha256::digest(bytes).iter().take(16).map(|b| format!("{b:02x}")).collect(); + let name = format!("{hash}.{ext}"); + let d = dir().join("media"); + let p = d.join(&name); + if !p.is_file() && std::fs::create_dir_all(&d).is_ok() { + let _ = std::fs::write(&p, bytes); + prune(MEDIA_CAP); + } + name +} + +/// A kept clip's bytes, by its name. +pub fn media(name: &str) -> Option> { + std::fs::read(dir().join("media").join(name)).ok() +} + +/// The kept media held to `cap` bytes, the least recently written going first. +pub fn prune(cap: u64) { + let Ok(rd) = std::fs::read_dir(dir().join("media")) else { return }; + let mut files: Vec<(std::time::SystemTime, u64, PathBuf)> = rd + .flatten() + .filter_map(|e| e.metadata().ok().and_then(|m| Some((m.modified().ok()?, m.len(), e.path())))) + .collect(); + let mut total: u64 = files.iter().map(|f| f.1).sum(); + files.sort(); + for (_, len, p) in files { + if total <= cap { + break; + } + if std::fs::remove_file(&p).is_ok() { + total -= len; + } + } +} + +/// One call kept; its id. +#[allow(clippy::too_many_arguments)] +pub fn call(file: &str, project: Option<&str>, from: f64, to: Option, state: &str, media: &[String], route: &str, cost: f64, cached: bool) -> String { + let id = new_id("c"); + append("calls.jsonl", &json!({"id": id, "at": now(), "file": file, "project": project, "from": from, "to": to, "state": state, "media": media, "route": route, "cost": cost, "cached": cached})); + id +} + +/// One answer kept; its id. +pub fn answer(call: &str, file: &str, q: &Question, r: &Reading, flagged: bool) -> String { + let id = new_id("a"); + let row = json!({ + "id": id, "at": now(), "call": call, "file": file, "question": q.id, "version": q.version(), "ask": q.ask, + "value": r.value, "problem": r.problem, "choice": r.choice, "says": r.says, "flagged": flagged, "calibrated": q.calibrated(), + }); + append("answers.jsonl", &row); + id +} + +/// Every row of one of the record's files. +pub fn rows(file: &str) -> Vec { + std::fs::read_to_string(dir().join(file)).unwrap_or_default().lines().filter_map(|l| serde_json::from_str(l).ok()).collect() +} diff --git a/apps/ask/src/runner.rs b/apps/ask/src/runner.rs new file mode 100644 index 000000000..4dfa4bffb --- /dev/null +++ b/apps/ask/src/runner.rs @@ -0,0 +1,342 @@ +//! One runner for every caller (docs/ask.feature AK3): a batch planned into calls (grouped by file, window, the files +//! beside it and what of the picture its questions look at, at most 64 questions a call, the voice question added to a +//! call whose questions depend on whether anyone speaks when that isn't known), each clip cut once, the calls +//! made in parallel, the answers streamed back as they come and kept. A pool that's spent partway keeps what came and +//! reports the rest as waiting for it. `dry` cuts the media and says what would be sent, and spends nothing. + +use std::collections::{BTreeSet, HashMap}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, mpsc}; + +use raea_core_ask::{Kind, Picks, Question, Sees, Window, limits}; +use raea_io_clef::{Answered, Call, Fail, Media, media}; +use serde::Serialize; +use serde_json::{Value, json}; + +use crate::prepare::{self, Piece, VOICE}; +use crate::record; + +/// What to ask of one file: its questions (prepared), those dropped with why, a window over all of them (else each +/// question's own), and files set beside it (to pick between, or before and after). +pub struct Entry { + pub piece: Arc, + pub window: Option, + pub questions: Vec, + pub dropped: Vec<(String, String)>, + pub with: Vec, +} + +/// One call's worth. +pub struct Group { + pub piece: Arc, + pub from: f64, + pub to: Option, + pub sees: Sees, + pub hears: bool, + pub with: Vec, + pub questions: Vec, +} + +/// One answer, or why a question wasn't asked or still waits: the line `raea ask --json` prints for each. +#[derive(Debug, Clone, Default, Serialize)] +pub struct Answer { + /// its id in the record (`raea ask verdict` takes it) + pub answer: Option, + pub file: String, + pub id: String, + pub version: String, + pub from: f64, + pub to: Option, + pub says: Option, + pub value: Option, + pub problem: Option, + pub choice: Option, + pub flagged: bool, + pub calibrated: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub not_asked: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub waits: Option, + pub route: Option, + pub cached: bool, +} + +/// What the run came to. +#[derive(Debug, Clone, Default, Serialize)] +pub struct Summary { + pub asked: usize, + pub answered: usize, + pub calls: usize, + pub cost: f64, + pub estimate: f64, + pub pool_left: Option, + pub waiting: usize, + pub flagged: usize, + pub flagged_calibrated: usize, + pub routes: BTreeSet, + /// the clips cut (each once, however many calls used it) + pub clips: usize, +} + +/// What the runner tells its caller as it goes: an answer, or (dry) a call it would make. +pub enum Event<'a> { + Answer(&'a Answer), + Would(&'a Value), +} + +pub struct Opts { + pub jobs: usize, + pub dry: bool, + /// how a call is made: `raea_io_clef::ask` (a test gives its own) + pub ask: fn(&Call) -> Result, +} + +impl Opts { + pub fn new(jobs: usize, dry: bool) -> Opts { + Opts { jobs, dry, ask: raea_io_clef::ask } + } +} + +const NAMES: [&str; 4] = ["first", "second", "third", "fourth"]; + +/// A pick's options: the files given, in order (all, or all but the first, the reference). +fn picks(p: Picks, files: usize) -> Vec { + let n = files.min(NAMES.len()); + let skip = usize::from(p == Picks::AfterFirst); + NAMES[skip.min(n)..n].iter().map(|s| s.to_string()).collect() +} + +/// The batch as calls. +pub fn plan(entries: &[Entry]) -> (Vec, Vec) { + let mut groups: Vec = Vec::new(); + let mut unasked = Vec::new(); + for e in entries { + let file = e.piece.file.display().to_string(); + for (id, why) in &e.dropped { + unasked.push(Answer { file: file.clone(), id: id.clone(), not_asked: Some(why.clone()), ..Default::default() }); + } + for q in &e.questions { + let w = e.window.or(q.window).unwrap_or_default(); + let (from, to) = (w.from.unwrap_or(0.0).max(0.0), w.to); + let mut q = q.clone(); + if let Some(p) = q.picks { + q.kind = Kind::Choose; + q.options = picks(p, 1 + e.with.len()); + } + let compares = q.picks.is_some_and(|_| q.options.len() < 2) || (q.uses.iter().any(|u| u == "with") && e.with.is_empty()); + if compares { + unasked.push(Answer { file: file.clone(), version: q.version(), not_asked: Some("it compares files: set one or more beside it (--with)".into()), id: q.id, ..Default::default() }); + continue; + } + // one place in each call is kept for the voice question + let same = |g: &&mut Group| Arc::ptr_eq(&g.piece, &e.piece) && g.from == from && g.to == to && g.sees == q.sees && g.with == e.with && g.questions.len() < limits::QUESTIONS - 1; + match groups.iter_mut().find(same) { + Some(g) => { + g.hears |= q.hears; + g.questions.push(q); + } + None => groups.push(Group { piece: e.piece.clone(), from, to, sees: q.sees, hears: q.hears, with: e.with.clone(), questions: vec![q] }), + } + } + } + for g in &mut groups { + g.hears &= g.piece.probe.sound || g.with.iter().any(|f| media::probe(f).is_ok_and(|p| p.sound)); + if g.hears && g.piece.voice.is_none() && g.questions.iter().any(|q| q.when.voice.is_some()) { + g.questions.push(prepare::voice_question()); + } + } + (groups, unasked) +} + +/// Clips cut once a run, by what they are. +type Cuts = Mutex>, String>>>; + +fn cut(cuts: &Cuts, key: String, make: impl FnOnce() -> Result, String>) -> Result>, String> { + if let Some(r) = cuts.lock().unwrap().get(&key) { + return r.clone(); + } + let r = make().map(Arc::new); + cuts.lock().unwrap().insert(key, r.clone()); + r +} + +/// A group's media, and its seconds of video and of sound (for the estimate). +fn media_of(g: &Group, cuts: &Cuts) -> Result<(Media, f64, f64), String> { + let files: Vec<&Path> = std::iter::once(g.piece.file.as_path()).chain(g.with.iter().map(PathBuf::as_path)).collect(); + let (from, to) = (g.from, g.to); + let span = |p: &media::Probe| (to.unwrap_or(p.seconds) - from).clamp(0.0, p.seconds.max(0.1)); + let mut m = Media::default(); + let (mut vs, mut ss) = (0.0, 0.0); + for f in &files { + let p = if *f == g.piece.file.as_path() { g.piece.probe } else { media::probe(f)? }; + let k = |what: &str| format!("{what}:{}:{from}:{to:?}", f.display()); + match g.sees { + Sees::Video if p.video => { + m.videos.push(cut(cuts, k("video"), || media::video(f, from, to))?.to_vec()); + vs += span(&p).min(limits::VIDEO_SECONDS); + } + Sees::Video | Sees::Stills if !p.video => m.stills.push(cut(cuts, k("still"), || media::still(f, None))?.to_vec()), + Sees::Stills if files.len() == 1 => { + let len = span(&p); + for i in 0..limits::STILLS { + let at = from + len * (i as f64 + 0.5) / limits::STILLS as f64; + m.stills.push(cut(cuts, k(&format!("frame{i}")), || media::still(f, Some(at)))?.to_vec()); + } + } + Sees::Stills => m.stills.push(cut(cuts, k("frame"), || media::still(f, Some(from + span(&p) / 2.0)))?.to_vec()), + _ => {} + } + if g.hears && p.sound { + m.sounds.push(cut(cuts, k("sound"), || media::sound(f, from, to))?.to_vec()); + ss += span(&p).min(limits::SOUND_SECONDS); + } + } + if m.videos.len() > limits::VIDEOS || m.stills.len() > limits::STILLS || m.sounds.len() > limits::SOUNDS { + return Err(format!("too many files for one call: at most {} videos, {} stills and {} sounds", limits::VIDEOS, limits::STILLS, limits::SOUNDS)); + } + Ok((m, vs, ss)) +} + +enum Done { + Asked(Answered, Vec, String), + Would(Value), + Failed(String), + Waits(String), +} + +/// Every entry asked, at most `jobs` calls at once, each event told to `out` as it comes (in the caller's thread). +pub fn run(entries: Vec, o: &Opts, out: &mut dyn FnMut(Event)) -> Summary { + let (groups, unasked) = plan(&entries); + let mut sum = Summary::default(); + for a in &unasked { + out(Event::Answer(a)); + } + let cuts: Cuts = Mutex::default(); + let (next, spent) = (AtomicUsize::new(0), AtomicBool::new(false)); + let spent_why = Mutex::new(String::new()); + let (tx, rx) = mpsc::channel::<(usize, Done)>(); + std::thread::scope(|s| { + for _ in 0..o.jobs.clamp(1, 16).min(groups.len().max(1)) { + let (tx, groups, cuts, next, spent, spent_why) = (tx.clone(), &groups, &cuts, &next, &spent, &spent_why); + s.spawn(move || { + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + let Some(g) = groups.get(i) else { break }; + let done = if spent.load(Ordering::Relaxed) { + Done::Waits(spent_why.lock().unwrap().clone()) + } else { + match media_of(g, cuts) { + Err(e) => Done::Failed(e), + Ok((m, vs, ss)) => { + let call = Call { state: g.piece.state(), questions: g.questions.clone(), media: m }; + let estimate = limits::estimate(vs, ss, call.media.stills.len(), call.state.len(), call.questions.len()); + if o.dry { + Done::Would(would(g, &call, estimate)) + } else { + match (o.ask)(&call) { + Ok(a) => Done::Asked(a, kept(&call.media), call.state), + Err(Fail::Spent(why)) => { + spent.store(true, Ordering::Relaxed); + *spent_why.lock().unwrap() = why.clone(); + Done::Waits(why) + } + Err(f) => Done::Failed(f.to_string()), + } + } + } + } + }; + if tx.send((i, done)).is_err() { + break; + } + } + }); + } + drop(tx); + for (i, done) in rx { + told(&groups[i], done, &mut sum, out); + } + }); + sum.clips = cuts.lock().unwrap().values().filter(|c| c.is_ok()).count(); + sum +} + +/// The clips a call sent, kept by hash for `raea ask try`. +fn kept(m: &Media) -> Vec { + let each = |v: &[Vec], ext: &str| v.iter().map(|b| record::keep_media(b, ext)).collect::>(); + [each(&m.videos, "mp4"), each(&m.stills, "jpg"), each(&m.sounds, "mp3")].concat() +} + +fn would(g: &Group, call: &Call, estimate: f64) -> Value { + let sizes = |v: &[Vec]| v.iter().map(Vec::len).collect::>(); + json!({ + "file": g.piece.file.display().to_string(), "from": g.from, "to": g.to, + "questions": call.questions.iter().map(|q| json!({"id": q.id, "ask": q.ask, "type": q.kind})).collect::>(), + "state": call.state, + "media": {"videos": sizes(&call.media.videos), "stills": sizes(&call.media.stills), "sounds": sizes(&call.media.sounds)}, + "estimate_usd": estimate, + }) +} + +/// A finished call's answers told, kept and counted. When the voice question went along, the variant that doesn't +/// apply (sound where no one speaks, sound-silent where someone does) is set aside. +fn told(g: &Group, done: Done, sum: &mut Summary, out: &mut dyn FnMut(Event)) { + let file = g.piece.file.display().to_string(); + // the voice question only decides between variants: it isn't told + let asked: Vec<&Question> = g.questions.iter().filter(|q| q.id != VOICE).collect(); + let base = |q: &Question| Answer { file: file.clone(), id: q.id.clone(), version: q.version(), from: g.from, to: g.to, calibrated: q.calibrated(), ..Default::default() }; + match done { + Done::Would(v) => { + sum.calls += 1; + sum.estimate += v["estimate_usd"].as_f64().unwrap_or(0.0); + out(Event::Would(&v)); + } + Done::Failed(why) | Done::Waits(why) if g.questions.is_empty() => drop(why), + Done::Failed(why) => asked.iter().for_each(|q| out(Event::Answer(&Answer { not_asked: Some(why.clone()), ..base(q) }))), + Done::Waits(why) => { + sum.waiting += asked.len(); + asked.iter().for_each(|q| out(Event::Answer(&Answer { waits: Some(format!("waits for the pool: {why}")), ..base(q) }))); + } + Done::Asked(a, clips, state) => { + sum.calls += 1; + sum.asked += asked.len(); + sum.cost += a.cost; + sum.pool_left = a.pool_left.or(sum.pool_left); + sum.routes.insert(a.route.clone()); + let project = g.piece.project.as_ref().map(|p| p.display().to_string()); + let call = record::call(&file, project.as_deref(), g.from, g.to, &state, &clips, &a.route, a.cost, a.cached); + let heard = a.answers.get(VOICE).and_then(|v| v["noul"].as_f64()); + for q in g.questions.iter().filter(|q| q.id != VOICE) { + let Some(ans) = a.answers.get(&q.id) else { + out(Event::Answer(&Answer { not_asked: Some("no answer came back for it".into()), ..base(q) })); + continue; + }; + if let (Some(p), Some(v)) = (heard, q.when.voice) { + if v != (p >= 0.5) { + let who = if p >= 0.5 { "someone speaks" } else { "no one speaks" }; + out(Event::Answer(&Answer { not_asked: Some(format!("set aside: {who} ({p:.2})")), ..base(q) })); + continue; + } + } + let r = q.read(ans); + let flagged = q.flagged(&r); + sum.answered += 1; + sum.flagged += usize::from(flagged); + sum.flagged_calibrated += usize::from(flagged && q.calibrated()); + let id = record::answer(&call, &file, q, &r, flagged); + out(Event::Answer(&Answer { + answer: Some(id), + says: Some(r.says), + value: Some(r.value), + problem: r.problem, + choice: r.choice, + flagged, + route: Some(a.route.clone()), + cached: a.cached, + ..base(q) + })); + } + } + } +} diff --git a/apps/ask/src/tests.rs b/apps/ask/src/tests.rs new file mode 100644 index 000000000..c3f33b8e5 --- /dev/null +++ b/apps/ask/src/tests.rs @@ -0,0 +1,174 @@ +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use raea_core_ask::{Facts, Kind, Picks, Question, Sees, Window, library}; +use raea_io_clef::media::Probe; +use raea_io_clef::{Answered, Call, Fail}; +use serde_json::{Map, Value, json}; + +use crate::prepare::{self, Piece, VOICE}; +use crate::runner::{self, Answer, Entry, Event, Opts}; + +fn rubric() -> String { + std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/../../sdk/guides/craft/rubric.md")).expect("the craft rubric") +} + +fn setup() { + static ONCE: std::sync::Once = std::sync::Once::new(); + // SAFETY: set once, before any test reads it, to the same value for every test + ONCE.call_once(|| unsafe { std::env::set_var("RAEA_ASK_DIR", std::env::temp_dir().join(format!("raea-ask-test-{}", std::process::id()))) }); +} + +fn ids(qs: &[Question]) -> Vec<&str> { + qs.iter().map(|q| q.id.as_str()).collect() +} + +fn piece(file: &str, voice: Option) -> Arc { + let probe = Probe { seconds: 10.0, video: true, still: false, sound: true }; + Arc::new(Piece { file: PathBuf::from(file), probe, voice, ..Default::default() }) +} + +fn entry(p: &Arc, window: Option, questions: Vec) -> Entry { + Entry { piece: p.clone(), window, questions, dropped: vec![], with: vec![] } +} + +#[test] +fn no_one_speaking_asks_sound_silent_and_a_file_without_sound_isnt_listened_to() { + let r = rubric(); + let sound = library::find("sound", Some(&r)).unwrap(); + let silent = Facts { voice: Some(false), sound: true, video: true, ..Default::default() }; + let (asked, dropped) = prepare::keep(vec![sound.clone()], &silent, Some(&r)); + assert_eq!(ids(&asked), ["sound-silent"]); + assert!(dropped[0].1.contains("no one speaks") && dropped[0].1.contains("sound-silent instead"), "{dropped:?}"); + let mut listen = Question::plain("loud", "Is it loud?"); + listen.sees = Sees::Nothing; + let (asked, dropped) = prepare::keep(vec![listen], &Facts { video: true, ..Default::default() }, None); + assert!(asked.is_empty() && dropped[0].1.contains("no sound"), "{dropped:?}"); + // not knowing whether anyone speaks, both are kept, and the plan adds the voice question to their call + let unknown = Facts { sound: true, video: true, ..Default::default() }; + let (asked, _) = prepare::keep(vec![sound], &unknown, Some(&r)); + assert_eq!(ids(&asked), ["sound", "sound-silent"]); + let (groups, _) = runner::plan(&[entry(&piece("a.mp4", None), None, asked)]); + assert_eq!(groups.len(), 1); + assert_eq!(ids(&groups[0].questions), ["sound", "sound-silent", VOICE]); +} + +#[test] +fn the_state_says_what_the_piece_is() { + let mut p = (*piece("a.mp4", Some(false))).clone(); + p.brief = json!({"title": "Spring launch", "mode": ["pitch"], "audience": "busy founders", "believe": "it saves them an hour."}); + p.extra = Some("raea measured -14 LUFS.".into()); + let s = p.state(); + for want in ["A pitch piece: \"Spring launch\".", "It's for: busy founders.", "believing: it saves them an hour.", "No one speaks", "runs 10 s", "-14 LUFS"] { + assert!(s.contains(want), "{want}: {s}"); + } + assert_eq!(p.facts().modes, ["pitch"]); + // no brief and the caller's own state: the caller says what the piece is + let own = Piece { extra: Some("A game: \"Rover\".".into()), ..(*piece("a.mp4", None)).clone() }; + assert_eq!(own.state(), "It runs 10 s. A game: \"Rover\"."); +} + +#[test] +fn calls_hold_64_questions_and_split_by_window_and_what_they_look_at() { + let p = piece("a.mp4", Some(true)); + let many: Vec = (0..65).map(|i| Question::plain(&format!("q{i}"), "Is it loud?")).collect(); + let (groups, _) = runner::plan(&[entry(&p, None, many)]); + assert_eq!(groups.iter().map(|g| g.questions.len()).collect::>(), [63, 2], "one place a call is kept for the voice question"); + let hook = Window { from: Some(0.0), to: Some(3.0) }; + let mut still = Question::plain("s", "Is the text readable?"); + still.sees = Sees::Stills; + let (groups, _) = runner::plan(&[entry(&p, None, vec![Question::plain("a", "Is it loud?"), still]), entry(&p, Some(hook), vec![Question::plain("b", "Is it loud?")])]); + assert_eq!(groups.len(), 3); + let mut pick = Question::plain("best", "Which suits the brief best?"); + pick.picks = Some(Picks::All); + let (groups, unasked) = runner::plan(&[entry(&p, None, vec![pick])]); + assert!(groups.is_empty() && unasked[0].not_asked.as_deref().is_some_and(|n| n.contains("--with")), "{unasked:?}"); + let (groups, unasked) = runner::plan(&[entry(&p, None, vec![library::find("decide.changed", None).unwrap()])]); + assert!(groups.is_empty() && !unasked[0].version.is_empty(), "{unasked:?}"); +} + +/// clef's answers as a fake gives them: yes 0.8 to every yes question (0.9 to the voice question), the lowest level of +/// a grade. +fn answers(call: &Call) -> Map { + call.questions + .iter() + .map(|q| { + let a = match q.kind { + Kind::Yes => json!({"noul": if q.id == VOICE { 0.9 } else { 0.8 }}), + _ => json!({"choice": q.levels()[0], "probabilities": {q.levels()[0].clone(): 1.0}}), + }; + (q.id.clone(), a) + }) + .collect() +} + +fn answered(call: &Call) -> Answered { + Answered { answers: answers(call), cost: 0.001, cached: false, route: "a test".into(), pool_left: Some(1.5) } +} + +/// A 2 s clip with a picture and a tone, made with FFmpeg. +fn clip(name: &str) -> PathBuf { + let out = std::env::temp_dir().join(format!("raea-ask-{}-{name}.mp4", std::process::id())); + let ff = raea_io_assets::ffmpeg_tool("ffmpeg", false).expect("FFmpeg"); + let ok = std::process::Command::new(ff) + .args(["-y", "-v", "error", "-f", "lavfi", "-i", "testsrc=size=320x240:rate=10:duration=2", "-f", "lavfi", "-i", "sine=frequency=440:duration=2"]) + .args(["-c:v", "mpeg4", "-c:a", "aac", "-shortest"]) + .arg(&out) + .status() + .expect("FFmpeg ran"); + assert!(ok.success()); + out +} + +fn run(entries: Vec, jobs: usize, ask: fn(&Call) -> Result) -> (runner::Summary, Vec) { + setup(); + let mut got = Vec::new(); + let sum = runner::run(entries, &Opts { jobs, dry: false, ask }, &mut |e| { + if let Event::Answer(a) = e { + got.push(a.clone()); + } + }); + (sum, got) +} + +#[test] +fn each_clip_is_cut_once_and_the_voice_answer_sets_a_variant_aside() { + let file = clip("once"); + let p = Arc::new(Piece::read(&file, None, None, None, None).unwrap()); + assert!(p.probe.video && p.probe.sound && p.voice.is_none()); + let r = rubric(); + let (asked, _) = prepare::keep(vec![library::find("sound", Some(&r)).unwrap()], &p.facts(), Some(&r)); + let mut listen = Question::plain("loud", "Is it too loud?"); + listen.sees = Sees::Nothing; + let (sum, got) = run(vec![entry(&p, None, [asked, vec![listen]].concat())], 4, |c| Ok(answered(c))); + assert_eq!((sum.calls, sum.clips), (2, 2), "the picture's call and the sound's call share one cut of the sound"); + let by = |id: &str| got.iter().find(|a| a.id == id).unwrap_or_else(|| panic!("{id}: {got:?}")); + assert!(by("sound").says.is_some() && by("loud").flagged); + assert!(by("sound-silent").not_asked.as_deref().is_some_and(|n| n.contains("someone speaks (0.90)")), "{:?}", by("sound-silent")); + assert!(!got.iter().any(|a| a.id == VOICE), "the voice question isn't told"); + assert_eq!((sum.answered, sum.cost, sum.pool_left), (2, 0.002, Some(1.5))); +} + +static CALLS: AtomicUsize = AtomicUsize::new(0); + +#[test] +fn a_pool_spent_partway_keeps_what_came_and_the_rest_waits() { + let file = clip("spent"); + let p = Arc::new(Piece::read(&file, None, None, Some(true), None).unwrap()); + let at = |s: f64| Some(Window { from: Some(s), to: Some(s + 0.5) }); + let entries = (0..4).map(|i| entry(&p, at(i as f64 * 0.4), vec![Question::plain(&format!("w{i}"), "Is it loud?")])).collect(); + let (sum, got) = run(entries, 1, |c| match CALLS.fetch_add(1, Ordering::SeqCst) { + 2 => Err(Fail::Spent("this month's pool is used up".into())), + _ => Ok(answered(c)), + }); + assert_eq!((sum.calls, sum.answered, sum.waiting), (2, 2, 2)); + assert_eq!(got.iter().filter(|a| a.waits.as_deref().is_some_and(|w| w.contains("used up"))).count(), 2); + assert_eq!(CALLS.load(Ordering::SeqCst), 3, "nothing is sent after the pool is spent"); +} + +#[test] +fn own_words_keep_their_id() { + assert_eq!(crate::verbs::own_id("Does the music fight the voice?"), "own-does-the-music-fight-the-voice"); + assert_eq!(crate::verbs::own_id("Does the music fight the voice?"), crate::verbs::own_id("does the MUSIC fight the voice")); +} diff --git a/apps/ask/src/verbs.rs b/apps/ask/src/verbs.rs new file mode 100644 index 000000000..3113e026b --- /dev/null +++ b/apps/ask/src/verbs.rs @@ -0,0 +1,324 @@ +//! `raea ask` (docs/ask.feature AK3): questions about a piece's picture and sound, from the command line or a batch, +//! prepared (prepare.rs), asked in parallel (runner.rs) and kept (record.rs). It prints as answers come: words and a +//! probability a line, or with `--json` one JSON object a line (an answer, a call `--dry` would make, then the summary). +//! `--gate` exits 1 when a calibrated question flags (`--gate any`: when any does), for a pipeline. + +use std::collections::{BTreeMap, HashMap}; +use std::io::Read as _; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use raea_core_ask::{Kind, Question, Window, library}; +use serde_json::{Value, json}; + +use crate::prepare::{self, Piece}; +use crate::runner::{self, Entry, Event, Opts}; + +pub const USAGE: &str = "raea ask FILE [QUESTION...] [--choose A,B,C] [--grade] [--rate N] [--flag ANSWERS] [--set NAME] [--std ID] [--from S] [--to S] [--at S] [--with FILE2] [--brief brief.ts|none] [--words words.json] [--voice yes|no] [--state TEXT] [--dry] [--json] [--gate [any]] [--jobs N] | ask --batch FILE [--dry] [--json] [--gate [any]] [--jobs N] | ask list [SET] [--json] (questions about a piece's picture and sound, answered by clef-omni and paid from your raea plan's pool when the app is signed in, else RAEA_AGENT_KEY's, else your own Cloudflare keys: your own words, or the standard sets (rubric, av, defects, platform, decide); every question about one clip goes in one call, --batch (a JSON list or JSON lines, - for stdin) asks many clips in parallel; --dry says what would be sent and spends nothing; answers are advice until a question is calibrated)"; + +const VALUED: [&str; 15] = ["--choose", "--rate", "--flag", "--set", "--std", "--from", "--to", "--at", "--with", "--brief", "--words", "--voice", "--state", "--jobs", "--batch"]; +const BARE: [&str; 4] = ["--grade", "--dry", "--json", "--gate"]; + +/// The arguments: the words that aren't flags, each flag's values, and whether `--gate` is `--gate any`. +#[derive(Default)] +struct Args { + words: Vec, + flags: BTreeMap>, + gate_any: bool, +} + +impl Args { + fn parse(args: &[String]) -> Result { + let mut a = Args::default(); + let mut i = 0; + while i < args.len() { + let s = args[i].as_str(); + if VALUED.contains(&s) { + let v = args.get(i + 1).ok_or_else(|| format!("ask: {s} takes a value\nusage: {USAGE}"))?; + a.flags.entry(s.into()).or_default().push(v.clone()); + i += 1; + } else if BARE.contains(&s) { + a.flags.entry(s.into()).or_default(); + if s == "--gate" && args.get(i + 1).is_some_and(|v| v == "any") { + a.gate_any = true; + i += 1; + } + } else if s.starts_with("--") { + return Err(format!("ask doesn't take {s}\nusage: {USAGE}")); + } else { + a.words.push(s.into()); + } + i += 1; + } + Ok(a) + } + fn has(&self, f: &str) -> bool { + self.flags.contains_key(f) + } + fn one(&self, f: &str) -> Option<&str> { + self.flags.get(f).and_then(|v| v.last()).map(String::as_str) + } + fn num(&self, f: &str) -> Result, String> { + self.one(f).map(|v| v.parse::().map_err(|_| format!("ask: {f} takes seconds, got {v:?}"))).transpose() + } + fn list(&self, f: &str) -> Vec { + self.flags.get(f).into_iter().flatten().flat_map(|v| v.split(',')).map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect() + } +} + +/// The craft rubric's text, where the guides are installed, for the `rubric` set. +pub fn rubric_md() -> Option { + std::fs::read_to_string(raea_io_home::home().join("sdk/guides/craft/rubric.md")).ok() +} + +/// A question's id from its words: `own-` and its first words, so the same words keep their id. +pub fn own_id(words: &str) -> String { + let slug: Vec = words.split(|c: char| !c.is_ascii_alphanumeric()).filter(|w| !w.is_empty()).take(6).map(str::to_lowercase).collect(); + let mut id = format!("own-{}", slug.join("-")); + id.truncate(raea_core_ask::limits::ID_LEN); + id +} + +/// The questions an entry names: its own words (shaped by the kind flags), its sets and its standard ids. +fn named(own: &[Question], sets: &[String], std: &[String], rubric: Option<&str>) -> Result, String> { + let mut qs = own.to_vec(); + for s in sets { + qs.extend(library::set(s, rubric)?); + } + for id in std { + qs.push(library::find(id, rubric).ok_or_else(|| format!("ask: no standard question {id} (raea ask list)"))?); + } + Ok(qs) +} + +fn shaped(words: &str, a: &Args) -> Result { + let mut q = Question::plain(&own_id(words), words); + if let Some(c) = a.one("--choose") { + q.kind = Kind::Choose; + q.options = c.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect(); + } else if let Some(n) = a.one("--rate") { + let n: usize = n.parse().map_err(|_| format!("ask: --rate takes how many levels, got {n:?}"))?; + q.kind = Kind::Rate; + q.options = (1..=n).map(|i| i.to_string()).collect(); + } else if a.has("--grade") { + q.kind = Kind::Grade; + } + q.flags = a.list("--flag"); + if let Some(why) = raea_core_ask::limits::check(&q) { + return Err(format!("ask: {why}")); + } + Ok(q) +} + +/// The window from --from, --to and --at (three seconds around it). +fn window(a: &Args) -> Result, String> { + if let Some(at) = a.num("--at")? { + return Ok(Some(Window { from: Some((at - 1.5).max(0.0)), to: Some(at + 1.5) })); + } + let (from, to) = (a.num("--from")?, a.num("--to")?); + Ok((from.is_some() || to.is_some()).then_some(Window { from, to })) +} + +fn voice(v: Option<&str>) -> Result, String> { + match v { + None => Ok(None), + Some("yes" | "true") => Ok(Some(true)), + Some("no" | "false") => Ok(Some(false)), + Some(v) => Err(format!("ask: --voice takes yes or no, got {v:?}")), + } +} + +/// Pieces read once a run, by what they were read with. +type Pieces = HashMap>; + +#[allow(clippy::too_many_arguments)] +fn piece(pieces: &mut Pieces, file: &str, brief: Option<&str>, words: Option<&str>, voice: Option, state: Option) -> Result, String> { + let key = format!("{file}\u{1f}{brief:?}\u{1f}{words:?}\u{1f}{voice:?}\u{1f}{state:?}"); + if let Some(p) = pieces.get(&key) { + return Ok(p.clone()); + } + let p = Arc::new(Piece::read(Path::new(file), brief.map(Path::new), words.map(Path::new), voice, state).map_err(|e| format!("ask: {e}"))?); + pieces.insert(key, p.clone()); + Ok(p) +} + +fn entry(p: Arc, window: Option, qs: Vec, with: Vec, rubric: Option<&str>) -> Entry { + let (questions, dropped) = prepare::keep(qs, &p.facts(), rubric); + Entry { piece: p, window, questions, dropped, with } +} + +/// The command line's one entry. +fn from_args(a: &Args, rubric: Option<&str>, pieces: &mut Pieces) -> Result, String> { + let Some((file, words)) = a.words.split_first() else { return Err(USAGE.into()) }; + let own = words.iter().map(|w| shaped(w, a)).collect::, _>>()?; + let qs = named(&own, &a.list("--set"), &a.list("--std"), rubric)?; + if qs.is_empty() { + return Err(format!("ask: nothing to ask: give a question in words, --set or --std (raea ask list)\nusage: {USAGE}")); + } + let p = piece(pieces, file, a.one("--brief"), a.one("--words"), voice(a.one("--voice"))?, a.one("--state").map(str::to_string))?; + let with = a.flags.get("--with").into_iter().flatten().map(PathBuf::from).collect(); + Ok(vec![entry(p, window(a)?, qs, with, rubric)]) +} + +/// A batch: a JSON list of entries or JSON lines, from a file or stdin (`-`). Each entry is {file, window?, questions? +/// (words or questions), set?, std?, with?, brief?, words?, voice?, state?}. +fn from_batch(path: &str, rubric: Option<&str>, pieces: &mut Pieces) -> Result, String> { + let text = match path { + "-" => { + let mut s = String::new(); + std::io::stdin().read_to_string(&mut s).map_err(|e| format!("ask: stdin: {e}"))?; + s + } + p => std::fs::read_to_string(p).map_err(|e| format!("ask: {p}: {e}"))?, + }; + let rows: Vec = match serde_json::from_str::(text.trim()) { + Ok(Value::Array(a)) => a, + Ok(v @ Value::Object(_)) => vec![v], + _ => text.lines().filter(|l| !l.trim().is_empty()).enumerate().map(|(i, l)| serde_json::from_str(l).map_err(|e| format!("ask: {path} line {}: {e}", i + 1))).collect::>()?, + }; + let strs = |v: &Value| -> Vec { + match v { + Value::String(s) => s.split(',').map(|s| s.trim().to_string()).collect(), + Value::Array(a) => a.iter().filter_map(Value::as_str).map(str::to_string).collect(), + _ => vec![], + } + }; + let mut out = Vec::new(); + for (n, r) in rows.iter().enumerate() { + let at = |e: String| format!("ask: {path} entry {}: {e}", n + 1); + let file = r["file"].as_str().ok_or_else(|| at("it has no file".into()))?; + let own = r["questions"].as_array().into_iter().flatten().map(|q| match q { + Value::String(w) => Ok(Question::plain(&own_id(w), w)), + q => serde_json::from_value::(q.clone()).map_err(|e| e.to_string()), + }); + let own = own.collect::, _>>().map_err(at)?; + let qs = named(&own, &strs(&r["set"]), &strs(&r["std"]), rubric).map_err(at)?; + let window: Option = serde_json::from_value(r["window"].clone()).ok(); + let v = voice(r["voice"].as_str()).map_err(at)?.or(r["voice"].as_bool()); + let p = piece(pieces, file, r["brief"].as_str(), r["words"].as_str(), v, r["state"].as_str().map(str::to_string)).map_err(at)?; + out.push(entry(p, window, qs, strs(&r["with"]).into_iter().map(PathBuf::from).collect(), rubric)); + } + Ok(out) +} + +/// `raea ask list [SET]`: the standard questions. +fn list(a: &Args, rubric: Option<&str>) -> Result { + let sets: Vec = match a.words.get(1) { + Some(s) => vec![s.clone()], + None => library::SETS.iter().map(|s| s.to_string()).collect(), + }; + let mut out = Vec::new(); + for s in &sets { + let qs = library::set(s, rubric)?; + if a.has("--json") { + out.push(json!({"set": s, "questions": qs}).to_string()); + continue; + } + out.push(format!("{s}:")); + for q in qs { + let how = if q.calibrated() { "calibrated" } else { "advice" }; + out.push(format!(" {:<28} {:<6} {:<10} {}", q.id, format!("{:?}", q.kind).to_lowercase(), how, q.ask)); + } + } + Ok(out.join("\n")) +} + +fn span(from: f64, to: Option) -> String { + match to { + Some(t) => format!("{from:.0}-{t:.0} s"), + None if from > 0.0 => format!("from {from:.0} s"), + None => "whole".into(), + } +} + +/// An event as a line of text. +fn line(e: &Event, files: usize) -> String { + match e { + Event::Would(v) => { + let qs: Vec<&str> = v["questions"].as_array().into_iter().flatten().filter_map(|q| q["id"].as_str()).collect(); + let m = &v["media"]; + let kb = |k: &str| m[k].as_array().map(|a| a.iter().filter_map(Value::as_u64).map(|b| format!("{} KB", b / 1024)).collect::>().join(", ")).unwrap_or_default(); + format!( + "would ask {} ({}) about {}:\n {}\n videos [{}], stills [{}], sounds [{}], about ${:.4}\n state: {}", + qs.len(), + span(v["from"].as_f64().unwrap_or(0.0), v["to"].as_f64()), + v["file"].as_str().unwrap_or(""), + qs.join(", "), + kb("videos"), + kb("stills"), + kb("sounds"), + v["estimate_usd"].as_f64().unwrap_or(0.0), + v["state"].as_str().unwrap_or("") + ) + } + Event::Answer(a) => { + let who = if files > 1 { format!("{}: ", a.file) } else { String::new() }; + let what = match (&a.says, &a.not_asked, &a.waits) { + (Some(s), ..) => format!("{s} [{}]{}", span(a.from, a.to), if a.flagged && a.calibrated { " <- finding" } else if a.flagged { " <- flagged" } else { "" }), + (_, Some(n), _) => format!("not asked: {n}"), + (.., Some(w)) => w.clone(), + _ => "no answer".into(), + }; + format!("{who}{:<34} {what}", a.id) + } + } +} + +/// `raea ask ...` with its exit code: 0, or 1 when it couldn't ask or a gate failed. +pub fn cli(args: &[String]) -> i32 { + match run(args) { + Ok(code) => code, + Err(e) => { + eprintln!("{e}"); + 1 + } + } +} + +fn run(args: &[String]) -> Result { + let a = Args::parse(args)?; + let rubric = rubric_md(); + if a.words.first().is_some_and(|w| w == "list") && !Path::new("list").exists() { + println!("{}", list(&a, rubric.as_deref())?); + return Ok(0); + } + let mut pieces = Pieces::new(); + let entries = match a.one("--batch") { + Some(b) => from_batch(b, rubric.as_deref(), &mut pieces)?, + None => from_args(&a, rubric.as_deref(), &mut pieces)?, + }; + let jobs = a.one("--jobs").map(|j| j.parse::().map_err(|_| format!("ask: --jobs takes a number, got {j:?}"))).transpose()?.unwrap_or(4); + let (json_out, dry) = (a.has("--json"), a.has("--dry")); + let files = pieces.len(); + let sum = runner::run(entries, &Opts::new(jobs, dry), &mut |e| { + match (&e, json_out) { + (Event::Answer(r), true) => println!("{}", serde_json::to_string(r).unwrap_or_default()), + (Event::Would(v), true) => println!("{}", json!({"would": v})), + (e, false) => println!("{}", line(e, files)), + } + }); + if json_out { + println!("{}", json!({"summary": sum})); + } else if dry { + println!("{} call{}, about ${:.4}; nothing was sent", sum.calls, if sum.calls == 1 { "" } else { "s" }, sum.estimate); + } else if sum.calls > 0 { + let routes: Vec<&str> = sum.routes.iter().map(String::as_str).collect(); + let left = sum.pool_left.map(|l| format!(", ${l:.2} left this month")).unwrap_or_default(); + let waiting = if sum.waiting > 0 { format!("; {} wait for the pool", sum.waiting) } else { String::new() }; + println!( + "asked {} question{} in {} call{} through {}: ${:.4}{left}{waiting}. Answers are advice until a question is calibrated (docs/ask.feature AK7).", + sum.asked, + if sum.asked == 1 { "" } else { "s" }, + sum.calls, + if sum.calls == 1 { "" } else { "s" }, + routes.join(" and "), + sum.cost + ); + } + if !dry && sum.calls == 0 { + return Err("ask: nothing was asked (each question's line says why)".into()); + } + // a gate fails on a flag (a calibrated one, or with `any` any), and on questions still waiting for the pool + let flagged = if a.gate_any { sum.flagged } else { sum.flagged_calibrated }; + Ok(i32::from(a.has("--gate") && (flagged > 0 || sum.waiting > 0))) +} diff --git a/apps/judge/Cargo.toml b/apps/judge/Cargo.toml index 223e36028..e130bb7d4 100644 --- a/apps/judge/Cargo.toml +++ b/apps/judge/Cargo.toml @@ -13,6 +13,6 @@ raea-io-home = { path = "../../io/home" } raea-core-facts = { path = "../../core/facts" } serde_json = "1" raea-runtime-bundle = { path = "../../runtime/bundle" } -ureq = { version = "2", features = ["json"] } -base64 = "0.22" -image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } +# the one clef client: the question format and the call, routed to the person's plan first (docs/ask.feature AK1b) +raea-core-ask = { path = "../../core/ask" } +raea-io-clef = { path = "../../io/clef" } diff --git a/apps/judge/src/clef.rs b/apps/judge/src/clef.rs deleted file mode 100644 index 0af664361..000000000 --- a/apps/judge/src/clef.rs +++ /dev/null @@ -1,37 +0,0 @@ -//! One clef-omni call on Workers AI (as apps/app/host/critique.ts makes it): the state paragraph, the questions by id, -//! the images as data URLs (each made a JPEG at most 1600 px wide first). Keys: CLOUDFLARE_ACCOUNT_ID and -//! CLOUDFLARE_API_TOKEN from raea's settings (the environment, then ~/.raea/env). - -use std::path::PathBuf; - -use base64::Engine as _; -use serde_json::{Value, json}; - -fn jpeg(p: &PathBuf) -> Result { - let img = image::open(p).map_err(|e| format!("{}: {e}", p.display()))?; - let img = if img.width() > 1600 { img.resize(1600, u32::MAX, image::imageops::FilterType::Triangle) } else { img }; - let mut buf = std::io::Cursor::new(Vec::new()); - img.to_rgb8().write_to(&mut buf, image::ImageFormat::Jpeg).map_err(|e| e.to_string())?; - Ok(format!("data:image/jpeg;base64,{}", base64::engine::general_purpose::STANDARD.encode(buf.into_inner()))) -} - -/// The answers by question id, and the call's cost in dollars ($0.15 a million input tokens). -pub fn ask(state: &str, questions: &Value, images: &[PathBuf]) -> Result<(Value, f64), String> { - let (Some(acct), Some(token)) = (raea_io_home::setting("CLOUDFLARE_ACCOUNT_ID"), raea_io_home::setting("CLOUDFLARE_API_TOKEN")) else { - return Err("judge: clef-omni on Workers AI needs CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN (the environment or ~/.raea/env)".into()); - }; - let imgs: Vec = images.iter().map(jpeg).collect::>()?; - let body = json!({"state": state, "questions": questions, "images": imgs}); - let url = format!("https://api.cloudflare.com/client/v4/accounts/{acct}/ai/run/@cf/cloudflare/clef-omni"); - let res = ureq::post(&url).set("authorization", &format!("Bearer {token}")).timeout(std::time::Duration::from_secs(180)).send_json(body); - let j: Value = match res { - Ok(r) => r.into_json().map_err(|e| format!("clef: {e}"))?, - Err(ureq::Error::Status(code, r)) => return Err(format!("clef: HTTP {code}: {}", r.into_string().unwrap_or_default().chars().take(300).collect::())), - Err(e) => return Err(format!("clef: {e}")), - }; - if j["success"] != true { - return Err(format!("clef: {}", j["errors"].to_string().chars().take(300).collect::())); - } - let cost = j["result"]["usage"]["input_tokens"].as_f64().unwrap_or(0.0) * 0.15 / 1e6; - Ok((j["result"]["answers"].clone(), cost)) -} diff --git a/apps/judge/src/lib.rs b/apps/judge/src/lib.rs index 93739a5b8..a2bebd5fe 100644 --- a/apps/judge/src/lib.rs +++ b/apps/judge/src/lib.rs @@ -10,7 +10,6 @@ //! each a fresh call) and the stall (the mean up by less than a point over two rounds) are read from it. mod agent; -mod clef; mod eye; pub mod rubric; @@ -24,6 +23,17 @@ use serde_json::{Value, json}; pub const USAGE: &str = "raea judge DIR [--by clef|kimi|claude] [--brief brief.ts] [--target T.png] [--json] [--dry] | judge eye DIR (CATEGORY=SCORE ... [--pass yes|no] | --by kimi|claude) | judge calibrate [--json] (a --review folder's sheets judged blind: clef-omni on the frozen rubric, or a fresh kimi or claude session on the judge prompt; the pass decided in code; clef advises until calibrated against your eye or an agent's)"; +/// One call through raea's one clef client (raea-io-clef: the person's plan first, then their own Cloudflare keys), +/// with the rubric's questions as written and each sheet as a JPEG still. +fn ask(state: &str, questions: &Value, images: &[PathBuf]) -> Result<(Value, f64), String> { + let qs = questions.as_object().ok_or("judge: the questions aren't a map")?; + let questions = qs.iter().map(|(id, w)| raea_core_ask::Question::from_wire(id, w).ok_or(format!("judge: {id} isn't one of clef's question types"))).collect::>()?; + let stills = images.iter().map(|p| raea_io_clef::media::still(p, None)).collect::>()?; + let call = raea_io_clef::Call { state: state.into(), questions, media: raea_io_clef::Media { stills, ..Default::default() } }; + let a = raea_io_clef::ask(&call).map_err(|f| f.to_string())?; + Ok((Value::Object(a.answers), a.cost)) +} + /// The brief's words, from `raea spec brief.ts --as brief` (look, logline, notDoing), or nothing. fn brief_of(path: Option<&str>) -> Result { let Some(p) = path else { return Ok(json!({})) }; @@ -104,7 +114,7 @@ pub fn cli(args: &[String]) -> Result { let mut photo = Value::Null; let mut cost = 0.0; for (sheet, st, q, images) in &calls { - let (answers, c) = clef::ask(st, q, images)?; + let (answers, c) = ask(st, q, images)?; cost += c; if *sheet == "blind" { photo = json!(answers["photo"]["noul"].as_f64().map(|p| (p * 1000.0).round() / 1000.0)); diff --git a/apps/solid/Cargo.toml b/apps/solid/Cargo.toml index e923db93c..af643c8d3 100644 --- a/apps/solid/Cargo.toml +++ b/apps/solid/Cargo.toml @@ -16,6 +16,7 @@ serde_json = "1" raea-runtime-bundle = { path = "../../runtime/bundle" } # `raea judge`, dispatched with `raea solid` (verb()) raea-app-judge = { path = "../judge" } +raea-app-ask = { path = "../ask" } # `raea solid export`: the solid built, written as .glb and read back (core/shape), named and coloured (core/meshcheck) raea-core-shape = { path = "../../core/shape" } raea-core-meshcheck = { path = "../../core/meshcheck" } diff --git a/apps/solid/src/lib.rs b/apps/solid/src/lib.rs index 2adfdefaa..60a4a001d 100644 --- a/apps/solid/src/lib.rs +++ b/apps/solid/src/lib.rs @@ -91,11 +91,13 @@ fn parse_set(s: &str) -> Result, String> { Ok(m) } -/// `raea solid ...` and `raea judge ...`, the scene loop's verbs (engine/cli hands them over), or None for another verb. +/// `raea solid ...`, `raea judge ...` and `raea ask ...` (docs/ask.feature AK3, which prints as it goes and exits with +/// its own code), the verbs that build and judge a piece (engine/cli hands them over), or None for another verb. pub fn verb(args: &[String]) -> Option> { match args.first().map(String::as_str) { Some("solid") => Some(cli(args)), Some("judge") => Some(raea_app_judge::cli(args)), + Some("ask") => std::process::exit(raea_app_ask::cli(&args[1..])), _ => None, } } diff --git a/core/ask/Cargo.toml b/core/ask/Cargo.toml new file mode 100644 index 000000000..72894f952 --- /dev/null +++ b/core/ask/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "raea-core-ask" +version = "0.1.0" +edition = "2024" +license = "FSL-1.1-ALv2" +publish = false +description = "Questions for the decision model (clef-omni): their types, how answers read, the standard sets and the model's limits" + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +# the TypeScript declarations are generated from these types, so the two can't drift +ts-rs = "12" diff --git a/core/ask/library/av.json b/core/ask/library/av.json new file mode 100644 index 000000000..03a9cc3e1 --- /dev/null +++ b/core/ask/library/av.json @@ -0,0 +1,9 @@ +[ + {"id": "av.together", "type": "grade", "ask": "How well do the sound and the picture work together: does what you hear belong to what you see, moment by moment?", "uses": ["brief"]}, + {"id": "av.sound-fits", "ask": "Does every sound fit what is on screen: an action that would make a sound has one, and each thing sounds like what it is made of?", "flags": ["no"]}, + {"id": "av.music-mood", "ask": "Does the music's mood match what the picture shows at each point?", "flags": ["no"], "uses": ["brief"]}, + {"id": "av.music-fights-voice", "ask": "At any point do the music or the effects make the voice hard to follow?", "when": {"voice": true}}, + {"id": "av.voice-suits", "ask": "Does the voice suit the person shown speaking: their age, their manner and their energy?", "flags": ["no"], "when": {"voice": true}}, + {"id": "av.ambience", "ask": "Does the background sound match the place the picture shows (a street, a kitchen, a room, outdoors)?", "flags": ["no"]}, + {"id": "av.effects-mark", "ask": "Does each sound effect mark something happening on screen, rather than filling space?", "flags": ["no"]} +] diff --git a/core/ask/library/decide.json b/core/ask/library/decide.json new file mode 100644 index 000000000..8f34131ad --- /dev/null +++ b/core/ask/library/decide.json @@ -0,0 +1,5 @@ +[ + {"id": "decide.best", "type": "choose", "ask": "Which of these best fits what the piece is for, for the audience it's for?", "picks": "all", "uses": ["brief"]}, + {"id": "decide.nearest", "type": "choose", "ask": "The first is the reference. Of the others, which looks and sounds most like it?", "picks": "after-first"}, + {"id": "decide.changed", "ask": "These are two versions, before and after a change. Does the second change what a viewer sees or hears?", "flags": ["no"], "uses": ["with"]} +] diff --git a/core/ask/library/defects.json b/core/ask/library/defects.json new file mode 100644 index 000000000..19b539419 --- /dev/null +++ b/core/ask/library/defects.json @@ -0,0 +1,9 @@ +[ + {"id": "defect.muffled-voice", "ask": "Is there a muffled, distant voice that is hard to understand?", "sees": "nothing", "when": {"voice": true}, "label_words": ["muffled", "can't hear", "hard to hear", "distant voice"]}, + {"id": "defect.harsh-music", "ask": "Is there harsh, shrill music that hurts to listen to?", "sees": "nothing", "label_words": ["harsh", "shrill", "piercing"]}, + {"id": "defect.glitch", "ask": "Is there a rendering glitch: broken or torn geometry, or corrupted, blocky video?", "hears": false, "label_words": ["glitch", "torn", "corrupted", "artifact"]}, + {"id": "defect.fake", "ask": "Does the scene look fake: plastic and computer generated, like a cheap video game render?", "hears": false, "label_words": ["fake", "plastic", "cgi", "video game"]}, + {"id": "defect.barren", "ask": "Is the scene empty and barren: a bare room or space with nothing in it and no detail?", "hears": false, "label_words": ["empty", "barren", "bare", "nothing in it"]}, + {"id": "defect.unreadable-text", "ask": "Is there small or cluttered text that is hard to read?", "hears": false, "label_words": ["can't read", "unreadable", "too small", "cluttered"]}, + {"id": "defect.shaky-camera", "ask": "Is the camera shaky, like an unsteady handheld shot?", "hears": false, "label_words": ["shaky", "wobbly", "unsteady", "jittery"]} +] diff --git a/core/ask/library/platform.json b/core/ask/library/platform.json new file mode 100644 index 000000000..21b083567 --- /dev/null +++ b/core/ask/library/platform.json @@ -0,0 +1,19 @@ +[ + {"id": "face.text_over", "ask": "At any point does on-screen text or a graphic cover the speaker's eyes, nose or mouth?", "label_words": ["over the face", "covers the face", "cover the face", "over the speaker's", "over his face", "over her face", "over their face", "mouth", "covering the face", "across the face", "on the face"], "calibration": {"auc": 0.85, "n": 126, "positives": 14, "threshold": 0.3487, "date": "2026-10-16", "by": "raea"}, "fix": "look at the frames where cards and captions sit and move them off the face"}, + {"id": "frame.too_tight", "ask": "Is the framing so tight that the speaker's face is cut off (forehead, chin, or part of the face out of frame) for much of the video?", "label_words": ["too tight", "forehead", "chin", "top of his head", "top of her head", "top of the head", "head is cut", "face is cut", "cuts off the head", "cut off the head", "cropped too tight", "tight crop", "face cut off"], "calibration": {"auc": 0.66, "n": 126, "positives": 32, "threshold": 0.2, "date": "2026-10-16", "by": "raea"}, "fix": "zoom the clips out (framing)"}, + {"id": "speech.uncaptioned", "ask": "While someone is talking, are their words missing from the screen, so a viewer with the sound off can't follow?", "uses": ["words", "screen-words"], "label_words": ["no captions", "no speech captions", "captions stop", "uncaptioned", "without captions"], "calibration": {"auc": 0.45, "n": 126, "positives": 22, "threshold": 0.5, "date": "2026-10-16", "by": "raea"}, "fix": "caption what's said (captions), unless the piece is meant to run without"}, + {"id": "text.hard_to_read", "ask": "Is any on-screen text hard to read: too faint against what's behind it, overlapping other text, or too small for a phone?", "uses": ["screen-words"], "label_words": ["low-contrast", "low contrast", "unreadable", "illegible", "hard to read", "overlap", "collide", "invisible", "blends into"], "calibration": {"auc": 0.77, "n": 126, "positives": 45, "threshold": 0.2184, "date": "2026-10-16", "by": "raea"}, "fix": "give the text contrast and room: a scrim or shadow, a stronger colour, one line at a time"}, + {"id": "end.unfinished", "ask": "Does the video stop before its point lands: mid-sentence, mid-step, or without the ending it set up?", "uses": ["words"], "label_words": ["cuts off", "cut off", "mid-sentence", "mid-thought", "abruptly", "no ending", "mid-word", "never finished"], "calibration": {"auc": 0.51, "n": 126, "positives": 55, "threshold": 0.5, "date": "2026-10-16", "by": "raea"}, "fix": "end on a finished sentence and the payoff or the ask the opening promised"}, + {"id": "screen.empty", "ask": "Is there a stretch of a second or more with nothing worth looking at: black, blank, frozen or washed out?", "label_words": ["black", "blank", "empty", "washed out", "dead air", "frozen", "static", "nothing usable"], "calibration": {"auc": 0.71, "n": 126, "positives": 88, "threshold": 0.5, "date": "2026-10-16", "by": "raea"}, "fix": "give those seconds a picture: a clip, b-roll, the product, or cut them"}, + {"id": "message.clear", "ask": "How clearly does the video get one point across to a viewer who has never heard of the brand?", "type": "grade", "uses": ["words", "screen-words", "brand"], "label_words": ["never explains", "no idea what", "never mentions", "confusing", "unclear", "structure is broken"], "calibration": {"auc": 0.39, "n": 126, "positives": 4, "threshold": 0.5, "date": "2026-10-16", "by": "raea"}, "fix": "say the one thing the piece is for earlier and plainer; cut what doesn't serve it"}, + {"id": "hook.someone_or_something", "ask": "In these opening seconds, is a person's face or the thing being sold clearly on screen?", "flags": ["no"], "window": {"to": 3}, "uses": ["brand"], "label_words": ["opens on a black", "black screen", "empty", "no face", "hook"], "calibration": {"auc": 0.42, "n": 127, "positives": 54, "threshold": 0.5, "date": "2026-10-16", "by": "raea"}, "fix": "open on the speaker's face or the product, not a title card or an empty frame"}, + {"id": "hook.headline", "ask": "Is there short on-screen text in these opening seconds that gives the viewer a reason to keep watching?", "flags": ["no"], "window": {"to": 3}, "uses": ["screen-words", "words"], "label_words": ["no text overlays to hook", "no hook", "weak hook", "killing the hook"], "calibration": {"auc": 0.25, "n": 127, "positives": 6, "threshold": 0.5, "date": "2026-10-16", "by": "raea"}, "fix": "put the hook line on screen in the first second, short and readable"}, + {"id": "hook.stops_scroll", "ask": "How likely is someone scrolling a feed to stop on this opening?", "type": "grade", "window": {"to": 3}, "uses": ["words", "screen-words", "platform"], "label_words": ["hook", "opening", "first second"], "calibration": {"auc": 0.48, "n": 127, "positives": 46, "threshold": 0.5, "date": "2026-10-16", "by": "raea"}, "fix": "lead with the most surprising line or picture the takes have; cut the warm-up"}, + {"id": "board.bar_over_face", "ask": "The panels are a storyboard in time order, with text drawn as solid bars. In any panel, does a bar or graphic sit over a person's eyes, nose or mouth?", "sees": "stills", "hears": false, "label_words": ["over the face", "covers the face", "mouth", "over the speaker's", "across the face"], "calibration": {"auc": 0.87, "n": 124, "positives": 15, "threshold": 0.4354, "date": "2026-10-16", "by": "raea"}, "fix": "move that text or graphic off the face before rendering"}, + {"id": "board.bars_collide", "ask": "The panels are a storyboard in time order, with text drawn as solid bars. In any panel, do two blocks of text overlap or touch each other?", "sees": "stills", "hears": false, "label_words": ["overlap", "stacked on the exact same line", "collide", "text over text"], "calibration": {"auc": 0.72, "n": 124, "positives": 27, "threshold": 0.5, "date": "2026-10-16", "by": "raea"}, "fix": "give each line its own band, or show them one after another"}, + {"id": "board.no_change", "ask": "The panels are a storyboard in time order. Do most panels look nearly the same, as if the picture barely changes through the video?", "sees": "stills", "hears": false, "label_words": ["static shot", "held for", "no movement", "monotonous", "same shot", "nothing changes"], "calibration": {"auc": 0.84, "n": 124, "positives": 2, "threshold": 0.5, "date": "2026-10-16", "by": "raea"}, "fix": "vary the picture: punch-ins, b-roll, the product, a designed moment"}, + {"id": "board.empty_panel", "ask": "The panels are a storyboard in time order. Is any panel empty: black, blank, or with nothing to look at?", "sees": "stills", "hears": false, "label_words": ["black", "blank", "empty", "nothing on screen", "dead air"], "calibration": {"auc": 0.58, "n": 124, "positives": 78, "threshold": 0.5, "date": "2026-10-16", "by": "raea"}, "fix": "cover that moment with a clip, b-roll or a card"}, + {"id": "brand.named", "ask": "Is the brand or product name said aloud or shown on screen at some point?", "flags": ["no"], "uses": ["brand", "words", "screen-words"], "label_words": ["never mentions", "no idea what the product", "never names", "brand name", "app name"], "calibration": {"auc": 0.4, "n": 127, "positives": 8, "threshold": 0.5, "date": "2026-10-16", "by": "raea"}, "fix": "say or show the name: on screen in the first seconds and again at the end"}, + {"id": "brand.logo", "ask": "Is the brand's logo or wordmark visible at some point in the video?", "flags": ["no"], "uses": ["brand"], "label_words": ["no logo", "logo", "wordmark", "no branding"], "calibration": {"auc": 0.2, "n": 127, "positives": 6, "threshold": 0.5, "date": "2026-10-16", "by": "raea"}, "fix": "place the logo from media/brand on the end card or as a small lockup"}, + {"id": "brand.look", "ask": "How well do the on-screen graphics (type, colours, cards) match the brand described?", "type": "grade", "uses": ["brand"], "label_words": ["off-brand", "generic", "template", "same branding", "doesn't match the brand"], "calibration": {"auc": 0.46, "n": 127, "positives": 6, "threshold": 0.5, "date": "2026-10-16", "by": "raea"}, "fix": "take type and colours from brand/brand.css and BRAND.md; drop the house look"} +] diff --git a/core/ask/src/lib.rs b/core/ask/src/lib.rs new file mode 100644 index 000000000..fd9c3b48a --- /dev/null +++ b/core/ask/src/lib.rs @@ -0,0 +1,27 @@ +//! Questions for the decision model (clef-omni), as every caller of `raea ask` writes them (docs/ask.feature AK1): the +//! question and how its answer reads, the sets that come standard, and the model's limits. Its TypeScript declarations +//! are generated from these types (`declarations`): sdk/raea/src/ask.gen.ts, checked by a test. + +pub mod library; +pub mod limits; +pub mod question; + +pub use question::{Calibration, Facts, Kind, LEVELS, Picks, Question, Reading, Sees, When, Window}; + +use ts_rs::{Config, TS}; + +macro_rules! decls { + ($cfg:ident; $($t:ty),* $(,)?) => { vec![$(format!("export {}", <$t as TS>::decl(&$cfg))),*] }; +} + +/// The TypeScript of every type, one file's worth: sdk/raea/src/ask.gen.ts is this, checked by a test. +pub fn declarations() -> String { + let cfg = Config::new(); + let mut out = vec!["// Generated from raea-core-ask (core/ask) by its `declarations`; edit the Rust, not this.".to_string()]; + out.extend(decls!(cfg; question::Question, question::Kind, question::Sees, question::Picks, question::When, question::Window, question::Calibration)); + out.push(format!("export const SETS = {:?} as const;", library::SETS)); + out.join("\n\n") + "\n" +} + +#[cfg(test)] +mod tests; diff --git a/core/ask/src/library.rs b/core/ask/src/library.rs new file mode 100644 index 000000000..ac0a5ae53 --- /dev/null +++ b/core/ask/src/library.rs @@ -0,0 +1,75 @@ +//! The questions that come standard (docs/ask.feature AK1), each set named: `rubric` (the craft rubric's own questions, +//! read from sdk/guides/craft/rubric.md, so there's one copy), `av` (picture with sound), `defects` (the fingerprint's +//! axes with no labels yet), `platform` (the check packs' questions with their calibration) and `decide` (picks and +//! comparisons). The JSON sets ship inside raea; the rubric is read from the guide the caller hands in. + +use crate::question::{Kind, Question}; + +/// Every set, in the order `raea ask list` shows them. +pub const SETS: [&str; 5] = ["rubric", "av", "defects", "platform", "decide"]; + +fn embedded(name: &str) -> Option<&'static str> { + Some(match name { + "av" => include_str!("../library/av.json"), + "defects" => include_str!("../library/defects.json"), + "platform" => include_str!("../library/platform.json"), + "decide" => include_str!("../library/decide.json"), + _ => return None, + }) +} + +/// The rubric's questions from its guide: each line under "## The questions" is `- id (modes): the question`. A grade +/// over the five levels, as critique asks it. `silent` is when no one speaks, not a mode; a question with an `ID-silent` +/// twin is asked only when someone speaks, and its twin in its place otherwise. +pub fn rubric(md: &str) -> Vec { + let body = md.split("## The questions").nth(1).unwrap_or(""); + let body = body.split("\n## ").next().unwrap_or(""); + let mut out: Vec = body + .lines() + .filter_map(|l| { + let l = l.strip_prefix("- ")?; + let (head, ask) = l.split_once("): ")?; + let (id, modes) = head.split_once(" (")?; + let mut q = Question::plain(id.trim(), ask.trim()); + q.kind = Kind::Grade; + q.uses = vec!["brief".into(), "measures".into()]; + for m in modes.split(',').map(str::trim) { + match m { + "all" => {} + "silent" => q.when.voice = Some(false), + m => q.when.modes.push(m.to_string()), + } + } + Some(q) + }) + .collect(); + let ids: Vec = out.iter().map(|q| q.id.clone()).collect(); + for q in &mut out { + let twin = format!("{}-silent", q.id); + if ids.contains(&twin) { + q.when.voice = Some(true); + q.swap = Some(twin); + } + } + out +} + +/// A set's questions. `rubric_md` is the rubric guide's text, needed only for `rubric`. +pub fn set(name: &str, rubric_md: Option<&str>) -> Result, String> { + if name == "rubric" { + let md = rubric_md.ok_or("the rubric set is read from the rubric guide (raea guide rubric), which wasn't found")?; + return Ok(rubric(md)); + } + let text = embedded(name).ok_or_else(|| format!("no set named {name:?}: {}", SETS.join(", ")))?; + serde_json::from_str(text).map_err(|e| format!("the {name} set doesn't read: {e}")) +} + +/// Every standard question. +pub fn all(rubric_md: Option<&str>) -> Vec { + SETS.iter().filter_map(|s| set(s, rubric_md).ok()).flatten().collect() +} + +/// A standard question by id. +pub fn find(id: &str, rubric_md: Option<&str>) -> Option { + all(rubric_md).into_iter().find(|q| q.id == id) +} diff --git a/core/ask/src/limits.rs b/core/ask/src/limits.rs new file mode 100644 index 000000000..4fd128526 --- /dev/null +++ b/core/ask/src/limits.rs @@ -0,0 +1,68 @@ +//! clef-omni's limits, and raea-api's where it's stricter (POST /v1/clef/ask), so a call that passes here passes on +//! every route (docs/ask.feature AK1). Verified against clef's schema and probe calls 2026-10-18 (sdk/app playtest.ts). + +use crate::question::{Kind, Question}; + +/// questions in one call (clef refuses 65) +pub const QUESTIONS: usize = 64; +/// a question id's length (raea-api's; clef takes 100) +pub const ID_LEN: usize = 64; +/// a choice's options (raea-api's; clef takes 255) +pub const OPTIONS: (usize, usize) = (2, 16); +/// a rate's levels +pub const LEVELS: (usize, usize) = (2, 10); +/// stills a call: 4 MiB each, 8 MiB in all +pub const STILLS: usize = 4; +pub const STILL_BYTES: usize = 4 << 20; +pub const STILLS_BYTES: usize = 8 << 20; +/// videos a call: 60 s and 16 MiB each, sampled at 2 fps +pub const VIDEOS: usize = 2; +pub const VIDEO_SECONDS: f64 = 60.0; +pub const VIDEO_BYTES: usize = 16 << 20; +/// sounds a call: 300 s and 8 MiB each +pub const SOUNDS: usize = 4; +pub const SOUND_SECONDS: f64 = 300.0; +pub const SOUND_BYTES: usize = 8 << 20; +/// sound and video together, decoded +pub const AV_BYTES: usize = 16 << 20; +/// the state paragraph (raea-api cuts it at 8,000 characters) +pub const STATE_CHARS: usize = 8000; +/// dollars a million input tokens +pub const USD_PER_M: f64 = 0.15; +/// input tokens, for the estimate `--dry` prints: a second of 480 px video (about 1,400 a two-second clip, 11-18), a +/// minute of sound, a still +pub const TOKENS_VIDEO_S: f64 = 700.0; +pub const TOKENS_SOUND_MIN: f64 = 780.0; +pub const TOKENS_STILL: f64 = 1024.0; + +/// Why a question can't be sent, or None. +pub fn check(q: &Question) -> Option { + let id_ok = !q.id.is_empty() && q.id.len() <= ID_LEN && q.id.bytes().all(|b| b.is_ascii_alphanumeric() || b"_.-".contains(&b)); + if !id_ok { + return Some(format!("{:?}: an id is 1 to {ID_LEN} letters, digits, _ . or -", q.id)); + } + if q.ask.trim().is_empty() { + return Some(format!("{}: it asks nothing", q.id)); + } + let n = q.levels().len(); + let (lo, hi) = match q.kind { + Kind::Yes => return None, + Kind::Choose | Kind::Grade => OPTIONS, + Kind::Rate => LEVELS, + }; + if q.picks.is_none() && !(lo..=hi).contains(&n) { + return Some(format!("{}: {lo} to {hi} {}, not {n}", q.id, if q.kind == Kind::Choose { "options" } else { "levels" })); + } + let mut seen = std::collections::BTreeSet::new(); + if let Some(o) = q.levels().into_iter().find(|o| !seen.insert(o.clone())) { + return Some(format!("{}: {o:?} is there twice", q.id)); + } + None +} + +/// The estimated cost of one call, in dollars. +pub fn estimate(video_s: f64, sound_s: f64, stills: usize, state_chars: usize, questions: usize) -> f64 { + let text = (state_chars as f64 + questions as f64 * 120.0) / 4.0; + let tokens = video_s * TOKENS_VIDEO_S + sound_s / 60.0 * TOKENS_SOUND_MIN + stills as f64 * TOKENS_STILL + text; + tokens * USD_PER_M / 1e6 +} diff --git a/core/ask/src/question.rs b/core/ask/src/question.rs new file mode 100644 index 000000000..264147e76 --- /dev/null +++ b/core/ask/src/question.rs @@ -0,0 +1,320 @@ +//! A question for the decision model (docs/ask.feature AK1): clef-omni's own types, what a question needs to be asked, +//! and how its answer reads as a problem score, so every question is calibrated the same way whatever its type. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use ts_rs::TS; + +/// The five levels a `grade` reads over unless it names its own (the rubric's, and checks.rs's "score"). +pub const LEVELS: [&str; 5] = ["poor", "weak", "acceptable", "strong", "excellent"]; + +/// clef's question types: `yes` is its noul, `choose` and `grade` its choice, `rate` its score. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, TS)] +#[serde(rename_all = "lowercase")] +pub enum Kind { + /// the probability of yes + #[default] + Yes, + /// one of its options, with a probability on each + Choose, + /// ordered levels, lowest first, read as the expected level + Grade, + /// clef's own rubric: 2 to 10 levels, its probability-weighted level + Rate, +} + +/// What of the picture a question looks at. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, TS)] +#[serde(rename_all = "lowercase")] +pub enum Sees { + /// the moving picture (2 frames a second, at most a minute) + #[default] + Video, + /// stills (a sheet, a storyboard's panels, frames picked out) + Stills, + /// no picture: the sound or the words alone + Nothing, +} + +/// Its options are the files it's given, in order (first, second ...), or all but the first, which is the reference. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TS)] +#[serde(rename_all = "kebab-case")] +pub enum Picks { + All, + AfterFirst, +} + +/// What a piece must be for a question to be asked of it. Unset is any. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, TS)] +#[ts(optional_fields)] +pub struct When { + /// the brief's modes it's for (pitch, story, explain, play, tool); empty is every mode + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(as = "Option>", optional)] + pub modes: Vec, + /// true: only when someone is heard speaking; false: only when no one is + #[serde(skip_serializing_if = "Option::is_none")] + pub voice: Option, + /// true: only for a game; false: never for one + #[serde(skip_serializing_if = "Option::is_none")] + pub game: Option, +} + +/// The seconds a question looks at; unset ends are the file's. +#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize, TS)] +#[ts(optional_fields)] +pub struct Window { + #[serde(skip_serializing_if = "Option::is_none")] + pub from: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub to: Option, +} + +/// How well a question's answers separate work with the problem from work without it, on labels (AK7). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, TS)] +#[ts(optional_fields)] +pub struct Calibration { + pub auc: f64, + pub n: u32, + pub positives: u32, + /// the problem score at which it flags (10% false positives on its labels) + #[serde(skip_serializing_if = "Option::is_none")] + pub threshold: Option, + /// YYYY-MM-DD + pub date: String, + /// whose labels: "raea" (raea's bench, shipped) or "you" + pub by: String, +} + +/// A question, as the standard sets, a taste line, a batch file or the command line write it. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, TS)] +#[serde(deny_unknown_fields)] +#[ts(optional_fields)] +pub struct Question { + /// letters, digits, `_`, `.` and `-` + pub id: String, + /// the question in words, as clef reads it + pub ask: String, + #[serde(rename = "type", default)] + #[ts(rename = "type", as = "Option", optional)] + pub kind: Kind, + /// choose: its options; grade and rate: its levels, lowest first (a grade's are LEVELS unless given) + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(as = "Option>", optional)] + pub options: Vec, + /// what an option (or yes and no) means, when its name alone doesn't say + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + #[ts(as = "Option>", optional)] + pub means: BTreeMap, + /// the answers that mean a problem: yes's "yes" (the default) or "no"; choose's options (none: a pick, no problem) + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(as = "Option>", optional)] + pub flags: Vec, + #[serde(default)] + #[ts(as = "Option", optional)] + pub sees: Sees, + /// whether it listens to the sound (true unless set) + #[serde(default = "yes", skip_serializing_if = "is_true")] + #[ts(as = "Option", optional)] + pub hears: bool, + /// what the state tells clef beside the media: brief, words (heard), screen-words, measures, brand, platform; and + /// with: it compares the file with the files set beside it, so it isn't asked without them + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(as = "Option>", optional)] + pub uses: Vec, + #[serde(default, skip_serializing_if = "When::is_any")] + #[ts(as = "Option", optional)] + pub when: When, + /// the question asked in its place when `when` doesn't hold (sound -> sound-silent) + #[serde(skip_serializing_if = "Option::is_none")] + pub swap: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub window: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub picks: Option, + /// words in a person's note that mean they saw this problem (AK6) + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[ts(as = "Option>", optional)] + pub label_words: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub calibration: Option, + /// what to change when it flags + #[serde(skip_serializing_if = "Option::is_none")] + pub fix: Option, +} + +fn yes() -> bool { + true +} +fn is_true(b: &bool) -> bool { + *b +} + +impl When { + pub fn is_any(&self) -> bool { + self.modes.is_empty() && self.voice.is_none() && self.game.is_none() + } +} + +/// What raea knows of a piece before asking (apps/ask prepare.rs reads it), for `when` and for what the media can be. +#[derive(Debug, Clone, Default)] +pub struct Facts { + pub modes: Vec, + /// someone is heard speaking: Some(true), no one: Some(false), unknown: None + pub voice: Option, + pub game: bool, + pub sound: bool, + pub video: bool, +} + +/// An answer read: its value (yes's probability, choose's top probability, a level), its problem score (0 to 1, higher +/// is likelier a problem; none for a pick), and a line in words. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Reading { + pub value: f64, + pub problem: Option, + pub choice: Option, + pub says: String, +} + +impl Question { + /// A yes or no question in plain words, as the command line and a taste line make one. + pub fn plain(id: &str, ask: &str) -> Question { + serde_json::from_value(json!({"id": id, "ask": ask})).expect("a plain question") + } + + /// A question written in clef's own wire format (`{type: noul | choice | score, instructions, criteria}`), as raea + /// judge's rubric and the playtest packs write them; None when it isn't one. + pub fn from_wire(id: &str, w: &Value) -> Option { + let mut q = Question::plain(id, w["instructions"].as_str()?); + match w["type"].as_str()? { + "noul" => { + if let (Some(y), Some(n)) = (w["criteria"]["true"].as_str(), w["criteria"]["false"].as_str()) { + q.means = BTreeMap::from([("yes".into(), y.into()), ("no".into(), n.into())]); + } + } + "choice" => { + q.kind = Kind::Choose; + for (k, v) in w["criteria"].as_object()? { + q.options.push(k.clone()); + if let Some(m) = v.as_str() { + q.means.insert(k.clone(), m.into()); + } + } + } + "score" => { + q.kind = Kind::Rate; + q.options = w["criteria"].as_array()?.iter().map(|l| l.as_str().map(str::to_string)).collect::>()?; + } + _ => return None, + } + Some(q) + } + + /// Its levels or options as clef is sent them. + pub fn levels(&self) -> Vec { + match (self.kind, self.options.is_empty()) { + (Kind::Grade, true) => LEVELS.iter().map(|s| s.to_string()).collect(), + _ => self.options.clone(), + } + } + + /// A hash of what clef reads (its words, type, options and their meanings): a new wording is a new version, with + /// its own calibration. FNV-1a, so it's the same on every build. + pub fn version(&self) -> String { + let text = format!("{:?}\u{1f}{}\u{1f}{}\u{1f}{:?}", self.kind, self.ask, self.levels().join("\u{1e}"), self.means); + let h = text.bytes().fold(0xcbf2_9ce4_8422_2325u64, |h, b| (h ^ b as u64).wrapping_mul(0x0100_0000_01b3)); + format!("{h:016x}") + } + + /// Why it can't be asked of this piece, or None when it can. + pub fn not_for(&self, f: &Facts) -> Option { + let w = &self.when; + if !w.modes.is_empty() && !w.modes.iter().any(|m| m == "all" || f.modes.contains(m)) { + return Some(format!("it's for {} work, and this is {}", w.modes.join(" or "), if f.modes.is_empty() { "unnamed".into() } else { f.modes.join(" and ") })); + } + match (w.voice, f.voice) { + (Some(true), Some(false)) => return Some("no one speaks in it".into()), + (Some(false), Some(true)) => return Some("someone speaks in it".into()), + _ => {} + } + if w.game.is_some_and(|g| g != f.game) { + return Some(if f.game { "it isn't asked of a game".into() } else { "it's asked of games".into() }); + } + if self.hears && self.sees == Sees::Nothing && !f.sound { + return Some("it listens, and the file has no sound".into()); + } + if self.sees == Sees::Video && !f.video && !(self.hears && f.sound) { + return Some("it watches the picture move, and the file is a still".into()); + } + None + } + + /// The question as clef is sent it. + pub fn wire(&self) -> Value { + match self.kind { + Kind::Yes => { + let mut q = json!({"type": "noul", "instructions": self.ask}); + if let (Some(y), Some(n)) = (self.means.get("yes"), self.means.get("no")) { + q["criteria"] = json!({"true": y, "false": n}); + } + q + } + Kind::Choose | Kind::Grade => { + let criteria: serde_json::Map = self.levels().into_iter().map(|o| { + let m = self.means.get(&o).map(|s| Value::from(s.as_str())).unwrap_or(Value::Null); + (o, m) + }).collect(); + json!({"type": "choice", "instructions": self.ask, "criteria": criteria}) + } + Kind::Rate => json!({"type": "score", "instructions": self.ask, "criteria": self.levels()}), + } + } + + /// clef's answer read (the reading checks.rs and the playtest DSL make). + pub fn read(&self, answer: &Value) -> Reading { + let levels = self.levels(); + let top = (levels.len().max(2) - 1) as f64; + let pr = |k: &str| answer["probabilities"][k].as_f64().unwrap_or(0.0); + match self.kind { + Kind::Yes => { + let p = answer["noul"].as_f64().unwrap_or(0.0); + let bad = if self.flags.first().is_some_and(|f| f == "no") { 1.0 - p } else { p }; + Reading { value: p, problem: Some(bad), choice: None, says: format!("{} ({p:.2} yes)", if p >= 0.5 { "yes" } else { "no" }) } + } + Kind::Choose => { + let choice = answer["choice"].as_str().map(str::to_string); + let p = choice.as_deref().map(pr).unwrap_or(0.0); + let problem = (!self.flags.is_empty()).then(|| self.flags.iter().map(|f| pr(f)).sum::()); + Reading { value: p, problem, says: format!("{} ({p:.2})", choice.as_deref().unwrap_or("no answer")), choice } + } + Kind::Grade | Kind::Rate => { + let v = if self.kind == Kind::Rate { + answer["score"].as_f64().unwrap_or(0.0) + } else { + let total: f64 = levels.iter().map(|l| pr(l)).sum(); + match total > 0.0 { + true => levels.iter().enumerate().map(|(i, l)| i as f64 * pr(l)).sum::() / total, + false => levels.iter().position(|l| answer["choice"].as_str() == Some(l)).unwrap_or(0) as f64, + } + }; + let name = levels.get(v.round() as usize).cloned(); + Reading { value: v, problem: Some((1.0 - v / top).clamp(0.0, 1.0)), says: format!("{} ({v:.1} of {top})", name.as_deref().unwrap_or("?")), choice: name } + } + } + } + + /// Calibrated: its labels separate (an AUC of 0.8 or more, the fingerprint's gate) and it has a threshold. Until + /// then its answers are advice. + pub fn calibrated(&self) -> bool { + self.calibration.as_ref().is_some_and(|c| c.auc >= 0.8 && c.threshold.is_some()) + } + + /// Whether a reading flags a problem: past its calibrated threshold, else past even odds (as advice). + pub fn flagged(&self, r: &Reading) -> bool { + let at = self.calibration.as_ref().and_then(|c| c.threshold).filter(|_| self.calibrated()).unwrap_or(0.5); + r.problem.is_some_and(|p| p >= at) + } +} diff --git a/core/ask/src/tests.rs b/core/ask/src/tests.rs new file mode 100644 index 000000000..0c05affca --- /dev/null +++ b/core/ask/src/tests.rs @@ -0,0 +1,132 @@ +use super::*; +use serde_json::json; + +const RUBRIC: &str = include_str!("../../../sdk/guides/craft/rubric.md"); + +#[test] +fn declarations_match_the_committed_file() { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../sdk/raea/src/ask.gen.ts"); + let now = declarations(); + if std::env::var_os("RAEA_WRITE").is_some() { + std::fs::write(path, &now).unwrap(); + } + let committed = std::fs::read_to_string(path).unwrap_or_default(); + assert!(committed == now, "sdk/raea/src/ask.gen.ts is out of date: RAEA_WRITE=1 cargo test -p raea-core-ask"); +} + +#[test] +fn every_standard_question_reads_and_fits_the_limits() { + let all = library::all(Some(RUBRIC)); + assert!(all.len() >= 45, "{} questions", all.len()); + for q in &all { + assert_eq!(limits::check(q), None, "{}", q.id); + } + let mut ids: Vec<&str> = all.iter().map(|q| q.id.as_str()).collect(); + ids.sort(); + let before = ids.len(); + ids.dedup(); + assert_eq!(before, ids.len(), "an id is used twice"); + for q in all.iter().filter(|q| q.swap.is_some()) { + assert!(library::find(q.swap.as_deref().unwrap(), Some(RUBRIC)).is_some(), "{} swaps to a question that isn't there", q.id); + } +} + +#[test] +fn the_rubric_comes_from_its_guide_and_sound_swaps_when_no_one_speaks() { + let r = library::rubric(RUBRIC); + let sound = r.iter().find(|q| q.id == "sound").unwrap(); + assert_eq!((sound.kind, sound.when.voice, sound.swap.as_deref()), (Kind::Grade, Some(true), Some("sound-silent"))); + let silent = r.iter().find(|q| q.id == "sound-silent").unwrap(); + assert_eq!(silent.when.voice, Some(false)); + let open = r.iter().find(|q| q.id == "open").unwrap(); + assert_eq!(open.when.modes, ["pitch", "story", "explain"]); + let facts = Facts { modes: vec!["pitch".into()], voice: Some(false), sound: true, video: true, ..Default::default() }; + assert_eq!(sound.not_for(&facts).as_deref(), Some("no one speaks in it")); + assert_eq!(silent.not_for(&facts), None); + assert!(r.iter().find(|q| q.id == "loop").unwrap().not_for(&facts).unwrap().contains("explain")); +} + +#[test] +fn a_question_needs_what_the_file_has() { + let mut q = Question::plain("hum", "Is there a hum under the voice?"); + q.sees = Sees::Nothing; + let still = Facts { video: false, sound: false, ..Default::default() }; + assert!(q.not_for(&still).unwrap().contains("no sound")); + let picture = Question::plain("cut", "Does the cut jump?"); + assert!(picture.not_for(&still).unwrap().contains("still")); + assert_eq!(picture.not_for(&Facts { video: true, ..Default::default() }), None); +} + +#[test] +fn each_type_goes_out_as_clefs_own_and_reads_back_as_a_problem_score() { + let mut q = Question::plain("fits", "Does the sound fit the picture?"); + q.flags = vec!["no".into()]; + assert_eq!(q.wire(), json!({"type": "noul", "instructions": "Does the sound fit the picture?"})); + let r = q.read(&json!({"noul": 0.2})); + assert_eq!((r.value, r.problem), (0.2, Some(0.8))); + assert!(q.flagged(&r), "advice flags past even odds"); + + let mut g = Question::plain("pace", "How well is it paced?"); + g.kind = Kind::Grade; + assert_eq!(g.wire()["criteria"].as_object().unwrap().len(), 5); + let r = g.read(&json!({"choice": "strong", "probabilities": {"strong": 0.5, "excellent": 0.5}})); + assert_eq!((r.value, r.problem, r.choice.as_deref()), (3.5, Some(0.125), Some("excellent"))); + + let mut rate = Question::plain("look", "How strong is the look?"); + rate.kind = Kind::Rate; + rate.options = vec!["none".into(), "some".into(), "strong".into()]; + assert_eq!(rate.wire()["criteria"], json!(["none", "some", "strong"]), "a score's levels are a list"); + assert_eq!(rate.read(&json!({"score": 1.0})).problem, Some(0.5)); + + let mut c = Question::plain("best", "Which is best?"); + c.kind = Kind::Choose; + c.options = vec!["first".into(), "second".into()]; + let r = c.read(&json!({"choice": "second", "probabilities": {"first": 0.3, "second": 0.7}})); + assert_eq!((r.choice.as_deref(), r.problem), (Some("second"), None), "a pick has no problem score"); +} + +#[test] +fn a_question_in_clefs_own_format_comes_back_as_it_went() { + for w in [ + json!({"type": "noul", "instructions": "Is it a photograph?"}), + json!({"type": "noul", "instructions": "Is it a photograph?", "criteria": {"true": "a real place", "false": "a render"}}), + json!({"type": "score", "instructions": "How is the light?", "criteria": ["none", "flat", "good"]}), + json!({"type": "choice", "instructions": "Which?", "criteria": {"a": null, "b": "the second"}}), + ] { + assert_eq!(Question::from_wire("q", &w).unwrap().wire(), w); + } + assert!(Question::from_wire("q", &json!({"type": "essay", "instructions": "Say"})).is_none()); +} + +#[test] +fn a_calibrated_question_flags_at_its_own_threshold() { + let q = library::find("face.text_over", None).unwrap(); + assert!(q.calibrated()); + assert!(q.flagged(&q.read(&json!({"noul": 0.36}))), "0.36 is past its 0.3487"); + let weak = library::find("speech.uncaptioned", None).unwrap(); + assert!(!weak.calibrated(), "an AUC of 0.45 is advice"); +} + +#[test] +fn a_new_wording_is_a_new_version_and_the_id_is_not_part_of_it() { + let a = Question::plain("a", "Is it loud?"); + let b = Question::plain("b", "Is it loud?"); + let c = Question::plain("a", "Is it too loud?"); + assert_eq!(a.version(), b.version()); + assert_ne!(a.version(), c.version()); + assert_eq!(a.version(), "3ab49a6dcc96d4ad", "the same on every build (FNV-1a)"); +} + +#[test] +fn limits_refuse_what_raea_api_would() { + let mut q = Question::plain("a b", "x"); + assert!(limits::check(&q).unwrap().contains("id")); + q.id = "ok".into(); + q.kind = Kind::Choose; + q.options = vec!["one".into()]; + assert!(limits::check(&q).unwrap().contains("2 to 16")); + q.options = vec!["one".into(), "one".into()]; + assert!(limits::check(&q).unwrap().contains("twice")); + let e = limits::estimate(60.0, 60.0, 0, 400, 3); + assert!(e > 0.005 && e < 0.008, "{e}"); +} diff --git a/docs/ask.feature b/docs/ask.feature new file mode 100644 index 000000000..766227738 --- /dev/null +++ b/docs/ask.feature @@ -0,0 +1,211 @@ +@spec +Feature: Ask: questions about a piece's picture and sound, answered by the decision model and paid from the person's plan + Shane, 2026-11-20: "a decision reviewer type api that calls and utilizes their raea subscription ... for checks and + things, or extending checks to be done quickly based on audio and video, maybe even being able to ask a question + about how does this sound based on the visuals". Planned "so that it sufficiently understands first what is + required of it, what types of options come standard, how ask works and can be improved over time". And: "the agent + or human may want to have a check automated in their own pipelines that asks, or ... fingerprint checks to also call + an ask call to double check, or ... a script that collates a bunch of ask calls in parallel and processes the needed + clips (which should be the case anyway)". + + The model this file sets: + 1. The decision model is clef-omni on Workers AI: typed questions (yes or no, a choice, a graded level) about text, + up to 4 stills, 4 sounds of 5 minutes and 2 videos of 60 s at 2 fps, answered in probabilities, never in free + text; the same request gets the same answer. It costs by the media it reads (about $0.006 a minute of video; + sound costs little), not by the questions, so every question about the same media goes in one call. + 2. One way to ask, for every caller: `raea ask` on the command line, a person's own pipeline or CI, a script that + asks many questions over many clips, `raea check`, the fingerprint double-checking its own flags, the app's + agents, its critique and game checks, and `raea judge`. One question format, one client, one parallel runner. + 3. Ask understands a question before it spends: what the question needs (picture, sound, both, the words heard, the + words on screen, the brief), whether this file can answer it, and what the piece is. A question that can't be + answered isn't asked; it says why, or its variant is asked instead. + 4. It's paid from the person's raea plan by default (Pro's and Studio's monthly pool, which stops when it's spent), + else from their own Cloudflare account; with neither it refuses and says how to get one. + 5. Local measures come first. Clef gets what can't be measured, only for windows that changed, cached by content. + Timing (sync, freezes, dropouts) stays local: clef samples video at 2 fps and was weakest there (11-18). + 6. An answer is advice until its question is calibrated on the person's own labels; every answer is kept, so each + question earns its threshold, a better wording can be measured against the old one, and a well-calibrated + question can later be taught to a free local head. + + Background: + Given clef's four callers today, each with its own client and all on the person's own Cloudflare keys: the app's + critique (apps/app host/critique.ts: the export's first minute at 480 px with its sound as mono MP3, the rubric's + questions for the brief's modes), its game check (host/playcheck.ts), `raea judge` (apps/judge clef.rs, review + sheets on a frozen rubric) and the playtest DSL (sdk/app playtest.ts, which holds clef's verified limits) + And raea-api's `POST /v1/clef/ask` (raea-platform PR #15, not deployed): an account's questions charged to its pool + (Pro $2, Studio $20 a month, stopping when spent), cached in R2 by request, under a daily ceiling across everyone; + images and videos only + And the check packs (render, hook, storyboard, brand: 22 checks), each clef question kept with its `label_words`, + its AUC, its threshold at 10% false positives and its status (advisory or warn), labelled from people's notes + And the fingerprint's evidence rows, where another eye's readings (`source: "eye:NAME"`) train a head and never + count as the person's (FP6d), its AUC and quantile (raea-sys-fingerprint head.rs) and its gates (16 labels of + each kind, 3 projects, a held-out AUC of 0.8: train.rs) + And taste lines (core/brief taste.rs): words, a check (one of six forms, a named check, a `.ts` check), values and a + level, held by `raea check PROJECT` (apps/direction check.rs) through raea's named checks (systems.rs `named()`) + And what clef has shown: its yes is soft (0.15 to 0.51 on real defects), so a threshold is per question; it answers + what's asked, so the rubric's sound question, which assumes a voice, called a promo with no voice "weak" until + `sound-silent` was asked in its place (docs/craft-system.md) + + Rule: AK1. One question format and one client + + @done + Scenario: AK1. Questions, and the standard library + Given the plan + | crate | file | lines | job | + | raea-core-ask | core/ask/src/lib.rs | 40 | the crate's exports | + | raea-core-ask | core/ask/src/question.rs | 220 | a question: id, ask, type (yes, choose, grade, rate) with its options or levels, sees and hears, uses, when, window, label_words, calibration; its version, a hash of its words, type and options; TypeScript by ts-rs | + | raea-core-ask | core/ask/src/library.rs | 200 | the standard sets: rubric (read from sdk/guides/craft/rubric.md, one source), av, defects, platform, decide; a set or a question by id | + | raea-core-ask | core/ask/src/limits.rs | 60 | clef's limits (questions, ids, options, levels, stills, videos, sounds, sizes, tokens, price), checked before anything is sent | + | raea-core-ask | core/ask/library/av.json | 80 | picture with sound: the sound fits what's on screen, the music's mood matches the picture, the music fights the voice, the voice suits who's on screen, the ambience matches the place, effects mark moments | + | raea-core-ask | core/ask/library/defects.json | 70 | the fingerprint's axes with no labels yet: muffled voice, harsh music, glitch, fake, barren, unreadable text, shaky camera | + | raea-core-ask | core/ask/library/platform.json | 160 | the check packs' clef questions, with their label words and calibration | + | raea-core-ask | core/ask/library/decide.json | 50 | the best of these for the brief; the nearest to a reference; whether a fix changed what a viewer sees | + Then a question is clef's own shape: `yes` (a probability of yes), `choose` (2 to 255 options), `grade` (ordered + levels, poor to excellent unless given, read as the expected level) or `rate` (2 to 10 levels) + And `when` says what a question needs to be asked (modes; a voice heard or none; a game), and `swap` names the + question asked in its place when it doesn't apply, so `sound` to `sound-silent` is data, not code + And every answer reads as a problem score from 0 to 1 whatever its type (a yes's probability on the answer that + flags, a choice's on its flagging options, one less a grade's level over its top), so one calibration fits all + And the limits are raea-api's where they're stricter than clef's (ids of 64 characters, 16 options), so a call that + passes them passes on every route + Status: done 11-20. raea-core-ask: four types, 49 standard questions in five sets (the rubric's read from + rubric.md), the platform packs' three calibrated questions keeping their thresholds; its tests read every + standard question, check the limits and the TypeScript declarations (sdk/raea/src/ask.gen.ts). + + @needs:AK1 @done + Scenario: AK1b. One clef client, routed to the person's plan + Given the plan + | crate | file | lines | job | + | raea-io-clef | io/clef/src/lib.rs | 40 | the crate's exports | + | raea-io-clef | io/clef/src/client.rs | 160 | one call: state, questions, stills, videos and sounds as data URLs; its answers and cost; apps/judge clef.rs moved here and widened | + | raea-io-clef | io/clef/src/media.rs | 180 | the media cut with raea's FFmpeg: a window as 480 px H.264, at most 60 s; its sound as mono 16 kHz MP3; a still as JPEG at most 1,600 px | + | raea-io-clef | io/clef/src/route.rs | 160 | the route: the app's host (~/.raea/app-host.json, /host/ask) when signed in with a pool; RAEA_AGENT_KEY to raea-api; the person's Cloudflare keys; else a refusal that says how to get one | + | raea-io-clef | io/clef/src/cache.rs | 80 | answers by request hash in ~/.raea/ask/cache, so an unchanged question on unchanged media costs nothing | + Then a call goes the first way that's open: the app (signing in stays the app's alone), an agent key with the + `ask` scope (a pipeline, CI, raea cloud), the person's own `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN` + And with none it refuses: "sign in to raea on Pro or Studio, set RAEA_AGENT_KEY, or add your Cloudflare keys + (docs/settings.md)" + And `raea judge` asks through it, and apps/judge's own clef.rs is gone + Status: done 11-20. raea-io-clef's tests, on a fake transport: a route that can't answer hands on, a spent pool + stops unless the person has their own keys, the plan's answer carries what's left and a repeat comes from the + cache, none open refuses with how to get one, a call over the limits is never sent. A real call went the + own-keys route (AK3's status); the app route answers once AK4 adds /host/ask. raea judge's tests pass on it. + + Rule: AK2-AK3. `raea ask` + + @needs:AK1b @done + Scenario: AK2. Ask understands the question before it spends + Given the plan + | crate | file | lines | job | + | raea-app-ask | apps/ask/src/prepare.rs | 260 | the file's facts (length, picture, sound, the words heard from a words file, whether anyone speaks) and the project's brief (title, mode, audience, believe, logline, voice); each question kept, swapped or dropped with why (not knowing whether anyone speaks, both variants kept); the state paragraph, with what the caller measured (`--state`; raea check passes its own, AK5) | + Then a question about sound on a file with none, or a stills-only render, isn't asked, and says it waits for one + And a piece where no one speaks is asked `sound-silent` in place of `sound`, and told it has no voice + And the state says what the piece is (its modes, the brief's audience and belief, its storyboard's logline, what + raea measured), as critique.ts writes it today + And `--dry` prints exactly what would be sent (the state, each call's questions, the media and its size) and an + estimated cost, and spends nothing + Status: done 11-20. prepare.rs reads the file (ffprobe), the nearest brief.ts (`raea spec --as brief`) and a words + file; apps/ask tests: a silent piece asks sound-silent with why, a file with no sound isn't listened to, the + state names the modes, audience, belief and what the caller measured. `--dry` on an 8 s clip printed one call + of 11 questions, its state, 26 KB of video and 63 KB of sound, about $0.0009, and sent nothing. + + @needs:AK2 @done + Scenario: AK3. The runner, the record and the verb + Given the plan + | crate | file | lines | job | + | raea-app-ask | apps/ask/src/lib.rs | 60 | the crate's exports and its usage | + | raea-app-ask | apps/ask/src/verbs.rs | 260 | `raea ask FILE [QUESTION]` (--choose, --grade, --rate, --set, --std, --from/--to, --at, --with, --brief, --dry, --json, --gate, --jobs), `--batch FILE`, `ask list` | + | raea-app-ask | apps/ask/src/runner.rs | 280 | the batch planned (grouped by file, window and media), each clip cut once, calls in parallel (4 at once unless --jobs), at most 64 questions each, answers streamed as they come; a pool spent midway keeps what came and reports the rest as waiting | + | raea-app-ask | apps/ask/src/record.rs | 160 | every answer in ~/.raea/ask/answers.jsonl (question and version, media hash, window, project, item, answer, cost, route); the cut media kept by hash in ~/.raea/ask/media, at most 2 GB, oldest out first | + When the person or an agent runs `raea ask out/piece.mp4 "Does the music fight the voice?"` + Then it answers in words with the probability, the window it looked at, what it cost and the pool left, and says + it's advice until the question is calibrated + And `raea ask --batch asks.json` (or JSON lines on stdin; each entry a file, a window, and questions, a set or + standard ids) cuts each clip once and asks everything in parallel, and `raea check`, the critique and the + fingerprint call the same runner + And `--json` is a stable schema for pipelines, and `--gate` exits 1 when a calibrated question fails (`--gate any`: + when any answer passes its threshold) + And sdk/raea's `raea/ask` gives scripts `ask` and `askMany`, typed from the generated questions, over + `raea ask --batch --json`, from a `.ts` check, a `raea run` script or plain Node + And docs/manual/reference/cli/ask.md is its reference, written in the same change + Status: done 11-20. One call on the person's own Cloudflare keys asked 4 questions of a test pattern with a tone + for $0.0003 (test pattern 0.88 yes, anyone speaking 0.00, the sound fits 0.06), and the same again came from the + cache for $0; calls.jsonl, answers.jsonl and the two cut clips were kept. apps/ask tests: 65 questions make two + calls, windows and what a question looks at split calls, a pick or a before-and-after without `--with` isn't + asked, a clip used by two calls is cut once, the voice answer sets the variant that doesn't apply aside, and a + pool spent on the third call keeps two calls' answers and sends nothing more. `raea check`, the critique and + the fingerprint move onto the runner in AK5, AK4 and AK8. docs/manual/reference/sdk/ask.md is `raea/ask`'s. + + Rule: AK4. On the person's plan + + @needs:AK3 + Scenario: AK4. The pool, through the app or an agent key + Then raea-api's `/v1/clef/ask` takes sounds (up to 4 MP3s) beside stills and videos, says the pool left with + each answer, and agent keys gain an `ask` scope that asks and can do nothing else + And the app's host answers `/host/ask` by adding its device token and passing the call on, with the API's answer + back as it came: a plan with no pool (402 plan_required) or signed out (401) sends the CLI on to the next route + And the app's critique and game check ask through `raea ask --json`, and their own clef code is gone + And the app's agents get an `ask` tool (HOST) with `raea ask`'s inputs + Status 11-20: built, in review. raea-platform #16 (on #15): audio, the criteria clef takes, pool_left_usd and the + `ask` scope, with test/pool.sh reading what a fake clef was sent. raea-feed's feed-ask branch (on #98): + /host/ask, the `ask` tool, and the critique's judge on `raea ask --batch - --json` with `--brief none`, its + tests, typecheck and the protocol and hub tests passing. It's done when they merge and raea-api is deployed, + which waits on Shane's go. + + Rule: AK5. Standing questions in `raea check` + + @needs:AK3 + Scenario: AK5. A taste line that asks + Then a taste line with `check: "ask"` is asked in its own words ("The music never fights the voice."), and + `check: "ask:ID"` asks a standard question; its `values` can set the type, options, sees, hears and window + And `raea check PROJECT` asks every such line of a composition in one batch on its newest render, waits without + a render, says what it cost, and asks nothing again on an unchanged render + And an answer is a finding at the line's level once its question is calibrated, and a note until then + And `--no-ask` leaves them out + + Rule: AK6-AK7. Better with use + + @needs:AK3 + Scenario: AK6. Labels + Given the plan + | crate | file | lines | job | + | raea-app-ask | apps/ask/src/labels.rs | 220 | an item's notes matched to the questions' label words (a match is yes, an approval with none is no); `raea ask verdict ANSWER yes|no`; an agent's own look kept as another eye | + Then the person's labels count toward calibration, and an agent's are kept as `eye:agent` and never counted + + @needs:AK6 + Scenario: AK7. Calibrate, and try new words + Given the plan + | crate | file | lines | job | + | raea-app-ask | apps/ask/src/calibrate.rs | 240 | per question version: its AUC and threshold at 10% false positives from the person's labels, through the fingerprint's gates; `ask try "new words" --like ID` replayed on the kept media | + Then a question with 16 labels of each kind from 3 projects and a held-out AUC of 0.8 or more is calibrated, at + its threshold + And one under 0.6 after 32 labels says it doesn't separate, with its misses, so its words can be fixed + And `raea ask try` asks the new words on the labelled answers' kept media and prints both AUCs, and a standard + question's words change only with a better AUC on raea's bench + And a standard question ships with raea's calibration from that bench, and the person's own replaces it once it + passes the gates + + Rule: AK8. The fingerprint asks too + + @needs:AK3 + Scenario: AK8. A double check on the fingerprint's flags + Given the plan + | crate | file | lines | job | + | raea-app-ask | apps/ask/src/double.rs | 160 | a score's flagged moments, each asked in its axis's own words on its own window; the answers beside the flags and kept as evidence | + | raea-app-fingerprint | apps/fingerprint/src/verbs.rs | 15 | `score --ask` and fingerprint.ts's `ask` hand the flagged moments to raea-app-ask | + Then `raea fingerprint score FILE --ask` (or `ask: "flags"` in fingerprint.ts) asks each flagged moment, and + `ask: "unsure"` also each moment between an axis's plausible and strong bands + And each flag says "clef agrees (0.71)" or "clef doesn't see it (0.08)", and each answer is an `eye:clef` + evidence row that trains the head and never counts as the person's + And it costs by the flags, never the whole piece + And the app's rule_check asks the rules nothing measures, and the fingerprint with `--ask unsure`, before it asks + the agent for its own verdicts, so the agent opens only the frames clef flags + + Rule: AK9. Handing a question to a local head + + @needs:AK7,AK8 + Scenario: AK9. A measurement: can a calibrated question be learned locally + Given one calibrated question with its kept answers and the person's labels + When a fingerprint head is trained on its windows' SigLIP or CLAP columns from clef's answers + Then its held-out AUC on the person's labels is set beside clef's + And only if it matches does `raea check` ask the local head first and clef only in its unsure band diff --git a/docs/manual/reference/cli/ask.md b/docs/manual/reference/cli/ask.md new file mode 100644 index 000000000..1cb0baf51 --- /dev/null +++ b/docs/manual/reference/cli/ask.md @@ -0,0 +1,77 @@ +--- +title: "raea ask" +summary: "Ask about a piece's picture and sound: clef-omni answers in words and a probability, paid from your raea plan, and every answer is kept so a question can be calibrated." +order: 130 +sources: ["apps/ask/src/verbs.rs", "apps/ask/src/prepare.rs", "apps/ask/src/runner.rs", "io/clef/src/route.rs", "core/ask/src/library.rs"] +--- + +`raea ask` puts questions about a video, a sound or a picture to clef-omni, the decision model raea's checks use, and +prints each answer in words with its probability, the window it looked at and what it cost. Ask in your own words, or +use the standard questions: the craft rubric's (`rubric`), picture and sound together (`av`), the defects the +fingerprint can't judge yet (`defects`), the platform checks (`platform`) and picking between versions (`decide`). +`raea ask list` lists them all. + +Before it spends anything it reads what it can about the piece: what the file holds, the project's brief (the nearest +`brief.ts` above the file, or `--brief`) and whether anyone speaks (`--words`, `--voice`). A question that doesn't +apply isn't asked, and its line says why: a sound question of a silent file, or the rubric's `sound` question of a piece +where no one speaks, which asks `sound-silent` instead. Every question about the same clip goes in one call, and +many questions cost about what one does. + +It pays from your raea plan's monthly pool when the raea app is signed in on Pro or Studio, or with an agent key +(`RAEA_AGENT_KEY` with `RAEA_API`) where the app isn't running, as in a pipeline or CI. Otherwise it uses your own +Cloudflare keys ([settings](../../settings.md)). When the pool is spent, the questions not yet asked wait for next +month, and their lines say so. The same question about the same media comes from a cache and costs nothing. + +```sh +raea ask out/piece.mp4 "Does the music fight the voice?" +raea ask out/piece.mp4 --set av --brief brief.ts +raea ask out/piece.mp4 "Which ending lands harder?" --with out/v2.mp4 --std decide.best +raea ask out/piece.mp4 "Is the logo readable?" --at 12.5 --flag no +raea ask out/piece.mp4 --set defects --dry # what would be sent and its cost; nothing is spent +raea ask --batch asks.jsonl --json --gate # a pipeline: one JSON line an answer, exit 1 on a finding +``` + +A batch is a JSON list, or JSON lines, of `{file, questions?, set?, std?, window?, with?, brief?, words?, voice?, +state?}`, with `-` reading stdin. Its clips are each cut once, and its calls go four at a time (`--jobs`). Scripts can +ask through [`raea/ask`](../sdk/ask.md) instead. + +Answers are advice until a question is calibrated against your own verdicts, and the answers say which they are. With +`--gate`, a calibrated question that flags exits 1; with `--gate any`, any question that flags does. Every call and +answer is kept in `~/.raea/ask` (`RAEA_ASK_DIR`), with the clips it cut, so that a question can be labelled and +calibrated later. + + + +## Usage + +```sh +raea ask FILE [QUESTION...] [--choose A,B,C] [--grade] [--rate N] [--flag ANSWERS] [--set NAME] [--std ID] [--from S] [--to S] [--at S] [--with FILE2] [--brief brief.ts|none] [--words words.json] [--voice yes|no] [--state TEXT] [--dry] [--json] [--gate [any]] [--jobs N] | ask --batch FILE [--dry] [--json] [--gate [any]] [--jobs N] | ask list [SET] [--json] (questions about a piece's picture and sound, answered by clef-omni and paid from your raea plan's pool when the app is signed in, else RAEA_AGENT_KEY's, else your own Cloudflare keys: your own words, or the standard sets (rubric, av, defects, platform, decide); every question about one clip goes in one call, --batch (a JSON list or JSON lines, - for stdin) asks many clips in parallel; --dry says what would be sent and spends nothing; answers are advice until a question is calibrated) +``` + +Headless: it needs nothing running and opens no window. Takes any number of arguments besides its flags. + +## Flags + +| Flag | Takes | What it does | +| --- | --- | --- | +| `--at` | `S` | The three seconds around this moment: every question looks at that window. | +| `--batch` | `FILE` | A file of entries to ask, a JSON list or JSON lines (`-` reads stdin); each is a file with its questions. | +| `--brief` | `brief.ts\|none` | The project's brief, so clef knows who it's for and what it should do (else the nearest brief.ts above the file; `none` for no brief, when `--state` says what the piece is). | +| `--choose` | `A,B,C` | Your questions pick one of these options, rather than answering yes or no. | +| `--dry` | – | Cut the media and print each call that would be made (its state, questions, media sizes) with an estimated cost, and send nothing. | +| `--flag` | `ANSWERS` | The answers that mean a problem: `no` for a question where no is bad ("Is the logo readable?"), or a choice's options. | +| `--from` | `S` | Where the window starts, in seconds. | +| `--gate` | – | Exit 1 when a calibrated question flags, or questions wait for the pool; `--gate any` exits 1 when any question flags. | +| `--grade` | – | Your questions answer on a graded scale (poor, weak, acceptable, strong, excellent). | +| `--jobs` | `N` | How many calls go at once (4). | +| `--json` | – | One JSON object a line: each answer, each call `--dry` would make, then the summary. | +| `--rate` | `N` | Your questions answer with a score from 1 to N. | +| `--set` | `NAME` | A standard set to ask: rubric, av, defects, platform or decide (comma-separated for several). | +| `--state` | `TEXT` | More for clef to read before the media: what you measured, the platform it's for. | +| `--std` | `ID` | Standard questions by id (`raea ask list`), comma-separated. | +| `--to` | `S` | Where the window ends, in seconds. | +| `--voice` | `yes\|no` | Whether anyone speaks in it, when no words file says: it decides which of the voice questions' variants is asked. | +| `--with` | `FILE2` | Another file beside this one, to pick between or compare (repeat it for up to four in all). | +| `--words` | `words.json` | A words file (a JSON list of the words heard): clef reads what is said, and none heard means no one speaks. | + + diff --git a/docs/manual/reference/cli/index.md b/docs/manual/reference/cli/index.md index c4e151daa..7a5e52479 100644 --- a/docs/manual/reference/cli/index.md +++ b/docs/manual/reference/cli/index.md @@ -131,5 +131,6 @@ What an agent reads and shows the person, and the tools raea installs. | [`raea skills`](skills.md) | Install raea's agent skill where Claude Code, Codex, Kimi or Gemini read skills, with its MCP server and hook. | | [`raea setup`](setup.md) | Find Blender, or download its current LTS, and install raea's add-on and the MCP add-on. | | [`raea bug`](bug.md) | Report a bug in raea itself to raea's server: a line saying what's wrong and what happened, with raea's version and the OS added, no account needed. | +| [`raea ask`](ask.md) | Ask about a piece's picture and sound: clef-omni answers in words and a probability, paid from your raea plan, and every answer is kept so a question can be calibrated. | diff --git a/docs/manual/reference/sdk/ask.md b/docs/manual/reference/sdk/ask.md new file mode 100644 index 000000000..2f5133b2c --- /dev/null +++ b/docs/manual/reference/sdk/ask.md @@ -0,0 +1,177 @@ +--- +title: "raea/ask" +summary: "Ask about a piece's picture and sound from a script: the same questions, runner and record as raea ask, typed." +order: 106 +sources: ["sdk/raea/src/ask.ts"] +--- + +`raea/ask` asks from a script what [`raea ask`](../cli/ask.md) asks from the command line: a script run with `raea run`, +a `.ts` check, or plain Node or Bun. It runs `raea ask --batch - --json`, so every question about one clip goes in one +call, the clips are cut once, the calls go in parallel, and every answer is kept for calibration. It pays the same way: +your raea plan when the app is signed in, an agent key, or your own Cloudflare keys. It throws when nothing could be +asked, and the error says why. + +```ts +import { ask, askMany } from "raea/ask"; + +const { answers } = ask("out/piece.mp4", "Does the music fight the voice?"); +const run = askMany([ + { file: "out/a.mp4", set: "av", brief: "brief.ts" }, + { file: "out/b.mp4", std: ["defect.glitch"], window: { from: 0, to: 3 } }, +]); +for (const a of run.answers) if (a.flagged) console.log(a.file, a.id, a.says, a.calibrated ? "finding" : "advice"); +``` + + + +```ts +import { ask, askMany, SETS } from "raea/ask" +``` + +`raea/ask` exports 2 functions, 1 value, 12 types. Source: [sdk/raea/src/ask.ts](https://github.com/shinyobjectz/raea/blob/main/sdk/raea/src/ask.ts). + +## Functions + +### `ask` + +Questions about one file: your own words (yes or no), written-out questions, or with `std`/`set` the standard ones. + +```ts +function ask(file: string, question?: string | Question | (string | Question)[], more?: Omit & AskOptions): Asked +``` + +### `askMany` + +Many files asked in one run, in parallel; throws when nothing could be asked (the error says why). + +```ts +function askMany(entries: AskEntry[], o?: AskOptions): Asked +``` + +## Values + +### `SETS` + +## Types + +### `Answer` + +One question's answer, or why it wasn't asked or still waits: a line of `raea ask --json`. + +| Field | Type | What it is | +| --- | --- | --- | +| `answer` | `string` | its id in the record, for `raea ask verdict` | +| `file` (required) | `string` | the file asked about | +| `id` (required) | `string` | the question's id | +| `version` (required) | `string` | a hash of its words, type and options: a new wording is a new version, calibrated on its own | +| `from` (required) | `number` | the window's start, in seconds | +| `to` | `number` | the window's end, in seconds (unset: the file's end) | +| `says` | `string` | the answer in words with its probability | +| `value` | `number` | yes's probability, a pick's probability, or a level | +| `problem` | `number` | 0 to 1, higher is likelier a problem (none for a pick) | +| `choice` | `string` | the option or level chosen | +| `flagged` (required) | `boolean` | past the question's threshold (its calibrated one, else even odds) | +| `calibrated` (required) | `boolean` | its labels separate well enough for its answers to be findings; until then they're advice | +| `not_asked` | `string` | why it wasn't asked (it doesn't apply to this piece, or the call failed) | +| `waits` | `string` | why it still waits (this month's pool is spent) | +| `route` | `string` | the way it was paid: your raea plan through the app, an agent key, or your Cloudflare account | +| `cached` (required) | `boolean` | answered from the cache, at no cost | + +### `Asked` + +A run's answers, the calls `dry` would make, and its summary. + +| Field | Type | What it is | +| --- | --- | --- | +| `answers` (required) | `Answer[]` | every question's answer, or why it wasn't asked, in the order they came | +| `would` (required) | `unknown[]` | with `dry`: each call that would be made (its state, questions, media sizes and estimate) | +| `summary` (required) | `Summary` | what it came to | + +### `AskEntry` + +One file and what to ask of it: a line of `raea ask --batch`. + +| Field | Type | What it is | +| --- | --- | --- | +| `file` (required) | `string` | the video, sound or picture asked about | +| `questions` | `(string \| Question)[]` | questions in your own words (yes or no) or written out | +| `set` | `string \| string[]` | standard sets by name: rubric, av, defects, platform, decide | +| `std` | `string[]` | standard questions by id (`raea ask list` lists them) | +| `window` | `Window` | the seconds to look at, for every question here (else each question's own, else the whole file) | +| `with` | `string[]` | files set beside it: to pick between, or a before and after | +| `brief` | `string` | the project's brief (else the nearest brief.ts above the file; "none": no brief, the state saying what the piece is) | +| `words` | `string` | a words file (a JSON list of the words heard), so ask knows what's said and whether anyone speaks | +| `voice` | `boolean` | whether anyone speaks in it, when no words file says | +| `state` | `string` | more for clef to read before the media: what raea measured, the platform | + +### `AskOptions` + +How a run goes. + +| Field | Type | What it is | +| --- | --- | --- | +| `jobs` | `number` | calls at once (4 unless set) | +| `dry` | `boolean` | cut the media and say what would be sent, spending nothing | +| `bin` | `string` | the raea to run (RAEA_BIN, else raea on the PATH) | + +### `Calibration` + +```ts +type Calibration = { auc: number, n: number, positives: number, /** * the problem score at which it flags (10% false positives on its labels) */ threshold?: number, /** * YYYY-MM-DD */ date: string, /** * whose labels: "raea" (raea's bench, shipped) or "you" */ by: string, } +``` + +### `Kind` + +```ts +type Kind = "yes" | "choose" | "grade" | "rate" +``` + +### `Picks` + +```ts +type Picks = "all" | "after-first" +``` + +### `Question` + +```ts +type Question = { /** * letters, digits, `_`, `.` and `-` */ id: string, /** * the question in words, as clef reads it */ ask: string, type?: Kind, /** * choose: its options; grade and rate: its levels, lowest first (a grade's are LEVELS unless given) */ options?: Array, /** * what an option (or yes and … +``` + +### `Sees` + +```ts +type Sees = "video" | "stills" | "nothing" +``` + +### `Summary` + +What a run came to. + +| Field | Type | What it is | +| --- | --- | --- | +| `asked` (required) | `number` | the questions sent | +| `answered` (required) | `number` | the answers read | +| `calls` (required) | `number` | the calls made (with `dry`, the calls that would be) | +| `cost` (required) | `number` | what the calls cost, in dollars | +| `estimate` (required) | `number` | with `dry`, what the calls would cost, in dollars | +| `pool_left` | `number` | what's left of this month's pool, in dollars, when the plan paid | +| `waiting` (required) | `number` | the questions waiting for the pool | +| `flagged` (required) | `number` | the answers past their threshold | +| `flagged_calibrated` (required) | `number` | the calibrated answers past their threshold: findings | +| `routes` (required) | `string[]` | the ways the calls were paid | +| `clips` (required) | `number` | the clips cut, each once | + +### `When` + +```ts +type When = { /** * the brief's modes it's for (pitch, story, explain, play, tool); empty is every mode */ modes?: Array, /** * true: only when someone is heard speaking; false: only when no one is */ voice?: boolean, /** * true: only for a game; false: never for one */ game?: boolean, } +``` + +### `Window` + +```ts +type Window = { from?: number, to?: number, } +``` + + diff --git a/docs/manual/reference/sdk/index.md b/docs/manual/reference/sdk/index.md index 41a086b8d..437a90c43 100644 --- a/docs/manual/reference/sdk/index.md +++ b/docs/manual/reference/sdk/index.md @@ -99,5 +99,6 @@ Checks, lints and a headless App you can drive. | [`raea/fingerprint`](fingerprint.md) | A fingerprint (fingerprint.ts, docs/fingerprint.md): a person's or a team's sense of good work as axes, each with a small head over on-device encoders that `raea fingerprint` trains from their labels. `raea fingerprint new NAME` writes one with raea's base taxonomy; add an axis, reword one, or remove one, and raea retrains. Rust reads the default export (systems/fingerprint/src/taxonomy.rs), so these types follow its fields. | 1 function, 4 types | | [`raea/dev_tools`](dev_tools.md) | Checks and lints you can run from code: a motion performance's lint report. | 2 functions, 3 types | | [`raea/remote`](remote.md) | A headless Bevy App you open, step and query from TypeScript, through Bevy's remote protocol. | 1 class, 1 type | +| [`raea/ask`](ask.md) | Ask about a piece's picture and sound from a script: the same questions, runner and record as raea ask, typed. | 2 functions, 1 value, 12 types | diff --git a/docs/settings.md b/docs/settings.md index 868c95e88..3674bd0af 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -41,7 +41,11 @@ libraries (pianos, General MIDI instruments, orchestral and percussion samples, | Setting | What | |---|---| -| `CLOUDFLARE_ACCOUNT_ID`, `CLOUDFLARE_API_TOKEN` | The checks' judge model (clef-omni on Workers AI) | +| `CLOUDFLARE_ACCOUNT_ID`, `CLOUDFLARE_API_TOKEN` | The decision model (clef-omni on Workers AI) on your own Cloudflare account, for `raea ask`, `raea judge` and the checks that ask. Signed in to raea on Pro or Studio, the app pays from your plan's monthly pool first (docs/ask.feature) | +| `RAEA_AGENT_KEY`, `RAEA_API` | An agent key with the `ask` scope, and the API address it came with: asks from your plan's pool with no app running (a pipeline, CI, a server) | +| `RAEA_ASK_ROUTE` | `app`, `agent` or `own`: ask only that way, rather than the first that answers | +| `RAEA_ASK_DIR` | Where `raea ask` keeps its answers, the clips it cut and its cache (`~/.raea/ask`) | +| `RAEA_CLEF_REPLAY` | A JSON file of answers by question id, given back for every call: for tests, which then never spend | | `RAEA_GUIDES` | A folder of craft references for the agent's `guide` tool (`captions.md`, `motion-rules.md`, ...) | ## Rendering diff --git a/engine/cli/Cargo.toml b/engine/cli/Cargo.toml index 70303b314..e09ab5343 100644 --- a/engine/cli/Cargo.toml +++ b/engine/cli/Cargo.toml @@ -70,6 +70,7 @@ raea-app-engines = { path = "../../apps/engines" } raea-app-solid = { path = "../../apps/solid" } # raea judge: review sheets judged by clef-omni raea-app-judge = { path = "../../apps/judge" } +raea-app-ask = { path = "../../apps/ask" } raea-app-media = { path = "../../apps/media" } # `raea app run|check|bundle` and `raea rune` on either target (docs/targets.feature T0b) raea-app-targets = { path = "../../apps/targets" } diff --git a/engine/cli/src/cli.rs b/engine/cli/src/cli.rs index 85191cf02..5f1aa4c51 100644 --- a/engine/cli/src/cli.rs +++ b/engine/cli/src/cli.rs @@ -145,7 +145,7 @@ pub fn run(args: &[String]) -> R { // a workspace's files and specs (docs/projects.feature), listed under render's usage like library let usage = format!("{usage}\n raea spec FILE.ts [--as brief|script|taste|plan|controller] (a spec's default export as JSON, checked against its schema: brief.ts, script.ts, taste.ts, plan.ts, controllers/ID.ts)\n raea workspace init [NAME] [--layout LAYOUT] | raea workspace adopt DIR [--name NAME] | raea workspace [--json] (a workspace: a git repo of projects and controllers under raea.root, ~/Documents/raea unless git config says)\n raea project new NAME [--storage inline|submodule] | raea project split NAME [--remote URL] | raea project join NAME | raea project [--json] (a project in the workspace: its folder, brief.ts, media/, out/; split and join keep its history)\n raea controller ls [--json] | raea controller get PATH [--json] | raea controller set PATH VALUE [--project] [--by NAME] | raea controller write FILE.ts [--by NAME] | raea controller rm ID [--by NAME] (the workspace's controllers, controllers/ID.ts; PATH is controller.group.field; every change a commit)\n raea check EDIT.tsx --quick [--shape S] [--platform P] [--taste T] [-o check.json] (three stills of a composition checked for text: clipped, cramped, safe areas, sizes; about a second)\n raea hook [claude|codex|gemini|kimi] (every agent's tool hooks, which raea skills install adds: before a shell command, media opened in the Mac's viewer while the raea app is installed is stopped once for raea show (the same command again goes through); after a save, the quick check on a composition)\n raea next [DIR] (the two or three useful next steps for the project, from its brief, compositions, renders and checks)\n raea doctor [--fix] [DIR] (what doesn't match this raea: old brief files, an older workspace format, an agent's skill from another version)\n raea app run|check|bundle APP.tsx [--target web|native] [--size WxH] [--script steps.json] [--png DIR] [-o App.app] (a TSX app on @raea/ui: its window, the headless check, a Mac app; web is Blitz on Bevy, native is Rune; the app's `export const app` sets the target, web when it doesn't; raea rune ... is --target native)\n {}\n {}\n {} ({})\n {}\n {}\n {}", skill::USAGE, tools::blender::USAGE, tools::motion::SETUP_USAGE, tools::motion::SETUP_ABOUT, raea_app_engines::godot::USAGE, raea_app_engines::shot::USAGE, raea_app_engines::round::USAGE); let (export_game, capcut, layers, xml, scene, physics, splat, wear) = (raea_app_game::USAGE, raea_app_capcut::USAGE, raea_app_layers::USAGE, raea_app_layers::USAGE_XML, export::SCENE_USAGE, motion::physics::USAGE, tools::splat::USAGE, raea_runtime_wear::USAGE); - let apps = format!(" {}\n {}\n {}\n {}\n {}\n {}\n {}\n {}\n {}\n", raea_app_direction::check::USAGE, raea_app_check::USAGE, raea_app_solid::USAGE, raea_app_judge::USAGE, raea_app_direction::USAGE, raea_app_direction::findings::USAGE, raea_app_notes::USAGE, raea_app_fingerprint::USAGE, raea_sys_index::cli::USAGE); + let apps = format!(" {}\n {}\n {}\n {}\n {}\n {}\n {}\n {}\n {}\n {}\n", raea_app_direction::check::USAGE, raea_app_check::USAGE, raea_app_solid::USAGE, raea_app_judge::USAGE, raea_app_ask::USAGE, raea_app_direction::USAGE, raea_app_direction::findings::USAGE, raea_app_notes::USAGE, raea_app_fingerprint::USAGE, raea_sys_index::cli::USAGE); return Ok(format!("raea: renders, plays and checks raea compositions (React and TypeScript) on the GPU.\n\nusage: {usage}\n raea init (a project's files where they're missing: edit.tsx, cuts.ts, components/, AGENTS.md)\n raea cut TAKE \"line\" ... | --at A-B ... [--append] (cuts.ts and words.json: a take under media/ cut in the pauses around each line it says, transcribed first if it has no TAKE.words.json; or at those seconds)\n raea frame TAKE [--at S] (how a take fits the edit: a talking take for , a screen for , from where its face is)\n raea component list | raea component add NAME ... [--shared] [--force] | raea part list | raea part add NAME ... [--shared] [--force] (a ready component or set copied into components/, or the workspace's shared/components/ with --shared, yours to change; raea part is its old name)\n raea brand DOMAIN|URL (the brand kit read from its website on this machine: brand/brand.css, brand/brand.json, BRAND.md, logos in media/brand; says when a plain fetch saw too little)\n raea guide [NAME] (craft, the floor read before any edit; the modes: pitch, explain, story, play, tool; motion, music, layout and the rest; bevy and bevy/NAME for 3D)\n raea context PROJECT (what to read before working on a project: taste, brief, look, the floor and its modes' guides)\n{apps} raea play EDIT.tsx [--input | --input-script KEYS.json] [--size WxH] [--seconds N|end] (a game, live; also: --input-log FILE --offscreen --snapshot T FILE --json FILE --seek F FILE --pick X,Y --view YAW,PITCH,DISTANCE --scrub N --baked --autoplay --no-vsync --master --3d)\n {export_game} (a game as a program of its own: a Mac app or a command)\n {capcut}\n {layers}\n {xml}\n {scene} (what each 3D part rests on: floating, sunk and clipping parts; light: the scene's lighting)\n {physics} (the bakes and make)\n {wear} (a .glb's weathering causes for raea/wear: the model with a second UV set and its cause maps, made once in the asset store)\n {splat} (footage trained into a Gaussian splat for ; setup installs its tools)\n raea run SCRIPT.ts (a TypeScript program on the engine, headless: its default export is called, what it prints is the answer; a check like a room's checkLayout; also: --timeout S)\n raea gen image|video|music|sfx|speech|mesh|text \"PROMPT\" | gen plan|run FLOW.ts|FLOW.json | gen comfy import|export|run|schema FILE | gen models | gen balance (generated media through made.app, fal, Replicate, OpenRouter, Modal or ComfyUI; ComfyUI workflows both ways: raea gen alone lists its flags)\n {neural_usage} (DLSS 5 neural rendering on a `render --aov` folder, with its motion and depth: your own MLX-DLSS build and weights, RAEA_MLXDLSS and RAEA_DLSS_MODEL)\n {light_usage} (each scene on each light path against its path-traced reference: distance in stops, the worst tile, a moving shot's flicker, a frame's cost; DIR/report.json)\n raea mezz [--check] PATH... (an intra-only ProRes copy of each long-GOP source over 1080p, for smooth scrubbing; --check says which would get one)\n raea traces [show [N|last|DIR]] (the sessions raea left, newest first: how each ended, its glitches; show tells one in full)")); } // `--version`: the release's version, which a skill's front matter names (docs/skills.feature SK6) @@ -279,7 +279,7 @@ pub fn run(args: &[String]) -> R { Some("gen") => return raea_app_tools::generate::generate(args), // the agent shell's own verbs ported here (`init`, `cut`, `frame`): engine/cli's handler, run by its native.ts (shell()) Some(v) if SHELL.contains(&v) => return shell(args), - _ => if let Some(r) = raea_app_solid::verb(args) { return r }, // `solid info|view FILE.ts`, `judge DIR` (apps/solid) + _ => if let Some(r) = raea_app_solid::verb(args) { return r }, // `solid info|view FILE.ts`, `judge DIR`, `ask FILE` (apps/solid) } // `master-gain BUNDLE.js DIR`: the gain and ceiling a composition's render masters its sound with, as JSON (live // play asks this of a child process, so its second React and the whole mix never sit in the game's memory) diff --git a/engine/cli/src/tests.rs b/engine/cli/src/tests.rs index 48cf1de5a..211d44dfe 100644 --- a/engine/cli/src/tests.rs +++ b/engine/cli/src/tests.rs @@ -32,7 +32,7 @@ fn help_lists_every_verb() { let listed = |v: &str| [format!("raea {v} "), format!("raea {v}\n"), format!("| {v} ")].iter().any(|l| help.contains(l.as_str())); let missing: Vec<_> = verbs.iter().filter(|v| !listed(v) && !unlisted.contains(&v.as_str())).collect(); assert!(missing.is_empty(), "not in raea help: {missing:?}"); - for usage in [raea_app_check::USAGE, raea_app_direction::check::USAGE, raea_app_direction::USAGE, raea_app_notes::USAGE, raea_app_game::USAGE, raea_runtime_export::SCENE_USAGE, motion::physics::USAGE, raea_runtime_wear::USAGE, tools::splat::USAGE, tools::sound::SAO_USAGE, tools::sound::TOKENSYNTH_USAGE, tools::song::USAGE, motion::track::USAGE, raea_app_solid::USAGE, raea_app_judge::USAGE, tools::blender::USAGE] { + for usage in [raea_app_check::USAGE, raea_app_ask::USAGE, raea_app_direction::check::USAGE, raea_app_direction::USAGE, raea_app_notes::USAGE, raea_app_game::USAGE, raea_runtime_export::SCENE_USAGE, motion::physics::USAGE, raea_runtime_wear::USAGE, tools::splat::USAGE, tools::sound::SAO_USAGE, tools::sound::TOKENSYNTH_USAGE, tools::song::USAGE, motion::track::USAGE, raea_app_solid::USAGE, raea_app_judge::USAGE, tools::blender::USAGE] { assert!(help.contains(usage), "{usage}"); } } diff --git a/io/clef/Cargo.toml b/io/clef/Cargo.toml new file mode 100644 index 000000000..8a9d78bc2 --- /dev/null +++ b/io/clef/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "raea-io-clef" +version = "0.1.0" +edition = "2024" +license = "FSL-1.1-ALv2" +publish = false +description = "The decision model (clef-omni) called one way: the call, the media cut to its limits, the route (the person's raea plan first) and the cache" + +[dependencies] +raea-core-ask = { path = "../../core/ask" } +raea-io-home = { path = "../home" } +# raea's own FFmpeg (fetched on Windows) and the H.264 encoder this machine has +raea-io-assets = { path = "../assets" } +raea-io-ffmpeg = { path = "../ffmpeg" } +serde_json = "1" +ureq = { version = "2", features = ["json"] } +base64 = "0.22" +sha2 = "0.10" diff --git a/io/clef/src/cache.rs b/io/clef/src/cache.rs new file mode 100644 index 000000000..9fbd0caec --- /dev/null +++ b/io/clef/src/cache.rs @@ -0,0 +1,38 @@ +//! Answers kept by what was asked (docs/ask.feature AK1b): the hash of everything the model reads (the state, the +//! questions as sent, the media), so the same questions on the same media cost nothing the second time. clef gives the +//! same answer to the same request, so a kept answer is the answer. Under RAEA_ASK_DIR, else ~/.raea/ask, in cache/. + +use std::path::PathBuf; + +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; + +use crate::client::Call; + +/// raea ask's own folder: answers, kept media and this cache. +pub fn dir() -> PathBuf { + raea_io_home::env_path("RAEA_ASK_DIR", ".raea/ask") +} + +/// The request's hash. +pub fn key(call: &Call) -> String { + let mut h = Sha256::new(); + h.update(call.clef_body().to_string().as_bytes()); + h.finalize().iter().map(|b| format!("{b:02x}")).collect() +} + +pub fn get(key: &str) -> Option> { + let text = std::fs::read_to_string(dir().join("cache").join(format!("{key}.json"))).ok()?; + serde_json::from_str(&text).ok() +} + +/// Kept, unless nothing came back (an empty answer is never cached, so a fixed problem is asked again). +pub fn put(key: &str, answers: &Map) { + if answers.is_empty() { + return; + } + let d = dir().join("cache"); + if std::fs::create_dir_all(&d).is_ok() { + let _ = std::fs::write(d.join(format!("{key}.json")), Value::Object(answers.clone()).to_string()); + } +} diff --git a/io/clef/src/client.rs b/io/clef/src/client.rs new file mode 100644 index 000000000..1ab111965 --- /dev/null +++ b/io/clef/src/client.rs @@ -0,0 +1,166 @@ +//! One call to the decision model (docs/ask.feature AK1b): the state, the questions by id and the media, sent to Workers +//! AI with the person's keys, or to raea-api (directly with an agent key, or through the app, which adds its sign-in). +//! The answers come back by question id with the call's cost. A transport is handed in, so tests replay recorded +//! answers and never spend. + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as B64; +use raea_core_ask::{Question, limits}; +use serde_json::{Map, Value, json}; + +/// The media one call reads: stills as JPEG, videos as MP4, sounds as MP3, each already cut to clef's limits. +#[derive(Debug, Clone, Default)] +pub struct Media { + pub stills: Vec>, + pub videos: Vec>, + pub sounds: Vec>, +} + +/// One call: what the piece is (the state), every question about this media, and the media. +#[derive(Debug, Clone, Default)] +pub struct Call { + pub state: String, + pub questions: Vec, + pub media: Media, +} + +/// What came back: the answers by question id (clef's own shape: `noul`, or `choice` and `probabilities`, or `score`), +/// what it cost, whether it came from a cache, the route it took and the pool left after it (raea's routes only). +#[derive(Debug, Clone, Default)] +pub struct Answered { + pub answers: Map, + pub cost: f64, + pub cached: bool, + pub route: String, + pub pool_left: Option, +} + +/// Why a route didn't answer: `Next` means try the next one (no plan, signed out, not running, no ask scope; from +/// `route::ask`, that none did); `Stop` means asking again won't help (the call itself is wrong); `Spent` means the +/// month's pool is used up and nothing else is set, so a batch's remaining calls wait for the pool. +#[derive(Debug, Clone, PartialEq)] +pub enum Fail { + Next(String), + Stop(String), + Spent(String), +} + +impl std::fmt::Display for Fail { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let (Fail::Next(s) | Fail::Stop(s) | Fail::Spent(s)) = self; + f.write_str(s) + } +} + +/// An HTTP POST of JSON: its status and body. `Http` is the real one; a test hands in its own. +pub trait Transport { + fn post(&self, url: &str, bearer: Option<&str>, body: &Value) -> Result<(u16, Value), String>; +} + +pub struct Http; + +impl Transport for Http { + fn post(&self, url: &str, bearer: Option<&str>, body: &Value) -> Result<(u16, Value), String> { + let mut r = ureq::post(url).timeout(std::time::Duration::from_secs(180)); + if let Some(b) = bearer { + r = r.set("authorization", &format!("Bearer {b}")); + } + match r.send_json(body.clone()) { + Ok(res) => Ok((res.status(), res.into_json().map_err(|e| format!("{url}: {e}"))?)), + Err(ureq::Error::Status(code, res)) => Ok((code, res.into_json().unwrap_or_else(|_| json!({})))), + Err(e) => Err(format!("{url}: {e}")), + } + } +} + +impl Call { + /// Why it can't be sent as it stands, or None: clef's counts and sizes, every question's own limits. + pub fn problem(&self) -> Option { + let m = &self.media; + if self.questions.is_empty() || self.questions.len() > limits::QUESTIONS { + return Some(format!("a call asks 1 to {} questions, not {}", limits::QUESTIONS, self.questions.len())); + } + if let Some(p) = self.questions.iter().find_map(limits::check) { + return Some(p); + } + if m.stills.is_empty() && m.videos.is_empty() && m.sounds.is_empty() { + return Some("a call needs something to look at or listen to".into()); + } + let over = |v: &[Vec], most: usize, each: usize| v.len() > most || v.iter().any(|b| b.len() > each); + if over(&m.stills, limits::STILLS, limits::STILL_BYTES) || m.stills.iter().map(Vec::len).sum::() > limits::STILLS_BYTES { + return Some(format!("at most {} stills of 4 MiB, 8 MiB in all", limits::STILLS)); + } + if over(&m.videos, limits::VIDEOS, limits::VIDEO_BYTES) || over(&m.sounds, limits::SOUNDS, limits::SOUND_BYTES) { + return Some(format!("at most {} videos of 16 MiB and {} sounds of 8 MiB", limits::VIDEOS, limits::SOUNDS)); + } + (m.videos.iter().chain(&m.sounds).map(Vec::len).sum::() > limits::AV_BYTES).then(|| "the videos and sounds together are over 16 MiB".into()) + } + + fn questions(&self) -> Value { + Value::Object(self.questions.iter().map(|q| (q.id.clone(), q.wire())).collect()) + } + + fn state(&self) -> String { + self.state.chars().take(limits::STATE_CHARS).collect() + } + + /// The body Workers AI takes: the media as data URLs. + pub fn clef_body(&self) -> Value { + let urls = |v: &[Vec], mime: &str| v.iter().map(|b| format!("data:{mime};base64,{}", B64.encode(b))).collect::>(); + let mut body = json!({"state": self.state(), "questions": self.questions()}); + for (k, v, mime) in [("images", &self.media.stills, "image/jpeg"), ("videos", &self.media.videos, "video/mp4"), ("audio", &self.media.sounds, "audio/mpeg")] { + if !v.is_empty() { + body[k] = json!(urls(v, mime)); + } + } + body + } + + /// raea-api's body: the media as bare base64 (it adds the data URL's type). + pub fn api_body(&self) -> Value { + let mut body = json!({"state": self.state(), "questions": self.questions()}); + for (k, v) in [("images", &self.media.stills), ("videos", &self.media.videos), ("audio", &self.media.sounds)] { + if !v.is_empty() { + body[k] = json!(v.iter().map(|b| B64.encode(b)).collect::>()); + } + } + body + } +} + +fn said(body: &Value, code: u16) -> String { + body["message"].as_str().or(body["error"].as_str()).map(str::to_string).unwrap_or_else(|| format!("HTTP {code}")) +} + +/// Workers AI, on the person's own Cloudflare account. +pub fn workers_ai(t: &dyn Transport, call: &Call, account: &str, token: &str) -> Result { + let url = format!("https://api.cloudflare.com/client/v4/accounts/{account}/ai/run/@cf/cloudflare/clef-omni"); + let (code, j) = t.post(&url, Some(token), &call.clef_body()).map_err(Fail::Stop)?; + if code == 401 || code == 403 { + return Err(Fail::Next(format!("your Cloudflare keys were refused ({code}): check CLOUDFLARE_API_TOKEN"))); + } + if j["success"] != true { + return Err(Fail::Stop(format!("clef: {}", j["errors"].to_string().chars().take(300).collect::()))); + } + let r = &j["result"]; + let cost = r["usage"]["input_tokens"].as_f64().unwrap_or(0.0) * limits::USD_PER_M / 1e6; + Ok(Answered { answers: r["answers"].as_object().cloned().unwrap_or_default(), cost, cached: false, route: "your Cloudflare account".into(), pool_left: None }) +} + +/// raea-api's /v1/clef/ask at `url` (the API itself with an agent key, or the app's /host/ask, which signs it), charged +/// to the account's pool. `own_keys` says whether a spent pool may fall through to the person's own account. +pub fn raea_api(t: &dyn Transport, call: &Call, url: &str, bearer: Option<&str>, route: &str, own_keys: bool) -> Result { + let (code, j) = t.post(url, bearer, &call.api_body()).map_err(|e| Fail::Next(format!("{route} didn't answer: {e}")))?; + match (code, j["error"].as_str()) { + (200..=299, _) => Ok(Answered { + answers: j["answers"].as_object().cloned().unwrap_or_default(), + cost: j["cost_usd"].as_f64().unwrap_or(0.0), + cached: j["cached"] == true, + route: route.into(), + pool_left: j["pool_left_usd"].as_f64(), + }), + (402, Some("pool_spent")) if !own_keys => Err(Fail::Spent(said(&j, code))), + (400 | 413, _) => Err(Fail::Stop(said(&j, code))), + _ => Err(Fail::Next(format!("{route}: {}", said(&j, code)))), + } +} diff --git a/io/clef/src/lib.rs b/io/clef/src/lib.rs new file mode 100644 index 000000000..aca516704 --- /dev/null +++ b/io/clef/src/lib.rs @@ -0,0 +1,19 @@ +//! The decision model, called one way for every caller (docs/ask.feature AK1b): a call's body and its answers +//! (`client`), the media cut to what it reads (`media`), the route it takes, the person's plan first (`route`), and the +//! answers kept by request (`cache`). `raea ask` (apps/ask) and `raea judge` call it. + +pub mod cache; +pub mod client; +pub mod media; +pub mod route; + +pub use client::{Answered, Call, Fail, Http, Media, Transport}; +pub use route::Route; + +/// One call, by the routes open on this computer. +pub fn ask(call: &Call) -> Result { + route::ask(&Http, call, &route::open()) +} + +#[cfg(test)] +mod tests; diff --git a/io/clef/src/media.rs b/io/clef/src/media.rs new file mode 100644 index 000000000..ba776829f --- /dev/null +++ b/io/clef/src/media.rs @@ -0,0 +1,135 @@ +//! The media cut to what the decision model reads (docs/ask.feature AK1b), with raea's own FFmpeg: a window of a video as +//! 480 px H.264 of at most 60 s, its sound as mono 16 kHz MP3 of at most 300 s, a still as JPEG of at most 1,600 px. +//! What a file has (its length, a picture, a sound) comes from ffprobe first, so a question that can't be answered +//! isn't asked. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use raea_core_ask::limits; + +/// What a file holds. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct Probe { + pub seconds: f64, + /// a moving picture (a video stream with more than one frame) + pub video: bool, + /// a picture that doesn't move (an image, or a video of one frame) + pub still: bool, + pub sound: bool, +} + +fn tool(name: &str, file: &Path) -> Result { + let video = file.extension().is_some_and(|e| !matches!(e.to_string_lossy().to_ascii_lowercase().as_str(), "png" | "jpg" | "jpeg" | "webp" | "mp3" | "wav" | "m4a")); + raea_io_assets::ffmpeg_tool(name, video).map(Command::new) +} + +fn run(mut c: Command, what: &str) -> Result, String> { + let out = c.output().map_err(|e| format!("{what}: {e}"))?; + if !out.status.success() { + let err = String::from_utf8_lossy(&out.stderr); + return Err(format!("{what}: {}", err.lines().last().unwrap_or("FFmpeg failed").trim())); + } + Ok(out.stdout) +} + +/// What `file` holds, from ffprobe. +pub fn probe(file: &Path) -> Result { + if !file.is_file() { + return Err(format!("{}: no such file", file.display())); + } + let mut c = tool("ffprobe", file)?; + c.args(["-v", "error", "-show_entries", "stream=codec_type,nb_frames,disposition:format=duration", "-of", "json"]).arg(file); + let j: serde_json::Value = serde_json::from_slice(&run(c, "ffprobe")?).map_err(|e| format!("ffprobe: {e}"))?; + let streams = j["streams"].as_array().cloned().unwrap_or_default(); + let seconds = j["format"]["duration"].as_str().and_then(|s| s.parse().ok()).unwrap_or(0.0); + let pictures: Vec<&serde_json::Value> = streams.iter().filter(|s| s["codec_type"] == "video" && s["disposition"]["attached_pic"] != 1).collect(); + let one_frame = |s: &&serde_json::Value| s["nb_frames"].as_str().is_some_and(|n| n == "1") || seconds == 0.0; + Ok(Probe { + seconds, + video: pictures.iter().any(|s| !one_frame(s)), + still: !pictures.is_empty() && pictures.iter().all(one_frame), + sound: streams.iter().any(|s| s["codec_type"] == "audio"), + }) +} + +/// A file in the temp folder, removed when dropped. +struct Temp(PathBuf); + +impl Temp { + fn new(ext: &str) -> Temp { + static N: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let n = N.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + Temp(std::env::temp_dir().join(format!("raea-ask-{}-{n}.{ext}", std::process::id()))) + } + fn read(&self) -> Result, String> { + std::fs::read(&self.0).map_err(|e| format!("{}: {e}", self.0.display())) + } +} + +impl Drop for Temp { + fn drop(&mut self) { + std::fs::remove_file(&self.0).ok(); + } +} + +fn window(from: f64, to: Option, most: f64) -> Vec { + let len = to.map(|t| (t - from).max(0.1)).unwrap_or(most).min(most); + vec!["-ss".into(), format!("{from:.3}"), "-t".into(), format!("{len:.3}")] +} + +/// The H.264 this FFmpeg has, small: libx264 at CRF 30, else the system's own at 600 kb/s (raea_io_ffmpeg's choice, which +/// for a clip a person plays is 8 Mb/s, too big for a minute under 16 MiB). +fn small_h264() -> Result, String> { + let mut args: Vec = raea_io_ffmpeg::h264().ok_or("this FFmpeg has no H.264 encoder (libx264, VideoToolbox or Media Foundation)")?.into_iter().map(str::to_string).collect(); + match args.iter().position(|a| a == "8M") { + Some(i) => args[i] = "600k".into(), + None => args.extend(["-crf", "30", "-preset", "veryfast"].map(String::from)), + } + Ok(args) +} + +/// The picture from `from` to `to` (at most 60 s), 480 px wide, no sound. +pub fn video(file: &Path, from: f64, to: Option) -> Result, String> { + let out = Temp::new("mp4"); + let mut c = tool("ffmpeg", file)?; + c.args(["-nostdin", "-v", "error", "-y"]).args(window(from, to, limits::VIDEO_SECONDS)).arg("-i").arg(file); + c.args(["-an", "-vf", "scale=480:-2", "-r", "10"]).args(small_h264()?).args(["-movflags", "+faststart"]).arg(&out.0); + run(c, "cutting the picture")?; + out.read() +} + +/// The sound from `from` to `to` (at most 300 s), mono 16 kHz MP3. +pub fn sound(file: &Path, from: f64, to: Option) -> Result, String> { + let out = Temp::new("mp3"); + let mut c = tool("ffmpeg", file)?; + c.args(["-nostdin", "-v", "error", "-y"]).args(window(from, to, limits::SOUND_SECONDS)).arg("-i").arg(file); + c.args(["-vn", "-ac", "1", "-ar", "16000", "-c:a", "libmp3lame", "-b:a", "64k"]).arg(&out.0); + run(c, "cutting the sound").map_err(|e| if e.contains("libmp3lame") { "this FFmpeg can't write MP3 (libmp3lame)".into() } else { e })?; + out.read() +} + +/// A still: an image file as it is, or a video's frame at `at` seconds, as JPEG at most 1,600 px wide. +pub fn still(file: &Path, at: Option) -> Result, String> { + let out = Temp::new("jpg"); + let mut c = tool("ffmpeg", file)?; + c.args(["-nostdin", "-v", "error", "-y"]); + if let Some(t) = at { + c.args(["-ss", &format!("{t:.3}")]); + } + c.arg("-i").arg(file).args(["-frames:v", "1", "-vf", "scale='min(1600,iw)':-2", "-q:v", "4"]).arg(&out.0); + run(c, "taking the still")?; + out.read() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_window_is_held_to_the_limit() { + assert_eq!(window(5.0, Some(20.0), 60.0), ["-ss", "5.000", "-t", "15.000"]); + assert_eq!(window(0.0, None, 60.0), ["-ss", "0.000", "-t", "60.000"]); + assert_eq!(window(0.0, Some(400.0), 300.0)[3], "300.000"); + } +} diff --git a/io/clef/src/route.rs b/io/clef/src/route.rs new file mode 100644 index 000000000..ac83ed9ed --- /dev/null +++ b/io/clef/src/route.rs @@ -0,0 +1,113 @@ +//! Which way a call goes (docs/ask.feature AK1b), the first that answers: the raea app on this computer when it's signed +//! in with a pool (it adds its sign-in, which stays the app's alone); an agent key (`RAEA_AGENT_KEY` with `RAEA_API`, the +//! address the key came with, for a pipeline, CI or raea cloud); the person's own Cloudflare account. Unchanged questions on unchanged media come from the cache +//! first. `RAEA_CLEF_REPLAY` names a file of recorded answers, for tests that must never spend. + +use std::path::PathBuf; + +use raea_io_home::setting; +use serde_json::{Map, Value}; + +use crate::cache; +use crate::client::{Answered, Call, Fail, Transport, raea_api, workers_ai}; + +#[derive(Debug, Clone, PartialEq)] +pub enum Route { + /// the raea app's host, at the address ~/.raea/app-host.json gives + App(String), + /// raea-api with an agent key that has the `ask` scope + Agent { api: String, key: String }, + /// the person's own Workers AI + Own { account: String, token: String }, + /// recorded answers by question id, for tests + Replay(PathBuf), +} + +impl Route { + pub fn name(&self) -> &'static str { + match self { + Route::App(_) => "your raea plan, through the app", + Route::Agent { .. } => "your raea plan, with an agent key", + Route::Own { .. } => "your Cloudflare account", + Route::Replay(_) => "recorded answers", + } + } +} + +/// The address the running raea app listens at, if it left one (RAEA_HOST_FILE, else ~/.raea/app-host.json). +fn app() -> Option { + let file = raea_io_home::env_path("RAEA_HOST_FILE", ".raea/app-host.json"); + let j: Value = serde_json::from_str(&std::fs::read_to_string(file).ok()?).ok()?; + j["url"].as_str().map(|u| u.trim_end_matches('/').to_string()) +} + +/// The routes open here, in the order they're tried. `RAEA_ASK_ROUTE` (app, agent or own) keeps one. +pub fn open() -> Vec { + if let Some(p) = setting("RAEA_CLEF_REPLAY") { + return vec![Route::Replay(p.into())]; + } + let mut out = Vec::new(); + if let Some(url) = app() { + out.push(Route::App(url)); + } + if let (Some(key), Some(api)) = (setting("RAEA_AGENT_KEY"), setting("RAEA_API")) { + out.push(Route::Agent { api: api.trim_end_matches('/').to_string(), key }); + } + if let (Some(account), Some(token)) = (setting("CLOUDFLARE_ACCOUNT_ID"), setting("CLOUDFLARE_API_TOKEN")) { + out.push(Route::Own { account, token }); + } + match setting("RAEA_ASK_ROUTE").as_deref() { + Some("app") => out.retain(|r| matches!(r, Route::App(_))), + Some("agent") => out.retain(|r| matches!(r, Route::Agent { .. })), + Some("own") => out.retain(|r| matches!(r, Route::Own { .. })), + _ => {} + } + out +} + +pub const NONE: &str = "sign in to raea on Pro or Studio (the app), set RAEA_AGENT_KEY and RAEA_API, or add CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN to ~/.raea/env (docs/settings.md)"; + +fn replay(call: &Call, file: &PathBuf) -> Result { + let read = || -> Result, Box> { Ok(serde_json::from_str(&std::fs::read_to_string(file)?)?) }; + let name = file.display(); + let all = read().map_err(|e| Fail::Stop(format!("{name} isn't a recorded answer: {e}")))?; + let answers = call.questions.iter().filter_map(|q| all.get(&q.id).map(|a| (q.id.clone(), a.clone()))).collect(); + Ok(Answered { answers, cost: 0.0, cached: false, route: Route::Replay(file.clone()).name().into(), pool_left: None }) +} + +/// One call answered: from the cache, else by the first route that answers. Every route's reason is kept for the +/// refusal when none does. +pub fn ask(t: &dyn Transport, call: &Call, routes: &[Route]) -> Result { + if let Some(p) = call.problem() { + return Err(Fail::Stop(p)); + } + if let Some(Route::Replay(f)) = routes.first() { + return replay(call, f); + } + let key = cache::key(call); + if let Some(answers) = cache::get(&key) { + return Ok(Answered { answers, cost: 0.0, cached: true, route: "asked before (cache)".into(), pool_left: None }); + } + if routes.is_empty() { + return Err(Fail::Next(format!("nothing to ask with: {NONE}"))); + } + let own = routes.iter().any(|r| matches!(r, Route::Own { .. })); + let mut why = Vec::new(); + for r in routes { + let got = match r { + Route::App(url) => raea_api(t, call, &format!("{url}/host/ask"), None, r.name(), own), + Route::Agent { api, key } => raea_api(t, call, &format!("{api}/v1/clef/ask"), Some(key), r.name(), own), + Route::Own { account, token } => workers_ai(t, call, account, token), + Route::Replay(f) => replay(call, f), + }; + match got { + Ok(a) => { + cache::put(&key, &a.answers); + return Ok(a); + } + Err(Fail::Next(e)) => why.push(e), + Err(f) => return Err(f), + } + } + Err(Fail::Next(format!("nothing answered ({}): {NONE}", why.join("; ")))) +} diff --git a/io/clef/src/tests.rs b/io/clef/src/tests.rs new file mode 100644 index 000000000..f64e0b330 --- /dev/null +++ b/io/clef/src/tests.rs @@ -0,0 +1,108 @@ +use std::cell::RefCell; + +use raea_core_ask::Question; +use serde_json::{Value, json}; + +use crate::client::{Call, Media, Transport}; +use crate::route::{self, Route}; + +/// Canned replies by URL ending, and every URL asked. +struct Fake { + replies: Vec<(&'static str, u16, Value)>, + seen: RefCell>, +} + +impl Transport for Fake { + fn post(&self, url: &str, _bearer: Option<&str>, _body: &Value) -> Result<(u16, Value), String> { + self.seen.borrow_mut().push(url.to_string()); + self.replies.iter().find(|(end, ..)| url.ends_with(end)).map(|(_, c, j)| (*c, j.clone())).ok_or_else(|| format!("{url}: connection refused")) + } +} + +fn setup() { + static ONCE: std::sync::Once = std::sync::Once::new(); + // SAFETY: set once, before any test reads it, to the same value for every test + ONCE.call_once(|| unsafe { std::env::set_var("RAEA_ASK_DIR", std::env::temp_dir().join(format!("raea-clef-test-{}", std::process::id()))) }); +} + +/// A call unique to its test (the cache is shared), with one question and a little media. +fn call(test: &str) -> Call { + setup(); + Call { state: format!("a test piece: {test}"), questions: vec![Question::plain("q", "Is it loud?")], media: Media { sounds: vec![vec![1, 2, 3]], ..Default::default() } } +} + +fn routes() -> Vec { + vec![Route::App("http://127.0.0.1:9".into()), Route::Agent { api: "https://api.test".into(), key: "k".into() }, Route::Own { account: "a".into(), token: "t".into() }] +} + +const ANSWER: &str = r#"{"success": true, "result": {"answers": {"q": {"noul": 0.7}}, "usage": {"input_tokens": 10000}}}"#; + +#[test] +fn each_route_that_cant_answer_hands_on_to_the_next() { + let f = Fake { + replies: vec![ + ("/host/ask", 402, json!({"error": "plan_required", "message": "checks come with Pro and Studio"})), + ("/v1/clef/ask", 403, json!({"error": "agent_scope", "message": "this key can't ask"})), + ("/clef-omni", 200, serde_json::from_str(ANSWER).unwrap()), + ], + seen: RefCell::default(), + }; + let a = route::ask(&f, &call("hand-on"), &routes()).unwrap(); + assert_eq!((a.route.as_str(), a.answers["q"]["noul"].as_f64()), ("your Cloudflare account", Some(0.7))); + assert!((a.cost - 0.0015).abs() < 1e-9, "{}", a.cost); + assert_eq!(f.seen.borrow().len(), 3); +} + +#[test] +fn a_spent_pool_stops_unless_the_person_has_their_own_keys() { + let spent = || Fake { replies: vec![("/host/ask", 402, json!({"error": "pool_spent", "message": "this month's checks are used up"}))], seen: RefCell::default() }; + let e = route::ask(&spent(), &call("spent"), &routes()[..1]).unwrap_err(); + assert!(matches!(&e, crate::Fail::Spent(m) if m.contains("used up")), "{e:?}"); + let mut f = spent(); + f.replies.push(("/clef-omni", 200, serde_json::from_str(ANSWER).unwrap())); + let a = route::ask(&f, &call("spent-own"), &[routes()[0].clone(), routes()[2].clone()]).unwrap(); + assert_eq!(a.route, "your Cloudflare account"); +} + +#[test] +fn the_plan_answers_with_whats_left_and_a_second_ask_costs_nothing() { + let f = Fake { replies: vec![("/host/ask", 200, json!({"answers": {"q": {"noul": 0.2}}, "cost_usd": 0.004, "cached": false, "pool_left_usd": 1.62}))], seen: RefCell::default() }; + let c = call("plan"); + let a = route::ask(&f, &c, &routes()).unwrap(); + assert_eq!((a.route.as_str(), a.cost, a.pool_left), ("your raea plan, through the app", 0.004, Some(1.62))); + let again = route::ask(&f, &c, &routes()).unwrap(); + assert!(again.cached && again.cost == 0.0); + assert_eq!(f.seen.borrow().len(), 1, "the second came from the cache"); +} + +#[test] +fn with_nothing_open_it_says_how_to_get_one() { + let f = Fake { replies: vec![], seen: RefCell::default() }; + let e = route::ask(&f, &call("none"), &[]).unwrap_err().to_string(); + assert!(e.contains("RAEA_AGENT_KEY") && e.contains("CLOUDFLARE_API_TOKEN"), "{e}"); + let e = route::ask(&f, &call("refused"), &routes()[..1]).unwrap_err().to_string(); + assert!(e.contains("connection refused") && e.contains("Pro or Studio"), "{e}"); +} + +#[test] +fn a_call_that_breaks_the_limits_is_refused_before_anything_is_sent() { + let f = Fake { replies: vec![], seen: RefCell::default() }; + let mut c = call("limits"); + c.media = Media::default(); + assert!(route::ask(&f, &c, &routes()).unwrap_err().to_string().contains("look at or listen to")); + let mut c = call("too-many"); + c.questions = (0..65).map(|i| Question::plain(&format!("q{i}"), "Is it loud?")).collect(); + assert!(route::ask(&f, &c, &routes()).unwrap_err().to_string().contains("1 to 64")); + assert!(f.seen.borrow().is_empty()); +} + +#[test] +fn workers_ai_gets_data_urls_and_raea_api_bare_base64() { + let c = Call { state: "s".into(), questions: vec![Question::plain("q", "Is it loud?")], media: Media { stills: vec![vec![0xff]], videos: vec![], sounds: vec![vec![1]] } }; + let clef = c.clef_body(); + assert_eq!(clef["images"][0], "data:image/jpeg;base64,/w=="); + assert_eq!(clef["audio"][0], "data:audio/mpeg;base64,AQ=="); + assert!(clef.get("videos").is_none()); + assert_eq!(c.api_body()["audio"][0], "AQ=="); + assert_eq!(clef["questions"]["q"]["type"], "noul"); +} diff --git a/sdk/raea/src/ask.gen.ts b/sdk/raea/src/ask.gen.ts new file mode 100644 index 000000000..4e2fa3a90 --- /dev/null +++ b/sdk/raea/src/ask.gen.ts @@ -0,0 +1,82 @@ +// Generated from raea-core-ask (core/ask) by its `declarations`; edit the Rust, not this. + +export type Question = { +/** + * letters, digits, `_`, `.` and `-` + */ +id: string, +/** + * the question in words, as clef reads it + */ +ask: string, type?: Kind, +/** + * choose: its options; grade and rate: its levels, lowest first (a grade's are LEVELS unless given) + */ +options?: Array, +/** + * what an option (or yes and no) means, when its name alone doesn't say + */ +means?: { [key in string]: string }, +/** + * the answers that mean a problem: yes's "yes" (the default) or "no"; choose's options (none: a pick, no problem) + */ +flags?: Array, sees?: Sees, +/** + * whether it listens to the sound (true unless set) + */ +hears?: boolean, +/** + * what the state tells clef beside the media: brief, words (heard), screen-words, measures, brand, platform; and + * with: it compares the file with the files set beside it, so it isn't asked without them + */ +uses?: Array, when?: When, +/** + * the question asked in its place when `when` doesn't hold (sound -> sound-silent) + */ +swap?: string, window?: Window, picks?: Picks, +/** + * words in a person's note that mean they saw this problem (AK6) + */ +label_words?: Array, calibration?: Calibration, +/** + * what to change when it flags + */ +fix?: string, }; + +export type Kind = "yes" | "choose" | "grade" | "rate"; + +export type Sees = "video" | "stills" | "nothing"; + +export type Picks = "all" | "after-first"; + +export type When = { +/** + * the brief's modes it's for (pitch, story, explain, play, tool); empty is every mode + */ +modes?: Array, +/** + * true: only when someone is heard speaking; false: only when no one is + */ +voice?: boolean, +/** + * true: only for a game; false: never for one + */ +game?: boolean, }; + +export type Window = { from?: number, to?: number, }; + +export type Calibration = { auc: number, n: number, positives: number, +/** + * the problem score at which it flags (10% false positives on its labels) + */ +threshold?: number, +/** + * YYYY-MM-DD + */ +date: string, +/** + * whose labels: "raea" (raea's bench, shipped) or "you" + */ +by: string, }; + +export const SETS = ["rubric", "av", "defects", "platform", "decide"] as const; diff --git a/sdk/raea/src/ask.ts b/sdk/raea/src/ask.ts new file mode 100644 index 000000000..03e57eed1 --- /dev/null +++ b/sdk/raea/src/ask.ts @@ -0,0 +1,151 @@ +// raea/ask: questions about a piece's picture and sound, answered by clef-omni and paid from the person's raea plan +// (docs/ask.feature AK3). It runs `raea ask --batch - --json`, so a script, a `.ts` check, a `raea run` script and plain +// Node all ask through the same runner as the command: every question about one clip goes in one call, the clips are +// cut once, the calls go in parallel, and every answer is kept for calibration. Answers are advice until a question is +// calibrated (`calibrated` on each answer). +// +// import { ask, askMany } from "raea/ask"; +// const { answers } = ask("out/piece.mp4", "Does the music fight the voice?"); +// const all = askMany([ +// { file: "out/a.mp4", set: "av" }, +// { file: "out/b.mp4", std: ["defect.glitch"], window: { from: 0, to: 3 } }, +// ]); + +import { spawnSync } from "child_process"; +import type { Question, Window } from "./ask.gen"; + +export type * from "./ask.gen"; +export { SETS } from "./ask.gen"; + +/** One file and what to ask of it: a line of `raea ask --batch`. */ +export interface AskEntry { + /** the video, sound or picture asked about */ + file: string; + /** questions in your own words (yes or no) or written out */ + questions?: (string | Question)[]; + /** standard sets by name: rubric, av, defects, platform, decide */ + set?: string | string[]; + /** standard questions by id (`raea ask list` lists them) */ + std?: string[]; + /** the seconds to look at, for every question here (else each question's own, else the whole file) */ + window?: Window; + /** files set beside it: to pick between, or a before and after */ + with?: string[]; + /** the project's brief (else the nearest brief.ts above the file; "none": no brief, the state saying what the piece is) */ + brief?: string; + /** a words file (a JSON list of the words heard), so ask knows what's said and whether anyone speaks */ + words?: string; + /** whether anyone speaks in it, when no words file says */ + voice?: boolean; + /** more for clef to read before the media: what raea measured, the platform */ + state?: string; +} + +/** One question's answer, or why it wasn't asked or still waits: a line of `raea ask --json`. */ +export interface Answer { + /** its id in the record, for `raea ask verdict` */ + answer?: string; + /** the file asked about */ + file: string; + /** the question's id */ + id: string; + /** a hash of its words, type and options: a new wording is a new version, calibrated on its own */ + version: string; + /** the window's start, in seconds */ + from: number; + /** the window's end, in seconds (unset: the file's end) */ + to?: number; + /** the answer in words with its probability */ + says?: string; + /** yes's probability, a pick's probability, or a level */ + value?: number; + /** 0 to 1, higher is likelier a problem (none for a pick) */ + problem?: number; + /** the option or level chosen */ + choice?: string; + /** past the question's threshold (its calibrated one, else even odds) */ + flagged: boolean; + /** its labels separate well enough for its answers to be findings; until then they're advice */ + calibrated: boolean; + /** why it wasn't asked (it doesn't apply to this piece, or the call failed) */ + not_asked?: string; + /** why it still waits (this month's pool is spent) */ + waits?: string; + /** the way it was paid: your raea plan through the app, an agent key, or your Cloudflare account */ + route?: string; + /** answered from the cache, at no cost */ + cached: boolean; +} + +/** What a run came to. */ +export interface Summary { + /** the questions sent */ + asked: number; + /** the answers read */ + answered: number; + /** the calls made (with `dry`, the calls that would be) */ + calls: number; + /** what the calls cost, in dollars */ + cost: number; + /** with `dry`, what the calls would cost, in dollars */ + estimate: number; + /** what's left of this month's pool, in dollars, when the plan paid */ + pool_left?: number; + /** the questions waiting for the pool */ + waiting: number; + /** the answers past their threshold */ + flagged: number; + /** the calibrated answers past their threshold: findings */ + flagged_calibrated: number; + /** the ways the calls were paid */ + routes: string[]; + /** the clips cut, each once */ + clips: number; +} + +/** A run's answers, the calls `dry` would make, and its summary. */ +export interface Asked { + /** every question's answer, or why it wasn't asked, in the order they came */ + answers: Answer[]; + /** with `dry`: each call that would be made (its state, questions, media sizes and estimate) */ + would: unknown[]; + /** what it came to */ + summary: Summary; +} + +/** How a run goes. */ +export interface AskOptions { + /** calls at once (4 unless set) */ + jobs?: number; + /** cut the media and say what would be sent, spending nothing */ + dry?: boolean; + /** the raea to run (RAEA_BIN, else raea on the PATH) */ + bin?: string; +} + +type Env = { process?: { env?: Record } }; + +/** Many files asked in one run, in parallel; throws when nothing could be asked (the error says why). */ +export function askMany(entries: AskEntry[], o: AskOptions = {}): Asked { + const bin = o.bin ?? (globalThis as Env).process?.env?.RAEA_BIN ?? "raea"; + const args = ["ask", "--batch", "-", "--json", ...(o.dry ? ["--dry"] : []), ...(o.jobs ? ["--jobs", String(o.jobs)] : [])]; + const r = spawnSync(bin, args, { input: JSON.stringify(entries), encoding: "utf8", maxBuffer: 64 << 20 }); + if (r.error) throw new Error(`raea ask: ${r.error.message}`); + const out: Asked = { answers: [], would: [], summary: undefined as unknown as Summary }; + for (const line of String(r.stdout ?? "").split("\n")) { + if (!line.trim().startsWith("{")) continue; + const j = JSON.parse(line); + if (j.summary) out.summary = j.summary; + else if (j.would) out.would.push(j.would); + else out.answers.push(j as Answer); + } + if (r.status !== 0 || !out.summary) throw new Error(String(r.stderr ?? "").trim() || `raea ask exited ${r.status}`); + return out; +} + +/** Questions about one file: your own words (yes or no), written-out questions, or with `std`/`set` the standard ones. */ +export function ask(file: string, question: string | Question | (string | Question)[] = [], more: Omit & AskOptions = {}): Asked { + const { jobs, dry, bin, ...entry } = more; + const questions = Array.isArray(question) ? question : [question]; + return askMany([{ file, questions, ...entry }], { jobs, dry, bin }); +} diff --git a/tools/layers.toml b/tools/layers.toml index d6cb16eb8..f33ae8d8b 100644 --- a/tools/layers.toml +++ b/tools/layers.toml @@ -26,6 +26,7 @@ raea-io-os = "io" raea-io-trace = "io" raea-io-media = "io" raea-io-assets = "io" +raea-io-clef = "io" raea-io-gen = "io" raea-io-gen-comfy = "io" raea-io-gen-core = "io" @@ -48,6 +49,7 @@ raea-gpu-device = "gpu" raea-core-k = "core" raea-core-calls = "core" raea-core-brief = "core" +raea-core-ask = "core" raea-sys-k = "systems" raea-sys-mix = "systems" raea-sys-tone = "systems" @@ -192,6 +194,7 @@ raea-app-motion = "apps" raea-app-engines = "apps" raea-app-interchange = "apps" raea-app-judge = "apps" +raea-app-ask = "apps" raea-app-solid = "apps" raea-app-media = "apps" raea-app-targets = "apps" diff --git a/tools/manual-ref/cli.ts b/tools/manual-ref/cli.ts index 4ec21161c..813e920ef 100644 --- a/tools/manual-ref/cli.ts +++ b/tools/manual-ref/cli.ts @@ -31,7 +31,7 @@ const GROUPS: [string, string, string[]][] = [ ["Footage", "Takes, cuts and the camera, from your own media.", ["media", "cut", "frame", "track", "splat", "mezz", "neural"]], ["Sound", "Music read, foley and instruments made on this machine.", ["song", "sounds", "sao", "tokensynth", "setup-audio"]], ["Generate and 3D tools", "Generated media, physics and weathering bakes, clothes and motion for SOMA's body, Blender and Godot.", ["gen", "physics", "wear", "clothes", "motion", "blender", "godot"]], - ["Agents and setup", "What an agent reads and shows the person, and the tools raea installs.", ["guide", "context", "notes", "show", "skill", "skills", "setup", "bug"]], + ["Agents and setup", "What an agent reads and shows the person, and the tools raea installs.", ["guide", "context", "notes", "show", "skill", "skills", "setup", "bug", "ask"]], ] /** A verb's page: VERB.md, but index.md is this section's own page, so raea index's is indexer.md */ const pageOf = (verb: string) => (verb === "index" ? "indexer.md" : `${verb}.md`) diff --git a/tools/manual-ref/sdk.ts b/tools/manual-ref/sdk.ts index c7143e519..6bea86ee8 100644 --- a/tools/manual-ref/sdk.ts +++ b/tools/manual-ref/sdk.ts @@ -27,7 +27,7 @@ const GROUPS: [string, string, string[]][] = [ ["Video, sound and text", "Footage, pictures, sound, words and colour, and the timeline they are edited on.", ["video", "audio", "text", "image", "asset", "color", "editor"]], ["3D", "Bevy's 3D: cameras, lights, materials, meshes and the frame after it's drawn.", ["3d", "camera", "light", "pbr", "mesh", "transform", "post_process", "picking", "gizmos"]], ["2D and UI", "Sprites and flat shapes, bevy_ui and Bevy's widgets.", ["sprite", "ui", "feathers"]], - ["Tools", "Checks, lints and a headless App you can drive.", ["check", "fingerprint", "dev_tools", "remote"]], + ["Tools", "Checks, lints and a headless App you can drive.", ["check", "fingerprint", "dev_tools", "remote", "ask"]], ] const groupAt = new Map(GROUPS.flatMap(([, , ps], g) => ps.map((p, i) => [p, g * 20 + i + 2] as const))) const BEGIN = ""