Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ members = [
"runtime/scene2d",
"runtime/shared",
"apps/ask",
"apps/double",
"apps/check",
"apps/check/flash",
"apps/check/inbox",
Expand Down
23 changes: 23 additions & 0 deletions apps/double/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
[package]
name = "raea-app-double"
version = "0.1.0"
edition = "2024"
license = "FSL-1.1-ALv2"
publish = false
description = "The fingerprint's flags asked again of the decision model (docs/ask.feature AK8): each flagged moment in its axis's own words on its own window, the answers beside the flags and kept as another eye's evidence"

[dependencies]
# the runner every ask goes through, and the question format
raea-app-ask = { path = "../ask" }
raea-core-ask = { path = "../../core/ask" }
# the score's moments, the axes' words and the evidence rows clef's answers become
raea-sys-fingerprint = { path = "../../systems/fingerprint" }
raea-sys-fingerprint-use = { path = "../../systems/fingerprint-use" }
raea-sys-columns = { path = "../../systems/columns" }
serde = { version = "1", features = ["derive"] }

[dev-dependencies]
# a clip made with FFmpeg, and clef's answers as the tests give them
raea-io-assets = { path = "../../io/assets" }
raea-io-clef = { path = "../../io/clef" }
serde_json = "1"
167 changes: 167 additions & 0 deletions apps/double/src/lib.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
//! The fingerprint's flags asked again (docs/ask.feature AK8). `raea fingerprint score FILE --ask flags` (or `ask:
//! "flags"` in fingerprint.ts) puts each flagged moment to the decision model in its axis's own words, on that moment's
//! window alone, so it costs by the flags and never by the piece; `--ask unsure` also asks each window an axis puts in
//! its plausible band without flagging it. Moments on the same window share a call. Each flag then says "clef agrees
//! (0.71)" or "clef doesn't see it (0.08)", and each answer becomes an `eye:clef` evidence row, which trains the axis's
//! head and never counts as the person's (FP6d). The question carries the axis's words as its label words, so the
//! person's notes calibrate it as they do any question (AK6, AK7).

use std::path::Path;
use std::sync::Arc;

use raea_app_ask::prepare::Piece;
use raea_app_ask::runner::{self, Entry, Event};
pub use raea_app_ask::runner::Opts;
use raea_core_ask::{Question, Sees, Window};
use raea_sys_columns::Columns;
use raea_sys_columns::window::Modality;
use raea_sys_fingerprint::evidence::{Row, features};
use raea_sys_fingerprint::folder::Folder;
use raea_sys_fingerprint::taxonomy::{Axis, Fingerprint};
use raea_sys_fingerprint_use::score::{Scored, clock};
use serde::Serialize;

/// One moment asked again.
#[derive(Debug, Clone, Serialize)]
pub struct Double {
pub a: f64,
pub b: f64,
pub axis: String,
/// the axis in the person's words
pub words: String,
/// whether the fingerprint flagged it (else it was unsure of it)
pub flagged: bool,
/// clef's problem score, when it answered
pub problem: Option<f64>,
/// clef agrees: past even odds, or past the question's threshold once the person's labels calibrated it
pub agrees: Option<bool>,
/// its answer's id in raea ask's record, for `raea ask verdict`
pub answer: Option<String>,
/// why it wasn't asked or still waits
pub not_asked: Option<String>,
}

impl Double {
/// What clef made of it, as a flag's line says it.
pub fn said(&self) -> String {
match (self.agrees, self.problem, &self.not_asked) {
(Some(true), Some(p), _) => format!("clef agrees ({p:.2})"),
(Some(false), Some(p), _) => format!("clef doesn't see it ({p:.2})"),
(_, _, Some(why)) => format!("clef wasn't asked: {why}"),
_ => "clef gave no answer".into(),
}
}
}

/// The moments asked again, a line each under the score's summary, as its flags are written (the app reads them).
pub fn lines(ds: &[Double]) -> String {
let each = ds.iter().map(|d| format!(" {}-{} {}: {}{}\n", clock(d.a), clock(d.b), d.axis, d.said(), if d.flagged { "" } else { " (the fingerprint was unsure)" }));
format!("asked again of clef ({} moment{}):\n{}", ds.len(), if ds.len() == 1 { "" } else { "s" }, each.collect::<String>())
}

/// The moments to ask: every flag, and with `unsure` each run of windows an axis puts in its plausible band unflagged.
pub fn moments(s: &Scored, fp: &Fingerprint, which: &str) -> Vec<Double> {
let words = |axis: &str| fp.axis(axis).and_then(|a| a.words.first().cloned()).unwrap_or_else(|| axis.to_string());
let one = |a: f64, b: f64, axis: &str, flagged: bool| Double { a, b, axis: axis.into(), words: words(axis), flagged, problem: None, agrees: None, answer: None, not_asked: None };
let mut out: Vec<Double> = s.moments.iter().map(|m| one(m.a, m.b, &m.axis, true)).collect();
if which == "unsure" {
for w in &s.windows {
for x in w.axes.iter().filter(|x| !x.flagged && x.band == Some("plausible")) {
match out.iter_mut().rev().find(|d| d.axis == x.axis && !d.flagged) {
Some(d) if w.a <= d.b => d.b = w.b,
_ => out.push(one(w.a, w.b, &x.axis, false)),
}
}
}
}
out
}

/// An axis as a question: is its defect there (or, for a quality, is it there), looking or listening as it does.
pub fn question(axis: &Axis) -> Question {
let words = axis.words.first().cloned().unwrap_or_else(|| axis.name.replace(['-', '_'], " "));
let good = axis.polarity == "good";
let ask = if good { format!("Is this {words}?") } else { format!("Is there {words} here?") };
let mut q = Question::plain(&format!("fp.{}", axis.name), &ask);
if good {
q.flags = vec!["no".into()];
}
let listens = matches!(axis.modality, Modality::Sound | Modality::Events);
q.sees = if listens { Sees::Nothing } else { Sees::Video };
q.hears = listens;
q.label_words = axis.words.clone();
q
}

/// Each moment asked of `file`, a call a window, at most `o.jobs` at once; the moments with clef's answers. A moment
/// that couldn't be asked (no route, the pool spent, the file unreadable) says why, and the score goes on without it.
pub fn ask(file: &Path, fp: &Fingerprint, mut ds: Vec<Double>, o: &Opts) -> Vec<Double> {
let piece = match Piece::read(file, None, None, None, None) {
Ok(p) => Arc::new(p),
Err(why) => {
for d in &mut ds {
d.not_asked = Some(format!("the file couldn't be read: {why}"));
}
return ds;
}
};
let entries = ds
.iter()
.filter_map(|d| {
let q = question(fp.axis(&d.axis)?);
Some(Entry { piece: piece.clone(), window: Some(Window { from: Some(d.a), to: Some(d.b) }), questions: vec![q], dropped: vec![], with: vec![] })
})
.collect();
runner::run(entries, o, &mut |e| {
let Event::Answer(r) = e else { return };
let Some(d) = ds.iter_mut().find(|d| format!("fp.{}", d.axis) == r.id && d.a == r.from && r.to == Some(d.b)) else { return };
d.problem = r.problem;
d.agrees = r.says.as_ref().map(|_| r.flagged);
d.answer = r.answer.clone();
d.not_asked = r.not_asked.clone().or_else(|| r.waits.clone());
});
ds
}

/// The answered moments kept in the fingerprint's ledger as clef's evidence, over the file's columns; a moment asked
/// again replaces clef's last row for it. How many it kept.
pub fn keep(dir: &Folder, c: &Columns, project: &str, ds: &[Double]) -> Result<usize, String> {
let rows: Vec<Row> = ds.iter().filter_map(|d| row(d, &c.media, project)).collect();
if rows.is_empty() {
return Ok(0);
}
let mut l = dir.ledger().map_err(|e| e.to_string())?;
let mut n = 0;
for r in rows {
l.retain(|x| !(x.source == r.source && x.media == r.media && x.axis == r.axis && x.a == r.a && x.b == r.b));
let f = features(c, r.a, r.b);
l.push(r, &f);
n += 1;
}
dir.write_ledger(&l).map_err(|e| e.to_string())?;
Ok(n)
}

/// An answered moment as another eye's evidence row.
pub fn row(d: &Double, media: &str, project: &str) -> Option<Row> {
Some(Row {
media: media.into(),
a: d.a,
b: d.b,
axis: d.axis.clone(),
bad: d.agrees?,
weight: 1.0,
who: "clef".into(),
source: "eye:clef".into(),
project: project.into(),
kind: None,
region: None,
words: None,
at: 0,
pin: None,
band: None,
})
}

#[cfg(test)]
mod tests;
88 changes: 88 additions & 0 deletions apps/double/src/tests.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
use std::path::PathBuf;

use raea_app_ask::runner::Opts;
use raea_core_ask::Sees;
use raea_io_clef::{Answered, Call, Fail};
use raea_sys_fingerprint::taxonomy::base;
use raea_sys_fingerprint_use::score::{AxisScore, Moment, Scored, Window};
use serde_json::json;

use crate::{ask, lines, moments, question, row};

fn moment(a: f64, b: f64, axis: &str) -> Moment {
Moment { a, b, axis: axis.into(), words: String::new(), by: "head", state: "gating".into(), band: Some("strong"), gating: true, peak: 1.0, at: None }
}

fn window(a: f64, axis: &str, band: &'static str, flagged: bool) -> Window {
let s = AxisScore { axis: axis.into(), value: 0.5, by: "head", state: "advice".into(), band: Some(band), flagged, gating: false };
Window { a, b: a + 2.0, axes: vec![s], seen: None, heard: None, level: -20.0, overall: None }
}

fn scored(moments: Vec<Moment>, windows: Vec<Window>) -> Scored {
Scored { file: "a.mp4".into(), media: "m".into(), dur: 10.0, fingerprint: "test".into(), moments, windows }
}

#[test]
fn flags_are_asked_and_unsure_adds_the_plausible_windows_run_together() {
let fp = base("test");
let s = scored(vec![moment(0.0, 3.0, "blur")], vec![window(4.0, "glitch", "plausible", false), window(5.0, "glitch", "plausible", false), window(8.0, "glitch", "weak", false)]);
assert_eq!(moments(&s, &fp, "flags").len(), 1);
let unsure = moments(&s, &fp, "unsure");
assert_eq!(unsure.iter().map(|d| (d.a, d.b, d.axis.as_str(), d.flagged)).collect::<Vec<_>>(), [(0.0, 3.0, "blur", true), (4.0, 7.0, "glitch", false)]);
assert!(unsure[0].words.contains("blurry"), "an axis is asked in its own words: {}", unsure[0].words);
}

#[test]
fn an_axis_asks_as_it_senses_and_a_quality_is_flagged_on_no() {
let fp = base("test");
let blur = question(fp.axis("blur").unwrap());
assert!(blur.ask.starts_with("Is there a blurry") && blur.sees == Sees::Video && !blur.hears && blur.flags.is_empty());
assert_eq!(blur.label_words, fp.axis("blur").unwrap().words, "the person's notes in the axis's words label it");
let muffled = question(fp.axis("muffled voice").unwrap());
assert!(muffled.sees == Sees::Nothing && muffled.hears);
let mut good = fp.axis("blur").unwrap().clone();
good.polarity = "good".into();
let q = question(&good);
assert!(q.ask.starts_with("Is this ") && q.flags == ["no"]);
}

/// clef as a fake: yes to the blur question, no to anything else.
fn blur_only(call: &Call) -> Result<Answered, Fail> {
let answers = call.questions.iter().map(|q| (q.id.clone(), json!({"noul": if q.id == "fp.blur" { 0.8 } else { 0.1 }}))).collect();
Ok(Answered { answers, cost: 0.001, cached: false, route: "a test".into(), pool_left: None })
}

/// A 6 s clip with a picture and a tone.
fn clip() -> PathBuf {
let out = std::env::temp_dir().join(format!("raea-double-{}.mp4", std::process::id()));
let ff = raea_io_assets::ffmpeg_tool("ffmpeg", false).expect("FFmpeg");
let ok = std::process::Command::new(ff)
.args(["-y", "-v", "error", "-f", "lavfi", "-i", "testsrc=size=320x240:rate=10:duration=6", "-f", "lavfi", "-i", "sine=frequency=440:duration=6"])
.args(["-c:v", "mpeg4", "-c:a", "aac", "-shortest"])
.arg(&out)
.status()
.expect("FFmpeg ran");
assert!(ok.success());
out
}

#[test]
fn each_flag_is_asked_on_its_own_window_and_said_beside_it_and_kept_as_clefs() {
// SAFETY: this crate's only test that reads it, set before the runner starts
unsafe { std::env::set_var("RAEA_ASK_DIR", std::env::temp_dir().join(format!("raea-double-ask-{}", std::process::id()))) };
let fp = base("test");
let file = clip();
let s = scored(vec![moment(0.0, 2.0, "blur"), moment(0.0, 2.0, "glitch"), moment(3.0, 5.0, "muffled voice")], vec![]);
let ds = ask(&file, &fp, moments(&s, &fp, "flags"), &Opts { jobs: 2, dry: false, ask: blur_only });
let said: Vec<String> = ds.iter().map(|d| format!("{}: {}", d.axis, d.said())).collect();
assert_eq!(said, ["blur: clef agrees (0.80)", "glitch: clef doesn't see it (0.10)", "muffled voice: clef doesn't see it (0.10)"]);
assert!(lines(&ds).starts_with("asked again of clef (3 moments):\n 0:00-0:02 blur: clef agrees (0.80)\n"), "{}", lines(&ds));
let gone = ask(&std::env::temp_dir().join("raea-double-none.mp4"), &fp, moments(&s, &fp, "flags"), &Opts { jobs: 2, dry: false, ask: blur_only });
assert!(gone.iter().all(|d| d.agrees.is_none() && d.said().starts_with("clef wasn't asked")), "a file it can't read asks nothing: {gone:?}");
let r = row(&ds[0], "m", "p").unwrap();
assert!(r.bad && r.source == "eye:clef" && r.who == "clef" && (r.a, r.b) == (0.0, 2.0));
assert!(!row(&ds[1], "m", "p").unwrap().bad);
let mut unanswered = ds[0].clone();
unanswered.agrees = None;
assert!(row(&unanswered, "m", "p").is_none(), "a moment clef didn't answer isn't evidence");
}
2 changes: 2 additions & 0 deletions apps/fingerprint/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ raea-io-home = { path = "../../io/home" }
raea-sys-fingerprint = { path = "../../systems/fingerprint" }
# scoring and the sheet, which one judges a piece, diff and profiles
raea-sys-fingerprint-use = { path = "../../systems/fingerprint-use" }
# score --ask: each flag asked again of the decision model (docs/ask.feature AK8)
raea-app-double = { path = "../double" }
raea-sys-fingerprint-things = { path = "../../systems/fingerprint-things" }
# the encoders and the columns they make
raea-sys-columns = { path = "../../systems/columns" }
Expand Down
Loading
Loading