Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion apps/media/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
//!
//! raea media probe FILE duration, size, fps, codec, audio (JSON)
//! raea media still FILE --at S [--width W] -o OUT.png|.jpg the frame showing at S (an image: itself), W px wide at most
//! raea media sheet FILE... [--n 12] [--width 240] -o OUT a contact sheet (n frames of one file, or each file's middle)
//! raea media sheet FILE... [--n 12] [--width 240] -o OUT a contact sheet (n frames of one file, or each file's middle),
//! and OUT.labels.json: each cell's time or file, for raea show
//! raea media audio FILE [--rate 16000] [--mono] [--from S --to S] -o OUT.wav|- (- is f32le on stdout)
//! raea media loudness FILE [--from S --to S] integrated LUFS, mean and peak dB, true peak
//! raea media motion FILE [--from S --to S] [--fps 6] how much the picture changes (mean luma difference, YDIF)
Expand Down
46 changes: 45 additions & 1 deletion apps/media/src/picture.rs
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,51 @@ pub fn sheet(files: &[PathBuf], n: usize, w: u32, out: &Path) -> R<Value> {
image::imageops::overlay(&mut sheet, &small, ((k as u32 % cols) * w) as i64, ((k as u32 / cols) * th) as i64);
}
save(&sheet, out)?;
Ok(json!({"wrote": out, "tiles": tiles.len(), "at": tiles.iter().map(|(t, _)| round3(*t)).collect::<Vec<_>>(), "width": sheet.width(), "height": sheet.height()}))
// OUT.labels.json beside it (docs/callouts.feature CO2c): each cell says what it is, so raea show OUT.png labels them
let heights: Vec<u32> = tiles.iter().map(|(_, i)| (i.height() as f64 * w as f64 / i.width() as f64).round().max(1.0) as u32).collect();
let texts: Vec<String> = if files.len() == 1 { tiles.iter().map(|(t, _)| clock(*t)).collect() } else { files.iter().map(|f| f.file_name().unwrap_or_default().to_string_lossy().into_owned()).collect() };
let labels = out.with_extension("labels.json");
std::fs::write(&labels, serde_json::to_string_pretty(&json!({"labels": cell_labels(&texts, &heights, w, th, cols, rows)})).unwrap_or_default()).map_err(|e| format!("{}: {e}", labels.display()))?;
Ok(json!({"wrote": out, "labels": labels, "tiles": tiles.len(), "at": tiles.iter().map(|(t, _)| round3(*t)).collect::<Vec<_>>(), "width": sheet.width(), "height": sheet.height()}))
}

/// Each cell's label: its text and its box in the sheet's own 0-1 (a cell `w` wide, its picture `heights[k]` tall in a row
/// `th` tall), in the sheet's order, left to right then down.
fn cell_labels(texts: &[String], heights: &[u32], w: u32, th: u32, cols: u32, rows: u32) -> Vec<Value> {
let (sw, sh) = ((cols * w) as f64, (rows * th) as f64);
let r4 = |v: f64| (v * 1e4).round() / 1e4;
texts.iter().zip(heights).enumerate().map(|(k, (text, h))| {
let (x, y) = (((k as u32 % cols) * w) as f64, ((k as u32 / cols) * th) as f64);
json!({"text": text, "box": [r4(x / sw), r4(y / sh), r4((x + w as f64) / sw), r4((y + *h as f64) / sh)]})
}).collect()
}

/// A moment as a cell's tag: 0:03.2, 1:05.0
fn clock(t: f64) -> String {
let d = (t.max(0.0) * 10.0).round() as u64;
format!("{}:{:04.1}", d / 600, (d % 600) as f64 / 10.0)
}

#[cfg(test)]
mod sheet_tests {
use super::*;

#[test]
fn a_sheets_cells_are_labelled_by_their_boxes() {
// five cells, four across: the second row's one cell, a shorter picture than its row
let texts: Vec<String> = ["a", "b", "c", "d", "e"].map(String::from).to_vec();
let l = cell_labels(&texts, &[135, 135, 135, 135, 90], 240, 135, 4, 2);
assert_eq!(l[0], json!({"text": "a", "box": [0.0, 0.0, 0.25, 0.5]}));
assert_eq!(l[3], json!({"text": "d", "box": [0.75, 0.0, 1.0, 0.5]}));
assert_eq!(l[4], json!({"text": "e", "box": [0.0, 0.5, 0.25, 0.8333]}));
}

#[test]
fn a_moment_reads_as_a_clock() {
assert_eq!(clock(3.24), "0:03.2");
assert_eq!(clock(59.96), "1:00.0");
assert_eq!(clock(65.0), "1:05.0");
}
}

// ---- sound
118 changes: 118 additions & 0 deletions docs/callouts.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
@spec
Feature: Callouts: an agent's labels on what it made, beside it, pointing in when looked at
Shane, 2026-11-18: "whenever it makes a image or video, it will like combine a bunch of frames and say the left is
this, the middle is this, the last is this ... label the frames within an image or within a video ... it can just
look like little tags if they were really short, but ... opening it up to have like a paragraph of information if
... the agent wants to. So instead of just like putting it in the title of the image of left this, bottom this, right
this, whatever, now it can have ... these labels show up like in the UI, not overlapping any of the imagery. Like we
would want it in the like periphery, like in the canvas area ... not an edge, like we don't want to use the edge
components, but like some sort of little like armature thing that just comes over and animates when you hover, just
to be sure that we're pointing at the right thing ... we could even utilize that same aspect for specific call out
to the elements within scenes ... link these little armature comments to like the 3D scene elements that can be ...
hovered on."

What there was (read 11-18): an agent that put several frames in one still (a contact sheet from `raea media sheet`,
a side by side, a before and after, a grid of props) said which was which in the item's title or in its message:
"left: v1, middle: v2, right: v3". The person's own notes have pins and marks on the picture (docs/fingerprint.feature
FP11a) and in a scene's space (docs/scene-feed.feature SF3), but those are comments, asking for something. Nothing let
the agent say what a part of its own picture is.

The callout:
- A label is the agent's: a tag of a few words, and a paragraph more when there is more to say. It points at a
place in what was made: a box or a point on a picture, a still of a storyboard, a moment or a span of a film, a
part of a 3D scene or a point in its space.
- It is never drawn over the picture. The tags stand in the canvas around it, each on the side nearest the place it
names, in that place's order, apart from each other.
- Pointing is on hover: the tag's armature, a thin line with one bend, draws out from the tag to the place, and the
place's corners are marked. Moving away, it draws back. Nothing is drawn on the picture until then.
- It isn't a comment: it asks nothing and goes to no agent. It is part of the item, as its title is.

Background:
Given the app's code is apps/app (raea-feed): src/model/labels.ts, src/detail/blocks/Callouts.tsx, the views that
draw them (src/detail/views Look.tsx, Watch.tsx, Scene.tsx), and how they come in (src/acp/post.ts, host/show.ts,
src/acp/protocol.ts feed_post)
And raea's own side: `raea show --label` (engine/cli/src/show.ts) and the sheet's labels (apps/media picture.rs)

Rule: CO1. A label, and what it can point at
# Status 11-18: in 0.1.8. The feed draws them (raea-feed labels: a still, a film, a scene's parts live, the card's
# count); raea show takes --label, --detail and --labels; raea media sheet writes OUT.labels.json; the raea skill says
# when to label (CO5a).

Scenario: CO1a. A tag and a paragraph
Given a label
Then its tag is its text, a few words (up to 60 characters; longer is cut, with the rest put first in its
paragraph)
And its detail, when it has one, is a paragraph (up to 1200 characters) the tag opens

Scenario: CO1b. Where it points
Then a label names one place: a box (x0, y0, x1, y1) or a point (x, y) in the picture's own 0-1 coordinates
And with frame N, the storyboard's still N (from 0, as a still's pin is) instead of the first
And with t (and to), a film's moment (or span) in seconds: it is drawn while the playhead is within half a second
of the moment, or in the span
And with part, a 3D scene's part by its source key or its name; with space, a point in the scene's space; with
entry, which of a board's scenes
And a label that names no place it can be drawn at is kept off the stage and listed under the picture instead

Rule: CO2. How labels come in
Scenario: CO2a. feed_post takes labels
When an agent posts with feed_post and labels: [{ text, detail?, box? | point? | part? | space?, frame?, t?, to? }]
Then the item has them, cleaned: up to 24, numbers held to 0-1, an empty tag dropped, a box's corners put in order
And a label it can't read is said back in the post's answer, not dropped quietly

Scenario: CO2b. raea show takes labels
When an agent runs raea show FILE --label "PLACE TAG" [--detail "PARAGRAPH"] ...
Then PLACE is x0,y0,x1,y1 or x,y, then #N for the Nth file shown (from 1), @T or @T-TO for a film's seconds, or
part=KEY for a scene's part, joined without spaces (0,0,.5,1#2@3-5); the tag is the rest
And a --detail is the paragraph of the --label before it
And --labels FILE.json gives them as feed_post does

Scenario: CO2c. A contact sheet comes labelled
When raea media sheet writes OUT.png
Then it writes OUT.labels.json beside it: each cell's box and its time ("0:03.2"), or its file's name for a sheet of
several files
And raea show OUT.png takes OUT.labels.json along when it is there, so the sheet's cells say what they are with
nothing more done; an agent that wants to say more edits the JSON's tags and paragraphs first

Rule: CO3. Drawn beside the picture, never on it
Scenario: CO3a. In the canvas around the picture
Given a still, a film or a scene with labels, opened
Then each label's tag stands in the canvas beside the picture, on the left or right, whichever the place it names
is nearer, or below when neither side has room
And tags on a side are in the order of their places from the top, and never overlap each other or the picture
And as the picture is zoomed or panned, or the stage resized, they keep to its sides

Scenario: CO3b. On a film, in time
Given a film's labels with moments or spans
Then a label stands while the playhead is in its time and fades out of it; one with no time stands throughout

Rule: CO4. Pointing, on hover
Scenario: CO4a. The armature
When the pointer is on a tag (or on the place it names, for a box)
Then the tag's armature draws out from the tag to the nearest edge of its place (a box) or to its point: along
the side's margin, one bend at the picture's edge, then in
And the place's corners are marked while it's out
And moving off, it draws back
And with reduced motion it appears and goes without drawing

Scenario: CO4b. The paragraph
When a tag with a paragraph is clicked
Then the paragraph opens under the tag, in the margin, and the armature stays out while it's open
And Esc or a click elsewhere closes it

Scenario: CO4c. A scene's part
Given a label on a scene's part
When its tag is hovered
Then the armature goes to the part as the stage draws it now, and the part is outlined as when the pointer is on it
And it follows the camera and the scene's time as they move

Rule: CO5. The agent's skill says when to label
Scenario: CO5a. Label, don't title
Then the raea skill says: when one picture holds several things (a sheet, a side by side, before and after, a grid
of props or characters), label each instead of naming them by position in the title or the message, with a
paragraph only where a tag isn't enough
And it shows raea show sheet.png --label "0,0,.5,1 v1, hook first" --label ".5,0,1,1 v2, product first"
--detail "Opens on the product..."

Rule: CO6. On the feed's card
Scenario: CO6a. Labels on the feed's card
Then the card shows how many labels the item has, and not the tags themselves, until a card has room for them
58 changes: 57 additions & 1 deletion engine/cli/conformance/show.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// R3), and where a note read from one stands (docs/direction.feature DR5, DR6): its asks, its badge and each ask's,
// and a clip under an ask for the person to sign. The host (apps/app/host/tasks.ts) is faked: this is the command's side.
import { describe, expect, test } from "bun:test";
import { showVerb } from "../src/show.ts";
import { labelsIn, showVerb } from "../src/show.ts";

/** showVerb with a host that keeps each op it was asked, and answers a task as the host does. */
function run(args: string[]) {
Expand Down Expand Up @@ -54,3 +54,59 @@ describe("raea show --working, for a note", () => {
refused(["--working", "cm-1", "--ask", "one", "--states", "to check,failing"], "one state for each --ask: 1 asks, 2 states");
});
});

// labels on what's shown (docs/callouts.feature CO2b): what the command sends the host, which reads each place
describe("raea show --label", () => {
const shown = (args: string[], listed?: unknown[]) => {
const ops: [string, Record<string, unknown>][] = [];
const ask = (op: string, b: Record<string, unknown>) => {
ops.push([op, b]);
if (op === "harness") return { name: "Claude Code", pid: 7 };
return { kind: "item", id: "i-1", type: "look", title: "Three cuts", project: "shop", workspace: "Work", inbox: "/inbox.jsonl" };
};
showVerb(ask, args, "/work/shop", { CLAUDECODE: "1", CLAUDE_CODE_SESSION_ID: "abcdef123" }, listed);
return ops.find(([op]) => op === "show")?.[1];
};

test("each --label is a place and its tag, a --detail the paragraph of the one before it", () => {
const b = shown(["out/three.png", "--label", "0,0,.5,1 v1, hook first", "--label", ".5,0,1,1#2@3-5 v2", "--detail", "Opens on the product.", "--title", "Three cuts"]);
expect(b?.files).toEqual(["/work/shop/out/three.png"]);
expect(b?.labels).toEqual([
{ place: "0,0,.5,1", text: "v1, hook first" },
{ place: ".5,0,1,1#2@3-5", text: "v2", detail: "Opens on the product." },
]);
expect(b?.title).toBe("Three cuts");
});

test("a scene part's key keeps its colons", () => {
expect(shown(["room.tsx", "--label", "part=room.tsx:46:7 the crate"])?.labels).toEqual([{ place: "part=room.tsx:46:7", text: "the crate" }]);
});

test("no labels: none sent, so a sheet's own OUT.labels.json comes along", () => {
expect(shown(["out/sheet.png"])).not.toHaveProperty("labels");
});

test("--labels sends the file's list as it is", () => {
const list = [{ text: "0:01.5", box: [0, 0, 0.25, 0.5] }];
expect(shown(["out/sheet.png", "--labels", "mine.json"], list)?.labels).toEqual(list);
});

test("what isn't a label is refused, saying what one is", () => {
expect(() => shown(["a.png", "--label", "v1"])).toThrow(/--label "v1" is a place, then its tag/);
expect(() => shown(["a.png", "--detail", "why"])).toThrow(/--detail goes after the --label/);
expect(() => shown(["a.png", "--label", "0,0 a", "--detail", "x", "--detail", "y"])).toThrow(/one --detail for the label "a"/);
expect(() => shown(["--question", "Which?", "--choice", "A", "--choice", "B", "--label", "0,0 a"])).toThrow(/labels go on the files you show/);
expect(() => shown(["a.png", "--label", "0,0 a", "--labels", "l.json"], [])).toThrow(/--label or --labels, not both/);
});
});

describe("labelsIn", () => {
test("a list, or { labels }", () => {
expect(labelsIn('[{"text":"a"}]', "l.json")).toEqual([{ text: "a" }]);
expect(labelsIn('{"labels":[{"text":"b"}]}', "l.json")).toEqual([{ text: "b" }]);
});
test("anything else is refused, naming the file", () => {
expect(() => labelsIn("{", "l.json")).toThrow(/--labels l.json isn't JSON/);
expect(() => labelsIn('{"text":"a"}', "l.json")).toThrow(/--labels l.json is a list of labels/);
});
});
Loading
Loading