diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 1f6e9d0..2bbecaa 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -7,14 +7,14 @@ }, "metadata": { "description": "Deep video and image work for coding agents: analyze, describe, search, caption, clip, detect faces, and publish via the tinycloud CLI.", - "version": "0.3.22" + "version": "0.3.23" }, "plugins": [ { "name": "tinycloud", "source": "./", "description": "Video and image understanding and editing via the tinycloud CLI (Cloudglue): watch/see/extract/search/ask/face over videos and images, captions, clips, collections, plus one-command workflows for sales coaching, blog posts, ad analysis, meeting breakdowns, and YouTube publishing.", - "version": "0.3.22", + "version": "0.3.23", "author": { "name": "Cloudglue" }, "homepage": "https://tinycloud.sh", "repository": "https://github.com/cloudglue/tinycloud", diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 336aec4..17ddfb0 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "name": "tinycloud", "displayName": "Tinycloud Video", "description": "Video and image understanding and editing via the tinycloud CLI (Cloudglue): watch/see/extract/search/ask/face over videos and images, captions, clips, collections, and one-command video workflows.", - "version": "0.3.22", + "version": "0.3.23", "author": { "name": "Cloudglue", "url": "https://cloudglue.dev" }, "homepage": "https://tinycloud.sh", "repository": "https://github.com/cloudglue/tinycloud", diff --git a/CLAUDE.md b/CLAUDE.md index 8789248..cb7f55c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -103,7 +103,8 @@ stdout (logs on stderr) with `status`: → exit codes 0/0/2/3/3/0/1. `tinycloud commands --json` is the authoritative flag list — verify doc claims against it, not memory (a doc bug shipped once because `--cached` only exists on watch/see/extract/caption/face/workflow). As -of 0.3.15 there are 16 verbs: `see` +of 0.3.23 there are 18 verbs (0.3.17 added `query`, 0.3.23 added +`moments`): `see` (0.3.7+) analyzes an **image** (file-level, JPEG/PNG/WebP — the image counterpart of `watch`) and `extract` also takes an image source (features `see.v1`, `extract.images.v1`); 0.3.8 adds @@ -383,6 +384,43 @@ STRING `"true"`/`"false"`, and tinycloud rejects any other spelling client-side. Because the skill teaches the media-import cost model and the new flags, the floor was raised to 0.3.22 (same merge-after-CDN gate — the dist PR merges only after CDN `channels.stable` = 0.3.22). +0.3.23 picks up SDK 0.7.29 (spec v0.7.23) and adds **Find Moments** — the +first new verb since `query` (verbs 17→18, features 49→52: `moments.v1`, +`moments.collections.v1`, `moments.search.v1`). Where `probe`/`ask` FIND +content semantically and `extract` pulls declared fields, `moments` sweeps a +WHOLE video against a rubric the caller wrote and persists every window that +qualifies. `tinycloud moments --name --instructions +""` runs one criterion over one video; `--criterion ''` +takes the full rubric (`moment_schema` typed fields, `finding_schema`, +`anchors`, one `scoring` key that populates each moment's `criterion_score`). +The criterion is snapshotted and hashed onto the run (`criterion_hash`), so +editing a rubric yields a different run rather than reinterpreting an old one; +runs reuse a compatible describe or create one (`--describe-job` pins), and +`--boundary` (sentence default), `--signals` (speech default), `--speakers` +and duration bounds tune acceptance. Criterion NAMES are lowercase snake_case +(`^[a-z][a-z0-9_]*$`, max 64) and tinycloud rejects anything else +client-side — the API answers a bad name with a bare "Field(s) in the request +are invalid" naming no field. `moments show|list|delete` manage run history +(deleting an in-flight run refunds it; a completed one does not), and +**`--limit`/`--min-score`/`--sort` are READ-TIME shaping, not selection**: +every accepted moment stays persisted and `data.run.total_moments` always +reports the full accepted count. Findings (`absence` | `observation`) are the +non-temporal counterpart. `library collections create --type moments` makes a +standing collection whose criteria run over every current AND future member — +at least one criterion is REQUIRED at create (the type alone is a 400), and +each entry is a criterion ATTACHMENT (the rubric wrapped in its run options), +not a bare rubric. `library moments attach|detach|list|findings` manage and +enumerate it: attaching is free while the per-file backfill runs bill as they +execute, `--sort criterion-score|rank` requires `--criterion ` (scores +compare only within one rubric), and detaching — or removing a file — drops +moments from COLLECTION ENUMERATION only, leaving the runs as job history. +`moments search` runs `/search` with `scope: "moment"` (the endpoint face +search uses); it is deliberately NOT `probe --scope moment`, because probe +runs on deep search whose scope levels are only file and segment. As of spec +v0.7.23 `query` has no moments virtual tables and `ask`/responses have no +moment awareness. Because the skill teaches the verb, the floor was raised to +0.3.23 (same merge-after-CDN gate — the dist PR merges only after CDN +`channels.stable` = 0.3.23). The host-level `profile` verb and the leading global flags `--home`/`--profile` (also `$TINYCLOUD_HOME`; 0.3.3+) relocate state and are intentionally absent from `commands --json` — like the launcher's install/update, they're CLI/host @@ -438,6 +476,10 @@ of printing JSON. Any script invoking the binary must redirect ` --json # poll: pen tinycloud library imports create col_meta --name "dbx tree" --connector --path /recordings --recursive --enrich-metadata --json # 0.3.22+ tinycloud library imports create col_desc --name "corpus" --connector --max-files 50 --json # 0.3.22+ MEDIA import — ask the user first, it bills per file +# Find Moments (0.3.23+) — sweep a WHOLE video against a rubric you wrote: +tinycloud moments ./earnings.mp4 --name guidance_changes --instructions "A moment where guidance is revised." --json +tinycloud moments show --limit 5 --sort start --json # read-time shaping; total_moments stays whole +# A standing rubric over many videos (criteria run over every current AND future member): +tinycloud library collections create "sales-calls" --type moments --name objection_handling --instructions "…" --json +tinycloud library moments list col_m --criterion objection_handling --sort criterion-score --json +tinycloud moments search "annual pricing" --in collection:col_m --json + # Publish an HTML artifact to Cloudglue Sites (manage with list / unpublish) tinycloud publish ./tinycloud-output/html/report.html --name report --visibility private --json tinycloud publish list --json @@ -256,6 +264,26 @@ Authoring your own recipes: [reference/workflow-authoring.md](reference/workflow backfills fields the connector listing omits — Gong parties + Call Spotlight (re-embedded, so they become searchable) and Dropbox `media_info` duration/dimensions. +- `moments` (0.3.23+) is the fourth retrieval shape and the easiest to confuse: + `probe`/`ask` FIND content semantically, `extract` pulls declared fields from + known places, `search` greps cached context locally, and **`moments` measures + a whole video against a rubric you wrote**, persisting every window that + qualifies. A criterion is `--name` (lowercase snake_case) + `--instructions`; + `--criterion ''` adds `moment_schema` / `finding_schema` / + `anchors` / one `scoring` key. Three things to get right: + **(1) `--limit`/`--min-score`/`--sort` shape the READ, not the result** — + every accepted moment stays persisted and `total_moments` reports the full + count, so re-read the same run with `moments show` rather than re-running. + **(2) On a collection, `--sort criterion-score|rank` REQUIRES `--criterion + `** — scores only compare within one rubric. **(3) Detaching a + criterion (or removing a file) drops moments from collection enumeration, + not from existence** — the runs persist as job history. + A moments COLLECTION needs at least one criterion at create. Attaching a + criterion is free; the per-file backfill runs it starts are billed as they + execute, so ask the user before attaching one to a large collection. Search + moments with `moments search`, NOT `probe --scope moment` (probe runs on deep + search, which has only file and segment scopes); `query` has no moments + tables yet. - `query` (0.3.17+) is for analytics, not search: when the task is to COUNT, GROUP, rank, or join across a collection ("how many…", "which … most", "total hours per host"), reach for `query`, not `probe`/`ask`. Run `query diff --git a/skills/tinycloud/reference/verbs.md b/skills/tinycloud/reference/verbs.md index 18b4e13..a1684fd 100644 --- a/skills/tinycloud/reference/verbs.md +++ b/skills/tinycloud/reference/verbs.md @@ -17,6 +17,7 @@ every verb. Regenerate doubts from it instead of trusting prose. | `clip` | local | no | Cuts, thumbs, audio, stitch, split, transcode, burn, explore | | `grab` | network | no | Download a remote video (YouTube, TikTok, Loom, direct) | | `face` | cloud | yes | Detect faces in a video, or match/search a query face (0.3.4+) | +| `moments` | cloud | yes | Sweep a whole video against a rubric you wrote, keeping every window that qualifies (0.3.23+) | | `library` | varies | no | Collections (incl. create/add/remove/delete), connectors, mirrors, sync | | `jobs` | network | yes | Poll/wait/forget tracked async jobs | | `workflow` | varies | no | Validate/plan/run workflow recipes | @@ -336,6 +337,93 @@ finds the query face across one or more collections (`--min-score`, `--group-by file`). `total` reports the server-available count across all modes (never rewritten by client `--min-*`/`--limit` filters). +### moments — find every moment that meets a rubric (cloud, 0.3.23+) + +```bash +tinycloud moments --name --instructions "" [options] --json +tinycloud moments --criterion '' [options] --json +tinycloud moments show [--limit ] [--min-score ] [--sort rank|start] --json +tinycloud moments list [--status ] [--url ] [--limit ] --json +tinycloud moments delete --json +tinycloud moments search "" --in collection: [--criterion ] [--limit ] --json +``` + +`moments` is rubric-driven **discovery**: it sweeps a WHOLE video against a +standard you wrote and persists every window that qualifies. It is the fourth +retrieval shape and the easiest to reach for by mistake — keep them apart: + +| Verb | What it does | +|---|---| +| `probe` / `ask` | FIND content semantically | +| `extract` | pull declared fields from known places | +| `search` | local keyword grep over cached context | +| **`moments`** | **measure a whole video against a rubric, keeping every hit** | + +**The criterion.** `--name` is a lowercase snake_case identifier +(`^[a-z][a-z0-9_]*$`, max 64) — anything else is rejected client-side, because +the API answers a bad name with a bare "Field(s) in the request are invalid" +that names no field. `--instructions` is the rubric prose. For a richer rubric, +`--criterion ''` takes `{name, instructions, moment_schema, +finding_schema, anchors, scoring}` — `moment_schema` declares typed fields on +every moment, and one `scoring` key populates each moment's `criterion_score`. +The criterion is snapshotted and hashed onto the run (`criterion_hash`), so +editing a rubric produces a different run rather than reinterpreting an old one. + +**Run options.** `--describe-job` pins a describe (otherwise a compatible one is +reused or created — a video with no describe is never an error); `--boundary +sentence|tight|loose` (default sentence) sets how generously edges are drawn; +`--signals` (default `speech`) sets the evidence a moment must rest on; +`--speakers`, `--min-duration`, `--max-duration` narrow acceptance; `--refresh` +forces a fresh sweep; `--background` returns `pending` with a poll hint. + +**`--limit` / `--min-score` / `--sort` are READ-TIME shaping, not selection.** +Every accepted moment stays persisted and `data.run.total_moments` always +reports the FULL accepted count — a narrowed read never means "that was all". +Re-read the same run with `moments show ` and different shaping instead +of re-running it. + +**Findings** are the non-temporal counterpart (`kind: absence | observation`): +what the rubric established about the video as a whole, including that an +expected thing never happened. + +`moments delete` on an IN-FLIGHT run cancels and refunds it; deleting a +COMPLETED run is not refunded. + +**Moments collections** — a standing rubric over many videos: + +```bash +tinycloud library collections create "sales-calls" --type moments \ + --name objection_handling --instructions "An objection and the rep's response." --json +tinycloud library moments attach --name pricing_pushback --instructions "…" [--boundary tight] --json +tinycloud library moments list [--criterion ] [--min-score ] [--sort position|criterion-score|rank] --json +tinycloud library moments findings [--criterion ] [--kind absence|observation] --json +tinycloud library moments detach --json +``` + +A moments collection's criteria run over every current **and future** member, so +adding a video sweeps the standing rubrics over it automatically. **At least one +criterion is REQUIRED at create** — the type alone is rejected; `--criterion` +also accepts an ARRAY of rubrics. + +**Billing:** attaching a criterion is free; the per-file backfill runs it starts +are billed as they execute, and a matching prior run (same rubric hash + file) +satisfies a pair with no extra execution. Ask the user before attaching to a +large collection. Track progress via `backfill_status` / `files_completed` / +`files_total` on `collections show`. + +`--sort criterion-score|rank` **requires `--criterion `** — scores only +compare within one rubric, and the guard is client-side so it never reaches the +API as an opaque 400. `moments detach`, and removing a file, drop the affected +moments from COLLECTION ENUMERATION only; the underlying runs persist as job +history and stay readable through `tinycloud moments`. + +**Searching moments** uses `moments search`, NOT `probe --scope moment` — probe +runs on deep search, whose scope levels are only `file` and `segment`. As of +spec v0.7.23 the `query` verb has no moments virtual tables and `ask`/responses +have no moment awareness, so moments are discoverable and searchable but not yet +countable via `query`. On pre-0.3.23 binaries the whole verb is an +unknown-command error. + ### library — collections, connectors, and bulk imports ```bash @@ -343,7 +431,7 @@ tinycloud library collections list --json tinycloud library collections show [--limit ] [--page-token ] --json # files[].status: pending|processing|completed (readiness) tinycloud library collections sync --artifacts descriptions,transcripts,thumbnails,metadata --json # Collection writes (0.3.4+) — the only write paths in library: -tinycloud library collections create [--type media-descriptions|entities|rich-transcripts|face-analysis|metadata] [--describe full|speech|light|] [--description ] [--prompt | --schema ] --json +tinycloud library collections create [--type media-descriptions|entities|rich-transcripts|face-analysis|metadata|moments] [--describe full|speech|light|] [--description ] [--prompt | --schema ] [--name --instructions "" | --criterion ] --json tinycloud library collections add --to [--metadata ''] [--no-upload] [--no-download] --json tinycloud library collections remove --from --json tinycloud library collections delete --json @@ -372,7 +460,9 @@ read-only `library` (gated by the `library.collections.create.v1` / `--type media-descriptions`; an `entities` collection also needs an extraction spec — `--prompt ` or `--schema ` — or `create` errors, and a `metadata` collection takes NO processing configs (`create` rejects -`--prompt`/`--schema`). +`--prompt`/`--schema`); a `moments` collection REQUIRES at least one criterion +(`--name`/`--instructions`, or `--criterion` with an object or array) — see +[moments](#moments--find-every-moment-that-meets-a-rubric-cloud-0323). **A media-descriptions collection indexes only the modalities chosen at create time, and the API default is speech+summary ONLY** — no visual scene diff --git a/skills/tinycloud/scripts/preflight.sh b/skills/tinycloud/scripts/preflight.sh index 43be71f..0c180fc 100755 --- a/skills/tinycloud/scripts/preflight.sh +++ b/skills/tinycloud/scripts/preflight.sh @@ -9,11 +9,11 @@ set -u # Mirror tinycloud-skill.json: min_version / supported_range upper bound # (CI diffs these against the manifest). -MIN_VERSION="0.3.22" +MIN_VERSION="0.3.23" MAX_VERSION_EXCLUSIVE="0.4.0" INSTALL_CMD='curl -fsSL https://app.cloudglue.dev/tinycloud.sh | bash' # Kept in sync with ../tinycloud-skill.json required_features (CI diffs them). -REQUIRED_FEATURES="envelope.v1 watch.v1 watch.speech.v1 see.v1 extract.v1 extract.images.v1 caption.v1 search.v1 probe.v1 probe.filters.v1 probe.entities.v1 probe.scope.auto.v1 ask.v1 query.v1 query.export.v1 clip.v1 grab.v1 face.v1 jobs.v1 library.collections.v1 library.collections.create.v1 library.collections.mutate.v1 library.collections.entities.v1 library.collections.metadata.v1 library.collections.describe.v1 library.collections.pagination.v1 library.collections.imports.v1 library.collections.imports.media.v1 library.collections.imports.enrich.v1 library.sync.url.v1 workflow.v1 publish.v1 publish.manage.v1 publish.video.v1 publish.link.preview.v1 publish.link.preview.player.v1 setup.v1" +REQUIRED_FEATURES="envelope.v1 watch.v1 watch.speech.v1 see.v1 extract.v1 extract.images.v1 caption.v1 search.v1 probe.v1 probe.filters.v1 probe.entities.v1 probe.scope.auto.v1 ask.v1 query.v1 query.export.v1 clip.v1 grab.v1 face.v1 jobs.v1 library.collections.v1 library.collections.create.v1 library.collections.mutate.v1 library.collections.entities.v1 library.collections.metadata.v1 library.collections.describe.v1 library.collections.pagination.v1 moments.v1 moments.collections.v1 moments.search.v1 library.collections.imports.v1 library.collections.imports.media.v1 library.collections.imports.enrich.v1 library.sync.url.v1 workflow.v1 publish.v1 publish.manage.v1 publish.video.v1 publish.link.preview.v1 publish.link.preview.player.v1 setup.v1" # 1) Binary present and responsive? if ! command -v tinycloud >/dev/null 2>&1; then diff --git a/skills/tinycloud/tinycloud-skill.json b/skills/tinycloud/tinycloud-skill.json index fb52d83..45b5790 100644 --- a/skills/tinycloud/tinycloud-skill.json +++ b/skills/tinycloud/tinycloud-skill.json @@ -1,8 +1,8 @@ { - "skill_version": "0.3.22", + "skill_version": "0.3.23", "tinycloud": { - "min_version": "0.3.22", - "supported_range": ">=0.3.22 <0.4.0", + "min_version": "0.3.23", + "supported_range": ">=0.3.23 <0.4.0", "required_features": [ "envelope.v1", "watch.v1", @@ -30,6 +30,9 @@ "library.collections.metadata.v1", "library.collections.describe.v1", "library.collections.pagination.v1", + "moments.v1", + "moments.collections.v1", + "moments.search.v1", "library.collections.imports.v1", "library.collections.imports.media.v1", "library.collections.imports.enrich.v1",