From 86352fb021a6af81aab11802683c75fb727825c8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 7 Nov 2026 21:29:15 -0500 Subject: [PATCH] B88: media moments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ⌘⇧K marks the start of a moment, then the end, saves it and pauses; with a loop set, one press saves the loop. ⌘⇧J saves the last 30 seconds. A moment is a card next to its page's card with its times and the captions shown during it. On the card: change the times, Play (opens the page at the start and loops the range) and Ask for ID (a question the moment raises). Trail files, Replay, the timeline and search past trails know moments. Also updates the B62 row: Mac signing is done; the passkey entitlement waits on Apple. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011V7Bv7WZ9Q92MZoNWYwevQ --- .github/workflows/ci.yml | 4 +- core/page-scripts.js | 85 ++++++++++++++++++++- core/recall.js | 5 +- core/replay.js | 6 ++ core/trail-file.js | 25 ++++++- core/trail.js | 46 +++++++++++- docs/02-roadmap-backlog.md | 4 +- docs/03-architecture.md | 7 ++ docs/05-changelog.md | 1 + docs/26-trail-file-format.md | 3 + main.js | 96 +++++++++++++++++++++++- package.json | 3 +- preload.js | 5 ++ renderer/app.js | 14 +++- renderer/canvas.js | 75 ++++++++++++++++++- renderer/styles.css | 7 ++ test/e2e/52-media-moments.js | 138 +++++++++++++++++++++++++++++++++++ test/e2e/README.md | 1 + test/e2e/site/moment.html | 12 +++ test/e2e/site/moment.vtt | 61 ++++++++++++++++ test/moment-demo.js | 86 ++++++++++++++++++++++ 21 files changed, 660 insertions(+), 24 deletions(-) create mode 100644 test/e2e/52-media-moments.js create mode 100644 test/e2e/site/moment.html create mode 100644 test/e2e/site/moment.vtt create mode 100644 test/moment-demo.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df1f71c..551c77f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: (cd test/e2e/site && python3 -m http.server 8765 >/dev/null 2>&1 &) sleep 1 mkdir -p e2e-out - for t in 01-basics 03-rules-and-fast-results 04-find-zoom-history 05-research-session 09-playback-limit 12-pip-bookmarks-adblock 13-ask-ai 15-agents-mcp 16-research-agent 17-talk 18-redteam 19-browser-basics 20-first-run-and-help 21-rules-editor 22-ui-refresh 23-beta-survey 24-mosaic-pick 25-video-loop 26-gestures-tour 27-privacy-visible 28-ask-across-tabs 29-blocker-onpage 30-tab-focus-sleep 31-passwords 32-passwords-more 33-sidebar-chrome 34-command-bar 35-frames-tabs 36-trail-tabs 37-threads 38-trail-share 39-trail-open 40-share-link 41-survey-newtab 42-trail-canvas 43-canvas-tools 44-share-open 45-mosaic-gallery 48-canvas-media 49-trail-rename-delete 50-provenance 51-trail-recall; do + for t in 01-basics 03-rules-and-fast-results 04-find-zoom-history 05-research-session 09-playback-limit 12-pip-bookmarks-adblock 13-ask-ai 15-agents-mcp 16-research-agent 17-talk 18-redteam 19-browser-basics 20-first-run-and-help 21-rules-editor 22-ui-refresh 23-beta-survey 24-mosaic-pick 25-video-loop 26-gestures-tour 27-privacy-visible 28-ask-across-tabs 29-blocker-onpage 30-tab-focus-sleep 31-passwords 32-passwords-more 33-sidebar-chrome 34-command-bar 35-frames-tabs 36-trail-tabs 37-threads 38-trail-share 39-trail-open 40-share-link 41-survey-newtab 42-trail-canvas 43-canvas-tools 44-share-open 45-mosaic-gallery 48-canvas-media 49-trail-rename-delete 50-provenance 51-trail-recall 52-media-moments; do echo "::group::$t" xvfb-run -a node test/e2e/$t.js "$PWD/e2e-out" 2>&1 | tee e2e-out/$t.log status=${PIPESTATUS[0]} @@ -71,7 +71,7 @@ jobs: (cd test/e2e/site && python -m http.server 8765 --bind 127.0.0.1 >/dev/null 2>&1 &) sleep 2 mkdir -p e2e-out - for t in 01-basics 05-research-session 19-browser-basics 20-first-run-and-help 22-ui-refresh 23-beta-survey 24-mosaic-pick 25-video-loop 26-gestures-tour 27-privacy-visible 28-ask-across-tabs 29-blocker-onpage 30-tab-focus-sleep 31-passwords 32-passwords-more 33-sidebar-chrome 34-command-bar 35-frames-tabs 36-trail-tabs 37-threads 38-trail-share 39-trail-open 40-share-link 41-survey-newtab 42-trail-canvas 43-canvas-tools 44-share-open 45-mosaic-gallery 48-canvas-media 49-trail-rename-delete 50-provenance 51-trail-recall; do + for t in 01-basics 05-research-session 19-browser-basics 20-first-run-and-help 22-ui-refresh 23-beta-survey 24-mosaic-pick 25-video-loop 26-gestures-tour 27-privacy-visible 28-ask-across-tabs 29-blocker-onpage 30-tab-focus-sleep 31-passwords 32-passwords-more 33-sidebar-chrome 34-command-bar 35-frames-tabs 36-trail-tabs 37-threads 38-trail-share 39-trail-open 40-share-link 41-survey-newtab 42-trail-canvas 43-canvas-tools 44-share-open 45-mosaic-gallery 48-canvas-media 49-trail-rename-delete 50-provenance 51-trail-recall 52-media-moments; do echo "::group::$t" node test/e2e/$t.js "$PWD/e2e-out" 2>&1 | tee e2e-out/$t.log status=${PIPESTATUS[0]} diff --git a/core/page-scripts.js b/core/page-scripts.js index a5d9ece..20da9c2 100644 --- a/core/page-scripts.js +++ b/core/page-scripts.js @@ -392,11 +392,12 @@ function pipScript() { * another address (YouTube reuses one video element for every video). * Returns { a, b, duration } or "none". */ -function loopScript(op, secs = 60) { +function loopScript(op, secs = 60, range = null) { return `(() => { const op = ${JSON.stringify(String(op))}; + const range = ${JSON.stringify(range && range.b > range.a ? { a: Number(range.a), b: Number(range.b) } : null)}; const secs = ${Math.max(5, Math.min(3600, Number(secs) || 60))}; - const vids = [...document.querySelectorAll("video")].filter((v) => v.readyState > 0); + const vids = [...document.querySelectorAll(op === "range" ? "video, audio" : "video")].filter((v) => v.readyState > 0); if (!vids.length) return "none"; const area = (v) => { const r = v.getBoundingClientRect(); return r.width * r.height; }; const v = vids.find((x) => x.__nodeLoop) || vids.find((x) => !x.paused) || vids.sort((a, b) => area(b) - area(a))[0]; @@ -410,6 +411,7 @@ function loopScript(op, secs = 60) { if (op === "next") { a = now; b = Math.min(now + secs, end); } else if (op === "start") { a = now; b = L && L.b > now + 1 ? L.b : Math.min(now + secs, end); } else if (op === "end") { b = now; a = L && L.a < now - 1 ? L.a : Math.max(0, now - secs); } + else if (op === "range" && range) { a = range.a; b = Math.min(range.b, end); } // a saved moment (B88) else return "none"; if (!(b > a + 0.5)) return "none"; v.__nodeLoop = { a, b, href: here() }; @@ -436,7 +438,8 @@ function loopScript(op, secs = 60) { v.addEventListener("ended", v.__nodeLoopFn); } v.__nodePrev = v.currentTime; - if (op === "next" || op === "start") { if (v.paused) v.play().catch(() => {}); } + if (op === "range") { v.currentTime = a; v.__nodePrev = a; v.play().catch(() => {}); } + else if (op === "next" || op === "start") { if (v.paused) v.play().catch(() => {}); } else v.currentTime = a; return { a, b, duration: v.duration }; })()`; @@ -469,4 +472,78 @@ function anyPlayingScript() { return `[...document.querySelectorAll("video, audio")].some((el) => !el.paused && !el.ended)`; } -module.exports = { extractionScript, autoplayScript, blockAutoplayScript, centerOnClickScript, pipScript, loopScript, readableTextScript, pauseForLimitScript, resumeScript, anyPlayingScript }; +// Media moments (B88). op: +// start mark the start (playback goes on) and collect the captions shown +// end the start mark to now, with the captions collected; pauses +// last the last secs seconds up to now, with the caption cues in that range; pauses +// loop the B70 loop's range, with its caption cues; pauses +// cancel forget the start mark +// state { marked, t } without changing anything +// Playing a moment again uses loopScript("range", 0, { a, b }). +// Returns "none" when the page has no started video or audio. +function momentScript(op, arg) { + return `(() => { + const op = ${JSON.stringify(String(op))}; + const arg = ${JSON.stringify(arg ?? null)}; + const media = [...document.querySelectorAll("video, audio")].filter((m) => m.readyState > 0 || (op === "play" && m.currentSrc)); + if (!media.length) return "none"; + const area = (v) => { const r = v.getBoundingClientRect(); return r.width * r.height; }; + const m = media.find((x) => x.__nodeMoment) || media.find((x) => !x.paused) || media.sort((a, b) => area(b) - area(a))[0]; + // Captions on screen now: text tracks, then players that draw their own. + const shown = () => { + const out = []; + for (const tr of m.textTracks || []) if (tr.mode !== "disabled" && tr.activeCues) for (const c of tr.activeCues) out.push(c.text); + if (!out.length) for (const el of document.querySelectorAll(".ytp-caption-segment, .vp-captions, .captions-text, [data-testid='caption-text']")) out.push(el.textContent); + return out.map((s) => s.replace(/<[^>]+>/g, "").replace(/\\s+/g, " ").trim()).filter(Boolean).join(" "); + }; + // Caption cues from a to b, for a range already played. + const cues = (a, b) => { + const out = []; + for (const tr of m.textTracks || []) { + if (!/^(subtitles|captions)$/.test(tr.kind) || !tr.cues) continue; + for (const c of tr.cues) if (c.endTime > a && c.startTime < b) out.push(c.text.replace(/<[^>]+>/g, "").replace(/\\s+/g, " ").trim()); + if (out.length) break; + } + return out; + }; + const join = (lines) => { const o = []; for (const l of lines) if (l && o[o.length - 1] !== l) o.push(l); return o.join(" ").slice(0, 1200); }; + const stop = () => { if (m.__nodeMoment) clearInterval(m.__nodeMoment.timer); m.__nodeMoment = null; }; + const info = (a, b, lines) => ({ a, b, duration: isFinite(m.duration) ? m.duration : null, transcript: join(lines), kind: m.tagName === "AUDIO" ? "audio" : "video" }); + if (op === "state") return { marked: Boolean(m.__nodeMoment), t: m.currentTime, a: m.__nodeMoment ? m.__nodeMoment.a : null }; + if (op === "cancel") { stop(); return { cancelled: true }; } + if (op === "start") { + stop(); + const mk = { a: m.currentTime, lines: [], href: location.href.split("#")[0] }; + const take = () => { const t = shown(); if (t && mk.lines[mk.lines.length - 1] !== t) mk.lines.push(t); }; + take(); + mk.timer = setInterval(take, 400); + m.__nodeMoment = mk; + if (m.paused) m.play().catch(() => {}); + return { started: true, a: mk.a }; + } + if (op === "end") { + const mk = m.__nodeMoment; + if (!mk || mk.href !== location.href.split("#")[0]) { stop(); return "none"; } + const b = m.currentTime; + const lines = mk.lines.length ? mk.lines : cues(mk.a, b); + stop(); + m.pause(); + return b > mk.a + 0.5 ? info(mk.a, b, lines) : { tooShort: true }; + } + if (op === "last") { + const b = m.currentTime, a = Math.max(0, b - Math.max(5, Math.min(600, Number(arg) || 30))); + stop(); + m.pause(); + return b > a + 0.5 ? info(a, b, cues(a, b)) : { tooShort: true }; + } + if (op === "loop") { + const L = m.__nodeLoop; + if (!L || L.b == null) return "none"; + m.pause(); + return info(L.a, L.b, cues(L.a, L.b)); + } + return "none"; + })()`; +} + +module.exports = { momentScript, extractionScript, autoplayScript, blockAutoplayScript, centerOnClickScript, pipScript, loopScript, readableTextScript, pauseForLimitScript, resumeScript, anyPlayingScript }; diff --git a/core/recall.js b/core/recall.js index bfc36d1..fea5181 100644 --- a/core/recall.js +++ b/core/recall.js @@ -23,6 +23,7 @@ const KINDS = { song: "audio", track: "audio", audio: "audio", podcast: "audio", listened: "audio", question: "question", questions: "question", thread: "question", threads: "question", note: "note", notes: "note", + moment: "moment", moments: "moment", search: "search", searches: "search", searched: "search", image: "image", images: "image", photo: "image", picture: "image", screenshot: "image", }; @@ -205,7 +206,7 @@ function recallIndex(store) { card: c ? c.id : null, url: it.url || c?.url || "", title: it.title || c?.title || it.url || "", - text: c ? [c.excerpt, ...(c.keywords || [])].filter(Boolean).join(" ") : "", + text: c ? [c.excerpt, c.transcript, c.text, ...(c.keywords || [])].filter(Boolean).join(" ") : "", at: it.at, timelineOnly: !c, }); @@ -213,7 +214,7 @@ function recallIndex(store) { } for (const c of cards.values()) { if (seenCards.has(c.id) || !c.placedAt) continue; - const text = [c.text, c.quote, c.query, c.excerpt, c.conclusion, ...(c.keywords || []), ...(c.replies || []).map((r) => r.text)].filter(Boolean).join(" "); + const text = [c.text, c.quote, c.query, c.excerpt, c.conclusion, c.transcript, ...(c.keywords || []), ...(c.replies || []).map((r) => r.text)].filter(Boolean).join(" "); rows.push({ kind: c.kind, media: c.media, trail: t.id, card: c.id, url: c.url || "", title: c.title || c.text || c.query || "", text, at: c.placedAt, timelineOnly: false }); } for (const g of snap.groups || []) { diff --git a/core/replay.js b/core/replay.js index da00d53..8d37379 100644 --- a/core/replay.js +++ b/core/replay.js @@ -28,6 +28,11 @@ // ids reached so far (this step included). const Prov = require("./provenance"); +const clockOf = (t) => { + t = Math.max(0, Math.round(Number(t) || 0)); + const h = Math.floor(t / 3600), m = Math.floor((t % 3600) / 60), s = String(t % 60).padStart(2, "0"); + return h ? `${h}:${String(m).padStart(2, "0")}:${s}` : `${m}:${s}`; +}; const SESSION_GAP_MS = 30 * 60 * 1000; const CARD_EVENTS = new Set(["opened", "placed", "captured", "asked", "reported", "noted"]); @@ -202,6 +207,7 @@ function stepCaption(st, ctx) { else if (c.kind === "note") t = `${w} wrote a note: ${q(title)}`; else if (c.kind === "voice") t = `${w} recorded a voice note.`; else if (c.kind === "search") t = `${w} searched: ${q(title)}.`; + else if (c.kind === "moment") t = `${w} saved a moment of ${q(title)}, ${clockOf(c.a)} to ${clockOf(c.b)}.`; else if (c.kind === "media") { const from = site(c.url) ? ` from ${site(c.url)}` : ""; t = { diff --git a/core/trail-file.js b/core/trail-file.js index 2f1bc9c..2d9bab1 100644 --- a/core/trail-file.js +++ b/core/trail-file.js @@ -515,7 +515,14 @@ function merge(base, incoming, answers = {}) { // payment pages, screenshots (only the capture's text and source go in). const FORMAT_V2 = 2; -const SIZE_V2 = { page: [260, 200], video: [260, 220], capture: [260, 210], report: [260, 240], question: [260, 170], note: [260, 150], voice: [260, 170], search: [260, 140], media: [300, 240] }; +const SIZE_V2 = { page: [260, 200], video: [260, 220], capture: [260, 210], report: [260, 240], question: [260, 170], note: [260, 150], voice: [260, 170], search: [260, 140], media: [300, 240], moment: [280, 190] }; +// Media moments (B88): "41:12" and back, in seconds. +const clockOf = (t) => { + t = Math.max(0, Math.round(Number(t) || 0)); + const h = Math.floor(t / 3600), m = Math.floor((t % 3600) / 60), s = String(t % 60).padStart(2, "0"); + return h ? `${h}:${String(m).padStart(2, "0")}:${s}` : `${m}:${s}`; +}; +const secsOf = (s) => String(s).split(":").reduce((n, p) => n * 60 + Number(p), 0); // Media cards (B114). The file never carries the media itself: it links to // where the media came from on the web (src), so the reader's NODE can show // a large preview from there. Screenshots, recordings and files added from @@ -552,6 +559,8 @@ function cardHeading(c) { return `### [${mdText(c.title || c.url)}](${c.url}) · video`; case "search": return `### [${mdText(c.title || `Search: ${c.query || ""}`)}](${c.url}) · search`; + case "moment": + return `### [${mdText(c.title || c.url)}](${c.url}) · moment ${clockOf(c.a)}–${clockOf(c.b)}`; case "capture": return `### > ${mdText(c.quote || c.title || c.source || "Capture")}`; case "report": @@ -627,6 +636,10 @@ function cardBody(c) { if (c.url) out.push(`From [${mdText(hostOf(c.url) || c.url)}](${c.url})`); if (!c.src) out.push("The file stayed on the sender's computer."); break; + case "moment": + // The captions shown during the moment, when the site had them. + if (c.transcript) out.push(`> ${oneLine(c.transcript)}`); + break; } return out; } @@ -645,7 +658,7 @@ const inGroup = (c, g) => { const x = c.x + c.w / 2, y = c.y + c.h / 2; return x >= g.x && x <= g.x + g.w && y >= g.y && y <= g.y + g.h; }; -const sendable = (c) => !c.removed && !(c.url && isSensitiveUrl(c.url)) && !(["page", "video", "search"].includes(c.kind) && !cleanUrl(c.url)); +const sendable = (c) => !c.removed && !(c.url && isSensitiveUrl(c.url)) && !(["page", "video", "search", "moment"].includes(c.kind) && !cleanUrl(c.url)); // Addresses leave without private tokens or tracking codes (cleanUrl). A // media card's file and Library id stay behind. const forFile = (c) => { @@ -856,6 +869,7 @@ function guessKind(head) { if (/^Voice note/i.test(head)) return "voice"; if (/^\[.*\]\(.*\)\s*·\s*video$/.test(head)) return "video"; if (/^\[.*\]\(.*\)\s*·\s*search$/.test(head)) return "search"; + if (/^\[.*\]\(.*\)\s*·\s*moment \d/.test(head)) return "moment"; if (/^\[.*\]\(.*\)/.test(head)) return "page"; return "note"; } @@ -891,6 +905,13 @@ function readCard(raw) { case "search": c.query = (c.title || "").replace(/^Search:\s*/, ""); break; + case "moment": { + const m = raw.head.match(/·\s*moment ([\d:]+)–([\d:]+)\s*$/); + if (m) Object.assign(c, { a: secsOf(m[1]), b: secsOf(m[2]) }); + const q = b.find((l) => /^> /.test(l)); + if (q) c.transcript = q.slice(2); + break; + } case "capture": c.quote = raw.head.replace(/^>\s*/, ""); c.fromText = b[0] || ""; diff --git a/core/trail.js b/core/trail.js index 8a9974f..862c0a7 100644 --- a/core/trail.js +++ b/core/trail.js @@ -24,8 +24,8 @@ const GROUP_SIDE = 24; // group padding around its cards: sides and bottom const GROUP_TOP = 48; // and top, where the name sits const SESSION_GAP_MS = 30 * 60 * 1000; // a gap of more than 30 minutes starts a new session const ENGAGED_MS = 20 * 1000; // the 20-second rule (doc 23); a starting guess -const EVENT_TYPES = new Set(["opened", "bounced", "captured", "asked", "reported", "noted", "placed", "removed", "linked", "unlinked", "grouped", "renamed", "resolved", "moved"]); -const CARD_KINDS = new Set(["page", "video", "capture", "report", "question", "note", "voice", "search", "media"]); +const EVENT_TYPES = new Set(["opened", "bounced", "captured", "asked", "reported", "noted", "placed", "removed", "linked", "unlinked", "grouped", "renamed", "resolved", "moved", "edited"]); +const CARD_KINDS = new Set(["page", "video", "capture", "report", "question", "note", "voice", "search", "media", "moment"]); const LINE_KINDS = new Set(["nav", "auto", "you", "suggested"]); const LINE_LABELS = new Set(["", "supports", "contradicts", "answers", "raises"]); // Card sizes at 100% zoom, by kind (handoff: cards 240-260 px wide). @@ -39,6 +39,7 @@ const SIZE = { voice: [260, 170], search: [260, 140], media: [300, 240], + moment: [280, 190], }; // Media cards (B114): image, screenshot, video and audio from the Library. const MEDIA_KINDS = new Set(["image", "screenshot", "video", "audio"]); @@ -542,6 +543,46 @@ class TrailStore { if (card && source && !source.removed) this.link(id, source.id, card.id, { kind: "auto", label: "captured from", at }); return card; } + // ---- media moments (B88) ---- + // A section of a video or audio page, a to b in seconds, with the caption + // text shown during it when the site had captions. It goes next to its + // page's card, with a grey line from the page. + momentCard(id, { url, title = "", a, b, transcript = "", near, at = this.now() } = {}) { + const d = this.get(id); + a = Math.max(0, Math.round(Number(a) * 10) / 10); + b = Math.round(Number(b) * 10) / 10; + if (!d || d.trail.readonly || !url || isSensitiveUrl(url) || !(b > a)) return null; + const have = d.cards.find((c) => c.kind === "moment" && !c.removed && c.url === url && Math.abs(c.a - a) < 1 && Math.abs(c.b - b) < 1); + if (have) return have; + const [w, h] = SIZE.moment; + const source = this.pageCard(id, url); + const pos = placeNear({ cards: d.cards.filter((c) => !c.removed), groups: d.groups }, source && !source.removed ? source : null, w, h, { near: near || this.below(id, w) }); + for (const m of pos.moves) { + const o = (m.kind === "group" ? d.groups : d.cards).find((k) => k.id === m.id); + if (!o) continue; + if (typeof m.y === "number") o.y = m.y; + if (typeof m.h === "number") o.h = m.h; + } + const fields = { kind: "moment", url, title: String(title || source?.title || url).slice(0, 300), a, b, placedAt: at, w, h, x: pos.x, y: pos.y }; + const text = String(transcript || "").replace(/\s+/g, " ").trim().slice(0, 1200); + if (text) fields.transcript = text; + const card = this.addCard(id, fields, { at, event: "captured", data: { url, a, b, moment: true } }); + if (card && source && !source.removed) this.link(id, source.id, card.id, { kind: "auto", label: "moment from", at }); + return card; + } + // New start and end for a moment, typed on its card. + momentTimes(id, cardId, a, b, { at = this.now() } = {}) { + const d = this.get(id); + const c = this.card(id, cardId); + a = Math.max(0, Math.round(Number(a) * 10) / 10); + b = Math.round(Number(b) * 10) / 10; + if (!d || d.trail.readonly || c?.kind !== "moment" || !(b > a)) return null; + c.a = a; + c.b = b; + this.event(id, { type: "edited", card: c.id, at, data: { a, b } }); + this.save(id); + return c; + } // ---- search cards (B106) ---- // A web search made from the canvas or the address bar: a card titled // "Search: " that opens the results. Pages opened from the results go @@ -791,6 +832,7 @@ class TrailStore { const item = { event: e.id, type: e.type, at: e.at, card: card && !card.removed ? card.id : null }; if (url) item.url = url; item.title = (card && (card.title || card.text || card.quote)) || e.data?.title || url || e.type; + if (card?.kind === "moment") item.moment = { a: card.a, b: card.b }; // B88: shown as "Moment 41:12–42:30" if (e.type === "opened") { // An opened page shows once per session, at its latest time. // Items from an opened file have a title and no address. diff --git a/docs/02-roadmap-backlog.md b/docs/02-roadmap-backlog.md index b561151..4b6f6b3 100644 --- a/docs/02-roadmap-backlog.md +++ b/docs/02-roadmap-backlog.md @@ -83,7 +83,7 @@ Effort: XS under 1 hour, S half a day, M 1-2 days, L 3+ days. Finding IDs (C1, K | B57 | 6 | Research agent v2: follow-up questions on a report (earlier sources and notes kept), a second round of searching when the notes leave gaps, a live view of the page being read | P1 | M | Done | | B58 | 6 | Prompt-injection red-team set (12 pages) run before each release; reader leaves out text a person cannot see; agent drops search queries that look like addresses or encoded data | P1 | S | Done | | B61 | 4 | Browser basics: right-click menu, downloads (Library → Downloads), reopen closed tab, tab shortcuts, print and Save as PDF, error and crashed-tab pages, reopen last session's tabs | P0 | M | Done | -| B62 | 4 | Passkeys on macOS: Apple Developer ID signing, Apple's web-browser passkey entitlement (com.apple.developer.web-browser.public-key-credential), and a native bridge to AuthenticationServices (as electron-webauthn-mac does) | P1 | L | Not started; passkeys off on Mac until then | +| B62 | 4 | Passkeys on macOS: Apple Developer ID signing, Apple's web-browser passkey entitlement (com.apple.developer.web-browser.public-key-credential), and a native bridge to AuthenticationServices (as electron-webauthn-mac does) | P1 | L | Signing done: Mac builds are signed with the Developer ID and checked by Apple since 0.9.6. Waiting on Apple: the entitlement has to be requested from Apple and approved. The native bridge can be built before then, switched off. Passkeys off on Mac until then | | B63 | 4 | Watch: tab name stayed the same after moving to another page (reported once on the Mac, 2026-10-14; cleared up by itself; not reproduced on Linux with link clicks, single-page sites, restored and sleeping tabs, or Mosaic). If it happens again, note the site and how the page was left, and copy diagnostics | P2 | S | Watching | | B64 | 4 | Usage summary for the beta survey: NODE counts locally (suggestions shown and opened, pins, hides, trail pages, sessions, agent runs, Ask AI questions, days used, version, OS); Settings → Help → Copy usage summary puts the counts on the clipboard. No pages, addresses or keywords; nothing sent automatically. See `10-beta-survey.md` | P1 | S | Done (0.9.1) | | B65 | 4 | Beta survey prompt: after NODE has been used on 3 different days, a one-time sidebar notice (Take survey / Later / Don't ask again) opens the survey in a tab; link also in Settings → Help. Survey address in `core/urls.js` (`SURVEY_URL`) | P1 | XS | Done (0.9.1). Too small to notice (Sven, 2026-10-30): see B98 | @@ -109,7 +109,7 @@ Effort: XS under 1 hour, S half a day, M 1-2 days, L 3+ days. Finding IDs (C1, K | B85 | 4 | "Searched, nothing found" entry: a reply type in a thread that records a search that found nothing | P2 | XS | Not started | | B86 | 4 | Check before creating a share link: warn when a trail has pages naming private people or showing social posts; trails private by default. Add a shared trail file with a prompt injection to the B58 red-team set | P1 | S | Not started | | B87 | 4 | Save a copy of each trail page (offered, not forced) with a SHA-256 hash and the time; kept on the computer, sent only through share links | P1 | M | Not started | -| B88 | 4 | Media moments: one key saves start and end time, transcript text if available, and the link, then pauses playback; reuses the B70 loop range. "ID at 41:12" requests in threads | P1 | M | Not started | +| B88 | 4 | Media moments: one key saves start and end time, transcript text if available, and the link, then pauses playback; reuses the B70 loop range. "ID at 41:12" requests in threads | P1 | M | Built 2026-11-07 (PR #68): ⌘⇧K marks the start, then the end (with a loop set, one press saves the loop), and ⌘⇧J saves the last 30 seconds (Sven chose both, 2026-11-07); a moment card next to the page's card with the captions shown during it; times can be changed on the card; Play opens the page at the start and loops the range; Ask for ID makes a question "ID at 41:12?" that the moment raises. Electron build only. Tests `test:moment`, e2e `52-media-moments` | | B89 | 4 | Search past trails by what the person remembers: order ("after the Discogs page"), time, thread, keyword | P1 | M | Built 2026-11-07 (PR #67): an "In your trails" group in the CommandBar and the canvas search bar's list; local rules only, no AI; picking a result goes to its card, or its timeline row (Sven, 2026-11-07). Electron build only. Tests `test:recall`, e2e `51-trail-recall` | | B90 | 4 | Close tabs into a trail with a reason (to read, reference, reminder). Possible answer to B75 | P2 | S | Not started | | B91 | 4 | Export citations as CSL-JSON and BibTeX (Zotero, Overleaf); README states trails always export to open formats | P2 | S | Not started | diff --git a/docs/03-architecture.md b/docs/03-architecture.md index f6d70ad..56b6836 100644 --- a/docs/03-architecture.md +++ b/docs/03-architecture.md @@ -274,6 +274,13 @@ The same core modules run in the built-in extension of the Chromium build (`brow - **Rename:** `rename(id, name)` records a `renamed` event; `applySession` now does the same when a session's name changes, so renames through research:\* are events too. - **Channels:** `trail:rename-id` (id, name; refused for read-only trails; the current trail goes through research:\*), `trail:delete` (closes the trail's tabs first, then `changeTrail` if it was current; returns `{deleted, name, tabs, current}`), `trail:restore` (via `changeTrail`). - **Switcher** (`renderer/app.js`): each trail row is `.sb-menu-row` with the item and a "…" button; the actions row (`.sb-menu-acts`) holds Rename and Delete…, then the question with Delete trail and Keep. `#sb-undo` under the menu shows Undo for 10 s. Keys on a trail in the menu: F2 renames, Delete or Backspace asks to delete, → reaches "…". Over a page the field is typed in the sidebar's layer (`chrome-overlay.html`); the real field does not save on losing focus there, and Enter or Esc come back through `overlayEvent`. +## Media moments (B88) + +- **Page script** `momentScript(op, secs)` (`core/page-scripts.js`), run in every frame until one has media: `state`, `start` (marks the time and collects the captions on screen every 400 ms: text tracks' active cues, else `.ytp-caption-segment`, `.vp-captions` and similar), `end` (the mark to now, with what it collected, else the text-track cues in that range; pauses), `last` (the last n seconds with their cues; pauses), `loop` (the B70 loop's range; pauses), `cancel`. `loopScript("range", 60, { a, b })` seeks to a, plays and loops a–b, for video or audio. +- **Main** `saveMoment(tab, "mark" | "last")`: refuses private tabs and read-only trails; makes the page a card (`engagePage(tab, "moment")`), then `trailStore.momentCard`. `trail:moment-play` finds a tab on the page in the current trail or opens one, then tries `loopScript("range")` every 500 ms for up to 15 s. `trail:moment-times` and `trail:moment-id` go through `canvasEdit`. +- **Store:** `momentCard(id, { url, title, a, b, transcript })` makes a `moment` card (280×190) next to the page's card with an auto line labelled "moment from" and a `captured` event (`data.moment`); the same range twice (within 1 s) is one card. `momentTimes` changes a and b and records an `edited` event. Timeline items for moments carry `moment: { a, b }`. +- **Canvas:** `momentParts(c)`: the times button (an input on click), the captions (two lines), Play and Ask for ID (`.cv-media-ctl`, so they don't start a drag). Double-click plays the moment. + ## Search past trails (B89) - `core/recall.js` (no platform imports): `parseRecall(text, { now, trails })` takes out a trail name ("in …"), times (`timeAt`: a day, part of a day, a week, a month, "n days ago"; local time), an order ("after", "before", "around", "next to" and the words that follow), kind words, and stop words; the rest are keywords. `recallIndex(store)` lists every trail's timeline items (cards and timeline-only visits) and its other cards (questions, notes, searches, media), oldest first in each trail with a position. `recall(entries, text)` scores keywords (title 3, site 2, other text 1; all words must match), keeps the time, kind and trail, and for an order finds the pages that match the order words, then the 6 entries after or before each in the same trail, nearer first. One result per page or card, best first, with `why`. diff --git a/docs/05-changelog.md b/docs/05-changelog.md index 8ff4b90..83284bd 100644 --- a/docs/05-changelog.md +++ b/docs/05-changelog.md @@ -4,6 +4,7 @@ Newest first. Commit hashes refer to the git history inside the delivered zip fi | Date | Commit | Change | | --- | --- | --- | +| 2026-11-07 | (PR #68) | **B88: media moments.** On a page with video or audio, ⌘⇧K (View → Save Moment: Start / End, or right-click a video) marks the start and playback goes on; pressing it again marks the end, saves the moment and pauses. With a loop set (B70), one press saves the loop's range. ⌘⇧J (Save the Last 30 Seconds) saves the 30 seconds up to now and pauses. A moment is a card next to its page's card (the page becomes a card if it wasn't), with a grey "moment from" line, the start and end, and the captions shown during it when the site had captions (text tracks, YouTube and Vimeo captions; up to 1,200 characters). On the card: click the times to change them ("41:12–42:30"), **Play** opens the page at the start and loops the range, and **Ask for ID** adds a question "ID at 41:12?" that the moment raises, for others to answer. The timeline shows "Moment 41:12–42:30 · title", Replay says "You saved a moment of “…”, 41:12 to 42:30.", trail files carry moment cards (doc 26), and search past trails (B89) finds moments by their captions or the word "moment". Private tabs and opened (read-only) trails save no moments. New `TrailStore.momentCard` and `momentTimes` (event `edited`), `momentScript` in `core/page-scripts.js`, `loopScript` op `range`, channels `tab:moment`, `trail:moment-play`, `trail:moment-times`, `trail:moment-id`. Electron build only. Also: the B62 row now says Mac signing is done and the passkey entitlement is waiting on Apple. Tests: `test/moment-demo.js` (`npm run test:moment`, in `test:unit`, 7 checks), new `test/e2e/52-media-moments.js` (17 checks, in CI). | | 2026-11-07 | (PR #67) | **B89: search past trails by what you remember.** Typing in the CommandBar (⌘L, ⌘T) or the canvas search bar now also lists up to four matches from all your trails under "In your trails", each with why it matched, the day and the trail ("after “Memex – Encyclopedia” · Tue 3 Nov · Reading list"). NODE reads, with its own rules and no AI: words (titles, sites, addresses, keywords, excerpts, question and note text, replies, group names); times (today, yesterday, this morning, last night, Tuesday, last Tuesday, this week, last week, this month, last month, in October, 3 days ago, 2 weeks ago, a few days ago, recently); order (after X, before X, around X, next to X: the pages just after or before X in the same trail); kinds (video, song, podcast, question, thread, note, search, image, screenshot); and a trail ("in Rye breads"). Pages that stayed in the timeline are found too. Picking a result switches to its trail and selects its card with a short ring (opacity only), or, for a page that never became a card, shows the canvas and marks its row in the sidebar's timeline. In the canvas search list, cards on the current trail are left out, since the search bar already marks them. Nothing leaves the computer. New `core/recall.js` (no platform imports), channel `trail:recall`. Electron build only. Tests: `test/recall-demo.js` (`npm run test:recall`, in `test:unit`, 9 checks, including 6,000 pages in 40 trails in about 30 ms), new `test/e2e/51-trail-recall.js` (13 checks, in CI). | | 2026-11-06 | (PR #65) | **B83: trail provenance.** NODE records how each page in a trail was reached: a link on another page (its site), a web search (the query; pages opened from Brave, Google, DuckDuckGo, Bing and other results pages, or after a search typed in NODE), an address you typed (address bar, CommandBar, canvas), NODE's suggestions (with the keyword or search they came from), the research agent, or the site moving on to the next video or track by itself (no click or key press in the tab for 5 s while the page before was playing). Separately, a page is marked "played by NODE" when NODE's autoplay started its video or audio with no click just before. Each page card shows one line under its title, such as "Suggested by NODE for “levain”" or "Link from reddit.com"; NODE's and the site's picks are violet. Replay captions say it ("NODE suggested “…” for “…”, and you opened it.", "youtube.com played “…” next by itself.", "The research agent found “…”."), and visits that stayed in the timeline keep it too. Trail files add `Reached: …` to page cards and `(timeline only · …)` to timeline items (doc 26); a file opened by someone else keeps it. Single-page sites that change address (YouTube's next video) now start a new page for the trail. New `core/provenance.js` (no platform imports; the window and the Chromium canvas page load it for the card line). `tab:create` and `tab:navigate` take an optional `via`. Electron build only for now. Tests: `test/provenance-demo.js` (`npm run test:provenance`, in `test:unit`), new `test/e2e/50-provenance.js` (14 checks, in CI). | | 2026-11-06 | (PR #64) | **B115: rename and delete trails from the switcher.** Each trail in the switcher menu has a "…" button with Rename and Delete…. Rename turns the row into a text field (Enter or clicking away saves, Esc cancels); double-clicking the switcher's name renames the current trail, and F2 renames the trail in focus. Delete… asks first: "Delete “X”? Its canvas, timeline and Replay go, and its n tabs close. Library files stay." with Delete trail and Keep (focus on Keep). Deleting closes the trail's tabs and, if it was current, makes the most recently used other trail current; deleting the only trail starts a new one. "Deleted “X”." with Undo shows in the sidebar for 10 s; Undo brings the trail back as the current trail (its tabs stay closed). The files are deleted when NODE next starts. The Delete key on a trail in the menu asks to delete it. Opened files (read only) can be deleted, not renamed. Over a page, renaming works in the sidebar's layer. A rename made through research:\* (the current trail, the Trail panel) is now recorded as a renamed event too. New `TrailStore.remove`, `restore`, `deletedIds`, `purge`; storage `remove()` in `fs-storage.js` and `memoryStorage`; channels `trail:rename-id`, `trail:delete`, `trail:restore`. Tests: 6 more checks in `test/trail-demo.js`, new `test/e2e/49-trail-rename-delete.js` (32 checks, in CI). | diff --git a/docs/26-trail-file-format.md b/docs/26-trail-file-format.md index 0cbab89..a190516 100644 --- a/docs/26-trail-file-format.md +++ b/docs/26-trail-file-format.md @@ -44,6 +44,9 @@ Status: draft, Nov 5 2026 (media cards added the same day, B114). Replaces the v | media, image from the web | `### ![Title](image address)` | `From [site](page address)` | | media, video or audio from the web | `### [Title](media address) · video clip` (or `· audio`) | `From [site](page address)` | | media, screenshot or a file from the computer | `### Screenshot: Title` (or `Image:`, `Video:`, `Audio:`) | `From [site](page address)` if any, then `The file stayed on the sender's computer.` | +| moment | `### [Title](page address) · moment 41:12–42:30` | `> captions shown during the moment`, if the site had captions | + +Moment cards (B88, added 2026-11-07) keep `node-trail: 2`: a section of a video or audio page, start and end in the heading (`m:ss` or `h:mm:ss`), the captions as one quoted line (at most 1,200 characters). A "moment from" line joins the page's card to the moment. An "ID at 41:12?" request is an ordinary question card with a "raises" line from the moment. Media cards (B114, added 2026-11-05) keep `node-trail: 2`: the layout comment adds `media:image`, `media:screenshot`, `media:video` or `media:audio`. The file never carries the media itself or the sender's Library id. Media saved from the web links to where it came from (`src`), without tracking codes, so the reader's NODE shows a large preview from there (the image, or a video or audio player). A screenshot links only to its page. Readers that don't know media cards see a normal Markdown image or link. diff --git a/main.js b/main.js index bf6fcf6..aa734d5 100644 --- a/main.js +++ b/main.js @@ -46,7 +46,7 @@ const { createAdblock } = require("./adblock"); const { createNet } = require("./core/net"); const { createSuggest } = require("./core/suggest"); const { createPreview } = require("./core/preview"); -const { extractionScript, autoplayScript, blockAutoplayScript, centerOnClickScript, pipScript, loopScript, readableTextScript, pauseForLimitScript, resumeScript, anyPlayingScript } = require("./core/page-scripts"); +const { momentScript, extractionScript, autoplayScript, blockAutoplayScript, centerOnClickScript, pipScript, loopScript, readableTextScript, pauseForLimitScript, resumeScript, anyPlayingScript } = require("./core/page-scripts"); const { PlaybackLimiter } = require("./core/playback"); const { fsStorage } = require("./fs-storage"); const { normalizeUrl, hostOf, SEARCH_URL, isSensitiveUrl } = require("./core/urls"); @@ -1905,11 +1905,11 @@ function contextMenuItems(tab, p, { video = p.mediaType === "video" } = {}) { if ((p.mediaType === "video" || p.mediaType === "audio") && http(p.srcURL)) { sep(); items.push({ label: `Open ${p.mediaType === "video" ? "Video" : "Audio"} in New Tab`, click: () => createTab(p.srcURL) }, { label: `Copy ${p.mediaType === "video" ? "Video" : "Audio"} Address`, click: () => clipboard.writeText(p.srcURL) }); - if (p.mediaType === "video") items.push({ label: "Picture in Picture", click: () => togglePip(tab.id) }, { label: "Loop Section", submenu: loopMenu(tab.id) }); + if (p.mediaType === "video") items.push({ label: "Picture in Picture", click: () => togglePip(tab.id) }, { label: "Loop Section", submenu: loopMenu(tab.id) }, { label: "Save Moment: Start / End", click: () => saveMoment(tab.id, "mark") }, { label: "Save the Last 30 Seconds", click: () => saveMoment(tab.id, "last", 30) }); } else if (video) { // A video the right-click did not land on directly (custom player, blob: address). sep(); - items.push({ label: "Picture in Picture", click: () => togglePip(tab.id) }, { label: "Loop Section", submenu: loopMenu(tab.id) }); + items.push({ label: "Picture in Picture", click: () => togglePip(tab.id) }, { label: "Loop Section", submenu: loopMenu(tab.id) }, { label: "Save Moment: Start / End", click: () => saveMoment(tab.id, "mark") }, { label: "Save the Last 30 Seconds", click: () => saveMoment(tab.id, "last", 30) }); } // On a Mac, a right-click selects the word under the pointer, which can be a // lone quote mark or bracket; only offer search and pin for real words. @@ -2549,6 +2549,8 @@ function buildMenu() { if (tab) loopVideo(tab.id, tab.loop ? "stop" : "next", 60); }, }, + { label: "Save Moment: Start / End", accelerator: "CmdOrCtrl+Shift+K", click: () => saveMoment(activeId, "mark") }, + { label: "Save the Last 30 Seconds", accelerator: "CmdOrCtrl+Shift+J", click: () => saveMoment(activeId, "last", 30) }, { label: "Show or Hide Suggestions", accelerator: "CmdOrCtrl+Shift+.", click: () => sendUi("toggle-suggest") }, { type: "separator" }, // Over a page the sidebar is out of the way; this brings it over the page, or sends it away. @@ -3365,6 +3367,94 @@ async function loopVideo(id, op, secs) { return tab.loop; } ipcMain.handle("tab:loop", (_e, id, op, secs) => loopVideo(id, op, secs)); + +// ---- Media moments (B88) ---- +// ⌘⇧K: the first press marks the start and playback goes on; the second +// marks the end, saves the moment and pauses. With a loop set (B70), one +// press saves the loop's range. ⌘⇧J saves the last 30 seconds. A moment is +// a card next to its page's card on the canvas, with the captions shown +// during it when the site had captions. +const MOMENT_KEY = process.platform === "darwin" ? "⌘⇧K" : "Ctrl+Shift+K"; +async function inMediaFrames(tab, script) { + let frames = []; + try { + frames = tab.view.webContents.mainFrame.framesInSubtree; + } catch (_) {} + for (const f of frames) { + try { + const r = await f.executeJavaScript(script, true); + if (r && r !== "none") return r; + } catch (_) {} + } + return null; +} +async function saveMoment(id, mode = "mark", secs = 30) { + const tab = tabs.get(id || activeId); + if (!tab || tab.sleeping) return null; + if (tab.private) return void sendUi("notice", { message: "Private tabs are in no trail, so moments can't be saved from them." }); + const trail = tab.page?.trail || tab.trail || currentTrail(); + if (trailStore?.get(trail)?.trail.readonly) return void sendUi("notice", { message: "This trail is read only. Save it as a new trail to add moments." }); + let op = mode === "last" ? "last" : "end"; + if (mode === "mark") { + const st = await inMediaFrames(tab, momentScript("state")); + if (!st) return void sendUi("notice", { message: "No video or audio on this page. Start it first." }); + if (!st.marked) op = tab.loop ? "loop" : "start"; + } + const r = await inMediaFrames(tab, momentScript(op, secs)); + if (!r) return void sendUi("notice", { message: "No video or audio on this page. Start it first." }); + if (r.started) { + tab.momentMark = true; + return void sendUi("notice", { message: `Moment started at ${clock(r.a)}. Press ${MOMENT_KEY} again to end it and save it.` }); + } + tab.momentMark = false; + if (r.tooShort) return void sendUi("notice", { message: "That moment was under a second. Nothing saved." }); + const wc = tab.view.webContents; + const url = tab.page?.url || wc.getURL().split("#")[0]; + research.saveNow(); + // The page becomes a card first, so the moment goes next to it. + engagePage(tab, "moment"); + const card = trailStore.momentCard(trail, { url, title: wc.getTitle(), a: r.a, b: r.b, transcript: r.transcript }); + research.reload(); + if (!card) return void sendUi("notice", { message: "This page can't be saved to a trail." }); + sendUi("trail-changed", { trail, card: card.id }); + sendUi("notice", { message: `Saved a moment, ${clock(card.a)} to ${clock(card.b)}, to the canvas.${r.transcript ? " Its captions came with it." : ""}` }); + return card; +} +ipcMain.handle("tab:moment", (_e, id, mode, secs) => saveMoment(id, mode === "last" ? "last" : "mark", Number(secs) || 30)); +// Play a moment card: its page, at its start, looping its range. +ipcMain.handle("trail:moment-play", async (_e, cardId) => { + const c = trailStore?.card(currentTrail(), String(cardId)); + if (c?.kind !== "moment" || !c.url) return null; + const base = (u) => String(u || "").split("#")[0]; + let tab = [...tabs.values()].find((t) => !t.private && t.trail === currentTrail() && base(t.page?.url || t.view.webContents.getURL()) === base(c.url)); + if (tab) activateTab(tab.id); + else tab = tabs.get(createTab(c.url)); + if (!tab) return null; + // The player may still be loading: try for up to 15 s. + for (let i = 0; i < 30; i++) { + if (tab.view.webContents.isDestroyed()) return null; + const r = await inMediaFrames(tab, loopScript("range", 60, { a: c.a, b: c.b })); + if (r && !r.stopped) { + tab.loop = { a: r.a, b: r.b, url: base(tab.view.webContents.getURL()) }; + emitList(); + return tab.loop; + } + await new Promise((res) => setTimeout(res, 500)); + } + sendUi("notice", { message: "The video didn't start here. Press play, then try again." }); + return null; +}); +ipcMain.handle("trail:moment-times", (_e, cardId, a, b) => canvasEdit((id) => trailStore.momentTimes(id, String(cardId), Number(a), Number(b)))); +// Ask for ID: a question "ID at 41:12?" next to the moment, with a "raises" line. +ipcMain.handle("trail:moment-id", (_e, cardId) => + canvasEdit((id) => { + const c = trailStore.card(id, String(cardId)); + if (c?.kind !== "moment") return null; + const q = trailStore.questionCard(id, { id: `t${Date.now().toString(36)}`, title: `ID at ${clock(c.a)}?`, status: "open", by: loadSettings().displayName || "", at: Date.now(), replies: [] }, { place: { x: Math.round(c.x + c.w + 60), y: Math.round(c.y) } }); + if (q) trailStore.link(id, c.id, q.id, { kind: "you", label: "raises" }); + return q; + }), +); function loopMenu(id) { const tab = tabs.get(id || activeId); const run = (op, secs) => () => loopVideo(tab?.id, op, secs); diff --git a/package.json b/package.json index 1205e03..1bb78ae 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "test:replay": "node test/replay-demo.js", "test:provenance": "node test/provenance-demo.js", "test:recall": "node test/recall-demo.js", + "test:moment": "node test/moment-demo.js", "test:trailfile": "node test/trail-file-v2-demo.js", "try:canvas": "node test/try-canvas.js", "test:keywords": "node test/keywords-demo.js", @@ -37,7 +38,7 @@ "dist:mac": "electron-builder --mac", "dist:win": "electron-builder --win", "test:redteam": "node test/e2e/18-redteam.js", - "test:unit": "node test/net-demo.js && node test/trail-demo.js && node test/canvas-demo.js && node test/placement-demo.js && node test/ladder-demo.js && node test/replay-demo.js && node test/provenance-demo.js && node test/recall-demo.js && node test/share-link-demo.js && node test/share-worker-demo.js && node test/trail-file-demo.js && node test/trail-file-v2-demo.js && node test/threads-demo.js && node test/trail-open-demo.js && node test/passwords-demo.js && node test/chat-demo.js && node test/usage-demo.js && node test/research-demo.js && node test/keywords-demo.js && node test/keywords-eval.js && node test/providers-demo.js && node test/rules-demo.js && node test/ai-demo.js && node test/controller-demo.js && node test/playback-demo.js && node test/bookmarks-demo.js && node test/google-demo.js && node test/agents-demo.js && node test/agent-demo.js && node test/talk-demo.js", + "test:unit": "node test/net-demo.js && node test/trail-demo.js && node test/canvas-demo.js && node test/placement-demo.js && node test/ladder-demo.js && node test/replay-demo.js && node test/provenance-demo.js && node test/recall-demo.js && node test/moment-demo.js && node test/share-link-demo.js && node test/share-worker-demo.js && node test/trail-file-demo.js && node test/trail-file-v2-demo.js && node test/threads-demo.js && node test/trail-open-demo.js && node test/passwords-demo.js && node test/chat-demo.js && node test/usage-demo.js && node test/research-demo.js && node test/keywords-demo.js && node test/keywords-eval.js && node test/providers-demo.js && node test/rules-demo.js && node test/ai-demo.js && node test/controller-demo.js && node test/playback-demo.js && node test/bookmarks-demo.js && node test/google-demo.js && node test/agents-demo.js && node test/agent-demo.js && node test/talk-demo.js", "release": "electron-builder --publish always", "build:browser-extension": "node browser/extension/build.js", "check:browser-patches": "python3 browser/check_patches.py", diff --git a/preload.js b/preload.js index c090ecf..240da00 100644 --- a/preload.js +++ b/preload.js @@ -115,6 +115,11 @@ contextBridge.exposeInMainWorld("scout", { historySearch: (q, limit) => ipcRenderer.invoke("history:search", q, limit), // B89: results from past trails for what the person remembers. trailRecall: (q, limit) => ipcRenderer.invoke("trail:recall", q, limit), + // B88: media moments. + tabMoment: (id, mode, secs) => ipcRenderer.invoke("tab:moment", id, mode, secs), + trailMomentPlay: (id) => ipcRenderer.invoke("trail:moment-play", id), + trailMomentTimes: (id, a, b) => ipcRenderer.invoke("trail:moment-times", id, a, b), + trailMomentId: (id) => ipcRenderer.invoke("trail:moment-id", id), historyClear: () => ipcRenderer.invoke("history:clear"), claudeDesktop: (mode) => ipcRenderer.invoke("agents:claude-desktop", mode), historyRemove: (url) => ipcRenderer.invoke("history:remove", url), diff --git a/renderer/app.js b/renderer/app.js index 3be5f5f..fd941dc 100644 --- a/renderer/app.js +++ b/renderer/app.js @@ -3880,7 +3880,7 @@ async function openCanvas({ focus = false, select = "" } = {}) { if (!api?.trailGet) return; const seq = ++canvasSeq; trailCanvas ||= new TrailCanvas(canvasEl, api, { - open: (card, tab) => (tab?.tabId ? api.activateTab(tab.tabId) : api.createTab(card.url)), + open: (card, tab) => (card.kind === "moment" ? api.trailMomentPlay(card.id) : tab?.tabId ? api.activateTab(tab.tabId) : api.createTab(card.url)), search: (text) => api.createTab(text, { how: "typed" }), pickTrail: (id) => api.trailSwitch(id), // B104: Put on canvas (a timeline row dropped on it) and Remove from canvas. @@ -3902,6 +3902,10 @@ async function openCanvas({ focus = false, select = "" } = {}) { webSearch: (q) => api.trailSearch(q), privateSearch: (q) => api.trailSearch(q, { private: true }), ask: (q) => askAboutTrail(q), + // B88: moment cards. + momentPlay: (id) => api.trailMomentPlay(id), + momentTimes: (id, a, b) => api.trailMomentTimes(id, a, b), + momentId: (id) => api.trailMomentId(id), // B89: results from past trails in the search bar's list. Cards on this // trail are left out: the search bar already marks them. recall: async (q) => ((await api.trailRecall?.(q, 6)) || []).filter((r) => !(r.card && r.trail === trailCanvas?.data?.trail?.id)).slice(0, 4), @@ -4619,6 +4623,12 @@ async function goRecall(r) { } } let timelineFound = null; +// "41:12", for moments in the timeline (B88). +function clockText(t) { + t = Math.max(0, Math.round(Number(t) || 0)); + const h = Math.floor(t / 3600), m = Math.floor((t % 3600) / 60), s = String(t % 60).padStart(2, "0"); + return h ? `${h}:${String(m).padStart(2, "0")}:${s}` : `${m}:${s}`; +} async function showFoundInTimeline(r) { timelineFound = { url: r.url, at: r.at, until: Date.now() + 4000 }; // The sidebar shows beside the canvas; over a page it is hidden. @@ -4938,7 +4948,7 @@ function paintTimeline() { }); } row.querySelector(".t").textContent = time; - row.querySelector(".nm").textContent = it.title || it.url || "Page"; + row.querySelector(".nm").textContent = (it.moment ? `Moment ${clockText(it.moment.a)}–${clockText(it.moment.b)} · ` : "") + (it.title || it.url || "Page"); const secs = typeof it.dwell === "number" ? Math.round(it.dwell / 1000) : null; row.querySelector(".k").textContent = it.timelineOnly ? (secs !== null ? `${secs} s` : it.removedFromCanvas ? "Removed" : "Timeline only") : ""; row.title = `${it.title || ""}\n${it.url || ""}${it.timelineOnly ? "\nTimeline only: not on the canvas" : ""}`; diff --git a/renderer/canvas.js b/renderer/canvas.js index 15d051d..c13ddb2 100644 --- a/renderer/canvas.js +++ b/renderer/canvas.js @@ -111,7 +111,14 @@ ]; const LABEL_TEXT = { supports: "Supports", contradicts: "Contradicts", answers: "Answers", raises: "Raises" }; const looksLikeAddress = (q) => /^[a-z][a-z0-9+.-]*:\/\//i.test(q) || /^(localhost|[\w-]+(\.[\w-]+)+)(:\d+)?(\/\S*)?$/i.test(q); - const KIND_LABEL = { page: "Page", video: "Video", capture: "Capture", report: "Report", question: "Question", note: "Note", voice: "Voice", search: "Search", media: "Media" }; + // "41:12" and back, for moments (B88). + const clockText = (t) => { + t = Math.max(0, Math.round(Number(t) || 0)); + const hh = Math.floor(t / 3600), m = Math.floor((t % 3600) / 60), s = String(t % 60).padStart(2, "0"); + return hh ? `${hh}:${String(m).padStart(2, "0")}:${s}` : `${m}:${s}`; + }; + const secsText = (s) => String(s).split(":").reduce((n, p) => n * 60 + Number(p), 0); + const KIND_LABEL = { page: "Page", video: "Video", capture: "Capture", report: "Report", question: "Question", note: "Note", voice: "Voice", search: "Search", media: "Media", moment: "Moment" }; const MEDIA_LABEL = { image: "Image", screenshot: "Screenshot", video: "Video", audio: "Audio" }; // B114: what a card shows as media, and from where: the Library file on // this computer, or (in an opened file) the address it came from. Older @@ -840,13 +847,13 @@ if (this.multi?.has(c.id)) el.classList.add("multi"); if (this.seenAt && c.placedAt > this.seenAt && c.id !== this.selected) el.classList.add("new"); if (c.kind === "question" && c.status === "resolved") el.classList.add("resolved"); - const pv = c.url ? this.previews[c.url] : null; + const pv = c.url && c.kind !== "moment" ? this.previews[c.url] : null; if (pv) el.classList.add("open"); const head = h("div", "cv-card-head"); const label = [c.kind === "media" ? MEDIA_LABEL[c.media] || "Media" : KIND_LABEL[c.kind] || "Card"]; if (c.kind === "media" && c.url) label.push(hostOf(c.url)); - if (c.kind === "page" || c.kind === "video") label.push(hostOf(c.url)); + if (c.kind === "page" || c.kind === "video" || c.kind === "moment") label.push(hostOf(c.url)); if (c.kind === "capture" && c.source) label.push(c.source); if (c.kind === "question") { const n = (c.replies || []).length; @@ -873,7 +880,8 @@ const title = c.kind === "note" ? c.text : c.kind === "capture" && c.quote ? `“${c.quote}”` : c.title || c.text || (c.url ? hostOf(c.url) : KIND_LABEL[c.kind]); el.append(h("div", "cv-card-title", title || "")); - const sub = md ? "" : this.cardSub(c); + if (c.kind === "moment") el.append(...this.momentParts(c)); + const sub = md || c.kind === "moment" ? "" : this.cardSub(c); if (sub && !pv?.thumb) el.append(h("div", "cv-card-sub", sub)); // How the page was reached (B83). NODE's own picks are marked. const via = c.kind === "page" && root.NodeProvenance ? root.NodeProvenance.viaText(c.via, c.played) : ""; @@ -888,6 +896,65 @@ return el; } + // B88: a moment's times (click to change them), the captions shown + // during it, Play and Ask for ID. + momentParts(c) { + const times = h("button", "cv-moment-times cv-media-ctl", `${clockText(c.a)} – ${clockText(c.b)}`); + times.type = "button"; + times.title = this.readonly ? "" : "Change the start and end"; + times.disabled = this.readonly; + times.addEventListener("click", (e) => { + e.stopPropagation(); + if (this.readonly) return; + const input = h("input", "cv-moment-edit cv-media-ctl"); + input.value = `${clockText(c.a)}–${clockText(c.b)}`; + input.setAttribute("aria-label", "Start and end, like 41:12–42:30"); + times.replaceWith(input); + input.focus(); + input.select(); + let done = false; + const finish = async (keep) => { + if (done) return; + done = true; + const m = input.value.match(/^\s*([\d:.]+)\s*[–-]\s*([\d:.]+)\s*$/); + const a = m && secsText(m[1]), b = m && secsText(m[2]); + if (keep && m && b > a) this.apply(await this.hooks.momentTimes?.(c.id, a, b)); + else { + if (keep) this.say("Type the start and end like 41:12–42:30."); + input.replaceWith(times); + } + }; + input.addEventListener("keydown", (e) => { + e.stopPropagation(); + if (e.key === "Enter") finish(true); + if (e.key === "Escape") finish(false); + }); + input.addEventListener("blur", () => finish(true)); + }); + const parts = [times]; + if (c.transcript) parts.push(h("div", "cv-moment-text", c.transcript)); + const acts = h("div", "cv-moment-acts cv-media-ctl"); + const play = h("button", "chip", "Play"); + play.type = "button"; + play.addEventListener("click", (e) => { + e.stopPropagation(); + this.hooks.momentPlay?.(c.id); + }); + const id = h("button", "chip", "Ask for ID"); + id.type = "button"; + id.disabled = this.readonly; + if (this.readonly) id.title = "Read only. Save as new trail to change it."; + id.addEventListener("click", async (e) => { + e.stopPropagation(); + const out = await this.hooks.momentId?.(c.id); + this.apply(out); + if (out?.result?.id) this.select(out.result.id, { reveal: true }); + }); + acts.append(play, id); + parts.push(acts); + return parts; + } + surveyEl() { const box = h("section", "cv-survey"); box.hidden = true; diff --git a/renderer/styles.css b/renderer/styles.css index 2a36acf..3e6ed8e 100644 --- a/renderer/styles.css +++ b/renderer/styles.css @@ -1361,6 +1361,13 @@ body.side #canvas { grid-column: 2; grid-row: 1; border: 1px solid var(--line); #canvas .cv-card.k-note .cv-card-title { font-weight: 400; -webkit-line-clamp: 5; } #canvas .cv-card.k-capture .cv-card-title { font-weight: 400; font-style: italic; } #canvas .cv-card-sub { font-size: 13px; line-height: 18px; color: var(--ink-muted); overflow: hidden; display: -webkit-box; -webkit-line-clamp: 4; -webkit-box-orient: vertical; } +/* Media moments (B88): times, captions, Play and Ask for ID. */ +#canvas .cv-moment-times { align-self: flex-start; margin: 2px 0 0; padding: 2px 6px; border: 1px solid var(--line); border-radius: var(--radius-sm); background: transparent; color: var(--ink); font: 14px/18px var(--mono); cursor: text; } +#canvas .cv-moment-times:disabled { cursor: default; border-color: transparent; padding-left: 0; } +#canvas .cv-moment-edit { align-self: flex-start; width: 150px; margin: 2px 0 0; padding: 2px 6px; border: 1px solid var(--signal); border-radius: var(--radius-sm); background: var(--surface); color: var(--ink); font: 14px/18px var(--mono); } +#canvas .cv-moment-text { font-size: 12px; line-height: 16px; color: var(--ink-muted); display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; } +#canvas .cv-moment-acts { display: flex; gap: 6px; margin-top: auto; } +#canvas .cv-moment-acts + .cv-card-foot { margin-top: 0; } /* How the page was reached (B83); violet when NODE or the site chose it. */ #canvas .cv-card-via { margin-top: auto; font: 11px/14px var(--mono); color: var(--ink-muted); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; flex: none; } #canvas .cv-card-via.node { color: var(--signal); } diff --git a/test/e2e/52-media-moments.js b/test/e2e/52-media-moments.js new file mode 100644 index 0000000..4ca7360 --- /dev/null +++ b/test/e2e/52-media-moments.js @@ -0,0 +1,138 @@ +// Media moments (B88): ⌘⇧K marks a start, then the end, and saves a moment +// card next to the page's card with the captions shown during it, then +// pauses. With a loop set, one press saves the loop. ⌘⇧J saves the last +// 30 seconds. On the canvas: change the times, Ask for ID, Play. +const { _electron: electron } = require(process.env.PLAYWRIGHT_PATH || "playwright"); +const path = require("path"), fs = require("fs"), os = require("os"); +const OUT = process.argv[2] || fs.mkdtempSync(path.join(os.tmpdir(), "node-e2e-")); +const BASE = "http://127.0.0.1:8765"; +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +let failed = 0; +const check = (name, ok, detail = "") => { + if (!ok) failed++; + console.log(ok ? "✓" : "✗", name, detail ? `(${detail})` : ""); +}; +const userData = fs.mkdtempSync(path.join(OUT, "ud-")); +fs.writeFileSync(path.join(userData, "settings.json"), JSON.stringify({ firstRunDone: true })); + +(async () => { + const app = await electron.launch({ executablePath: require("electron"), args: ["--no-sandbox", "--autoplay-policy=no-user-gesture-required", `--user-data-dir=${userData}`, path.join(__dirname, "..", "..")] }); + await app.evaluate(() => { + const o = globalThis.fetch; + globalThis.fetch = async (u, opt) => (/wikipedia|duckduckgo|brave\.com|x\.ai|11434/.test(String(u)) ? new Response("{}", { status: 200 }) : o(u, opt)); + }); + const ui = await app.firstWindow(); + await sleep(1800); + const menuItem = (label) => app.evaluate(({ Menu }, l) => Menu.getApplicationMenu().items.flatMap((m) => m.submenu?.items || []).find((i) => i.label === l).click(), label); + const page = (js) => + app.evaluate(async ({ BrowserWindow }, code) => { + const v = BrowserWindow.getAllWindows()[0].contentView.children.find((x) => x.webContents && /moment\.html/.test(x.webContents.getURL())); + return v ? v.webContents.executeJavaScript(code) : null; + }, js); + const moments = async () => (await ui.evaluate(() => window.scout.trailGet())).cards.filter((c) => c.kind === "moment").sort((a, b) => a.placedAt - b.placedAt); + const notices = []; + await ui.exposeFunction("__notice", (m) => notices.push(m)); + await ui.evaluate(() => window.scout.onCommand?.((cmd, msg) => cmd === "notice" && window.__notice(msg.message))); + + // Seek once the whole video is loaded (the test server has no range requests). + const seekPlay = (t) => + page(`new Promise((done) => { + const v = document.getElementById("v"); + const loaded = () => v.buffered.length && v.buffered.end(v.buffered.length - 1) >= v.duration - 0.5; + const go = () => { v.addEventListener("seeked", () => v.play().then(() => done(true), () => done(false)), { once: true }); v.currentTime = ${t}; }; + if (loaded()) go(); + else { const t0 = setInterval(() => { if (loaded()) { clearInterval(t0); go(); } }, 100); setTimeout(() => done(false), 15000); } + })`); + await ui.evaluate((u) => window.scout.createTab(u), `${BASE}/moment.html`); + await sleep(2000); + check("the test video loads and seeks", await seekPlay(4)); + await sleep(800); + + // ⌘⇧K: start, then end. + await menuItem("Save Moment: Start / End"); + await sleep(3200); + check("the first press keeps it playing", (await page(`!document.getElementById("v").paused`)) === true); + await menuItem("Save Moment: Start / End"); + await sleep(1500); + let ms = await moments(); + const first = ms[0]; + check("the second press saves a moment card", ms.length === 1 && first.b - first.a > 2 && first.b - first.a < 5 && first.a >= 4, JSON.stringify(first && { a: first.a, b: first.b })); + check("and pauses", (await page(`document.getElementById("v").paused`)) === true); + check("the moment keeps the captions shown during it", /Line \d+: /.test(first?.transcript || ""), first?.transcript); + const snap = await ui.evaluate(() => window.scout.trailGet()); + const pageCard = snap.cards.find((c) => c.kind === "page" && /moment\.html/.test(c.url)); + check("its page became a card, with a line to the moment", Boolean(pageCard) && snap.lines.some((l) => l.from === pageCard.id && l.to === first.id && l.label === "moment from")); + + // ⌘⇧J: the last 30 seconds (from the start of the video here). + await seekPlay(12); + await sleep(1200); + await menuItem("Save the Last 30 Seconds"); + await sleep(1500); + ms = await moments(); + const last = ms[1]; + check("“Save the Last 30 Seconds” saves up to now", ms.length === 2 && last.a === 0 && last.b >= 12 && last.b < 15, JSON.stringify(last && { a: last.a, b: last.b })); + check("with the caption cues in that range", /Line 1: the starter/.test(last?.transcript || "") && /Line 6: after two/.test(last?.transcript || ""), last?.transcript); + + // With a loop set, one press saves the loop. + await seekPlay(20); + await sleep(500); + const tabId = await ui.evaluate(() => activeId); + await ui.evaluate((id) => window.scout.loopTab?.(id, "next", 5) ?? window.scout.tabLoop?.(id, "next", 5), tabId); + await sleep(800); + await menuItem("Save Moment: Start / End"); + await sleep(1500); + ms = await moments(); + const looped = ms[2]; + check("with a loop set, one press saves the loop's range", ms.length === 3 && Math.abs(looped.b - looped.a - 5) < 0.6 && looped.a >= 19.5, JSON.stringify(looped && { a: looped.a, b: looped.b })); + + // The canvas: the card, change the times, Ask for ID, Play. + await menuItem("Tab 1"); + await sleep(1500); + await ui.evaluate((id) => trailCanvas.select(id, { reveal: true }), first.id); + await sleep(600); + const card = `#canvas .cv-card[data-id="${first.id}"]`; + const shown = await ui.evaluate((sel) => { + const el = document.querySelector(sel); + return el && { kind: el.querySelector(".cv-card-kind").textContent, times: el.querySelector(".cv-moment-times").textContent, text: el.querySelector(".cv-moment-text")?.textContent || "", acts: [...el.querySelectorAll(".cv-moment-acts .chip")].map((b) => b.textContent) }; + }, card); + check("the card shows Moment, its times, its captions, Play and Ask for ID", shown && shown.kind === "Moment · 127.0.0.1" && /^0:0\d – 0:0\d$/.test(shown.times) && /Line/.test(shown.text) && shown.acts.join() === "Play,Ask for ID", JSON.stringify(shown)); + await ui.screenshot({ path: path.join(OUT, "52-card.png") }); + const rows = await ui.evaluate(() => [...document.querySelectorAll("#sb-timeline-list .sb-tl-row .nm")].map((r) => r.textContent)); + check("the timeline says Moment and its times", rows.some((r) => /^Moment 0:\d\d–0:\d\d · A long talk about levain$/.test(r)), JSON.stringify(rows)); + await ui.click(`${card} .cv-moment-times`); + await sleep(200); + await ui.keyboard.press("ControlOrMeta+a"); + await ui.keyboard.type("0:02–0:06"); + await ui.keyboard.press("Enter"); + await sleep(1000); + const edited = (await moments()).find((c) => c.id === first.id); + check("clicking the times lets you change them", edited.a === 2 && edited.b === 6, JSON.stringify({ a: edited.a, b: edited.b })); + await ui.click(`${card} .cv-moment-acts .chip:nth-child(2)`); + await sleep(1000); + const s2 = await ui.evaluate(() => window.scout.trailGet()); + const q = s2.cards.find((c) => c.kind === "question" && c.title === "ID at 0:02?"); + check("Ask for ID makes a question “ID at 0:02?” that the moment raises", Boolean(q) && s2.lines.some((l) => l.from === first.id && l.to === q.id && l.label === "raises"), JSON.stringify(s2.cards.filter((c) => c.kind === "question").map((c) => c.title))); + await ui.screenshot({ path: path.join(OUT, "52-id.png") }); + await ui.click(`${card} .cv-moment-acts .chip:nth-child(1)`); + await sleep(2500); + const playing = await page(`(() => { const v = document.getElementById("v"); return { t: v.currentTime, paused: v.paused }; })()`); + check("Play goes to the page and plays the moment", playing && !playing.paused && playing.t >= 2 && playing.t <= 6.5, JSON.stringify(playing)); + check("looping its range", (await ui.evaluate(() => tabs.find((t) => t.id === activeId)?.loop?.label)) === "0:02–0:06"); + await sleep(4500); + const again = await page(`document.getElementById("v").currentTime`); + check("and goes back to the start at the end", again >= 2 && again <= 6.5, String(again)); + + // A page with no video or audio. + await ui.evaluate((u) => window.scout.createTab(u), `${BASE}/article.html`); + await sleep(1500); + await menuItem("Save Moment: Start / End"); + await sleep(600); + check("no video: nothing saved", (await moments()).length === 3); + + await app.close(); + console.log(failed ? `\n${failed} FAILED` : "\nall passed"); + process.exit(failed ? 1 : 0); +})().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/test/e2e/README.md b/test/e2e/README.md index 5b8948d..8a0df22 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -32,6 +32,7 @@ On Linux without a display, run each script under `xvfb-run -a`. | 49-trail-rename-delete | Rename and delete trails from the switcher (B115): the … button on each trail, inline rename (Enter saves, Esc cancels), double-click and F2, rename over a page in the sidebar's layer, a rename is an event, delete asks first and says what goes, its tabs close, the next trail becomes current, Undo, the Delete key, files purged at the next start, deleting the only trail starts a new one, an opened file can be deleted but not renamed | | 50-provenance | Trail provenance (B83): a typed address, a link clicked on a page, a suggestion with its keyword, a page the agent found, NODE's autoplay, a site moving to the next item by itself (in-page navigation), unknown ways ignored; the line on canvas cards with NODE's picks in violet; Replay captions | | 51-trail-recall | Search past trails (B89): the "In your trails" group in the CommandBar ("after the memex page" finds the next page in another trail), picking switches trail and selects the card with a ring; the canvas search list ("memex today") under its heading, after the web choices; a timeline-only page shows its row marked; words that say nothing find nothing | +| 52-media-moments | Media moments (B88): ⌘⇧K start then end saves a moment card with captions and pauses; ⌘⇧J saves the last 30 seconds with the caption cues; with a loop set one press saves the loop; the card (Moment, times, captions, Play, Ask for ID); the timeline says Moment; changing the times; Ask for ID makes a "raises" question; Play seeks, plays and loops the range; no video, nothing saved | | 21-rules-editor | Site rules editor: list with plain summaries, Rule for this site, saving and the rule taking effect (autoplay off after reload), invalid pattern refused with the file unchanged, hand-written fields kept, move and delete | | 20-first-run-and-help | Welcome screen on first start (page views hidden), choices saved with keys encrypted, not shown again; new tab page with most visited pages and its search box; a custom new tab page; Copy diagnostics without keys; Help menu | | 19-browser-basics | Error page for a failed load, page-stopped page after a crash, reopen closed tab, Tab 1 and Next Tab, right-click menu items for links, images, text and the page, two downloads (second renamed), Library → Downloads, Save as PDF, sleeping idle tabs and waking one, tabs restored after a restart (asleep except the active one) | diff --git a/test/e2e/site/moment.html b/test/e2e/site/moment.html new file mode 100644 index 0000000..bde8f04 --- /dev/null +++ b/test/e2e/site/moment.html @@ -0,0 +1,12 @@ +A long talk about levain +

A long talk about levain

+ +

A video with captions, for media moments (B88).

+ + diff --git a/test/e2e/site/moment.vtt b/test/e2e/site/moment.vtt new file mode 100644 index 0000000..cb78956 --- /dev/null +++ b/test/e2e/site/moment.vtt @@ -0,0 +1,61 @@ +WEBVTT + +00:00:00.000 --> 00:00:02.000 +Line 1: the starter + +00:00:02.000 --> 00:00:04.000 +Line 2: rises + +00:00:04.000 --> 00:00:06.000 +Line 3: in rye + +00:00:06.000 --> 00:00:08.000 +Line 4: at room + +00:00:08.000 --> 00:00:10.000 +Line 5: temperature + +00:00:10.000 --> 00:00:12.000 +Line 6: after two + +00:00:12.000 --> 00:00:14.000 +Line 7: feeds + +00:00:14.000 --> 00:00:16.000 +Line 8: then + +00:00:16.000 --> 00:00:18.000 +Line 9: it doubles + +00:00:18.000 --> 00:00:20.000 +Line 10: in four hours + +00:00:20.000 --> 00:00:22.000 +Line 11: and smells + +00:00:22.000 --> 00:00:24.000 +Line 12: sour + +00:00:24.000 --> 00:00:26.000 +Line 13: like yogurt + +00:00:26.000 --> 00:00:28.000 +Line 14: not vinegar + +00:00:28.000 --> 00:00:30.000 +Line 15: which is + +00:00:30.000 --> 00:00:32.000 +Line 16: a good sign + +00:00:32.000 --> 00:00:34.000 +Line 17: for baking + +00:00:34.000 --> 00:00:36.000 +Line 18: the next + +00:00:36.000 --> 00:00:38.000 +Line 19: morning + +00:00:38.000 --> 00:00:40.000 +Line 20: early diff --git a/test/moment-demo.js b/test/moment-demo.js new file mode 100644 index 0000000..da8575b --- /dev/null +++ b/test/moment-demo.js @@ -0,0 +1,86 @@ +// Media moments (B88): moment cards in the trail store, the trail file, +// Replay and search. Run: npm run test:moment +const assert = require("assert"); +const { TrailStore } = require("../core/trail"); +const { memoryStorage } = require("../core/storage"); +const { replaySteps, stepCaption } = require("../core/replay"); +const { recallIndex, recall } = require("../core/recall"); +const TF = require("../core/trail-file"); +const { momentScript, loopScript } = require("../core/page-scripts"); + +let n = 0; +const ok = (name) => console.log(`ok ${++n} - ${name}`); +let clock = new Date(2026, 10, 7, 20, 0).getTime(); +const now = () => clock; +const store = new TrailStore(memoryStorage(), { now }); +const id = store.current().trail.id; +const V = "https://www.youtube.com/watch?v=mix1"; + +store.visit(id, V, { title: "Boiler Room mix" }); +clock += 20000; +const page = store.placePage(id, V, { title: "Boiler Room mix", x: 0, y: 0 }); +clock += 60000; +const m = store.momentCard(id, { url: V, title: "Boiler Room mix", a: 2472.4, b: 2550, transcript: " this one is unreleased\n" }); +{ + assert.strictEqual(m.kind, "moment"); + assert.deepStrictEqual([m.a, m.b, m.w, m.h], [2472.4, 2550, 280, 190]); + assert.strictEqual(m.transcript, "this one is unreleased"); + const snap = store.snapshot(id); + const line = snap.lines.find((l) => l.from === page.id && l.to === m.id); + assert.ok(line && line.kind === "auto" && line.label === "moment from", "a grey line from the page"); + assert.ok(Math.abs(m.x - page.x) < 700 && Math.abs(m.y - page.y) < 700, "next to the page's card"); + assert.ok(store.events(id).some((e) => e.type === "captured" && e.card === m.id && e.data.moment), "saving is an event"); + assert.strictEqual(store.momentCard(id, { url: V, a: 2472.6, b: 2550.3 }).id, m.id, "the same moment twice is one card"); + assert.strictEqual(store.momentCard(id, { url: V, a: 10, b: 5 }), null, "the end must be after the start"); + assert.strictEqual(store.momentCard(id, { url: "https://accounts.google.com/signin", a: 1, b: 5 }), null, "never sign-in pages (B66)"); + ok("a moment is a card next to its page, with a grey line and an event"); +} +{ + clock += 1000; + assert.strictEqual(store.momentTimes(id, m.id, 2470, 2560).b, 2560); + assert.ok(store.events(id).some((e) => e.type === "edited" && e.card === m.id && e.data.a === 2470)); + assert.strictEqual(store.momentTimes(id, m.id, 30, 20), null); + assert.strictEqual(store.momentTimes(id, page.id, 1, 2), null, "only moments have times"); + ok("its start and end can be changed, and that is an event"); +} +{ + const snap = store.snapshot(id); + const md = TF.toTrailMd(snap, { by: "Sven", at: clock }); + assert.ok(md.includes(`### [Boiler Room mix](${V}) · moment 41:10–42:40`), md); + assert.ok(md.includes("> this one is unreleased")); + const back = TF.parseTrailMd(md); + const c = back.cards.find((k) => k.kind === "moment"); + assert.deepStrictEqual([c.a, c.b, c.url, c.transcript], [2470, 2560, V, "this one is unreleased"]); + assert.strictEqual(TF.toTrailMd(back, { by: "Sven", at: back.trail.at }), md, "the file reads back the same"); + const reader = new TrailStore(memoryStorage(), { now }); + const opened = reader.importFile(back); + assert.ok(reader.snapshot(opened.trail.id).cards.some((k) => k.kind === "moment" && k.a === 2470)); + ok("a trail file carries the moment: times in the heading, captions as a quote"); +} +{ + const snap = store.snapshot(id); + const byId = new Map(snap.cards.map((c) => [c.id, c])); + const caps = replaySteps(snap, store.events(id)).map((st) => stepCaption(st, { byId, lines: snap.lines, seen: new Set() })); + assert.ok(caps.includes("You saved a moment of “Boiler Room mix”, 41:10 to 42:40."), caps.join("\n")); + ok("Replay says when a moment was saved"); +} +{ + const r = recall(recallIndex(store), "unreleased moment", { now: clock, trails: store.list() }); + assert.strictEqual(r[0]?.card, m.id, JSON.stringify(r)); + ok("search past trails finds a moment by its captions"); +} +{ + const ro = new TrailStore(memoryStorage(), { now }); + const f = ro.importFile(TF.parseTrailMd(TF.toTrailMd(store.snapshot(id), { by: "Sven", at: clock }))).trail.id; + ro.visit(f, V, { title: "x" }); + assert.strictEqual(ro.momentCard(f, { url: V, a: 1, b: 9 }), null); + const c = ro.snapshot(f).cards.find((k) => k.kind === "moment"); + assert.strictEqual(ro.momentTimes(f, c.id, 1, 9), null); + ok("an opened file (read only) gets no new moments or times"); +} +{ + for (const op of ["start", "end", "last", "loop", "state", "cancel"]) new Function(momentScript(op, 30)); + new Function(loopScript("range", 60, { a: 1, b: 4 })); + ok("the page scripts parse"); +} +console.log(`\n${n} checks passed`);