Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ jobs:
(cd test/e2e/site && python3 -m http.server 8765 >/dev/null 2>&1 &)
sleep 1
mkdir -p e2e-out
for t in 01-basics 03-rules-and-fast-results 04-find-zoom-history 05-research-session 09-playback-limit 12-pip-bookmarks-adblock 13-ask-ai 15-agents-mcp 16-research-agent 17-talk 18-redteam 19-browser-basics 20-first-run-and-help 21-rules-editor 22-ui-refresh 23-beta-survey 24-mosaic-pick 25-video-loop 26-gestures-tour 27-privacy-visible 28-ask-across-tabs 29-blocker-onpage 30-tab-focus-sleep 31-passwords 32-passwords-more 33-sidebar-chrome 34-command-bar 35-frames-tabs 36-trail-tabs 37-threads 38-trail-share 39-trail-open 40-share-link 41-survey-newtab 42-trail-canvas 43-canvas-tools 44-share-open 45-mosaic-gallery 48-canvas-media 49-trail-rename-delete 50-provenance 51-trail-recall; do
for t in 01-basics 03-rules-and-fast-results 04-find-zoom-history 05-research-session 09-playback-limit 12-pip-bookmarks-adblock 13-ask-ai 15-agents-mcp 16-research-agent 17-talk 18-redteam 19-browser-basics 20-first-run-and-help 21-rules-editor 22-ui-refresh 23-beta-survey 24-mosaic-pick 25-video-loop 26-gestures-tour 27-privacy-visible 28-ask-across-tabs 29-blocker-onpage 30-tab-focus-sleep 31-passwords 32-passwords-more 33-sidebar-chrome 34-command-bar 35-frames-tabs 36-trail-tabs 37-threads 38-trail-share 39-trail-open 40-share-link 41-survey-newtab 42-trail-canvas 43-canvas-tools 44-share-open 45-mosaic-gallery 48-canvas-media 49-trail-rename-delete 50-provenance 51-trail-recall 52-media-moments; do
echo "::group::$t"
xvfb-run -a node test/e2e/$t.js "$PWD/e2e-out" 2>&1 | tee e2e-out/$t.log
status=${PIPESTATUS[0]}
Expand Down Expand Up @@ -71,7 +71,7 @@ jobs:
(cd test/e2e/site && python -m http.server 8765 --bind 127.0.0.1 >/dev/null 2>&1 &)
sleep 2
mkdir -p e2e-out
for t in 01-basics 05-research-session 19-browser-basics 20-first-run-and-help 22-ui-refresh 23-beta-survey 24-mosaic-pick 25-video-loop 26-gestures-tour 27-privacy-visible 28-ask-across-tabs 29-blocker-onpage 30-tab-focus-sleep 31-passwords 32-passwords-more 33-sidebar-chrome 34-command-bar 35-frames-tabs 36-trail-tabs 37-threads 38-trail-share 39-trail-open 40-share-link 41-survey-newtab 42-trail-canvas 43-canvas-tools 44-share-open 45-mosaic-gallery 48-canvas-media 49-trail-rename-delete 50-provenance 51-trail-recall; do
for t in 01-basics 05-research-session 19-browser-basics 20-first-run-and-help 22-ui-refresh 23-beta-survey 24-mosaic-pick 25-video-loop 26-gestures-tour 27-privacy-visible 28-ask-across-tabs 29-blocker-onpage 30-tab-focus-sleep 31-passwords 32-passwords-more 33-sidebar-chrome 34-command-bar 35-frames-tabs 36-trail-tabs 37-threads 38-trail-share 39-trail-open 40-share-link 41-survey-newtab 42-trail-canvas 43-canvas-tools 44-share-open 45-mosaic-gallery 48-canvas-media 49-trail-rename-delete 50-provenance 51-trail-recall 52-media-moments; do
echo "::group::$t"
node test/e2e/$t.js "$PWD/e2e-out" 2>&1 | tee e2e-out/$t.log
status=${PIPESTATUS[0]}
Expand Down
85 changes: 81 additions & 4 deletions core/page-scripts.js
Original file line number Diff line number Diff line change
Expand Up @@ -392,11 +392,12 @@ function pipScript() {
* another address (YouTube reuses one video element for every video).
* Returns { a, b, duration } or "none".
*/
function loopScript(op, secs = 60) {
function loopScript(op, secs = 60, range = null) {
return `(() => {
const op = ${JSON.stringify(String(op))};
const range = ${JSON.stringify(range && range.b > range.a ? { a: Number(range.a), b: Number(range.b) } : null)};
const secs = ${Math.max(5, Math.min(3600, Number(secs) || 60))};
const vids = [...document.querySelectorAll("video")].filter((v) => v.readyState > 0);
const vids = [...document.querySelectorAll(op === "range" ? "video, audio" : "video")].filter((v) => v.readyState > 0);
if (!vids.length) return "none";
const area = (v) => { const r = v.getBoundingClientRect(); return r.width * r.height; };
const v = vids.find((x) => x.__nodeLoop) || vids.find((x) => !x.paused) || vids.sort((a, b) => area(b) - area(a))[0];
Expand All @@ -410,6 +411,7 @@ function loopScript(op, secs = 60) {
if (op === "next") { a = now; b = Math.min(now + secs, end); }
else if (op === "start") { a = now; b = L && L.b > now + 1 ? L.b : Math.min(now + secs, end); }
else if (op === "end") { b = now; a = L && L.a < now - 1 ? L.a : Math.max(0, now - secs); }
else if (op === "range" && range) { a = range.a; b = Math.min(range.b, end); } // a saved moment (B88)
else return "none";
if (!(b > a + 0.5)) return "none";
v.__nodeLoop = { a, b, href: here() };
Expand All @@ -436,7 +438,8 @@ function loopScript(op, secs = 60) {
v.addEventListener("ended", v.__nodeLoopFn);
}
v.__nodePrev = v.currentTime;
if (op === "next" || op === "start") { if (v.paused) v.play().catch(() => {}); }
if (op === "range") { v.currentTime = a; v.__nodePrev = a; v.play().catch(() => {}); }
else if (op === "next" || op === "start") { if (v.paused) v.play().catch(() => {}); }
else v.currentTime = a;
return { a, b, duration: v.duration };
})()`;
Expand Down Expand Up @@ -469,4 +472,78 @@ function anyPlayingScript() {
return `[...document.querySelectorAll("video, audio")].some((el) => !el.paused && !el.ended)`;
}

module.exports = { extractionScript, autoplayScript, blockAutoplayScript, centerOnClickScript, pipScript, loopScript, readableTextScript, pauseForLimitScript, resumeScript, anyPlayingScript };
// Media moments (B88). op:
// start mark the start (playback goes on) and collect the captions shown
// end the start mark to now, with the captions collected; pauses
// last the last secs seconds up to now, with the caption cues in that range; pauses
// loop the B70 loop's range, with its caption cues; pauses
// cancel forget the start mark
// state { marked, t } without changing anything
// Playing a moment again uses loopScript("range", 0, { a, b }).
// Returns "none" when the page has no started video or audio.
function momentScript(op, arg) {
return `(() => {
const op = ${JSON.stringify(String(op))};
const arg = ${JSON.stringify(arg ?? null)};
const media = [...document.querySelectorAll("video, audio")].filter((m) => m.readyState > 0 || (op === "play" && m.currentSrc));
if (!media.length) return "none";
const area = (v) => { const r = v.getBoundingClientRect(); return r.width * r.height; };
const m = media.find((x) => x.__nodeMoment) || media.find((x) => !x.paused) || media.sort((a, b) => area(b) - area(a))[0];
// Captions on screen now: text tracks, then players that draw their own.
const shown = () => {
const out = [];
for (const tr of m.textTracks || []) if (tr.mode !== "disabled" && tr.activeCues) for (const c of tr.activeCues) out.push(c.text);
if (!out.length) for (const el of document.querySelectorAll(".ytp-caption-segment, .vp-captions, .captions-text, [data-testid='caption-text']")) out.push(el.textContent);
return out.map((s) => s.replace(/<[^>]+>/g, "").replace(/\\s+/g, " ").trim()).filter(Boolean).join(" ");
};
// Caption cues from a to b, for a range already played.
const cues = (a, b) => {
const out = [];
for (const tr of m.textTracks || []) {
if (!/^(subtitles|captions)$/.test(tr.kind) || !tr.cues) continue;
for (const c of tr.cues) if (c.endTime > a && c.startTime < b) out.push(c.text.replace(/<[^>]+>/g, "").replace(/\\s+/g, " ").trim());
if (out.length) break;
}
return out;
};
const join = (lines) => { const o = []; for (const l of lines) if (l && o[o.length - 1] !== l) o.push(l); return o.join(" ").slice(0, 1200); };
const stop = () => { if (m.__nodeMoment) clearInterval(m.__nodeMoment.timer); m.__nodeMoment = null; };
const info = (a, b, lines) => ({ a, b, duration: isFinite(m.duration) ? m.duration : null, transcript: join(lines), kind: m.tagName === "AUDIO" ? "audio" : "video" });
if (op === "state") return { marked: Boolean(m.__nodeMoment), t: m.currentTime, a: m.__nodeMoment ? m.__nodeMoment.a : null };
if (op === "cancel") { stop(); return { cancelled: true }; }
if (op === "start") {
stop();
const mk = { a: m.currentTime, lines: [], href: location.href.split("#")[0] };
const take = () => { const t = shown(); if (t && mk.lines[mk.lines.length - 1] !== t) mk.lines.push(t); };
take();
mk.timer = setInterval(take, 400);
m.__nodeMoment = mk;
if (m.paused) m.play().catch(() => {});
return { started: true, a: mk.a };
}
if (op === "end") {
const mk = m.__nodeMoment;
if (!mk || mk.href !== location.href.split("#")[0]) { stop(); return "none"; }
const b = m.currentTime;
const lines = mk.lines.length ? mk.lines : cues(mk.a, b);
stop();
m.pause();
return b > mk.a + 0.5 ? info(mk.a, b, lines) : { tooShort: true };
}
if (op === "last") {
const b = m.currentTime, a = Math.max(0, b - Math.max(5, Math.min(600, Number(arg) || 30)));
stop();
m.pause();
return b > a + 0.5 ? info(a, b, cues(a, b)) : { tooShort: true };
}
if (op === "loop") {
const L = m.__nodeLoop;
if (!L || L.b == null) return "none";
m.pause();
return info(L.a, L.b, cues(L.a, L.b));
}
return "none";
})()`;
}

module.exports = { momentScript, extractionScript, autoplayScript, blockAutoplayScript, centerOnClickScript, pipScript, loopScript, readableTextScript, pauseForLimitScript, resumeScript, anyPlayingScript };
5 changes: 3 additions & 2 deletions core/recall.js
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ const KINDS = {
song: "audio", track: "audio", audio: "audio", podcast: "audio", listened: "audio",
question: "question", questions: "question", thread: "question", threads: "question",
note: "note", notes: "note",
moment: "moment", moments: "moment",
search: "search", searches: "search", searched: "search",
image: "image", images: "image", photo: "image", picture: "image", screenshot: "image",
};
Expand Down Expand Up @@ -205,15 +206,15 @@ function recallIndex(store) {
card: c ? c.id : null,
url: it.url || c?.url || "",
title: it.title || c?.title || it.url || "",
text: c ? [c.excerpt, ...(c.keywords || [])].filter(Boolean).join(" ") : "",
text: c ? [c.excerpt, c.transcript, c.text, ...(c.keywords || [])].filter(Boolean).join(" ") : "",
at: it.at,
timelineOnly: !c,
});
}
}
for (const c of cards.values()) {
if (seenCards.has(c.id) || !c.placedAt) continue;
const text = [c.text, c.quote, c.query, c.excerpt, c.conclusion, ...(c.keywords || []), ...(c.replies || []).map((r) => r.text)].filter(Boolean).join(" ");
const text = [c.text, c.quote, c.query, c.excerpt, c.conclusion, c.transcript, ...(c.keywords || []), ...(c.replies || []).map((r) => r.text)].filter(Boolean).join(" ");
rows.push({ kind: c.kind, media: c.media, trail: t.id, card: c.id, url: c.url || "", title: c.title || c.text || c.query || "", text, at: c.placedAt, timelineOnly: false });
}
for (const g of snap.groups || []) {
Expand Down
6 changes: 6 additions & 0 deletions core/replay.js
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@
// ids reached so far (this step included).

const Prov = require("./provenance");
const clockOf = (t) => {
t = Math.max(0, Math.round(Number(t) || 0));
const h = Math.floor(t / 3600), m = Math.floor((t % 3600) / 60), s = String(t % 60).padStart(2, "0");
return h ? `${h}:${String(m).padStart(2, "0")}:${s}` : `${m}:${s}`;
};
const SESSION_GAP_MS = 30 * 60 * 1000;
const CARD_EVENTS = new Set(["opened", "placed", "captured", "asked", "reported", "noted"]);

Expand Down Expand Up @@ -202,6 +207,7 @@ function stepCaption(st, ctx) {
else if (c.kind === "note") t = `${w} wrote a note: ${q(title)}`;
else if (c.kind === "voice") t = `${w} recorded a voice note.`;
else if (c.kind === "search") t = `${w} searched: ${q(title)}.`;
else if (c.kind === "moment") t = `${w} saved a moment of ${q(title)}, ${clockOf(c.a)} to ${clockOf(c.b)}.`;
else if (c.kind === "media") {
const from = site(c.url) ? ` from ${site(c.url)}` : "";
t = {
Expand Down
25 changes: 23 additions & 2 deletions core/trail-file.js
Original file line number Diff line number Diff line change
Expand Up @@ -515,7 +515,14 @@ function merge(base, incoming, answers = {}) {
// payment pages, screenshots (only the capture's text and source go in).

const FORMAT_V2 = 2;
const SIZE_V2 = { page: [260, 200], video: [260, 220], capture: [260, 210], report: [260, 240], question: [260, 170], note: [260, 150], voice: [260, 170], search: [260, 140], media: [300, 240] };
const SIZE_V2 = { page: [260, 200], video: [260, 220], capture: [260, 210], report: [260, 240], question: [260, 170], note: [260, 150], voice: [260, 170], search: [260, 140], media: [300, 240], moment: [280, 190] };
// Media moments (B88): "41:12" and back, in seconds.
const clockOf = (t) => {
t = Math.max(0, Math.round(Number(t) || 0));
const h = Math.floor(t / 3600), m = Math.floor((t % 3600) / 60), s = String(t % 60).padStart(2, "0");
return h ? `${h}:${String(m).padStart(2, "0")}:${s}` : `${m}:${s}`;
};
const secsOf = (s) => String(s).split(":").reduce((n, p) => n * 60 + Number(p), 0);
// Media cards (B114). The file never carries the media itself: it links to
// where the media came from on the web (src), so the reader's NODE can show
// a large preview from there. Screenshots, recordings and files added from
Expand Down Expand Up @@ -552,6 +559,8 @@ function cardHeading(c) {
return `### [${mdText(c.title || c.url)}](${c.url}) · video`;
case "search":
return `### [${mdText(c.title || `Search: ${c.query || ""}`)}](${c.url}) · search`;
case "moment":
return `### [${mdText(c.title || c.url)}](${c.url}) · moment ${clockOf(c.a)}–${clockOf(c.b)}`;
case "capture":
return `### > ${mdText(c.quote || c.title || c.source || "Capture")}`;
case "report":
Expand Down Expand Up @@ -627,6 +636,10 @@ function cardBody(c) {
if (c.url) out.push(`From [${mdText(hostOf(c.url) || c.url)}](${c.url})`);
if (!c.src) out.push("The file stayed on the sender's computer.");
break;
case "moment":
// The captions shown during the moment, when the site had them.
if (c.transcript) out.push(`> ${oneLine(c.transcript)}`);
break;
}
return out;
}
Expand All @@ -645,7 +658,7 @@ const inGroup = (c, g) => {
const x = c.x + c.w / 2, y = c.y + c.h / 2;
return x >= g.x && x <= g.x + g.w && y >= g.y && y <= g.y + g.h;
};
const sendable = (c) => !c.removed && !(c.url && isSensitiveUrl(c.url)) && !(["page", "video", "search"].includes(c.kind) && !cleanUrl(c.url));
const sendable = (c) => !c.removed && !(c.url && isSensitiveUrl(c.url)) && !(["page", "video", "search", "moment"].includes(c.kind) && !cleanUrl(c.url));
// Addresses leave without private tokens or tracking codes (cleanUrl). A
// media card's file and Library id stay behind.
const forFile = (c) => {
Expand Down Expand Up @@ -856,6 +869,7 @@ function guessKind(head) {
if (/^Voice note/i.test(head)) return "voice";
if (/^\[.*\]\(.*\)\s*·\s*video$/.test(head)) return "video";
if (/^\[.*\]\(.*\)\s*·\s*search$/.test(head)) return "search";
if (/^\[.*\]\(.*\)\s*·\s*moment \d/.test(head)) return "moment";
if (/^\[.*\]\(.*\)/.test(head)) return "page";
return "note";
}
Expand Down Expand Up @@ -891,6 +905,13 @@ function readCard(raw) {
case "search":
c.query = (c.title || "").replace(/^Search:\s*/, "");
break;
case "moment": {
const m = raw.head.match(/·\s*moment ([\d:]+)–([\d:]+)\s*$/);
if (m) Object.assign(c, { a: secsOf(m[1]), b: secsOf(m[2]) });
const q = b.find((l) => /^> /.test(l));
if (q) c.transcript = q.slice(2);
break;
}
case "capture":
c.quote = raw.head.replace(/^>\s*/, "");
c.fromText = b[0] || "";
Expand Down
46 changes: 44 additions & 2 deletions core/trail.js
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ const GROUP_SIDE = 24; // group padding around its cards: sides and bottom
const GROUP_TOP = 48; // and top, where the name sits
const SESSION_GAP_MS = 30 * 60 * 1000; // a gap of more than 30 minutes starts a new session
const ENGAGED_MS = 20 * 1000; // the 20-second rule (doc 23); a starting guess
const EVENT_TYPES = new Set(["opened", "bounced", "captured", "asked", "reported", "noted", "placed", "removed", "linked", "unlinked", "grouped", "renamed", "resolved", "moved"]);
const CARD_KINDS = new Set(["page", "video", "capture", "report", "question", "note", "voice", "search", "media"]);
const EVENT_TYPES = new Set(["opened", "bounced", "captured", "asked", "reported", "noted", "placed", "removed", "linked", "unlinked", "grouped", "renamed", "resolved", "moved", "edited"]);
const CARD_KINDS = new Set(["page", "video", "capture", "report", "question", "note", "voice", "search", "media", "moment"]);
const LINE_KINDS = new Set(["nav", "auto", "you", "suggested"]);
const LINE_LABELS = new Set(["", "supports", "contradicts", "answers", "raises"]);
// Card sizes at 100% zoom, by kind (handoff: cards 240-260 px wide).
Expand All @@ -39,6 +39,7 @@ const SIZE = {
voice: [260, 170],
search: [260, 140],
media: [300, 240],
moment: [280, 190],
};
// Media cards (B114): image, screenshot, video and audio from the Library.
const MEDIA_KINDS = new Set(["image", "screenshot", "video", "audio"]);
Expand Down Expand Up @@ -542,6 +543,46 @@ class TrailStore {
if (card && source && !source.removed) this.link(id, source.id, card.id, { kind: "auto", label: "captured from", at });
return card;
}
// ---- media moments (B88) ----
// A section of a video or audio page, a to b in seconds, with the caption
// text shown during it when the site had captions. It goes next to its
// page's card, with a grey line from the page.
momentCard(id, { url, title = "", a, b, transcript = "", near, at = this.now() } = {}) {
const d = this.get(id);
a = Math.max(0, Math.round(Number(a) * 10) / 10);
b = Math.round(Number(b) * 10) / 10;
if (!d || d.trail.readonly || !url || isSensitiveUrl(url) || !(b > a)) return null;
const have = d.cards.find((c) => c.kind === "moment" && !c.removed && c.url === url && Math.abs(c.a - a) < 1 && Math.abs(c.b - b) < 1);
if (have) return have;
const [w, h] = SIZE.moment;
const source = this.pageCard(id, url);
const pos = placeNear({ cards: d.cards.filter((c) => !c.removed), groups: d.groups }, source && !source.removed ? source : null, w, h, { near: near || this.below(id, w) });
for (const m of pos.moves) {
const o = (m.kind === "group" ? d.groups : d.cards).find((k) => k.id === m.id);
if (!o) continue;
if (typeof m.y === "number") o.y = m.y;
if (typeof m.h === "number") o.h = m.h;
}
const fields = { kind: "moment", url, title: String(title || source?.title || url).slice(0, 300), a, b, placedAt: at, w, h, x: pos.x, y: pos.y };
const text = String(transcript || "").replace(/\s+/g, " ").trim().slice(0, 1200);
if (text) fields.transcript = text;
const card = this.addCard(id, fields, { at, event: "captured", data: { url, a, b, moment: true } });
if (card && source && !source.removed) this.link(id, source.id, card.id, { kind: "auto", label: "moment from", at });
return card;
}
// New start and end for a moment, typed on its card.
momentTimes(id, cardId, a, b, { at = this.now() } = {}) {
const d = this.get(id);
const c = this.card(id, cardId);
a = Math.max(0, Math.round(Number(a) * 10) / 10);
b = Math.round(Number(b) * 10) / 10;
if (!d || d.trail.readonly || c?.kind !== "moment" || !(b > a)) return null;
c.a = a;
c.b = b;
this.event(id, { type: "edited", card: c.id, at, data: { a, b } });
this.save(id);
return c;
}
// ---- search cards (B106) ----
// A web search made from the canvas or the address bar: a card titled
// "Search: <query>" that opens the results. Pages opened from the results go
Expand Down Expand Up @@ -791,6 +832,7 @@ class TrailStore {
const item = { event: e.id, type: e.type, at: e.at, card: card && !card.removed ? card.id : null };
if (url) item.url = url;
item.title = (card && (card.title || card.text || card.quote)) || e.data?.title || url || e.type;
if (card?.kind === "moment") item.moment = { a: card.a, b: card.b }; // B88: shown as "Moment 41:12–42:30"
if (e.type === "opened") {
// An opened page shows once per session, at its latest time.
// Items from an opened file have a title and no address.
Expand Down
Loading
Loading