From f0ea2136dd275e5d8c81a9d41ffdf6c38201c47b Mon Sep 17 00:00:00 2001 From: dzucconi Date: Thu, 22 Oct 2026 13:19:43 -0400 Subject: [PATCH 1/2] refactor(cli): share transfer progress and timeout helpers --- src/commands/import.tsx | 9 +-------- src/lib/format.ts | 11 +++++++++++ src/lib/network.ts | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/commands/import.tsx b/src/commands/import.tsx index ceb22dd..51eae40 100644 --- a/src/commands/import.tsx +++ b/src/commands/import.tsx @@ -1,7 +1,7 @@ import React, { useEffect, useMemo, useState } from "react"; import { Box, Text, useApp, useInput } from "ink"; import { flag, intFlag, requireArg, type Flags } from "../lib/args"; -import { plural, truncate } from "../lib/format"; +import { progressBar, truncate } from "../lib/format"; import { KeyHints, Panel, ScreenFrame } from "../components/ScreenChrome"; import { discoverImportFiles, @@ -45,13 +45,6 @@ export function parseImportOptions( }; } -function progressBar(completed: number, total: number, width = 28): string { - if (total <= 0) return `[${"░".repeat(width)}]`; - const ratio = Math.max(0, Math.min(1, completed / total)); - const filled = Math.round(ratio * width); - return `[${"█".repeat(filled)}${"░".repeat(width - filled)}]`; -} - interface UploadState { completed: number; total: number; diff --git a/src/lib/format.ts b/src/lib/format.ts index 60c16ba..1cfd8c0 100644 --- a/src/lib/format.ts +++ b/src/lib/format.ts @@ -28,6 +28,17 @@ export function truncate(str: string, max: number): string { return clean.slice(0, max - 1) + "…"; } +export function progressBar( + completed: number, + total: number, + width = 28, +): string { + if (total <= 0) return `[${"░".repeat(width)}]`; + const ratio = Math.max(0, Math.min(1, completed / total)); + const filled = Math.round(ratio * width); + return `[${"█".repeat(filled)}${"░".repeat(width - filled)}]`; +} + export function plural(n: number, word: string): string { return `${n.toLocaleString()} ${word}${n === 1 ? "" : "s"}`; } diff --git a/src/lib/network.ts b/src/lib/network.ts index b6afe11..0908914 100644 --- a/src/lib/network.ts +++ b/src/lib/network.ts @@ -4,7 +4,7 @@ const cancellationController = new AbortController(); let cancellationInitialized = false; let forcedExitTimer: NodeJS.Timeout | undefined; -function configuredTimeoutMs(): number { +export function configuredTimeoutMs(): number { const raw = process.env["ARENA_REQUEST_TIMEOUT_MS"]; if (!raw) return DEFAULT_REQUEST_TIMEOUT_MS; const parsed = Number(raw); From e8787b90be3c1acc20c2effb71f57c0dbd66b94c Mon Sep 17 00:00:00 2001 From: dzucconi Date: Thu, 22 Oct 2026 13:19:43 -0400 Subject: [PATCH 2/2] feat(cli): add channel export and one-way sync --- README.md | 51 +- package-lock.json | 32 +- package.json | 4 +- src/cli.contract.test.ts | 17 + src/cli.tsx | 3 +- src/commands/download.tsx | 201 ++++++ src/lib/args.ts | 2 + src/lib/download-format.ts | 237 ++++++ src/lib/download.test.ts | 1389 ++++++++++++++++++++++++++++++++++++ src/lib/download.ts | 1327 ++++++++++++++++++++++++++++++++++ src/lib/registry.tsx | 85 +++ 11 files changed, 3338 insertions(+), 10 deletions(-) create mode 100644 src/commands/download.tsx create mode 100644 src/lib/download-format.ts create mode 100644 src/lib/download.test.ts create mode 100644 src/lib/download.ts diff --git a/README.md b/README.md index 5ac1174..7e49369 100644 --- a/README.md +++ b/README.md @@ -60,18 +60,18 @@ arena channel contents --help ## Global Flags -| Flag | Description | -| --------- | ----------------------------------------------- | -| `--json` | Output as JSON (`import --json` streams NDJSON) | -| `--quiet` | Compact JSON output when supported | -| `--yes` | Bypass destructive confirmation prompts | -| `--help` | Show help | +| Flag | Description | +| --------- | --------------------------------------------------------- | +| `--json` | Output as JSON (`import`/`download --json` stream NDJSON) | +| `--quiet` | Compact JSON output when supported | +| `--yes` | Bypass destructive confirmation prompts | +| `--help` | Show help | ## Output & Errors - `stdout`: - Successful command output. - - `--json` returns JSON objects; `import --json` returns NDJSON progress events. + - `--json` returns JSON objects; `import --json` and `download --json` return NDJSON progress events. - `--quiet` only changes JSON formatting (compact, single-line); it does **not** change response fields. - `stderr`: - Human-readable failures in non-interactive mode. @@ -440,6 +440,43 @@ Options: - `--upload-concurrency ` - `--poll-interval ` +#### `download` (aliases: `export`, `sync`) + +Download a channel to a local directory. Run the same command again to sync it from Are.na. + +```bash +arena download worldmaking ./refs +arena sync worldmaking ./refs +arena sync worldmaking ./refs --prune +arena export worldmaking ./refs --zip ./worldmaking.zip +arena download worldmaking --type Image --size large --json +``` + +Each export contains: + +- Images and attachments, `.txt` files of Markdown for text, and `.webloc` shortcuts for links, embeds, and connected channels. Files are named like Are.na's web channel export: `_.<extension>` (for example `3_brutalist-housing.jpg`). When blocks are reordered, removed, or retitled, sync renames the existing files instead of downloading them again. +- `<channel-slug>.csv`, with the same columns as the web export: ID, filename, title, description, created and updated dates, and source URL. +- `index.html`, an offline index in channel order with images, text, descriptions, and links to each item's source and Are.na page. +- `manifest.json`, a snapshot of the channel and its items (including connection positions and metadata) plus a SHA-256 checksum for every file. Keep it with the directory: it is the sync state. + +Sync is **one way, from Are.na**. Unchanged files are skipped without a request, changed or missing files are downloaded again, and a file you edited locally is reported as a conflict instead of being overwritten. Files that leave the export, such as removed blocks, stay on disk until you pass `--prune`, which deletes only files the export created and you haven't edited. If another block now needs a kept file's name, the kept file is renamed to `<name>~block-<id>.<ext>`. Pruning is skipped if any download fails or the channel changes during the sync. + +Options: + +- `--dir <path>` — destination, instead of the second argument; defaults to the channel slug +- `--size <original|large|medium|small|square>` — image size; default `original` +- `--type <Image|Attachment|Text|Link|Embed|Channel|PendingBlock>` — only write files of one type (the manifest still lists every item); cannot be combined with `--prune` +- `--link-format <webloc|url>` — macOS (`webloc`, default) or Windows/Linux (`url`) shortcuts +- `--concurrency <n>` — concurrent transfers, 1–16; default `4` +- `--prune` — remove files from earlier syncs that are no longer part of the export +- `--overwrite` — download everything again and replace local edits, or replace a non-ZIP file at the `--zip` path +- `--zip <path>` — also write the export to a ZIP; an existing ZIP at that path is replaced +- `--json` — stream NDJSON events; the final `completed` event has `downloaded`, `unchanged`, `renamed`, `failed`, `pruned`, `retained`, and `cancelled` + +Exit code is `1` if anything failed and `130` if cancelled. Failed items are recorded in the manifest and left out of the ZIP. Ctrl+C keeps finished files, and the next run picks up where it left off; a lock, temporary files, or renames left unfinished by a killed process are cleaned up or completed automatically. A transfer is retried if no data arrives for `ARENA_REQUEST_TIMEOUT_MS` (default 30s). + +Private channels use your saved login. Connected channels are saved as shortcuts, not exported recursively; embeds are saved as source shortcuts, not videos; blocks still processing are recorded in the manifest only. Changes are detected from asset URLs and revision timestamps, so use `--overwrite` if a file changed upstream without either. + ### Connections #### `connect` diff --git a/package-lock.json b/package-lock.json index 9e895ab..8f09097 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,8 @@ "openapi-fetch": "^0.17.0", "react": "^19.0.0", "swr": "^2.4.1", - "terminal-image": "^4.2.0" + "terminal-image": "^4.2.0", + "yazl": "^3.3.1" }, "bin": { "arena": "dist/cli.js" @@ -24,6 +25,7 @@ "devDependencies": { "@types/node": "^25.3.3", "@types/react": "^19.0.0", + "@types/yazl": "^3.3.1", "husky": "^9.1.7", "lint-staged": "^16.3.2", "openapi-typescript": "^7.13.0", @@ -1900,6 +1902,16 @@ "csstype": "^3.2.2" } }, + "node_modules/@types/yazl": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@types/yazl/-/yazl-3.3.1.tgz", + "integrity": "sha512-DIWfCKpsTp6hE5BDBHV3+fIL/bLUF9Bv13iDrWnMlmhQpH67buNvI291ZauQ1xcccxK3FqQ9honnXpq4R8NMuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@xmldom/xmldom": { "version": "0.8.11", "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.11.tgz", @@ -2141,6 +2153,15 @@ "ieee754": "^1.2.1" } }, + "node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/buffer-equal": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-0.0.1.tgz", @@ -4826,6 +4847,15 @@ "node": ">=12" } }, + "node_modules/yazl": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/yazl/-/yazl-3.3.1.tgz", + "integrity": "sha512-BbETDVWG+VcMUle37k5Fqp//7SDOK2/1+T7X8TD96M3D9G8jK5VLUdQVdVjGi8im7FGkazX7kk5hkU8X4L5Bng==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "^1.0.0" + } + }, "node_modules/yoga-layout": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/yoga-layout/-/yoga-layout-3.2.1.tgz", diff --git a/package.json b/package.json index 3ecdbb9..487536f 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,8 @@ "openapi-fetch": "^0.17.0", "react": "^19.0.0", "swr": "^2.4.1", - "terminal-image": "^4.2.0" + "terminal-image": "^4.2.0", + "yazl": "^3.3.1" }, "overrides": { "simple-xml-to-json": "1.2.3" @@ -48,6 +49,7 @@ "devDependencies": { "@types/node": "^25.3.3", "@types/react": "^19.0.0", + "@types/yazl": "^3.3.1", "husky": "^9.1.7", "lint-staged": "^16.3.2", "openapi-typescript": "^7.13.0", diff --git a/src/cli.contract.test.ts b/src/cli.contract.test.ts index d95d773..bb65001 100644 --- a/src/cli.contract.test.ts +++ b/src/cli.contract.test.ts @@ -80,3 +80,20 @@ test("add command reads value from piped stdin", () => { `expected stdin to supply the value, got: ${output}`, ); }); + +test("download aliases share help and reject invalid options before requesting the API", () => { + for (const command of ["download", "export", "sync"]) { + const help = runCli([command, "--help"]); + assert.equal(help.status, 0); + assert.match(help.stdout, /--prune/); + const invalid = runCli([ + command, + "channel", + "--concurrency", + "17", + "--json", + ]); + assert.equal(invalid.status, 1); + assert.match(JSON.parse(invalid.stderr).error, /concurrency/); + } +}); diff --git a/src/cli.tsx b/src/cli.tsx index 3a28464..184e13e 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -53,6 +53,7 @@ function TopLevelHelp() { <Text> add Add text/URLs to a channel</Text> <Text> upload Upload local files</Text> <Text> import Bulk import from a directory</Text> + <Text> download Download or sync a channel to disk</Text> </Box> <Box flexDirection="column" marginBottom={1}> @@ -66,7 +67,7 @@ function TopLevelHelp() { <Box flexDirection="column" marginBottom={1}> <Text dimColor>Global Flags</Text> - <Text> --json Output JSON (import streams NDJSON events)</Text> + <Text> --json Output JSON (import/download stream NDJSON events)</Text> <Text> --quiet Compact JSON output when supported</Text> <Text> --yes Bypass destructive confirmation prompts</Text> <Text> --version Show CLI version</Text> diff --git a/src/commands/download.tsx b/src/commands/download.tsx new file mode 100644 index 0000000..2a6ab02 --- /dev/null +++ b/src/commands/download.tsx @@ -0,0 +1,201 @@ +import React, { useEffect, useState } from "react"; +import { join } from "node:path"; +import { Box, Text, useApp } from "ink"; +import { flag, intFlag, requireArg, type Flags } from "../lib/args"; +import { ScreenFrame } from "../components/ScreenChrome"; +import { Spinner } from "../components/Spinner"; +import { cancellationSignal } from "../lib/network"; +import { exitCodeFromError } from "../lib/exit-codes"; +import { plural, progressBar, truncate } from "../lib/format"; +import { DOWNLOAD_TYPES, IMAGE_SIZES } from "../lib/download-format"; +import { + executeDownload, + downloadExitCode, + validateDownloadOptions, + type DownloadOptions, + type DownloadSummary, + type DownloadEvent, +} from "../lib/download"; + +function valueFlag(flags: Flags, name: string): string | undefined { + if ( + flags[name] !== undefined && + (typeof flags[name] !== "string" || !String(flags[name]).trim()) + ) + throw new Error(`--${name} requires a value`); + return flag(flags, name); +} +function booleanFlag(flags: Flags, name: string): boolean { + const value = flags[name]; + if (value === undefined || value === false || value === "false") return false; + if (value === true || value === "true") return true; + throw new Error(`--${name} expects true or false`); +} +function choiceFlag<T extends string>( + flags: Flags, + name: string, + choices: readonly T[], +): T | undefined { + const raw = valueFlag(flags, name); + if (raw === undefined) return undefined; + const choice = choices.find( + (value) => value.toLowerCase() === raw.toLowerCase(), + ); + if (!choice) + throw new Error(`--${name} must be one of: ${choices.join(", ")}`); + return choice; +} +export function parseDownloadOptions( + args: string[], + flags: Flags, +): DownloadOptions { + if (args.length > 2) + throw new Error("Usage: arena download <channel> [directory] [flags]"); + const directory = valueFlag(flags, "dir"); + if (directory && args[1]) + throw new Error("Specify a directory either positionally or with --dir"); + const options: DownloadOptions = { + channel: requireArg(args, 0, "channel"), + directory: directory ?? args[1], + size: choiceFlag(flags, "size", IMAGE_SIZES) ?? "original", + type: choiceFlag(flags, "type", DOWNLOAD_TYPES), + linkFormat: + choiceFlag(flags, "link-format", ["webloc", "url"] as const) ?? "webloc", + concurrency: intFlag(flags, "concurrency") ?? 4, + overwrite: booleanFlag(flags, "overwrite"), + prune: booleanFlag(flags, "prune"), + zip: valueFlag(flags, "zip"), + }; + validateDownloadOptions(options); + return options; +} + +interface Progress { + label: string; + completed: number; + total: number; +} + +export function DownloadCommand(options: DownloadOptions) { + const { exit } = useApp(); + const [progress, setProgress] = useState<Progress>({ + label: "Listing channel contents", + completed: 0, + total: 0, + }); + const [summary, setSummary] = useState<DownloadSummary | null>(null); + const [error, setError] = useState<string | null>(null); + useEffect(() => { + const controller = new AbortController(); + const signal = AbortSignal.any([cancellationSignal(), controller.signal]); + let mounted = true; + void executeDownload({ + ...options, + signal, + onEvent(event) { + if (!mounted) return; + if (event.type === "list_progress") + setProgress({ + label: `Listing ${event.fetched}/${event.total} items`, + completed: 0, + total: 0, + }); + if (event.type === "file_progress") + setProgress({ + label: `${event.status} · ${truncate(event.file, 64)}`, + completed: event.completed, + total: event.total, + }); + if (event.type === "archive_started") + setProgress((current) => ({ + ...current, + label: "Creating ZIP archive", + })); + }, + }) + .then((result) => { + if (!mounted) return; + process.exitCode = downloadExitCode(result); + setSummary(result); + }) + .catch((failure: unknown) => { + if (!mounted) return; + process.exitCode = signal.aborted ? 130 : exitCodeFromError(failure); + setError(failure instanceof Error ? failure.message : String(failure)); + }); + return () => { + mounted = false; + controller.abort(); + }; + }, []); + useEffect(() => { + if (summary || error) exit(); + }, [summary, error, exit]); + if (error) return <Text color="red">✕ {error}</Text>; + if (!summary) { + if (progress.total === 0) return <Spinner label={progress.label} />; + return ( + <Box flexDirection="column"> + <Text> + {progress.completed}/{progress.total}{" "} + {progressBar(progress.completed, progress.total)} + </Text> + <Spinner label={progress.label} /> + </Box> + ); + } + const ok = downloadExitCode(summary) === 0; + const details = [ + summary.metadata_only ? `${summary.metadata_only} metadata only` : null, + summary.retained + ? options.type + ? `${plural(summary.retained, "earlier file")} kept outside this filtered export` + : `${plural(summary.retained, "earlier file")} not in this export (remove with --prune)` + : null, + ].filter(Boolean); + return ( + <ScreenFrame title={`download ${options.channel}`}> + <Box flexDirection="column"> + <Text color={ok ? "green" : "yellow"}> + {ok ? "✓" : "!"}{" "} + {summary.cancelled + ? "Sync cancelled" + : summary.failed + ? "Sync finished with errors" + : "Sync complete"} + </Text> + <Text dimColor> + {summary.downloaded} downloaded · {summary.unchanged} unchanged ·{" "} + {summary.renamed ? `${summary.renamed} renamed · ` : ""} + {summary.failed} failed + {options.prune ? ` · ${summary.pruned} pruned` : ""} + </Text> + {details.length ? <Text dimColor>{details.join(" · ")}</Text> : null} + <Text>→ {join(summary.directory, "index.html")}</Text> + {summary.zip ? <Text>→ {summary.zip}</Text> : null} + {summary.failures.slice(0, 5).map((failure, index) => ( + <Text color="red" key={index}> + {truncate(failure.file, 64)} · {failure.error} + </Text> + ))} + {summary.failures.length > 5 ? ( + <Text dimColor>…and {summary.failures.length - 5} more</Text> + ) : null} + </Box> + </ScreenFrame> + ); +} +export async function runDownloadJsonStream( + options: DownloadOptions, + write: (event: DownloadEvent) => void, +): Promise<number> { + const signal = cancellationSignal(); + try { + return downloadExitCode( + await executeDownload({ ...options, signal, onEvent: write }), + ); + } catch (error) { + if (signal.aborted) return 130; + throw error; + } +} diff --git a/src/lib/args.ts b/src/lib/args.ts index 5401e43..da66b0e 100644 --- a/src/lib/args.ts +++ b/src/lib/args.ts @@ -14,6 +14,8 @@ const KNOWN_BOOLEAN_LONG_FLAGS = new Set([ "recursive", "interactive", "unread", + "overwrite", + "prune", ]); const KNOWN_BOOLEAN_SHORT_FLAGS = new Set(["h", "j", "q", "v", "y"]); diff --git a/src/lib/download-format.ts b/src/lib/download-format.ts new file mode 100644 index 0000000..59f4476 --- /dev/null +++ b/src/lib/download-format.ts @@ -0,0 +1,237 @@ +import { extname } from "node:path"; +import type { Connectable } from "../api/types"; + +export const IMAGE_SIZES = [ + "original", + "large", + "medium", + "small", + "square", +] as const; +export type ImageSize = (typeof IMAGE_SIZES)[number]; +export const DOWNLOAD_TYPES = [ + "Image", + "Attachment", + "Text", + "Link", + "Embed", + "Channel", + "PendingBlock", +] as const; +export type DownloadType = (typeof DOWNLOAD_TYPES)[number]; +export type LinkFormat = "webloc" | "url"; +export type DownloadTarget = + | { kind: "fetch"; filename: string; url: string } + | { kind: "text"; filename: string; content: string }; + +/** Keep names portable, bounded in UTF-8 bytes, and confined to one directory. */ +export function safeFilename(value: string): string { + const clean = value + .normalize("NFC") + .replace(/[<>:"/\\|?*\x00-\x1f\x7f]/g, "_") + .replace(/[. ]+$/g, "") + .trim(); + let result = ""; + for (const char of clean) { + if (Buffer.byteLength(result + char) > 150) break; + result += char; + } + result = result.replace(/[. ]+$/g, ""); + if (!result || /^\.+$/.test(result)) return "untitled"; + return /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(result) + ? `_${result}` + : result; +} + +export function escapeMarkup(value: string): string { + return value.replace( + /[&<>"']/g, + (char) => + ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[ + char + ]!, + ); +} + +export function httpUrl(value: string): string { + const url = new URL(value); + if ( + !["https:", "http:"].includes(url.protocol) || + url.username || + url.password + ) { + throw new Error("Expected an HTTP(S) URL without credentials"); + } + return url.href; +} + +export function arenaUrl(item: Connectable): string { + return item.type === "Channel" + ? `https://www.are.na/${encodeURIComponent(item.owner.slug)}/${encodeURIComponent(item.slug)}` + : `https://www.are.na/block/${item.id}`; +} + +export function urlShortcut(url: string, format: LinkFormat): string { + const safeUrl = httpUrl(url); + return format === "url" + ? `[InternetShortcut]\r\nURL=${safeUrl}\r\n` + : `<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n<plist version="1.0"><dict><key>URL</key><string>${escapeMarkup(safeUrl)}</string></dict></plist>\n`; +} + +// Mirrors the API's Scraper::ContentTypes, which picks the first listed extension. +const IMAGE_EXTENSIONS: Record<string, string> = { + "image/avif": "avif", + "image/bmp": "bmp", + "image/x-ms-bmp": "bmp", + "image/gif": "gif", + "image/heic": "heic", + "image/heif": "heif", + "image/jpeg": "jpg", + "image/jpg": "jpg", + "image/png": "png", + "image/svg+xml": "svg", + "image/tiff": "tif", + "image/webp": "webp", +}; + +const TRANSLITERATIONS: Record<string, string> = { + ß: "ss", + æ: "ae", + Æ: "AE", + œ: "oe", + Œ: "OE", + ø: "o", + Ø: "O", + đ: "d", + Đ: "D", + ð: "d", + Ð: "D", + ł: "l", + Ł: "L", + þ: "th", + Þ: "TH", +}; + +/** Rails' `String#parameterize`: ASCII-transliterated, hyphenated, lowercase. */ +export function parameterize(value: string): string { + return value + .replace(/[ßæÆœŒøØđĐðÐłŁþÞ]/g, (char) => TRANSLITERATIONS[char]!) + .normalize("NFKD") + .replace(/[\u0300-\u036f]/g, "") + .replace(/[^a-z0-9\-_]+/gi, "-") + .replace(/-{2,}/g, "-") + .replace(/^-|-$/g, "") + .toLowerCase(); +} + +/** Rails' `String#truncate`, which counts the omission toward the length. */ +function truncateWithOmission(value: string, length: number): string { + return value.length > length ? `${value.slice(0, length - 3)}...` : value; +} + +/** The API's `Block#generated_title`. */ +function generatedTitle(item: Connectable): string { + if (item.title?.trim()) return item.title; + const content = + item.type === "Text" + ? item.content.markdown + : item.type === "Link" + ? item.content?.markdown + : undefined; + const text = [content, item.description?.markdown].find((value) => + value?.trim(), + ); + if (!text) return "Untitled"; + return escapeMarkup(truncateWithOmission(text.replace(/<[^>]*>/g, ""), 25)); +} + +/** + * The API's GraphQL `file_name`, so exports are named like Are.na's web + * channel export (`<position>_<file_name>.<extension>`). + */ +export function blockFileName(item: Connectable): string { + return truncateWithOmission(parameterize(generatedTitle(item)), 50); +} + +export function itemKey(item: Connectable): string { + return `${item.type === "Channel" ? "channel" : "block"}-${item.id}`; +} + +function resizedImageExtension( + url: string, + contentType: string | undefined, +): string { + const parsed = new URL(url); + const explicit = extname(parsed.pathname).slice(1).toLowerCase(); + if (/^(jpe?g|png|gif|webp|avif|tiff?)$/.test(explicit)) return explicit; + // images.are.na's default (non-WebP) response is JPEG, except animated GIFs. + // Its path is an opaque transform token, not a usable filename. + if (parsed.hostname === "images.are.na") + return contentType === "image/gif" ? "gif" : "jpg"; + return IMAGE_EXTENSIONS[contentType ?? ""] ?? "jpg"; +} + +/** `position` is the 1-based index in the channel's manual order. */ +export function resolveTarget( + item: Connectable, + position: number, + options: { size: ImageSize; linkFormat: LinkFormat }, +): DownloadTarget | null { + const stem = `${position}_${blockFileName(item)}`; + switch (item.type) { + case "Image": { + const url = + options.size === "original" + ? item.image.src + : item.image[options.size]?.src; + if (!url) throw new Error(`No ${options.size} image is available`); + const extension = + options.size === "original" + ? (IMAGE_EXTENSIONS[item.image.content_type ?? ""] ?? + (extname(item.image.filename ?? "") + .slice(1) + .toLowerCase() || + "jpg")) + : resizedImageExtension(url, item.image.content_type); + return { + kind: "fetch", + url: httpUrl(url), + filename: `${stem}.${extension}`, + }; + } + case "Attachment": { + const extension = safeFilename( + item.attachment.file_extension?.trim() || "bin", + ).toLowerCase(); + return { + kind: "fetch", + url: httpUrl(item.attachment.url), + filename: `${stem}.${extension}`, + }; + } + case "Text": + return { + kind: "text", + content: item.content.markdown ?? item.content.plain ?? "", + filename: `${stem}.txt`, + }; + case "Link": + case "Embed": + return { + kind: "text", + content: urlShortcut( + item.source?.url || arenaUrl(item), + options.linkFormat, + ), + filename: `${stem}.${options.linkFormat}`, + }; + case "Channel": + return { + kind: "text", + content: urlShortcut(arenaUrl(item), options.linkFormat), + filename: `${position}_${safeFilename(item.slug)}.${options.linkFormat}`, + }; + case "PendingBlock": + return null; + } +} diff --git a/src/lib/download.test.ts b/src/lib/download.test.ts new file mode 100644 index 0000000..041fceb --- /dev/null +++ b/src/lib/download.test.ts @@ -0,0 +1,1389 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + link, + mkdtemp, + readFile, + readdir, + rename, + rm, + writeFile, + symlink, + utimes, +} from "node:fs/promises"; +import crypto from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { hostname, tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; +import { inflateRawSync } from "node:zlib"; +import { ArenaError } from "../api/client"; +import type { Block, Channel, Connectable } from "../api/types"; +import { + buildDownloadCsv, + executeDownload, + downloadExitCode, + defaultDownloadAdapter, + type DownloadAdapter, + type DownloadManifest, + type DownloadOptions, + type DownloadEvent, + type ManagedFile, +} from "./download"; +import { + blockFileName, + resolveTarget, + safeFilename, + urlShortcut, +} from "./download-format"; +import { parseDownloadOptions } from "../commands/download"; +import { parseArgs, type Flags } from "./args"; + +const user = { + id: 1, + type: "User" as const, + slug: "owner", + name: "Owner", + avatar: null, + initials: "O", +}; +const channel = { + id: 10, + type: "Channel", + slug: "test-channel", + title: "Collection", + state: "available", + visibility: "public", + created_at: "2026-01-01", + updated_at: "2026-01-02", + owner: user, + counts: { contents: 1 }, + _links: { self: { href: "https://api.are.na/v3/blocks/1" } }, +} as Channel; +const base = { + id: 1, + base_type: "Block" as const, + title: "Note", + state: "available" as const, + visibility: "public" as const, + comment_count: 0, + created_at: "2026-01-01", + updated_at: "2026-01-02", + user, + _links: { self: { href: "https://api.are.na/v3/blocks/1" } }, +}; +function textBlock( + id = 1, + content = "hello", +): Extract<Block, { type: "Text" }> { + return { + ...base, + id, + type: "Text", + content: { markdown: content, plain: content, html: `<p>${content}</p>` }, + }; +} +function imageBlock(id = 2): Block { + const version = { src: `https://cdn.example/${id}/large.jpg`, src_2x: "" }; + return { + ...base, + id, + title: "photo.png?w=800", + type: "Image", + image: { + filename: "photo.png", + content_type: "image/png", + src: `https://cdn.example/${id}/original.png`, + small: version, + medium: version, + large: version, + square: version, + }, + }; +} +function adapter( + items: Connectable[], + overrides: Partial<DownloadAdapter> = {}, +): DownloadAdapter { + return { + async getChannel() { + return channel; + }, + async listContents() { + return { + data: items, + meta: { + current_page: 1, + per_page: 100, + total_pages: 1, + total_count: items.length, + has_more_pages: false, + }, + }; + }, + async openAsset(url) { + return Readable.from([`bytes:${url}`]); + }, + async sleep() {}, + ...overrides, + }; +} +const options: DownloadOptions = { + channel: "test-channel", + size: "original", + linkFormat: "webloc", + concurrency: 2, + overwrite: false, + prune: false, +}; +async function directory(t: test.TestContext) { + const path = await mkdtemp(join(tmpdir(), "arena-export-")); + t.after(() => rm(path, { force: true, recursive: true })); + return path; +} +async function manifest(path: string): Promise<DownloadManifest> { + return JSON.parse(await readFile(join(path, "manifest.json"), "utf8")); +} +function filename(item: Connectable, position = 1) { + return resolveTarget(item, position, options)!.filename; +} + +// A small reader for our archive assertions, using the ZIP central directory rather than a shell dependency. +function unzip(buffer: Buffer): Map<string, Buffer> { + const eocd = buffer.lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06])); + assert.ok(eocd >= 0); + let offset = buffer.readUInt32LE(eocd + 16); + const files = new Map<string, Buffer>(); + for (let i = 0; i < buffer.readUInt16LE(eocd + 10); i++) { + assert.equal(buffer.readUInt32LE(offset), 0x02014b50); + const compression = buffer.readUInt16LE(offset + 10); + const size = buffer.readUInt32LE(offset + 20); + const nameLength = buffer.readUInt16LE(offset + 28); + const local = buffer.readUInt32LE(offset + 42); + const name = buffer + .subarray(offset + 46, offset + 46 + nameLength) + .toString(); + const start = + local + + 30 + + buffer.readUInt16LE(local + 26) + + buffer.readUInt16LE(local + 28); + const data = buffer.subarray(start, start + size); + files.set(name, compression === 8 ? inflateRawSync(data) : data); + offset += + 46 + + nameLength + + buffer.readUInt16LE(offset + 30) + + buffer.readUInt16LE(offset + 32); + } + return files; +} + +test("exports every content type, connection metadata, escaped offline index, and real ZIP contents", async (t) => { + const dir = await directory(t); + const text = textBlock(1, '<script>alert("x")</script>'); + const items: Connectable[] = [ + text, + imageBlock(), + { + ...base, + id: 3, + type: "Attachment", + attachment: { url: "https://cdn.example/file.pdf", filename: "file.pdf" }, + }, + { + ...base, + id: 4, + type: "Link", + source: { url: "https://example.com/?a=1&b=2" }, + }, + { + ...base, + id: 5, + type: "Embed", + embed: { html: "<script>bad()</script>", type: "video" }, + } as Block, + { ...channel, id: 6, slug: "child" }, + { ...base, id: 7, type: "PendingBlock" }, + ]; + text.connection = { + id: 22, + position: 4, + pinned: true, + connected_at: "2026-01-01", + connected_by: null, + metadata: { note: "context" }, + }; + const zip = join(dir, "export.zip"); + const events: DownloadEvent[] = []; + const result = await executeDownload({ + ...options, + directory: dir, + zip, + adapter: adapter(items), + onEvent: (event) => events.push(event), + }); + assert.equal(result.downloaded, 6); + assert.equal(result.metadata_only, 1); + assert.equal(result.failed, 0); + const data = await manifest(dir); + assert.equal(data.complete, true); + assert.deepEqual(data.entries[0].item.connection, text.connection); + assert.ok(data.files.every((file) => /^[a-f0-9]{64}$/.test(file.sha256))); + const html = await readFile(join(dir, "index.html"), "utf8"); + assert.ok(!html.includes("<script>")); + assert.ok(html.includes("<script>")); + assert.ok( + (await readFile(join(dir, filename(items[3], 4)), "utf8")).includes( + "&b=2", + ), + ); + const zipped = unzip(await readFile(zip)); + assert.equal(zipped.size, 9); + const csv = zipped.get("test-channel.csv")!.toString(); + assert.equal(csv, await readFile(join(dir, "test-channel.csv"), "utf8")); + const [header, first] = csv.split("\n"); + assert.equal( + header, + "ID,Filename,Title,Description,Created At,Updated At,Source", + ); + assert.equal(first, `1,${filename(text)},Note,,2026-01-01,2026-01-02,`); + assert.equal(csv.split("\n").length, 7); + assert.equal(zipped.get(filename(text))?.toString(), text.content.markdown); + assert.deepEqual(JSON.parse(zipped.get("manifest.json")!.toString()), data); + assert.equal(events.at(-1)?.type, "completed"); +}); + +test("sync skips unchanged assets, updates changed text, and preserves local edits unless forced", async (t) => { + const dir = await directory(t); + let fetches = 0; + const network = { + async openAsset() { + fetches++; + return Readable.from(["image"]); + }, + }; + await executeDownload({ + ...options, + directory: dir, + adapter: adapter([imageBlock(), textBlock()], network), + }); + const unchanged = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([textBlock(), imageBlock()], network), + }); + assert.equal(unchanged.unchanged, 2); + assert.equal(fetches, 1); + const changed = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([imageBlock(), textBlock(1, "changed")], network), + }); + assert.equal(changed.downloaded, 1); + assert.equal(changed.unchanged, 1); + await writeFile(join(dir, filename(textBlock())), "local edit"); + const conflict = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([textBlock(1, "remote edit")]), + }); + assert.equal(downloadExitCode(conflict), 1); + assert.match(conflict.failures[0].error, /modified/); + assert.equal( + await readFile(join(dir, filename(textBlock())), "utf8"), + "local edit", + ); + const forced = await executeDownload({ + ...options, + directory: dir, + overwrite: true, + adapter: adapter([textBlock(1, "remote edit")]), + }); + assert.equal(forced.downloaded, 1); + assert.equal( + await readFile(join(dir, filename(textBlock())), "utf8"), + "remote edit", + ); +}); + +test("sync detects changed assets at the same URL and restores missing managed files", async (t) => { + const dir = await directory(t); + const image = imageBlock(); + await executeDownload({ + ...options, + directory: dir, + adapter: adapter([image]), + }); + image.updated_at = "2026-02-01"; + const changed = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([image], { + async openAsset() { + return Readable.from(["new bytes"]); + }, + }), + }); + assert.equal(changed.downloaded, 1); + await rm(join(dir, filename(image))); + const restored = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([image]), + }); + assert.equal(restored.downloaded, 1); +}); + +test("retains removed files by default; prune deletes only unchanged tracked files and includes prior retained exports", async (t) => { + const dir = await directory(t); + await executeDownload({ + ...options, + directory: dir, + adapter: adapter([textBlock(1), textBlock(2), textBlock(3)]), + }); + await writeFile(join(dir, "unrelated.txt"), "keep"); + await writeFile(join(dir, filename(textBlock(3), 3)), "local edit"); + const kept = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([textBlock(1)]), + }); + assert.equal(kept.retained, 2); + const pruned = await executeDownload({ + ...options, + directory: dir, + prune: true, + adapter: adapter([textBlock(1)]), + }); + assert.equal(pruned.pruned, 1); + assert.equal(pruned.failed, 1); + assert.equal(pruned.retained, 1); + assert.ok((await readdir(dir)).includes("unrelated.txt")); + assert.equal( + await readFile(join(dir, filename(textBlock(3), 3)), "utf8"), + "local edit", + ); +}); + +test("never prunes after a failed download", async (t) => { + const dir = await directory(t); + await executeDownload({ + ...options, + directory: dir, + adapter: adapter([textBlock()]), + }); + const result = await executeDownload({ + ...options, + directory: dir, + prune: true, + adapter: adapter([imageBlock()], { + async openAsset() { + throw new ArenaError("gone", 404); + }, + }), + }); + assert.equal(result.failed, 1); + assert.equal(result.pruned, 0); + assert.equal(result.retained, 1); + assert.ok((await readdir(dir)).includes(filename(textBlock()))); +}); + +test("retries interrupted streams from scratch and leaves no partial destination", async (t) => { + const dir = await directory(t); + let attempts = 0; + const result = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([imageBlock()], { + async openAsset() { + attempts++; + if (attempts === 1) + return Readable.from( + (async function* () { + yield "partial"; + throw Object.assign(new Error("socket reset"), { + code: "ECONNRESET", + }); + })(), + ); + return Readable.from(["complete"]); + }, + }), + }); + assert.equal(result.downloaded, 1); + assert.equal(attempts, 2); + assert.equal( + await readFile(join(dir, filename(imageBlock())), "utf8"), + "complete", + ); + assert.ok(!(await readdir(dir)).some((name) => name.endsWith(".part"))); +}); + +test("failed replacement preserves the old file; empty assets are failures and empty text is valid", async (t) => { + const dir = await directory(t); + await executeDownload({ + ...options, + directory: dir, + adapter: adapter([imageBlock()]), + }); + const old = await readFile(join(dir, filename(imageBlock())), "utf8"); + let attempts = 0; + const result = await executeDownload({ + ...options, + directory: dir, + overwrite: true, + adapter: adapter([imageBlock(), textBlock(1, "")], { + async openAsset() { + attempts++; + return Readable.from([]); + }, + }), + }); + assert.equal(result.failed, 1); + assert.equal(result.downloaded, 1); + assert.equal(attempts, 3); + assert.equal(await readFile(join(dir, filename(imageBlock())), "utf8"), old); + assert.equal((await manifest(dir)).complete, false); +}); + +test("recovers an identical untracked file but refuses unrelated content and symlinks", async (t) => { + const dir = await directory(t); + await writeFile(join(dir, filename(textBlock())), "hello"); + const recovered = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([textBlock()]), + }); + assert.equal(recovered.unchanged, 1); + await writeFile(join(dir, filename(textBlock(2), 2)), "unrelated"); + await symlink( + join(dir, filename(textBlock())), + join(dir, filename(textBlock(3), 3)), + ); + const result = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([textBlock(), textBlock(2), textBlock(3)]), + }); + assert.equal(result.failed, 2); + assert.equal( + await readFile(join(dir, filename(textBlock(2), 2)), "utf8"), + "unrelated", + ); +}); + +test("paginates in order and rejects duplicate or looping pages before modifying the destination", async (t) => { + const dir = await directory(t); + const calls: number[] = []; + const paginated = adapter([], { + async listContents(_id, page) { + calls.push(page); + return { + data: [textBlock(page)], + meta: { + current_page: page, + per_page: 1, + total_pages: 2, + total_count: 2, + has_more_pages: page === 1, + next_page: 2, + }, + }; + }, + }); + const result = await executeDownload({ + ...options, + directory: dir, + adapter: paginated, + }); + assert.equal(result.downloaded, 2); + assert.deepEqual(calls, [1, 2]); + const before = await readFile(join(dir, "manifest.json"), "utf8"); + await assert.rejects( + executeDownload({ + ...options, + directory: dir, + adapter: adapter([], { + async listContents() { + return { + data: [textBlock()], + meta: { + current_page: 1, + per_page: 1, + total_count: 2, + total_pages: 2, + has_more_pages: true, + next_page: 1, + }, + }; + }, + }), + }), + /pagination/, + ); + assert.equal(await readFile(join(dir, "manifest.json"), "utf8"), before); +}); + +test("bounded concurrency and cancellation settle active workers and preserve a resumable manifest", async (t) => { + const dir = await directory(t); + const controller = new AbortController(); + let active = 0; + let peak = 0; + const result = await executeDownload({ + ...options, + directory: dir, + signal: controller.signal, + adapter: adapter([imageBlock(1), imageBlock(2), imageBlock(3)], { + async openAsset(_url, signal) { + active++; + peak = Math.max(active, peak); + return Readable.from( + (async function* () { + try { + yield "part"; + await new Promise<void>((resolve) => { + signal.addEventListener("abort", () => resolve(), { + once: true, + }); + setTimeout(() => controller.abort(new Error("cancelled")), 20); + }); + signal.throwIfAborted(); + } finally { + active--; + } + })(), + ); + }, + }), + }); + assert.equal(peak, 2); + assert.equal(active, 0); + assert.equal(downloadExitCode(result), 130); + assert.equal((await manifest(dir)).complete, false); + assert.ok( + !(await readdir(dir)).some( + (name) => name.endsWith(".part") || name.endsWith(".lock"), + ), + ); +}); + +test("validates flags before any work, including booleans before positional arguments", () => { + const parsed = parseArgs([ + "--prune", + "--overwrite", + "test-channel", + "./export", + ]); + assert.equal(parseDownloadOptions(parsed.args, parsed.flags).prune, true); + assert.equal( + parseDownloadOptions(["channel"], { overwrite: "false" }).overwrite, + false, + ); + const invalidFlags: Flags[] = [ + { size: "huge" }, + { concurrency: "17" }, + { type: "Imag" }, + { dir: true }, + { zip: true }, + { prune: true, type: "Image" }, + { "link-format": "html" }, + ]; + for (const flags of invalidFlags) + assert.throws(() => parseDownloadOptions(["channel"], flags)); + assert.throws(() => + parseDownloadOptions(["channel", "here"], { dir: "there" }), + ); +}); + +test("filenames are portable, bounded, distinct across types; shortcuts escape XML and reject executable URLs", () => { + assert.equal(safeFilename("../../a:b?.png"), ".._.._a_b_.png"); + assert.ok(Buffer.byteLength(safeFilename("😀".repeat(200))) <= 150); + assert.equal(safeFilename("CON"), "_CON"); + assert.match( + urlShortcut("https://example.com/?x=1&y=2", "webloc"), + /&y=2/, + ); + assert.match( + urlShortcut("https://example.com", "url"), + /^\[InternetShortcut\]\r\nURL=/, + ); + assert.throws(() => urlShortcut("javascript:alert(1)", "webloc")); + assert.throws(() => + urlShortcut("https://user:password@example.com", "webloc"), + ); + assert.notEqual(filename(textBlock(10)), filename(channel)); + assert.equal( + resolveTarget(imageBlock(), 1, { ...options, size: "large" })!.filename, + "1_photo-png-w-800.jpg", + ); +}); + +test("rejects foreign manifests and malicious managed paths without touching files", async (t) => { + const dir = await directory(t); + await executeDownload({ + ...options, + directory: dir, + adapter: adapter([textBlock()]), + }); + const data = await manifest(dir); + data.files[0].filename = "../escape"; + await writeFile(join(dir, "manifest.json"), JSON.stringify(data)); + await assert.rejects( + executeDownload({ + ...options, + directory: dir, + prune: true, + adapter: adapter([]), + }), + /incompatible/, + ); + assert.ok((await readdir(dir)).includes(filename(textBlock()))); +}); + +test("CDN adapter requires HTTP 200, sends a user agent, and honors Retry-After", async (t) => { + let init: RequestInit | undefined; + t.mock.method( + globalThis, + "fetch", + async (_url: string, request: RequestInit) => { + init = request; + return new Response(null, { + status: 429, + headers: { "Retry-After": "2" }, + }); + }, + ); + const network = defaultDownloadAdapter(); + await assert.rejects( + network.openAsset( + "https://cdn.example/image", + new AbortController().signal, + ), + (error: unknown) => + error instanceof ArenaError && error.rateLimit?.retryAfterMs === 2000, + ); + assert.ok(new Headers(init?.headers).get("user-agent")); + assert.equal(new Headers(init?.headers).get("authorization"), null); +}); + +test("rate limiting uses the server delay and permanent HTTP errors do not retry", async (t) => { + const dir = await directory(t); + let attempts = 0; + const delays: number[] = []; + const result = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([imageBlock()], { + async openAsset() { + if (++attempts === 1) + throw new ArenaError("limited", 429, { + rateLimit: { retryAfterMs: 2000 }, + }); + return Readable.from(["bytes"]); + }, + async sleep(ms) { + delays.push(ms); + }, + }), + }); + assert.equal(result.downloaded, 1); + assert.deepEqual(delays, [2000]); + attempts = 0; + const failed = await executeDownload({ + ...options, + directory: dir, + overwrite: true, + adapter: adapter([imageBlock()], { + async openAsset() { + attempts++; + throw new ArenaError("gone", 404); + }, + }), + }); + assert.equal(failed.failed, 1); + assert.equal(attempts, 1); +}); + +test("prune rechecks actual channel contents even when channel timestamps do not change", async (t) => { + const dir = await directory(t); + await executeDownload({ + ...options, + directory: dir, + adapter: adapter([textBlock(1), textBlock(2)]), + }); + let listings = 0; + const changing = adapter([], { + async listContents(id, page, signal) { + return adapter( + ++listings === 1 ? [textBlock(1)] : [textBlock(1), textBlock(2)], + ).listContents(id, page, signal); + }, + }); + const result = await executeDownload({ + ...options, + directory: dir, + prune: true, + adapter: changing, + }); + assert.equal(result.pruned, 0); + assert.equal(result.failed, 1); + assert.ok((await readdir(dir)).includes(filename(textBlock(2), 2))); +}); + +test("prune preserves assets for pending blocks, and filtering preserves metadata without fetching excluded files", async (t) => { + const dir = await directory(t); + await executeDownload({ + ...options, + directory: dir, + adapter: adapter([imageBlock(2)]), + }); + const pending: Block = { ...base, id: 2, type: "PendingBlock" }; + const result = await executeDownload({ + ...options, + directory: dir, + prune: true, + adapter: adapter([pending]), + }); + assert.equal(result.pruned, 0); + assert.equal(result.retained, 1); + const filtered = await executeDownload({ + ...options, + directory: dir, + type: "Text", + adapter: adapter([imageBlock(2), textBlock()], { + async openAsset() { + throw new Error("should not fetch"); + }, + }), + }); + assert.equal(filtered.downloaded, 1); + assert.equal((await manifest(dir)).entries[0].status, "filtered"); +}); + +test("missing requested image sizes are explicit failures; ZIP cannot overwrite the manifest", async (t) => { + const dir = await directory(t); + const image = imageBlock(); + if (image.type === "Image") image.image.src = undefined; + const result = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([image]), + }); + assert.equal(result.failed, 1); + assert.match(result.failures[0].error, /No original image/); + const before = await readFile(join(dir, "manifest.json"), "utf8"); + await assert.rejects( + executeDownload({ + ...options, + directory: dir, + zip: join(dir, "manifest.json"), + overwrite: true, + adapter: adapter([]), + }), + /conflicts/, + ); + assert.equal(await readFile(join(dir, "manifest.json"), "utf8"), before); +}); + +test("asset adapter rejects accepted, empty, and truncated responses", async (t) => { + const responses = [ + new Response(null, { status: 202 }), + new Response("", { status: 200 }), + new Response("short", { status: 200, headers: { "Content-Length": "50" } }), + ]; + t.mock.method(globalThis, "fetch", async () => responses.shift()!); + const network = defaultDownloadAdapter(); + const signal = new AbortController().signal; + await assert.rejects( + network.openAsset("https://cdn.example/a", signal), + /202/, + ); + for (const expected of [/empty/, /truncated/]) { + const body = await network.openAsset("https://cdn.example/a", signal); + await assert.rejects(async () => { + for await (const _chunk of body) { + /* Drain the response. */ + } + }, expected); + } +}); + +test("filenames match Are.na's web channel export: <position>_<file_name>.<extension>", () => { + const image = imageBlock(); + assert.equal(image.type, "Image"); + assert.equal( + resolveTarget(image, 3, options)?.filename, + "3_photo-png-w-800.png", + ); + image.title = "Ünïcödé Straße / Æsthetics & Co."; + assert.equal(blockFileName(image), "unicode-strasse-aesthetics-co"); + image.title = + "A very long title that keeps going well beyond fifty characters"; + assert.equal( + blockFileName(image), + "a-very-long-title-that-keeps-going-well-beyond-...", + ); + image.title = "Строка"; + assert.equal(blockFileName(image), ""); + image.title = null; + image.image.content_type = "image/tiff"; + assert.equal(resolveTarget(image, 1, options)?.filename, "1_untitled.tif"); + image.image.small.src = "https://images.are.na/eyJvcGFxdWUiOnRydWV9"; + assert.equal( + resolveTarget(image, 1, { ...options, size: "small" })?.filename, + "1_untitled.jpg", + ); + image.image.content_type = "image/gif"; + assert.equal( + resolveTarget(image, 1, { ...options, size: "small" })?.filename, + "1_untitled.gif", + ); + assert.equal( + resolveTarget( + { + ...textBlock(5, "Some <b>bold</b> & longer text content"), + title: null, + }, + 2, + options, + )?.filename, + "2_some-bold-amp-longer-tex.txt", + ); + assert.equal( + resolveTarget( + { + ...base, + type: "Attachment", + attachment: { url: "https://cdn.example/a", file_extension: "PDF" }, + }, + 4, + options, + )?.filename, + "4_note.pdf", + ); + assert.equal(filename(channel, 6), "6_test-channel.webloc"); + assert.match( + (resolveTarget(channel, 6, options) as { content: string }).content, + /www.are.na\/owner\/test-channel/, + ); +}); + +test("transfers time out only when data stops arriving, then retry", async (t) => { + const previous = process.env["ARENA_REQUEST_TIMEOUT_MS"]; + process.env["ARENA_REQUEST_TIMEOUT_MS"] = "60"; + t.after(() => { + if (previous === undefined) delete process.env["ARENA_REQUEST_TIMEOUT_MS"]; + else process.env["ARENA_REQUEST_TIMEOUT_MS"] = previous; + }); + const dir = await directory(t); + let attempts = 0; + const result = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([imageBlock(1), imageBlock(2)], { + async openAsset(url, signal) { + if (url.includes("/1/")) + // A slow but steady transfer that outlasts the idle timeout overall. + return Readable.from( + (async function* () { + for (let i = 0; i < 6; i++) { + await new Promise((resolve) => setTimeout(resolve, 25)); + yield "chunk"; + } + })(), + ); + if (++attempts > 1) return Readable.from(["complete"]); + return Readable.from( + (async function* () { + yield "partial"; + await new Promise((resolve) => + signal.addEventListener("abort", resolve, { once: true }), + ); + signal.throwIfAborted(); + })(), + ); + }, + }), + }); + assert.equal(result.failed, 0); + assert.equal(attempts, 2); + assert.equal( + await readFile(join(dir, filename(imageBlock(1))), "utf8"), + "chunk".repeat(6), + ); + assert.equal( + await readFile(join(dir, filename(imageBlock(2), 2)), "utf8"), + "complete", + ); +}); + +test("reclaims a lock left by a dead process and removes its partial files, but respects a live lock", async (t) => { + const dir = await directory(t); + const lock = join(dir, ".arena-download.lock"); + const dead = spawnSync(process.execPath, ["-e", ""]).pid; + await writeFile(lock, `${dead}@${hostname()}`); + const partial = join(dir, ".arena-00000000-0000-0000-0000-000000000000.part"); + await writeFile(partial, "partial"); + const result = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([textBlock()]), + }); + assert.equal(result.downloaded, 1); + const names = await readdir(dir); + assert.ok(!names.some((name) => name.endsWith(".part"))); + assert.ok(!names.includes(".arena-download.lock")); + await writeFile(lock, `${process.pid}@${hostname()}`); + await assert.rejects( + executeDownload({ + ...options, + directory: dir, + adapter: adapter([textBlock()]), + }), + /locked/, + ); + await writeFile(lock, `${dead}@another-host`); + await assert.rejects( + executeDownload({ + ...options, + directory: dir, + adapter: adapter([textBlock()]), + }), + /locked/, + ); +}); + +test("re-exporting replaces an earlier ZIP without forcing downloads, but never clobbers other files", async (t) => { + const dir = await directory(t); + const zip = `${dir}.zip`; + t.after(() => rm(zip, { force: true })); + let fetches = 0; + const network = { + async openAsset() { + fetches++; + return Readable.from(["image"]); + }, + }; + await executeDownload({ + ...options, + directory: dir, + zip, + adapter: adapter([imageBlock()], network), + }); + const result = await executeDownload({ + ...options, + directory: dir, + zip, + adapter: adapter([imageBlock(), textBlock()], network), + }); + assert.equal(result.zip, zip); + assert.equal(fetches, 1); + assert.equal(unzip(await readFile(zip)).size, 5); + const unrelated = join(dir, "notes.txt"); + await writeFile(unrelated, "keep"); + await assert.rejects( + executeDownload({ + ...options, + directory: dir, + zip: unrelated, + adapter: adapter([textBlock()]), + }), + /not a ZIP archive/, + ); + assert.equal(await readFile(unrelated, "utf8"), "keep"); +}); + +test("detects same-size local edits and reports a corrupt manifest clearly", async (t) => { + const dir = await directory(t); + await executeDownload({ + ...options, + directory: dir, + adapter: adapter([textBlock()]), + }); + await writeFile(join(dir, filename(textBlock())), "HELLO"); + const result = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([textBlock(1, "remote")]), + }); + assert.match(result.failures[0].error, /modified/); + await writeFile(join(dir, "manifest.json"), "{"); + await assert.rejects( + executeDownload({ + ...options, + directory: dir, + adapter: adapter([textBlock()]), + }), + /manifest.json is not valid JSON/, + ); +}); + +test("offline index links to the original source of links", async (t) => { + const dir = await directory(t); + await executeDownload({ + ...options, + directory: dir, + adapter: adapter([ + { + ...base, + id: 4, + type: "Link", + source: { url: "https://example.com/article" }, + }, + { + ...base, + id: 5, + type: "Link", + source: { url: "javascript:alert(1)" }, + }, + ]), + }); + const html = await readFile(join(dir, "index.html"), "utf8"); + assert.ok(html.includes('href="https://example.com/article">Source</a>')); + assert.ok(!html.includes("javascript:")); +}); + +test("reordering, removing, and retitling blocks renames files without downloading them", async (t) => { + const dir = await directory(t); + let fetches = 0; + const network = { + async openAsset() { + fetches++; + return Readable.from(["image"]); + }, + }; + const first = { ...textBlock(1, "first"), title: null }; + const second = { ...textBlock(2, "second"), title: null }; + await executeDownload({ + ...options, + directory: dir, + adapter: adapter([first, second, imageBlock(3)], network), + }); + // Two blocks trade places, so each file's new name is the other's old name. + const twin = { ...textBlock(4, "same"), title: "Same" }; + const other = { ...textBlock(5, "different"), title: "Same" }; + await executeDownload({ + ...options, + directory: dir, + adapter: adapter([twin, other], network), + }); + const swapped = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([other, twin, imageBlock(3)], network), + }); + assert.equal(swapped.downloaded, 0); + assert.equal(swapped.renamed, 2); + assert.equal(await readFile(join(dir, "1_same.txt"), "utf8"), "different"); + assert.equal(await readFile(join(dir, "2_same.txt"), "utf8"), "same"); + assert.equal(fetches, 1); + const retitled = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([{ ...imageBlock(3), title: "Renamed" }], network), + }); + assert.equal(retitled.downloaded, 0); + assert.equal(retitled.renamed, 1); + assert.ok((await readdir(dir)).includes("1_renamed.png")); + assert.equal(fetches, 1); + assert.ok(!(await readdir(dir)).some((name) => name.endsWith(".part"))); +}); + +test("CSV quotes commas, quotes, and newlines", () => { + const manifest = { + entries: [ + { + item: { + ...textBlock(), + title: 'Say "hi", then', + description: { markdown: "two\nlines", plain: "", html: "" }, + source: { url: "https://example.com" }, + }, + filename: "1_say-hi-then.txt", + status: "downloaded", + }, + ], + } as unknown as DownloadManifest; + assert.equal( + buildDownloadCsv(manifest).split("\n").slice(1).join("\n"), + '1,1_say-hi-then.txt,"Say ""hi"", then","two\nlines",2026-01-01,2026-01-02,https://example.com', + ); +}); + +test("moving a block onto a removed block's name keeps the removed file until --prune", async (t) => { + const dir = await directory(t); + const removed = { ...textBlock(11, "removed"), title: "Same" }; + const kept = { ...textBlock(12, "kept"), title: "Same" }; + await executeDownload({ + ...options, + directory: dir, + adapter: adapter([removed, kept]), + }); + const events: DownloadEvent[] = []; + const result = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([kept]), + onEvent: (event) => events.push(event), + }); + assert.equal(result.renamed, 1); + assert.equal(result.pruned, 0); + assert.equal(result.retained, 1); + assert.equal(await readFile(join(dir, "1_same.txt"), "utf8"), "kept"); + assert.equal( + await readFile(join(dir, "1_same~block-11.txt"), "utf8"), + "removed", + ); + assert.ok( + events.some( + (event) => + event.type === "file_retained" && event.file === "1_same~block-11.txt", + ), + ); + // A new block taking a removed block's name is the same hazard without a rename. + const added = { ...textBlock(13, "added"), title: "Same" }; + await executeDownload({ + ...options, + directory: dir, + adapter: adapter([added]), + }); + assert.equal(await readFile(join(dir, "1_same.txt"), "utf8"), "added"); + assert.equal( + await readFile(join(dir, "1_same~block-12.txt"), "utf8"), + "kept", + ); + const pruned = await executeDownload({ + ...options, + directory: dir, + prune: true, + adapter: adapter([added]), + }); + assert.equal(pruned.pruned, 2); + assert.deepEqual( + (await readdir(dir)).filter((name) => name.includes("same")), + ["1_same.txt"], + ); +}); + +test("an interrupted rename is completed from its journal, even when assets are unavailable", async (t) => { + const dir = await directory(t); + await executeDownload({ + ...options, + directory: dir, + adapter: adapter([imageBlock(1), imageBlock(2)], { + async openAsset(url) { + return Readable.from([`bytes:${url}`]); + }, + }), + }); + const [first, second] = (await manifest(dir)).files; + // Simulate a run killed after staging both files of a swap. + const moves = [ + { record: first, from: first.filename, to: second.filename }, + { record: second, from: second.filename, to: first.filename }, + ].map((move) => ({ + ...move, + staged: `.arena-move-${crypto.randomUUID()}`, + })); + for (const move of moves) + await rename(join(dir, move.from), join(dir, move.staged)); + await writeFile(join(dir, ".arena-moves.json"), JSON.stringify({ moves })); + const result = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([imageBlock(2), imageBlock(1)], { + async openAsset() { + throw new ArenaError("gone", 404); + }, + }), + }); + assert.equal(result.failed, 0); + assert.equal(result.downloaded, 0); + assert.equal(result.unchanged, 2); + assert.equal( + await readFile(join(dir, filename(imageBlock(2), 1)), "utf8"), + "bytes:https://cdn.example/2/original.png", + ); + assert.equal( + await readFile(join(dir, filename(imageBlock(1), 2)), "utf8"), + "bytes:https://cdn.example/1/original.png", + ); + assert.ok( + !(await readdir(dir)).some((name) => name.startsWith(".arena-move")), + ); + assert.ok(!(await readdir(dir)).includes(".arena-moves.json")); +}); + +test("a rename stopped between any link() and rm() is recovered", async (t) => { + const assets = { + async openAsset(url: string) { + return Readable.from([`bytes:${url}`]); + }, + }; + const gone = { + async openAsset(): Promise<Readable> { + throw new ArenaError("gone", 404); + }, + }; + const staged = () => `.arena-move-${crypto.randomUUID()}`; + const scenarios = [ + { + name: "swap", + before: [imageBlock(1), imageBlock(2)], + after: [imageBlock(2), imageBlock(1)], + expected: { + [filename(imageBlock(2), 1)]: + "bytes:https://cdn.example/2/original.png", + [filename(imageBlock(1), 2)]: + "bytes:https://cdn.example/1/original.png", + }, + journal: ([first, second]: ManagedFile[]) => [ + { + record: first, + from: first.filename, + to: second.filename, + staged: staged(), + }, + { + record: second, + from: second.filename, + to: first.filename, + staged: staged(), + }, + ], + }, + { + name: "displacement", + before: [ + { ...textBlock(11, "removed"), title: "Same" }, + { ...textBlock(12, "kept"), title: "Same" }, + ], + after: [{ ...textBlock(12, "kept"), title: "Same" }], + expected: { "1_same.txt": "kept", "1_same~block-11.txt": "removed" }, + journal: ([removed, kept]: ManagedFile[]) => [ + { record: removed, from: removed.filename, to: "1_same~block-11.txt" }, + { + record: kept, + from: kept.filename, + to: removed.filename, + staged: staged(), + }, + ], + }, + ]; + // Replays a run's renames: displacements, then staging, then publishing. + const steps = ( + moves: Array<{ from: string; to: string; staged?: string }>, + ) => { + const all: Array<[string, string?]> = []; + for (const move of moves.filter((move) => !move.staged)) + all.push([move.from, move.to], [move.from]); + for (const move of moves.filter((move) => move.staged)) + all.push([move.from, move.staged], [move.from]); + for (const move of moves.filter((move) => move.staged)) + all.push([move.staged!, move.to], [move.staged!]); + return all; + }; + for (const scenario of scenarios) { + for (let stop = 0, total = 0; stop <= total; stop++) { + const dir = await directory(t); + await executeDownload({ + ...options, + directory: dir, + adapter: adapter(scenario.before, assets), + }); + // Records are saved in completion order; the journals expect channel order. + const records = (await manifest(dir)).files.sort( + (a, b) => Number(a.item.split("-")[1]) - Number(b.item.split("-")[1]), + ); + const moves = scenario.journal(records); + total = steps(moves).length; + const label = `${scenario.name}, stopped after ${stop} of ${total} steps`; + await writeFile( + join(dir, ".arena-moves.json"), + JSON.stringify({ moves }), + ); + for (const [from, to] of steps(moves).slice(0, stop)) + if (to) await link(join(dir, from), join(dir, to)); + else await rm(join(dir, from)); + const recovered = await executeDownload({ + ...options, + directory: dir, + adapter: adapter(scenario.after, gone), + }); + assert.equal(recovered.failed, 0, label); + assert.equal(recovered.downloaded, 0, label); + const names = (await readdir(dir)).filter( + (name) => !name.startsWith(".") && name.includes("_"), + ); + assert.deepEqual( + names.sort(), + Object.keys(scenario.expected).sort(), + label, + ); + for (const [name, text] of Object.entries(scenario.expected)) + assert.equal(await readFile(join(dir, name), "utf8"), text, label); + assert.ok( + !(await readdir(dir)).some((name) => name.startsWith(".arena-move")), + label, + ); + const resync = await executeDownload({ + ...options, + directory: dir, + adapter: adapter(scenario.after, gone), + }); + assert.equal(resync.failed, 0, label); + assert.equal(resync.unchanged, scenario.after.length, label); + } + } +}); + +test("an edit that preserves size and mtime is never pruned or replaced", async (t) => { + const dir = await directory(t); + const one = textBlock(1, "one"); + const two = textBlock(2, "two"); + await executeDownload({ + ...options, + directory: dir, + adapter: adapter([one, two]), + }); + const pinned = new Date(1_000_000_000_000); + for (const position of [1, 2]) + await utimes(join(dir, `${position}_note.txt`), pinned, pinned); + // Record the pinned mtime so the size-and-mtime fast path matches. + await executeDownload({ + ...options, + directory: dir, + adapter: adapter([one, two]), + }); + assert.ok( + (await manifest(dir)).files.every( + (file) => file.mtime_ms === pinned.getTime(), + ), + ); + for (const [position, edit] of [ + [1, "ONE"], + [2, "TWO"], + ] as const) { + await writeFile(join(dir, `${position}_note.txt`), edit); + await utimes(join(dir, `${position}_note.txt`), pinned, pinned); + } + const pruned = await executeDownload({ + ...options, + directory: dir, + prune: true, + adapter: adapter([one]), + }); + assert.equal(pruned.pruned, 0); + assert.match(pruned.failures[0].error, /modified/); + const replaced = await executeDownload({ + ...options, + directory: dir, + adapter: adapter([textBlock(1, "remote")]), + }); + assert.equal(replaced.downloaded, 0); + assert.match(replaced.failures[0].error, /modified/); + assert.equal(await readFile(join(dir, "1_note.txt"), "utf8"), "ONE"); + assert.equal(await readFile(join(dir, "2_note.txt"), "utf8"), "TWO"); +}); diff --git a/src/lib/download.ts b/src/lib/download.ts new file mode 100644 index 0000000..fd4b3a8 --- /dev/null +++ b/src/lib/download.ts @@ -0,0 +1,1327 @@ +import { createReadStream, createWriteStream, type Stats } from "node:fs"; +import { + link, + lstat, + mkdir, + open, + readdir, + readFile, + rename, + rm, + writeFile, +} from "node:fs/promises"; +import { createHash, randomUUID } from "node:crypto"; +import { hostname } from "node:os"; +import { basename, dirname, extname, join, resolve } from "node:path"; +import { Readable, Transform } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { setTimeout as sleep } from "node:timers/promises"; +import { ZipFile } from "yazl"; +import { ArenaError, client, getData } from "../api/client"; +import type { Channel, Connectable, PaginationMeta } from "../api/types"; +import { cancellationSignal, configuredTimeoutMs } from "./network"; +import { + arenaUrl, + escapeMarkup, + httpUrl, + safeFilename, + itemKey, + resolveTarget, + IMAGE_SIZES, + DOWNLOAD_TYPES, + type DownloadTarget, + type DownloadType, + type ImageSize, + type LinkFormat, +} from "./download-format"; + +export interface DownloadOptions { + channel: string; + directory?: string; + size: ImageSize; + linkFormat: LinkFormat; + concurrency: number; + type?: DownloadType; + overwrite: boolean; + prune: boolean; + zip?: string; +} +export interface DownloadAdapter { + getChannel(id: string, signal: AbortSignal): Promise<Channel>; + listContents( + id: string, + page: number, + signal: AbortSignal, + ): Promise<{ data: Connectable[]; meta: PaginationMeta }>; + openAsset(url: string, signal: AbortSignal): Promise<Readable>; + sleep(ms: number, signal: AbortSignal): Promise<void>; +} +export interface ManagedFile { + filename: string; + /** The block or channel the file was exported from, e.g. `block-123`. */ + item: string; + bytes: number; + sha256: string; + source_key: string; + /** Lets later syncs skip hashing files whose size and mtime are unchanged. */ + mtime_ms?: number; +} +export interface DownloadEntry { + item: Connectable; + filename: string | null; + status: + | "pending" + | "downloaded" + | "unchanged" + | "filtered" + | "metadata_only" + | "failed"; + error?: string; +} +export interface DownloadManifest { + version: 1; + channel: Channel; + exported_at: string; + complete: boolean; + options: { size: ImageSize; link_format: LinkFormat; type?: DownloadType }; + entries: DownloadEntry[]; + /** Includes retained files from earlier runs so later --prune can find them. */ + files: ManagedFile[]; +} +export interface DownloadSummary { + channel: string; + directory: string; + listed: number; + downloaded: number; + unchanged: number; + failed: number; + pruned: number; + /** Unchanged files moved because their position or title changed. */ + renamed: number; + retained: number; + metadata_only: number; + cancelled: boolean; + zip?: string; + failures: Array<{ file: string; error: string }>; +} +export type DownloadEvent = + | { type: "list_started"; channel: string } + | { type: "list_progress"; fetched: number; total: number } + | { type: "list_completed"; total: number } + | { + type: "file_progress"; + file: string; + status: DownloadEntry["status"]; + completed: number; + total: number; + } + | { type: "file_failed"; file: string; error: string } + | { type: "file_renamed"; file: string; from: string } + /** A file of another block was moved aside to make room; --prune removes it. */ + | { type: "file_retained"; file: string; from: string } + | { type: "file_pruned"; file: string } + | { type: "archive_started"; path: string } + | { type: "completed"; summary: DownloadSummary }; + +export function validateDownloadOptions(options: DownloadOptions): void { + if (!options.channel.trim()) + throw new Error("A channel ID or slug is required"); + if (!IMAGE_SIZES.includes(options.size)) + throw new Error(`Invalid --size: ${options.size}`); + if (!["webloc", "url"].includes(options.linkFormat)) + throw new Error("--link-format must be webloc or url"); + if (options.type && !DOWNLOAD_TYPES.includes(options.type)) + throw new Error(`Invalid --type: ${options.type}`); + if ( + !Number.isInteger(options.concurrency) || + options.concurrency < 1 || + options.concurrency > 16 + ) + throw new Error("--concurrency must be an integer from 1 to 16"); + if (options.prune && options.type) + throw new Error( + "--prune cannot be combined with --type; pruning requires a full channel sync", + ); +} + +export function defaultDownloadAdapter(): DownloadAdapter { + return { + getChannel: (id, signal) => + getData( + client.GET("/v3/channels/{id}", { params: { path: { id } }, signal }), + ), + listContents: (id, page, signal) => + getData( + client.GET("/v3/channels/{id}/contents", { + params: { + path: { id }, + query: { page, per: 100, sort: "position_asc" }, + }, + signal, + }), + ), + async openAsset(url, signal) { + // Use a standalone request: never forward the API's bearer token to assets. + const response = await fetch(url, { + signal, + headers: { + "User-Agent": "@aredotna/cli (arena download)", + "Accept-Encoding": "identity", + }, + }); + if (response.status !== 200 || !response.body) { + await response.body?.cancel(); + const raw = response.headers.get("retry-after"); + const seconds = raw === null ? NaN : Number(raw); + const delay = Number.isFinite(seconds) + ? seconds * 1000 + : raw + ? Date.parse(raw) - Date.now() + : undefined; + throw new ArenaError( + `Asset returned HTTP ${response.status}`, + response.status, + { rateLimit: { retryAfterMs: delay } }, + ); + } + const expected = response.headers.get("content-encoding") + ? null + : response.headers.get("content-length"); + let bytes = 0; + const body = Readable.fromWeb( + response.body as Parameters<typeof Readable.fromWeb>[0], + ); + return Readable.from( + (async function* () { + for await (const chunk of body) { + bytes += chunk.length; + yield chunk; + } + if (bytes === 0) + throw new ArenaError("Asset returned an empty body", 502); + if (expected !== null && bytes !== Number(expected)) + throw new ArenaError("Asset download was truncated", 502); + })(), + ); + }, + sleep: async (ms, signal) => { + await sleep(ms, undefined, { signal }); + }, + }; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} +function isTransient(error: unknown): boolean { + if (error instanceof ArenaError) + return [202, 408, 425, 429].includes(error.status) || error.status >= 500; + if (!(error instanceof Error)) return false; + const code = (error as NodeJS.ErrnoException).code; + return ( + error.name === "TimeoutError" || + error instanceof TypeError || + [ + "ECONNRESET", + "ETIMEDOUT", + "EAI_AGAIN", + "ECONNREFUSED", + "ERR_STREAM_PREMATURE_CLOSE", + ].includes(code ?? "") || + /fetch failed|terminated|socket/i.test(error.message) + ); +} +async function retry<T>( + run: () => Promise<T>, + adapter: DownloadAdapter, + signal: AbortSignal, +): Promise<T> { + for (let attempt = 0; ; attempt++) { + signal.throwIfAborted(); + try { + return await run(); + } catch (error) { + signal.throwIfAborted(); + if (attempt >= 2 || !isTransient(error)) throw error; + const requested = + error instanceof ArenaError ? error.rateLimit?.retryAfterMs : undefined; + // If the server asks for a long pause, surface it instead of retrying early. + if (requested !== undefined && requested > 60_000) throw error; + await adapter.sleep( + Math.max( + 300 * 2 ** attempt, + Number.isFinite(requested) ? requested! : 0, + ), + signal, + ); + } + } +} + +async function fileInfo(path: string) { + try { + return await lstat(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } +} +async function digestFile( + path: string, + signal?: AbortSignal, +): Promise<{ bytes: number; sha256: string }> { + const hash = createHash("sha256"); + let bytes = 0; + const stream = createReadStream(path, { signal }); + for await (const chunk of stream) { + bytes += chunk.length; + hash.update(chunk); + } + return { bytes, sha256: hash.digest("hex") }; +} +/** + * Size and mtime are enough to skip an unchanged file, but anything that + * deletes or replaces a file must `verify` its hash: an edit can preserve both. + */ +async function matchesRecord( + path: string, + info: Stats, + record: ManagedFile, + signal?: AbortSignal, + verify = false, +): Promise<boolean> { + if (info.size !== record.bytes) return false; + if (!verify && info.mtimeMs === record.mtime_ms) return true; + return (await digestFile(path, signal)).sha256 === record.sha256; +} +const NO_HARD_LINK_CODES = [ + "EPERM", + "ENOTSUP", + "EOPNOTSUPP", + "ENOSYS", + "EXDEV", +]; +/** Move a finished temporary file into place without replacing a file created meanwhile. */ +async function publishExclusive( + temporary: string, + destination: string, +): Promise<void> { + const conflict = () => + new Error( + `${basename(destination)} was created by another process; rerun to resolve it`, + ); + try { + await link(temporary, destination); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code ?? ""; + if (code === "EEXIST") throw conflict(); + if (!NO_HARD_LINK_CODES.includes(code)) throw error; + // Filesystems such as exFAT lack hard links; a checked rename is the closest equivalent. + if (await fileInfo(destination)) throw conflict(); + await rename(temporary, destination); + } +} +/** Rename without ever replacing an existing file. */ +async function moveExclusive(from: string, to: string): Promise<void> { + await publishExclusive(from, to); + await rm(from, { force: true }); +} +async function atomicText(path: string, text: string): Promise<void> { + const temporary = join(dirname(path), `.arena-${randomUUID()}.part`); + try { + await writeFile(temporary, text, { flag: "wx" }); + await rename(temporary, path); + } finally { + await rm(temporary, { force: true }); + } +} +const LOCK_FILENAME = ".arena-download.lock"; +const TEMPORARY_FILENAME = /^\.arena-[0-9a-f-]{36}\.part$/; +function processIsRunning(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} +/** Take the directory lock, reclaiming one left by a crashed process on this machine. */ +async function acquireLock(lockPath: string) { + const owner = `${process.pid}@${hostname()}`; + for (let attempt = 0; ; attempt++) { + try { + const lock = await open(lockPath, "wx"); + try { + await lock.writeFile(owner); + } catch (error) { + await lock.close(); + await rm(lockPath, { force: true }); + throw error; + } + return lock; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + const [pid, host] = ( + await readFile(lockPath, "utf8").catch(() => "") + ).split("@"); + const stale = + attempt === 0 && + host === hostname() && + Number.isSafeInteger(Number(pid)) && + Number(pid) > 0 && + !processIsRunning(Number(pid)); + if (!stale) + throw new Error( + `Export directory is locked by another sync. If none is running, remove ${lockPath} and retry.`, + ); + await rm(lockPath, { force: true }); + } + } +} +function validManagedFile(value: unknown): value is ManagedFile { + if (!value || typeof value !== "object") return false; + const file = value as Partial<ManagedFile>; + return ( + typeof file.filename === "string" && + /^\d+_/.test(file.filename) && + typeof file.item === "string" && + /^(block|channel)-\d+$/.test(file.item) && + !/[\\/\x00-\x1f]/.test(file.filename) && + file.filename === basename(file.filename) && + typeof file.sha256 === "string" && + /^[a-f0-9]{64}$/.test(file.sha256) && + typeof file.bytes === "number" && + Number.isSafeInteger(file.bytes) && + file.bytes >= 0 && + typeof file.source_key === "string" && + (file.mtime_ms === undefined || Number.isFinite(file.mtime_ms)) + ); +} +async function readPrevious( + directory: string, + channel: Channel, +): Promise<ManagedFile[]> { + const path = join(directory, "manifest.json"); + const info = await fileInfo(path); + if (!info) { + for (const generated of ["index.html", csvFilename(channel)]) + if (await fileInfo(join(directory, generated))) + throw new Error( + `Destination has a ${generated} without an export manifest; choose another directory`, + ); + return []; + } + if (!info.isFile()) throw new Error("manifest.json must be a regular file"); + let previous: Partial<DownloadManifest> | null; + try { + previous = JSON.parse(await readFile(path, "utf8")); + } catch (error) { + if (!(error instanceof SyntaxError)) throw error; + throw new Error( + "manifest.json is not valid JSON; restore it or choose another directory", + ); + } + if ( + !previous || + previous.version !== 1 || + previous.channel?.id !== channel.id || + !Array.isArray(previous.files) || + !previous.files.every(validManagedFile) + ) { + throw new Error( + "Destination contains an incompatible manifest or belongs to another channel; choose another directory", + ); + } + return previous.files; +} +function sourceKey(item: Connectable, target: DownloadTarget): string { + const revision = + item.type === "Image" + ? (item.image.updated_at ?? item.updated_at) + : item.type === "Attachment" + ? (item.attachment.updated_at ?? item.updated_at) + : ""; + return createHash("sha256") + .update( + target.kind === "text" ? target.content : `${target.url}\n${revision}`, + ) + .digest("hex"); +} + +async function syncTarget( + target: DownloadTarget, + item: Connectable, + directory: string, + previous: ManagedFile | undefined, + options: DownloadOptions, + adapter: DownloadAdapter, + signal: AbortSignal, +): Promise<{ status: "downloaded" | "unchanged"; file: ManagedFile }> { + const destination = join(directory, target.filename); + const info = await fileInfo(destination); + if (info && !info.isFile()) + throw new Error("Destination is not a regular file"); + const key = sourceKey(item, target); + const owner = itemKey(item); + const modified = () => + new Error("Local file has been modified; use --overwrite to replace it"); + if ( + info && + previous?.item === owner && + previous.source_key === key && + !options.overwrite && + (await matchesRecord(destination, info, previous, signal)) + ) + return { + status: "unchanged", + file: { ...previous, mtime_ms: info.mtimeMs }, + }; + // Files of other blocks were moved aside already; one still here was edited. + if ( + info && + previous && + !options.overwrite && + (previous.item !== owner || + !(await matchesRecord(destination, info, previous, signal, true))) + ) + throw modified(); + + return retry( + async () => { + const temporary = join(directory, `.arena-${randomUUID()}.part`); + // Large files may legitimately take a long time, so time out only when data stops arriving. + const idleMs = configuredTimeoutMs(); + const stalled = new AbortController(); + let idleTimer: NodeJS.Timeout | undefined; + const resetIdleTimer = () => { + clearTimeout(idleTimer); + idleTimer = setTimeout( + () => + stalled.abort( + new DOMException( + `Transfer stalled for ${idleMs}ms`, + "TimeoutError", + ), + ), + idleMs, + ); + }; + const transferSignal = AbortSignal.any([signal, stalled.signal]); + try { + resetIdleTimer(); + let bytes = 0; + const hash = createHash("sha256"); + const meter = new Transform({ + transform(chunk, _encoding, callback) { + resetIdleTimer(); + bytes += chunk.length; + hash.update(chunk); + callback(null, chunk); + }, + }); + const source = + target.kind === "text" + ? Readable.from([Buffer.from(target.content)]) + : await adapter.openAsset(target.url, transferSignal); + await pipeline( + source, + meter, + createWriteStream(temporary, { flags: "wx" }), + { signal: transferSignal }, + ); + clearTimeout(idleTimer); + if (target.kind === "fetch" && bytes === 0) + throw new ArenaError("Asset returned an empty body", 502); + const sha256 = hash.digest("hex"); + const record = (mtimeMs: number): ManagedFile => ({ + filename: target.filename, + bytes, + sha256, + item: itemKey(item), + source_key: key, + mtime_ms: mtimeMs, + }); + signal.throwIfAborted(); + // Recover completed files left by an interrupted run, but never adopt different bytes. + if (info && !previous && !options.overwrite) { + if ((await digestFile(destination, signal)).sha256 !== sha256) + throw new Error( + "An unmanaged file already exists; use --overwrite or another directory", + ); + return { status: "unchanged" as const, file: record(info.mtimeMs) }; + } + if (info) { + // Check again before replacing: a local edit made during transfer wins. + const current = await fileInfo(destination); + const unchanged = + current?.isFile() && + (previous && !options.overwrite + ? await matchesRecord( + destination, + current, + previous, + signal, + true, + ) + : current.size === info.size && current.mtimeMs === info.mtimeMs); + if (!unchanged) + throw new Error( + "Local file changed during download; rerun to resolve it", + ); + await rename(temporary, destination); + } else { + await publishExclusive(temporary, destination); + } + const published = await lstat(destination); + return { + status: "downloaded" as const, + file: record(published.mtimeMs), + }; + } catch (error) { + if (transferSignal.aborted && !signal.aborted) + throw transferSignal.reason; + throw error; + } finally { + clearTimeout(idleTimer); + await rm(temporary, { force: true }); + } + }, + adapter, + signal, + ); +} + +const MOVE_JOURNAL = ".arena-moves.json"; +const STAGED_FILENAME = /^\.arena-move-[0-9a-f-]{36}$/; + +interface Move { + record: ManagedFile; + from: string; + to: string; + /** Moves of files trading places pass through a staged name. */ + staged?: string; +} + +/** A name that no export target can take: targets never contain "~". */ +async function retainedName( + directory: string, + record: ManagedFile, + taken: (name: string) => boolean, +): Promise<string> { + const extension = extname(record.filename); + const stem = record.filename.slice(0, -extension.length || undefined); + for (let n = 1; ; n++) { + const name = `${stem}~${record.item}${n > 1 ? `-${n}` : ""}${extension}`; + if (!taken(name) && !(await fileInfo(join(directory, name)))) return name; + } +} + +function validMove(value: unknown): value is Move { + if (!value || typeof value !== "object") return false; + const move = value as Partial<Move>; + return ( + validManagedFile(move.record) && + move.from === move.record.filename && + typeof move.to === "string" && + validManagedFile({ ...move.record, filename: move.to }) && + (move.staged === undefined || + (typeof move.staged === "string" && STAGED_FILENAME.test(move.staged))) + ); +} + +/** + * Another name for the same file, left by a move stopped between `link()` and + * `rm()`: the move's source or destination, or a retained name it fell back to. + */ +async function linkedName( + directory: string, + name: string, + move: Move, +): Promise<string | undefined> { + const info = await fileInfo(join(directory, name)); + if (!info?.isFile() || info.nlink < 2) return undefined; + const extension = extname(move.from); + const retained = `${move.from.slice(0, -extension.length || undefined)}~${move.record.item}`; + const candidates = [ + move.from, + move.to, + ...(await readdir(directory)).filter((other) => other.startsWith(retained)), + ]; + for (const other of candidates) { + if (other === name) continue; + const twin = await fileInfo(join(directory, other)); + if (twin?.isFile() && twin.dev === info.dev && twin.ino === info.ino) + return other; + } + return undefined; +} + +/** + * Finish moves interrupted by a terminated run, locating each file by its + * hash. Returns whether a journal was found; the caller saves the manifest and + * then removes the journal. + */ +async function recoverMoves( + directory: string, + files: Map<string, ManagedFile>, +): Promise<boolean> { + const path = join(directory, MOVE_JOURNAL); + if (!(await fileInfo(path))) return false; + let moves: unknown; + try { + moves = (JSON.parse(await readFile(path, "utf8")) as { moves?: unknown }) + .moves; + } catch { + moves = undefined; + } + if (!Array.isArray(moves) || !moves.every(validMove)) + throw new Error( + `${MOVE_JOURNAL} is damaged; restore it or choose another directory`, + ); + const holds = async (name: string, record: ManagedFile) => { + const info = await fileInfo(join(directory, name)); + return Boolean( + info?.isFile() && + (await digestFile(join(directory, name))).sha256 === record.sha256, + ); + }; + for (const move of moves) { + for (const name of [move.from, move.to]) { + const current = files.get(name); + if ( + current?.item === move.record.item && + current.sha256 === move.record.sha256 + ) + files.delete(name); + } + let location: string | undefined; + const staged = move.staged && join(directory, move.staged); + if (staged && (await fileInfo(staged))) { + const twin = await linkedName(directory, move.staged!, move); + if (twin) { + // Staging had not finished or publishing had: keep the other name. + await rm(staged, { force: true }); + location = twin; + } else { + location = !(await fileInfo(join(directory, move.to))) + ? move.to + : !(await fileInfo(join(directory, move.from))) + ? move.from + : await retainedName(directory, move.record, (name) => + files.has(name), + ); + await moveExclusive(staged, join(directory, location)); + } + } else if ((await linkedName(directory, move.from, move)) === move.to) { + await rm(join(directory, move.from), { force: true }); + location = move.to; + } else if (await holds(move.to, move.record)) location = move.to; + else if (await holds(move.from, move.record)) location = move.from; + if (location) files.set(location, { ...move.record, filename: location }); + } + return true; +} + +/** + * Filenames start with the channel position, so reordering, removing, or + * retitling blocks changes them. Move unchanged files to their new names + * instead of downloading them again, and move files of other blocks out of the + * way under a retained name so that nothing is replaced without --prune. + * + * Everything is verified before the first rename, and the plan is journaled so + * an interrupted run can be completed. Returns whether anything moved; the + * caller saves the manifest and then removes the journal. + */ +async function reconcileNames( + directory: string, + planned: Array<{ item: Connectable; target: DownloadTarget }>, + files: Map<string, ManagedFile>, + signal: AbortSignal, + onMove: (move: Move, retained: boolean) => void, +): Promise<boolean> { + const unchanged = async (name: string, record: ManagedFile) => { + const info = await fileInfo(join(directory, name)); + return Boolean( + info?.isFile() && + (await matchesRecord(join(directory, name), info, record, signal)), + ); + }; + const moving = new Map<string, Move>(); + for (const { item, target } of planned) { + const owner = itemKey(item); + if (files.get(target.filename)?.item === owner) continue; + const key = sourceKey(item, target); + const record = [...files.values()].find( + (file) => + file.item === owner && + file.source_key === key && + !moving.has(file.filename), + ); + if (record && (await unchanged(record.filename, record))) + moving.set(record.filename, { + record, + from: record.filename, + to: target.filename, + }); + } + const displaced: Move[] = []; + const reserved = new Set<string>(); + for (const { item, target } of planned) { + const occupant = files.get(target.filename); + if ( + !occupant || + occupant.item === itemKey(item) || + moving.has(occupant.filename) || + !(await unchanged(occupant.filename, occupant)) + ) + continue; + const to = await retainedName( + directory, + occupant, + (name) => files.has(name) || reserved.has(name), + ); + reserved.add(to); + displaced.push({ record: occupant, from: occupant.filename, to }); + } + // Moving onto an unmanaged or edited file would destroy it, so skip such + // moves, and any move that depended on a skipped one vacating its name. + let moves = [...moving.values()]; + for (let changed = true; changed; ) { + const vacated = new Set([ + ...moves.map((move) => move.from), + ...displaced.map((move) => move.from), + ]); + const kept: Move[] = []; + for (const move of moves) + if (vacated.has(move.to) || !(await fileInfo(join(directory, move.to)))) + kept.push(move); + changed = kept.length !== moves.length; + moves = kept; + } + if (!moves.length && !displaced.length) return false; + for (const move of moves) move.staged = `.arena-move-${randomUUID()}`; + await atomicText( + join(directory, MOVE_JOURNAL), + JSON.stringify({ moves: [...displaced, ...moves] }, null, 2) + "\n", + ); + // No cancellation checks from here: the renames are quick and all-or-nothing. + for (const move of displaced) { + await moveExclusive(join(directory, move.from), join(directory, move.to)); + files.delete(move.from); + files.set(move.to, { ...move.record, filename: move.to }); + onMove(move, true); + } + for (const move of moves) { + await moveExclusive( + join(directory, move.from), + join(directory, move.staged!), + ); + files.delete(move.from); + } + for (const move of moves) { + // Something created at the destination meanwhile is kept; so is this file. + const to = (await fileInfo(join(directory, move.to))) + ? await retainedName(directory, move.record, (name) => files.has(name)) + : move.to; + await moveExclusive(join(directory, move.staged!), join(directory, to)); + files.set(to, { ...move.record, filename: to }); + onMove({ ...move, to }, to !== move.to); + } + return true; +} + +function sourceUrl(item: Connectable): string | null { + if (item.type === "Channel" || !item.source?.url) return null; + try { + return httpUrl(item.source.url); + } catch { + return null; + } +} + +function csvFilename(channel: Channel): string { + return `${safeFilename(channel.slug)}.csv`; +} + +function csvField(value: string | number | null | undefined): string { + const text = value?.toString() ?? ""; + return /[",\r\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text; +} + +/** The same columns as Are.na's web channel export. */ +export function buildDownloadCsv(manifest: DownloadManifest): string { + const header = [ + "ID", + "Filename", + "Title", + "Description", + "Created At", + "Updated At", + "Source", + ]; + const rows = manifest.entries + .filter((entry) => entry.filename) + .map(({ item, filename }) => [ + item.id, + filename, + item.title, + item.description?.markdown, + item.created_at, + item.updated_at, + item.type === "Channel" ? null : item.source?.url, + ]); + return [header, ...rows].map((row) => row.map(csvField).join(",")).join("\n"); +} + +export function buildDownloadIndex(manifest: DownloadManifest): string { + const esc = escapeMarkup; + const items = manifest.entries + .map((entry) => { + const item = entry.item; + const local = + entry.filename && ["downloaded", "unchanged"].includes(entry.status) + ? encodeURIComponent(entry.filename) + : null; + const source = sourceUrl(item); + const text = + item.type === "Text" + ? `<pre>${esc(item.content.plain ?? item.content.markdown ?? "")}</pre>` + : ""; + const image = + item.type === "Image" && local + ? `<a href="${local}"><img loading="lazy" src="${local}" alt="${esc(item.image.alt_text ?? item.title ?? "")}"></a>` + : ""; + const description = item.description?.plain + ? `<p>${esc(item.description.plain)}</p>` + : ""; + return `<li><article><h2>${esc(item.title || `${item.type} ${item.id}`)}</h2><p class="meta">${esc(item.type)} · ${esc(entry.status)} · <a href="${esc(arenaUrl(item))}">View on Are.na</a>${source ? ` · <a href="${esc(source)}">Source</a>` : ""}${local ? ` · <a href="${local}">Open local file</a>` : ""}</p>${image}${text}${description}${entry.error ? `<p class="error">${esc(entry.error)}</p>` : ""}</article></li>`; + }) + .join("\n"); + return `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><meta http-equiv="Content-Security-Policy" content="default-src 'none'; img-src 'self' file:; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'"><title>${esc(manifest.channel.title)}

${esc(manifest.channel.title)}

${esc(manifest.channel.description?.plain ?? "")}

Exported ${esc(manifest.exported_at)} · ${manifest.entries.length} items · ${manifest.complete ? "Complete" : "Incomplete"} · JSON manifest

    ${items}
\n`; +} + +/** Whether an existing file at the ZIP path may be replaced; throws if it must be protected. */ +async function archiveReplaceable( + path: string, + overwrite: boolean, +): Promise { + const info = await fileInfo(path); + if (!info) return false; + if (!info.isFile()) throw new Error("--zip must point to a file"); + if (overwrite) return true; + const handle = await open(path, "r"); + try { + const { buffer, bytesRead } = await handle.read(Buffer.alloc(4), 0, 4, 0); + const magic = buffer.subarray(0, bytesRead).toString("latin1"); + if (magic === "PK\x03\x04" || magic === "PK\x05\x06") return true; + } finally { + await handle.close(); + } + throw new Error( + "--zip path exists and is not a ZIP archive; use --overwrite or choose another path", + ); +} + +async function createArchive( + path: string, + directory: string, + filenames: string[], + replace: boolean, + signal: AbortSignal, +): Promise { + await mkdir(dirname(path), { recursive: true }); + // Distinct from download temporaries, which a sync of this directory may clean up. + const temporary = join(dirname(path), `.arena-zip-${randomUUID()}.part`); + const zip = new ZipFile(); + const output = zip.outputStream as Readable; + let activeInput: Readable | undefined; + zip.on("error", (error: Error) => output.destroy(error)); + const abort = () => { + activeInput?.destroy(); + output.destroy(new Error("Archive cancelled")); + }; + signal.addEventListener("abort", abort, { once: true }); + const writing = pipeline( + output, + createWriteStream(temporary, { flags: "wx" }), + { signal }, + ); + // Attach immediately so an asynchronous file error cannot become an unhandled rejection. + void writing.catch(() => {}); + try { + for (const filename of filenames) { + zip.addReadStreamLazy( + filename, + { compress: /\.(json|html|csv|txt|url|webloc)$/.test(filename) }, + (callback) => { + if (signal.aborted) { + callback(new Error("Archive cancelled"), Readable.from([])); + return; + } + activeInput = createReadStream(join(directory, filename), { signal }); + callback(null, activeInput); + }, + ); + } + zip.end(); + await writing; + signal.throwIfAborted(); + if (replace) await rename(temporary, path); + else await publishExclusive(temporary, path); + } finally { + signal.removeEventListener("abort", abort); + activeInput?.destroy(); + output.destroy(); + await writing.catch(() => {}); + await rm(temporary, { force: true }); + } +} + +export function downloadExitCode(summary: DownloadSummary): number { + return summary.cancelled ? 130 : summary.failed > 0 ? 1 : 0; +} + +async function listContents( + channel: Channel, + adapter: DownloadAdapter, + signal: AbortSignal, + onPage?: (fetched: number, total: number) => void, +): Promise { + const contents: Connectable[] = []; + const seen = new Set(); + let page = 1; + while (true) { + const response = await retry( + () => adapter.listContents(String(channel.id), page, signal), + adapter, + signal, + ); + for (const item of response.data) { + const key = itemKey(item); + if (seen.has(key)) + throw new Error( + "Channel contents changed while listing; rerun the sync", + ); + seen.add(key); + contents.push(item); + } + onPage?.(contents.length, response.meta.total_count); + if (!response.meta.has_more_pages) { + if (contents.length !== response.meta.total_count) + throw new Error( + "Channel contents changed while listing; rerun the sync", + ); + break; + } + const next = response.meta.next_page ?? page + 1; + if (response.data.length === 0 || next <= page) + throw new Error( + "Invalid channel pagination; export stopped before modifying files", + ); + page = next; + } + return contents; +} + +export async function executeDownload( + options: DownloadOptions & { + adapter?: DownloadAdapter; + signal?: AbortSignal; + onEvent?: (event: DownloadEvent) => void; + }, +): Promise { + validateDownloadOptions(options); + const adapter = options.adapter ?? defaultDownloadAdapter(); + const signal = options.signal ?? cancellationSignal(); + const emit = (event: DownloadEvent) => options.onEvent?.(event); + signal.throwIfAborted(); + const archivePath = options.zip ? resolve(options.zip) : undefined; + const replaceArchive = archivePath + ? await archiveReplaceable(archivePath, options.overwrite) + : false; + emit({ type: "list_started", channel: options.channel }); + const channel = await retry( + () => adapter.getChannel(options.channel, signal), + adapter, + signal, + ); + const directory = resolve(options.directory ?? safeFilename(channel.slug)); + const contents = await listContents( + channel, + adapter, + signal, + (fetched, total) => emit({ type: "list_progress", fetched, total }), + ); + // Reject a moving snapshot before any disk changes, especially before pruning. + const currentChannel = await retry( + () => adapter.getChannel(String(channel.id), signal), + adapter, + signal, + ); + if (currentChannel.updated_at !== channel.updated_at) + throw new Error("Channel changed while listing; rerun the sync"); + signal.throwIfAborted(); + await mkdir(directory, { recursive: true }); + const lockPath = join(directory, LOCK_FILENAME); + const lock = await acquireLock(lockPath); + try { + // Holding the lock means any temporary files here were left by a terminated run. + for (const name of await readdir(directory)) + if (TEMPORARY_FILENAME.test(name)) + await rm(join(directory, name), { force: true }); + const previous = await readPrevious(directory, channel); + const files = new Map(previous.map((file) => [file.filename, file])); + const manifest: DownloadManifest = { + version: 1, + channel, + exported_at: new Date().toISOString(), + complete: false, + options: { + size: options.size, + link_format: options.linkFormat, + type: options.type, + }, + entries: [], + files: previous, + }; + const summary: DownloadSummary = { + channel: channel.slug, + directory, + listed: contents.length, + downloaded: 0, + unchanged: 0, + failed: 0, + pruned: 0, + renamed: 0, + retained: 0, + metadata_only: 0, + cancelled: false, + failures: [], + }; + const targets = new Map(); + for (const [index, item] of contents.entries()) { + const entry: DownloadEntry = { item, filename: null, status: "pending" }; + manifest.entries.push(entry); + if (options.type && item.type !== options.type) { + entry.status = "filtered"; + continue; + } + try { + const target = resolveTarget(item, index + 1, options); + if (target) { + entry.filename = target.filename; + targets.set(entry, target); + } else { + entry.status = "metadata_only"; + summary.metadata_only++; + } + } catch (error) { + entry.status = "failed"; + entry.error = errorMessage(error); + } + } + const activeNames = new Set( + [...targets.values()].map((target) => target.filename), + ); + if ( + archivePath && + (((activeNames.has(basename(archivePath)) || + files.has(basename(archivePath))) && + dirname(archivePath) === directory) || + [ + join(directory, "manifest.json"), + join(directory, "index.html"), + join(directory, csvFilename(channel)), + join(directory, MOVE_JOURNAL), + lockPath, + ].includes(archivePath)) + ) + throw new Error("ZIP path conflicts with an export file"); + const saveManifest = async () => { + manifest.files = [...files.values()]; + await atomicText( + join(directory, "manifest.json"), + JSON.stringify(manifest, null, 2) + "\n", + ); + }; + // The journal may only be removed once the manifest records where files are. + if (await recoverMoves(directory, files)) { + await saveManifest(); + await rm(join(directory, MOVE_JOURNAL)); + } + const moved = await reconcileNames( + directory, + [...targets].map(([entry, target]) => ({ item: entry.item, target })), + files, + signal, + (move, retained) => { + if (!retained) summary.renamed++; + emit({ + type: retained ? "file_retained" : "file_renamed", + file: move.to, + from: move.from, + }); + }, + ); + await saveManifest(); + if (moved) await rm(join(directory, MOVE_JOURNAL)); + emit({ type: "list_completed", total: contents.length }); + const recordFailure = (file: string, error: string) => { + summary.failed++; + summary.failures.push({ file, error }); + emit({ type: "file_failed", file, error }); + }; + let cursor = 0; + let completed = 0; + const workers = Array.from( + { length: Math.min(options.concurrency, contents.length) }, + async () => { + while (!signal.aborted) { + const entry = manifest.entries[cursor++]; + if (!entry) return; + const target = targets.get(entry); + if (target) { + try { + const result = await syncTarget( + target, + entry.item, + directory, + files.get(target.filename), + options, + adapter, + signal, + ); + entry.status = result.status; + files.set(target.filename, result.file); + summary[result.status]++; + } catch (error) { + if (signal.aborted) return; + entry.status = "failed"; + entry.error = errorMessage(error); + } + } + if (entry.status === "failed") + recordFailure( + entry.filename ?? `${entry.item.type} ${entry.item.id}`, + entry.error!, + ); + emit({ + type: "file_progress", + file: entry.filename ?? `${entry.item.type} ${entry.item.id}`, + status: entry.status, + completed: ++completed, + total: contents.length, + }); + } + }, + ); + await Promise.all(workers); + summary.cancelled = signal.aborted; + // Never remove files following an incomplete sync or a channel modified during downloads. + if (options.prune && !summary.cancelled && summary.failed === 0) { + const latest = await retry( + () => adapter.getChannel(String(channel.id), signal), + adapter, + signal, + ).catch((error) => { + recordFailure("prune", errorMessage(error)); + return null; + }); + if (latest && latest.updated_at !== channel.updated_at) + recordFailure( + "prune", + "Channel changed during download; rerun before pruning", + ); + if (latest && summary.failed === 0) { + try { + const refreshed = await listContents(channel, adapter, signal); + if (JSON.stringify(refreshed) !== JSON.stringify(contents)) { + recordFailure( + "prune", + "Channel contents changed during download; rerun before pruning", + ); + } + } catch (error) { + if (!signal.aborted) recordFailure("prune", errorMessage(error)); + } + } + if (latest && summary.failed === 0 && !signal.aborted) { + for (const [filename, file] of files) { + if ( + activeNames.has(filename) || + contents.some( + (item) => + item.type === "PendingBlock" && file.item === itemKey(item), + ) + ) + continue; + try { + signal.throwIfAborted(); + const path = join(directory, filename); + const info = await fileInfo(path); + if (info) { + if ( + !info.isFile() || + !(await matchesRecord(path, info, file, signal, true)) + ) + throw new Error("Retained locally modified file"); + await rm(path); + summary.pruned++; + emit({ type: "file_pruned", file: filename }); + } + files.delete(filename); + } catch (error) { + if (signal.aborted) break; + recordFailure(filename, errorMessage(error)); + } + } + } + } + summary.cancelled = signal.aborted; + summary.retained = [...files.keys()].filter( + (filename) => !activeNames.has(filename), + ).length; + manifest.files = [...files.values()]; + manifest.complete = !summary.cancelled && summary.failed === 0; + await atomicText( + join(directory, "manifest.json"), + JSON.stringify(manifest, null, 2) + "\n", + ); + await atomicText( + join(directory, "index.html"), + buildDownloadIndex(manifest), + ); + await atomicText( + join(directory, csvFilename(channel)), + buildDownloadCsv(manifest), + ); + if (archivePath && !summary.cancelled) { + emit({ type: "archive_started", path: archivePath }); + const names = manifest.entries + .filter( + (entry) => + entry.filename && + ["downloaded", "unchanged"].includes(entry.status), + ) + .map((entry) => entry.filename!); + try { + await createArchive( + archivePath, + directory, + ["manifest.json", "index.html", csvFilename(channel), ...names], + replaceArchive, + signal, + ); + summary.zip = archivePath; + } catch (error) { + if (signal.aborted) summary.cancelled = true; + else recordFailure(archivePath, errorMessage(error)); + } + } + emit({ type: "completed", summary }); + return summary; + } finally { + await lock.close(); + await rm(lockPath, { force: true }); + } +} diff --git a/src/lib/registry.tsx b/src/lib/registry.tsx index 862d5d6..2e85691 100644 --- a/src/lib/registry.tsx +++ b/src/lib/registry.tsx @@ -76,6 +76,11 @@ import { } from "../commands/group"; import { LoginCommand } from "../commands/login"; import { LogoutCommand } from "../commands/logout"; +import { + DownloadCommand, + parseDownloadOptions, + runDownloadJsonStream, +} from "../commands/download"; import { ImportCommand, parseImportOptions, @@ -879,6 +884,28 @@ export const commands: CommandDefinition[] = [ }, }, + { + name: "download", + aliases: ["export", "sync"], + group: "Channels", + help: [ + { + usage: "download [directory] [flags]", + description: "Download or sync a channel to disk", + }, + { + usage: "download worldmaking --dir ./refs --zip ./refs.zip", + description: "Example", + }, + ], + render(args, flags) { + return ; + }, + async jsonStream(args, flags, write) { + return runDownloadJsonStream(parseDownloadOptions(args, flags), write); + }, + }, + { name: "import", aliases: [], @@ -2213,6 +2240,64 @@ export const commandHelpDocs: Record = { ], seeAlso: ["add", "import"], }, + download: { + summary: + "Download or sync a channel's files, text, links, and metadata. Aliases: export, sync.", + usage: ["arena download [directory] [flags]"], + options: [ + { + flag: "--dir ", + description: "Destination directory (default: channel slug)", + }, + { + flag: "--size ", + description: "original (default), large, medium, small, or square", + }, + { + flag: "--type ", + description: + "Image, Attachment, Text, Link, Embed, Channel, or PendingBlock", + }, + { + flag: "--link-format ", + description: "Link shortcut format (default: webloc)", + }, + { + flag: "--concurrency ", + description: "Concurrent transfers, 1–16 (default: 4)", + }, + { + flag: "--prune", + description: + "Remove unchanged managed files absent from this full export", + }, + { + flag: "--overwrite", + description: "Re-download everything and replace local edits", + }, + { + flag: "--zip ", + description: "Also write a ZIP (replaces an existing ZIP)", + }, + { flag: "--json", description: "Stream NDJSON progress events" }, + ], + examples: [ + "arena download worldmaking ./refs", + "arena sync worldmaking ./refs --prune", + "arena export worldmaking --zip ./worldmaking.zip", + "arena download worldmaking --type Image --size large --json", + ], + notes: [ + "Re-running syncs new and changed content; unchanged files are skipped.", + "Local edits are protected unless --overwrite is used. --prune always protects local edits.", + "A lock or partial files left by a killed sync are cleaned up on the next run.", + "Files are named _.<ext>, like Are.na's web channel export; reordered blocks are renamed, not re-downloaded.", + "Text, links, and connected-channel shortcuts are included by default; connected channels are not recursively exported.", + "manifest.json preserves channel and item metadata, order, connections, and file outcomes; <slug>.csv matches the web export; index.html is an offline browser.", + "Removed files are retained unless --prune is specified. --prune cannot be combined with --type.", + ], + seeAlso: ["channel", "import"], + }, import: { summary: "Bulk import local files into a channel.", usage: ["arena import <channel> [flags]"],