diff --git a/changes/feat-file-mirror.md b/changes/feat-file-mirror.md new file mode 100644 index 000000000..ca7d85e88 --- /dev/null +++ b/changes/feat-file-mirror.md @@ -0,0 +1,29 @@ +### Added + +- **`syncFileMirror` and the `FileTarget` port in `@wirecat/cli-core/documents`: two-way sync of plain files.** A CLI + for a file store (Yandex Disk next) keeps a local folder and a remote folder the same, whole files at a time, + binary and large files included, streamed both ways. A change on one side is copied, the same content on both is + adopted, different changes are a conflict that `keep` (`local`, `remote`, `both`) settles; deletions go to the + remote's trash and to `/.wirecat-trash/`, and a run that would delete more than 50 files or a quarter of the + mirror stops first. Every write is conditional on the last listing, so a crash at any step is settled by the next + run. `syncFileMirrors`, `fileSyncState`, `fileMirrorStatus` and `fileRemoteChanges` (the watch lists only when the + remote's revision moved) complete it ([DOCUMENTS.md](../docs/dev/DOCUMENTS.md#sync-plain-files-with-a-provider)). +- **A file mirror deletes only what it can prove was deleted.** The last sync saw the file, the side's listing is + complete, and the side's root is the same folder (the remote root's id; the local folder's device and inode, and + not empty while the state has files, as an unmounted drive is). Anything else is reported as `unknown` and the run + is partial. +- **`fileVersions`: a local version store for mirrors.** Before a sync overwrites or deletes a local file, its content + is kept by sha256 under the profile's state folder, with an append-only log of every change both ways; versions stay + 30 days, and a deleted file's last version stays for good. `restoreFileVersion` puts one back. +- **`watchMirrors` takes `localChanges` instead of `kindOf` and `stateOf`**, for an engine with its own state. A + document mirror passes `kindOf` and `stateOf` as before. +- **`scanLocal` reports each file's `size` and `mtimeMs`** and takes an optional lookup that reuses a known hash when + both match, so a sync does not read unchanged files again. `conflictReport` adds `local_size` and `_size` + when a conflict has them. A mirror guard's `action` can also be `delete` and `move`. + +### Changed — may break callers + +- **`PushConflict.url` is optional, and `MirrorWatchOptions` is a type, not an interface.** A file conflict has no + URL, so `conflictReport` leaves `url` out when there is none; a document conflict still has one. Code that reads + `conflict.url` as a string, or extends `MirrorWatchOptions` with `interface … extends`, needs a small change. drive + needs none. diff --git a/docs/dev/DOCUMENTS.md b/docs/dev/DOCUMENTS.md index 4e2f91359..0b48273b5 100644 --- a/docs/dev/DOCUMENTS.md +++ b/docs/dev/DOCUMENTS.md @@ -310,6 +310,81 @@ for file events the system dropped; and all of them at the full run (`fullMs`, 3 late. It yields one event per cycle with its triggers and mirrors, and a failed cycle as an error whose mirrors are tried again at the next poll. The CLI takes the lock inside `sync`, so a manual sync can run between cycles. + + +### Plain files, both ways + +A remote folder that keeps files as they are (Yandex Disk, a WebDAV share) uses the whole-file engine, +`syncFileMirror`, instead of `pushDocuments`: it copies bytes, never merges text, and reads the remote with one +listing per run instead of one read per file. The provider implements `FileTarget`: + +```ts +interface FileTarget { + identity(): Promise + list(root: string, signal?: AbortSignal): Promise<{ rootId: string; files: RemoteFile[]; complete: boolean } | undefined> + download(path: string, into: string, signal?: AbortSignal): Promise + upload(path: string, from: string, expect: Expect, signal?: AbortSignal): Promise + trash(path: string, expect: { sha256: string; md5?: string }, signal?: AbortSignal): Promise + ensureFolder(path: string, signal?: AbortSignal): Promise + revision?(signal?: AbortSignal): Promise +} +type Expect = { absent: true } | { sha256: string; md5?: string } +``` + +`list` returns each file's path relative to `root`, its `id` (kept across a move and an overwrite), `size`, `sha256` +(the plain hash of the content), `md5` and `modifiedAt`, plus the root folder's own id and `complete`, false when a page was skipped or a cap was hit. The other calls take the +remote path, `root` joined with the relative path. `download` and `upload` stream a local file, so a file of any +size takes the same memory. Every write is conditional on what the listing saw: `upload` and `trash` refuse with a +`conflict` error when the remote file is not as `expect` says; a provider without an atomic check reads the file +just before the write. An editing command reuses the same call, `upload(path, file, { sha256: whatWasDownloaded })`. + +1. Each path is compared three ways: the local file now, the remote file now and the last sync (`FileEntry` in the + state). A change on one side is copied to the other. The same content on both sides is adopted without a + transfer, so a first sync of two identical trees moves nothing. Different content on both sides, or a deletion + on one side against a change on the other, is a conflict: never overwritten, reported with each side's time and + size, and the run ends with a `mirror.partial` issue. `keep` settles conflicts on a later run: `local` and + `remote` pick a side; `both` downloads the remote version beside the local one as + `name (conflict YYYY-MM-DD ).ext`, then sends both. +2. A deletion crosses sides into a trash: the remote's (`trash`, with the expected hashes) or + `/.wirecat-trash/YYYY-MM-DD/`, which is hidden and so never synced. A run that would delete more than 50 + files, or more than a quarter of the mirror, stops before any write with `mirror.too-many-deletions` unless + `allowDeletions` covers the count. +3. A remote file found at a new path with the same `id` is renamed locally. A local rename is a deletion and a new + file. +4. A local file whose size and mtime match the state is not hashed again (`scanLocal`'s `known` lookup). A local + write goes through a hidden temporary file and lands only if the file still has the hash the run read, or is + still absent for a new file; a download whose content does not match the listing's `sha256` never lands. An + upload that reads back with another hash is a conflict. +5. A crash at any step is settled by the next run's listing: the same content on both sides is adopted, a file gone + from both is forgotten, so nothing is written twice and nothing is lost. `pending` in the state only names a + download's temporary file, removed at the next start, and a write that ended `outcome_unknown`, which is never + retried within the run. The state is saved at most once a second and at the end. +6. A file counts as deleted on a side only when the last sync saw it, the side's listing is complete, and the side's + root is the same folder: the remote root has the id it had at the first sync; the local root exists, is a folder + with the device and inode recorded at the first sync, and is not empty while the state has files (an unmounted + drive reads as empty). A file missing from an incomplete listing is reported in `unknown` and nothing is deleted + for it. When a root fails, the run does nothing and reports every file of the state in `unknown`. Either way the + run is partial. A file the last sync did not see is never a deletion. Hidden remote files are skipped as local + ones are. +7. `dryRun` lists the remote and returns the `plan`; it writes nothing on either side and saves no state. + +`versions` (`fileVersions(dir)`, with `dir` under the profile's state folder, one per mirror, never inside the +mirrored folder) keeps the content of every local file before the sync overwrites or deletes it (a pull, `keep: +"remote"`, a crossed deletion) in `/objects/`, and appends each change to `/log.jsonl`: path, +event (`created`, `changed`, `deleted`, `renamed`), direction, old and new hash, size and time. An upload logs the +remote version it replaced as `old`. Retention is one rule, applied after each sync: a content stays while a log +event newer than 30 days names it, and the last content of a path whose newest event is a deletion stays for good. +`versions.list(path)` lists a path's events with `available` (bytes still kept); `restoreFileVersion(root, path, +sha256, versions)` puts one back, keeping the current file first, and the next sync sends it. + +`guard` wraps every write with its `action` (`create`, `update`, `folder`, `delete`, `move`) and `side` (`local` or +`remote`), so the CLI maps overwrites and both trashes to its delete permission. `syncFileMirrors` runs the engine +for each mirror under one lock, as `syncMirrors` does; `fileSyncState(dir, mirror, principal)` keeps the state in +`/.json`; `fileMirrorStatus` lists local changes and deletions without asking the remote. For +`watchMirrors`, pass `localChanges` (the mirrors whose `fileMirrorStatus` lists anything) instead of `kindOf` and +`stateOf`, and `remoteChanges: fileRemoteChanges(target)`: it reads the target's `revision` at each poll and lists +only when it moved; a target without one is listed at every poll. + For documents with structure, `markdownBlocks` (async; the parser loads on first use) parses Markdown into `Block`s (headings, paragraphs, bullet, numbered and checklist items, code, quotes, tables, images, rules) with inline marks. The provider maps its own document into the same blocks and writes only `diffBlocks(current, next)`, so unchanged diff --git a/src/documents/file-mirror-model.ts b/src/documents/file-mirror-model.ts new file mode 100644 index 000000000..158379dc5 --- /dev/null +++ b/src/documents/file-mirror-model.ts @@ -0,0 +1,90 @@ +/** A file in the remote folder, as one listing returns it. */ +export interface RemoteFile { + /** Relative to the listed folder, with `/` separators. */ + path: string + /** Stays the same across a move or an overwrite, so a remote rename is found by it. */ + id: string + size: number + /** The plain sha256 of the content. A file without one (still being processed) is left for the next run. */ + sha256?: string + md5?: string + modifiedAt: string +} + +/** + * What a remote write expects to find at the path, from the last listing: nothing, or the file with these hashes. + * A target that cannot check the condition atomically checks it just before the write. + */ +export type Expect = { absent: true } | { sha256: string; md5?: string } + +/** + * A remote folder of plain files. Writes take the remote path (`root` joined with the relative path) and refuse with + * a `conflict` CliError when the file is not what `expect` says. + */ +export interface FileTarget { + identity(): Promise + /** + * Every file under `root`, with the root folder's own id; undefined when the folder does not exist. `complete` is + * false when any page was skipped or a cap was hit: a file missing from such a listing is not taken as deleted. + */ + list( + root: string, + signal?: AbortSignal, + ): Promise<{ rootId: string; files: RemoteFile[]; complete: boolean } | undefined> + /** Streams the file into `into`, a new local file. */ + download(path: string, into: string, signal?: AbortSignal): Promise + /** Streams the local file `from` to `path` and returns the file as the remote now has it. */ + upload(path: string, from: string, expect: Expect, signal?: AbortSignal): Promise + /** Moves the file to the remote's trash, where the owner can restore it. */ + trash(path: string, expect: Exclude, signal?: AbortSignal): Promise + /** Creates the folder and its parents; a folder that exists is fine. */ + ensureFolder(path: string, signal?: AbortSignal): Promise + /** A value that moves with every change on the remote, when the provider has one: a watch reads it before listing. */ + revision?(signal?: AbortSignal): Promise +} + +/** The last sync of one path: the base of the three-way comparison. */ +export interface FileEntry { + id: string + size: number + sha256: string + md5?: string + remoteModified: string + localMtimeMs: number + localSize: number + syncedAt: string +} + +export interface FileMirrorState { + version: 1 + principal: string + root: string + target: string + /** Set at the first sync; a later listing with another id means the remote folder was moved or replaced. */ + rootId?: string + /** The local folder's device and inode at the first sync: another folder at the same path is not this mirror. */ + localRoot?: { dev: number; ino: number } + entries: Record + /** + * Saved before each write. After a crash the next listing settles the write (same content on both sides is + * adopted); this only names the temporary file to remove and the operation to look up. + */ + pending: Record +} + +export interface FileMirrorStatePort { + read(): FileMirrorState + save(state: FileMirrorState): void +} + +export type FileStepKind = + | "upload" + | "download" + | "adopt" + | "trash-remote" + | "trash-local" + | "rename-local" + | "keep-both" + | "forget" + | "conflict" + | "unknown" diff --git a/src/documents/index.ts b/src/documents/index.ts index 3c0549494..82d27349d 100644 --- a/src/documents/index.ts +++ b/src/documents/index.ts @@ -11,6 +11,7 @@ export { documentIngestion, type IngestionOptions } from "../services/documents/ export { type DocumentKnowledge, documentKnowledge } from "../services/documents/knowledge.js" export { blocksMarkdown, inlineMarkdown, spliceMarkdown } from "../services/documents/markdown-splice.js" export { + type MirrorLocalCheck, type MirrorTrigger, type MirrorWatchEvent, type MirrorWatchOptions, @@ -28,6 +29,7 @@ export { export type { SyncOptions } from "../services/documents/ports.js" export { extractDocumentMemories, type ImportedSource, type ProposalOptions } from "../services/documents/proposals.js" export { + type MirrorAction, type PushConflict, type PushOptions, type PushResult, @@ -36,8 +38,36 @@ export { scanLocal, } from "../services/documents/push.js" export { type SyncResult, syncDocuments } from "../services/documents/sync.js" +export { + type FileMirrorOptions, + type FileMirrorResult, + type FileMirrorSyncOptions, + type FileStep, + fileMirrorStatus, + fileRemoteChanges, + fileSyncState, + syncFileMirror, + syncFileMirrors, +} from "../services/files/mirror.js" +export { + type FileVersion, + type FileVersions, + fileVersions, + restoreFileVersion, + type VersionDirection, + type VersionEvent, +} from "../services/files/versions.js" export { type Block, blockKey, blockShape, type Inline, type ListKind, normalizeInlines } from "./blocks.js" export { catalogFiles } from "./catalog.js" +export type { + Expect, + FileEntry, + FileMirrorState, + FileMirrorStatePort, + FileStepKind, + FileTarget, + RemoteFile, +} from "./file-mirror-model.js" export { htmlToMarkdown, markdownToHtml } from "./html-markdown.js" export { markdownBlocks, type PlacedBlock, placedMarkdownBlocks } from "./markdown-blocks.js" export type { diff --git a/src/documents/push-model.ts b/src/documents/push-model.ts index 0c3ed53cf..7e15d74fd 100644 --- a/src/documents/push-model.ts +++ b/src/documents/push-model.ts @@ -4,6 +4,9 @@ export interface LocalDocument { absolute: string hash: string kind: string + /** Set by `scanLocal`, so the next scan can skip hashing a file whose size and mtime did not change. */ + size?: number + mtimeMs?: number } export interface Pushed { id: string diff --git a/src/errors/reasons.ts b/src/errors/reasons.ts index c0ec8a78a..49528fe8b 100644 --- a/src/errors/reasons.ts +++ b/src/errors/reasons.ts @@ -240,6 +240,13 @@ const OTHERS: Entry[] = [ ["mirror.none", "not_found", "the profile has no mirrors; add one with mirrors add"], ["mirror.not-found", "not_found", "no such mirror in this profile; see mirrors list"], validation("mirror.not-folder", "the path to mirror is not a folder"), + [ + "mirror.too-many-deletions", + "confirmation_required", + "the sync would delete more files than the guard allows; check them with --dry-run and allow them explicitly", + ], + ["upload.mismatch", "conflict", "the uploaded file reads back with other content; the local file changed meanwhile"], + ["download.mismatch", "invalid_response", "the downloaded file has other content than the listing said"], ["catalog.unavailable", "configuration_error", "the store has no shared catalog for this account"], ["catalog.not-found", "not_found", "the file or folder is not in the local catalog; run catalog import or sync"], validation("catalog.depth-invalid", "tree depth must be between 0 and 50"), diff --git a/src/services/documents/mirror-watch.test.ts b/src/services/documents/mirror-watch.test.ts index 25c094f0a..71348e56c 100644 --- a/src/services/documents/mirror-watch.test.ts +++ b/src/services/documents/mirror-watch.test.ts @@ -20,7 +20,7 @@ const mirrorAt = (id: string, files: Record): Mirror => { * A watch on a clock that jumps to each wake-up. `at(ms, act)` runs `act` once the clock passes `ms`; the watch stops * after `cycles` cycles. The sync records the local files as pushed, as a real sync leaves them. */ -const harness = (mirrors: Mirror[], cycles: number) => { +const harness = (mirrors: Mirror[], cycles: number, localChanges?: (mirrors: Mirror[]) => Promise) => { let clock = 0 const stop = new AbortController() const stateDir = mkdtempSync(join(tmpdir(), "mirror-watch-state-")) @@ -54,8 +54,7 @@ const harness = (mirrors: Mirror[], cycles: number) => { const options = { mirrors, signal: stop.signal, - kindOf, - stateOf, + ...(localChanges ? { localChanges } : { kindOf, stateOf }), remoteChanges: async () => { const found = remote remote = [] @@ -180,6 +179,23 @@ describe("watchMirrors", () => { expect(watch.clock()).toBe(1_800_000) }) + it("asks the engine's own check, when given one, whether saved files differ", async () => { + const a = mirrorAt("a", { "one.md": "one" }) + const b = mirrorAt("b", { "two.md": "two" }) + const asked: string[][] = [] + const watch = harness([a, b], 2, async (chosen) => { + asked.push(chosen.map((mirror) => mirror.id)) + return chosen.filter((mirror) => mirror.id === "b").map((mirror) => mirror.id) + }) + watch.at(1_000, () => watch.save(a, "one.md", "one, edited")) + watch.at(10_000, () => watch.save(b, "two.md", "two, edited")) + expect(await watch.run()).toEqual([ + { triggers: ["start"], mirrors: ["a", "b"] }, + { triggers: ["local"], mirrors: ["b"] }, + ]) + expect(asked).toEqual([["a"], ["b"]]) + }) + it("hears a saved file through the real file watcher", async () => { const a = mirrorAt("a", { "one.md": "one" }) const stop = new AbortController() diff --git a/src/services/documents/mirror-watch.ts b/src/services/documents/mirror-watch.ts index fda495596..12e27e105 100644 --- a/src/services/documents/mirror-watch.ts +++ b/src/services/documents/mirror-watch.ts @@ -11,11 +11,17 @@ export type MirrorWatchEvent = | { triggers: MirrorTrigger[]; mirrors: string[]; result: T } | { triggers: MirrorTrigger[]; mirrors: string[]; error: CliError } -export interface MirrorWatchOptions { +/** + * How a watch tells which mirrors' local files differ from their state: a document push's kinds and state, or the + * engine's own check. + */ +export type MirrorLocalCheck = + | { kindOf(path: string): string | undefined; stateOf(mirror: Mirror): PushStatePort } + | { localChanges(mirrors: Mirror[]): Promise } + +export type MirrorWatchOptions = MirrorLocalCheck & { mirrors: Mirror[] signal: AbortSignal - kindOf(path: string): string | undefined - stateOf(mirror: Mirror): PushStatePort /** Ids of the mirrors whose remote files changed since the last call, or `all` when the provider cannot tell. */ remoteChanges?(signal: AbortSignal): Promise pollMs?: number @@ -87,6 +93,7 @@ export async function* watchMirrors( const differs = async (ids: Iterable) => { const chosen = [...ids].flatMap((id) => byId.get(id) ?? []) try { + if ("localChanges" in options) return await options.localChanges(chosen) const items = await mirrorStatus(chosen, options.kindOf, options.stateOf) return items.filter((item) => item.create.length || item.update.length).map((item) => item.id) } catch { diff --git a/src/services/documents/mirrors.ts b/src/services/documents/mirrors.ts index 0525b5d49..136084a35 100644 --- a/src/services/documents/mirrors.ts +++ b/src/services/documents/mirrors.ts @@ -1,10 +1,10 @@ import { createHash } from "node:crypto" import { mkdirSync, readFileSync } from "node:fs" -import { join } from "node:path" +import { dirname, join } from "node:path" import { CliError, writeSecurely } from "@wirecat/cli-base" import * as v from "valibot" import type { DocumentTarget, Keep, PushState, PushStatePort } from "../../documents/push-model.js" -import { type PushConflict, type PushOptions, type PushResult, planPush, pushDocuments, scanLocal } from "./push.js" +import { type PushConflict, type PushOptions, planPush, pushDocuments, scanLocal } from "./push.js" /** A local folder mirrored into a remote folder, as the app's profile config keeps it. */ export interface Mirror { @@ -44,50 +44,49 @@ const stateSchema = (kinds: [string, ...string[]]) => }) /** - * A mirror's sync state in `/.json`. A missing file is a first sync; an unreadable or invalid one is - * refused rather than reset, since a reset would push every file again as new. + * A state file read against `schema`, saved 0600 in a 0700 folder. A missing file is `initial`, a first sync; an + * unreadable or invalid one is refused rather than reset, since a reset would treat every file as new. */ +export const jsonState = (path: string, schema: v.GenericSchema, initial: () => T) => ({ + read: (): T => { + let value: unknown + try { + value = JSON.parse(readFileSync(path, "utf8")) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return initial() + throw new CliError("configuration_error", "mirror state is unreadable; refusing to reset it", { + reason: "state.invalid", + }) + } + const parsed = v.safeParse(schema, value) + if (!parsed.success) + throw new CliError("configuration_error", "mirror state is invalid; refusing to reset it", { + reason: "state.invalid", + }) + return parsed.output as T + }, + save: (state: T) => { + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }) + writeSecurely(path, `${JSON.stringify(state)}\n`, 0o600) + }, +}) + +/** A mirror's push state in `/.json`. */ export const fileMirrorState = ( dir: string, mirror: Mirror, principal: string, kinds: [string, ...string[]], -): PushStatePort => { - const path = join(dir, `${mirror.id}.json`) - const schema = stateSchema(kinds) - return { - read: () => { - let value: unknown - try { - value = JSON.parse(readFileSync(path, "utf8")) - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") - return { - version: 1, - principal, - root: mirror.path, - target: mirror.folder, - folders: {}, - entries: {}, - pending: {}, - } - throw new CliError("configuration_error", "mirror state is unreadable; refusing to reset it", { - reason: "state.invalid", - }) - } - const parsed = v.safeParse(schema, value) - if (!parsed.success) - throw new CliError("configuration_error", "mirror state is invalid; refusing to reset it", { - reason: "state.invalid", - }) - return parsed.output as unknown as PushState - }, - save: (state) => { - mkdirSync(dir, { recursive: true, mode: 0o700 }) - writeSecurely(path, `${JSON.stringify(state)}\n`, 0o600) - }, - } -} +): PushStatePort => + jsonState>(join(dir, `${mirror.id}.json`), stateSchema(kinds), () => ({ + version: 1, + principal, + root: mirror.path, + target: mirror.folder, + folders: {}, + entries: {}, + pending: {}, + })) /** What the next sync of each mirror would create or send, from local files and the state alone. */ export const mirrorStatus = async ( @@ -117,7 +116,7 @@ export const mirrorStatus = async ( /** A conflict as the user reads it, with the remote side named the way the app names it (`google`). */ export const conflictReport = (conflict: PushConflict, remote: string) => ({ path: conflict.path, - url: conflict.url, + ...(conflict.url ? { url: conflict.url } : {}), reason: conflict.reason, ...(conflict.parts ? { parts: conflict.parts.map((part) => ({ last_sync: part.base, local: part.local, [remote]: part.remote })) } @@ -125,53 +124,39 @@ export const conflictReport = (conflict: PushConflict, remote: string) => ({ ...(conflict.localChangedAt ? { local_changed_at: conflict.localChangedAt } : {}), ...(conflict.remoteChangedAt ? { [`${remote}_changed_at`]: conflict.remoteChangedAt } : {}), ...(conflict.remoteChangedBy ? { [`${remote}_changed_by`]: conflict.remoteChangedBy } : {}), + ...(conflict.localSize !== undefined ? { local_size: conflict.localSize } : {}), + ...(conflict.remoteSize !== undefined ? { [`${remote}_size`]: conflict.remoteSize } : {}), }) -export interface MirrorSyncOptions - extends Pick, "principal" | "recreate" | "guard" | "operationId" | "errorOf" | "signal"> { - mirrors: Mirror[] - target: DocumentTarget - stateOf(mirror: Mirror): PushStatePort - keep?: Keep - /** Taken once for all the mirrors, so a second sync or a watcher's next cycle waits its turn. */ - lock(): Promise<{ assertHeld(): void; release(): Promise | void }> - /** How the app names the remote side in conflict reports. */ - remote: string +interface MirrorLock { + assertHeld(): void + release(): Promise | void } /** - * Syncs each mirror both ways. A mirror with conflicts, missing remote files or errors does not stop the others; - * it gets one `mirror.partial` issue, so the caller can answer `complete: false` and fail after printing. + * Runs `sync` for each mirror under one lock. A mirror with conflicts, missing remote files or errors does not stop + * the others; it gets one `mirror.partial` issue, so the caller can answer `complete: false` and fail after printing. */ -export const syncMirrors = async (options: MirrorSyncOptions) => { - const { mirrors, keep, remote } = options - const lock = await options.lock() - const results: ({ id: string; path: string } & PushResult)[] = [] +export const eachMirror = async < + R extends { conflicts: PushConflict[]; errors: unknown[]; missing?: unknown[]; unknown?: unknown[] }, +>( + mirrors: Mirror[], + lock: () => Promise, + remote: string, + sync: (mirror: Mirror, assertHeld: () => void) => Promise, +) => { + const held = await lock() + const results: ({ id: string; path: string } & R)[] = [] try { for (const mirror of mirrors) - results.push({ - id: mirror.id, - path: mirror.path, - ...(await pushDocuments({ - root: mirror.path, - folder: mirror.folder, - target: options.target, - state: options.stateOf(mirror), - principal: options.principal, - ...(keep ? { keep } : {}), - recreate: options.recreate === true, - guard: options.guard, - operationId: options.operationId, - errorOf: options.errorOf, - assertHeld: () => lock.assertHeld(), - ...(options.signal ? { signal: options.signal } : {}), - })), - }) + results.push({ id: mirror.id, path: mirror.path, ...(await sync(mirror, () => held.assertHeld())) }) } finally { - await lock.release() + await held.release() } const issues = results - .filter((result) => result.errors.length || result.conflicts.length || result.missing.length) + .filter( + (result) => result.errors.length || result.conflicts.length || result.missing?.length || result.unknown?.length, + ) .map((result) => ({ code: "provider_error" as const, reason: "mirror.partial", source: `mirror ${result.id}` })) return { items: results.map(({ conflicts, ...result }) => ({ @@ -181,3 +166,34 @@ export const syncMirrors = async (options: MirrorSyncOptions) => { issues, } } + +export interface MirrorSyncOptions + extends Pick, "principal" | "recreate" | "guard" | "operationId" | "errorOf" | "signal"> { + mirrors: Mirror[] + target: DocumentTarget + stateOf(mirror: Mirror): PushStatePort + keep?: Keep + /** Taken once for all the mirrors, so a second sync or a watcher's next cycle waits its turn. */ + lock(): Promise + /** How the app names the remote side in conflict reports. */ + remote: string +} + +/** Syncs each mirror both ways, as `eachMirror` runs them. */ +export const syncMirrors = (options: MirrorSyncOptions) => + eachMirror(options.mirrors, options.lock, options.remote, (mirror, assertHeld) => + pushDocuments({ + root: mirror.path, + folder: mirror.folder, + target: options.target, + state: options.stateOf(mirror), + principal: options.principal, + ...(options.keep ? { keep: options.keep } : {}), + recreate: options.recreate === true, + guard: options.guard, + operationId: options.operationId, + errorOf: options.errorOf, + assertHeld, + ...(options.signal ? { signal: options.signal } : {}), + }), + ) diff --git a/src/services/documents/push.ts b/src/services/documents/push.ts index 11fde3e67..9ff5839c7 100644 --- a/src/services/documents/push.ts +++ b/src/services/documents/push.ts @@ -1,6 +1,6 @@ import { createHash, randomUUID } from "node:crypto" import { createReadStream } from "node:fs" -import { chmod, readdir, rename, rm, stat, writeFile } from "node:fs/promises" +import { chmod, link, readdir, rename, rm, stat, writeFile } from "node:fs/promises" import { basename, dirname, join, posix } from "node:path" import { CliError, isCliError } from "@wirecat/cli-base" import type { @@ -28,10 +28,7 @@ export interface PushOptions { recreate?: boolean dryRun?: boolean /** Runs one remote write behind the provider's permission checks and write journal. */ - guard( - attempt: { action: "create" | "update" | "folder"; path: string; operationId: string }, - act: () => Promise, - ): Promise + guard(attempt: { action: MirrorAction; path: string; operationId: string }, act: () => Promise): Promise operationId(): string errorOf(error: unknown): CliError assertHeld(): void @@ -39,14 +36,19 @@ export interface PushOptions { maxFiles?: number now?(): number } +/** What a mirror's write does; the guard maps it to the app's permission keys. */ +export type MirrorAction = "create" | "update" | "folder" | "delete" | "move" export interface PushConflict { path: string - url: string + /** The remote file's address, when the provider has one. */ + url?: string reason: string parts?: ConflictPart[] localChangedAt?: string remoteChangedAt?: string remoteChangedBy?: string + localSize?: number + remoteSize?: number } export interface PushResult { created: string[] @@ -68,41 +70,76 @@ const SKIPPED = /^(\.|node_modules$|~\$)|(~|\.swp|\.tmp)$/ /** Hidden files, `node_modules` and editor leftovers: never mirrored, and never a reason to sync. */ export const skippedName = (name: string): boolean => SKIPPED.test(name) -const hashFile = (path: string): Promise => +/** The sha256 and md5 of a file, read as a stream: a file of any size takes the same memory. */ +export const hashFile = (path: string): Promise<{ sha256: string; md5: string }> => new Promise((done, fail) => { - const hash = createHash("sha256") + const sha256 = createHash("sha256") + const md5 = createHash("md5") createReadStream(path) - .on("data", (chunk) => hash.update(chunk)) + .on("data", (chunk) => { + sha256.update(chunk) + md5.update(chunk) + }) .on("error", fail) - .on("end", () => done(hash.digest("hex"))) + .on("end", () => done({ sha256: sha256.digest("hex"), md5: md5.digest("hex") })) }) -const hashText = (text: string) => createHash("sha256").update(text).digest("hex") +/** A hidden temporary file beside `absolute`; `tag` lets a caller find a crash's leftover again. */ +export const localTemp = (absolute: string, tag: string) => join(dirname(absolute), `.${basename(absolute)}.${tag}.tmp`) /** - * Replaces the file through a temporary file beside it, keeping its mode. Undefined, and nothing written, when - * the file no longer has `hash`: someone saved it during the sync. + * Writes `absolute` through a temporary file beside it. With `hash`, the file is replaced only while it still has + * that sha256 (someone may have saved it during the sync) and keeps its mode; without, it is created only while + * absent. `content` is text, bytes, or a writer that fills the temporary file, for a stream of any size; with + * `sha256`, content with another hash is refused before it takes the file's place. `tag` names the temporary file, so + * a crash's leftover can be found. Returns the new file's hashes, or undefined, with nothing written, when the file + * was not as `hash` said. */ -const replaceLocal = async (file: LocalDocument, content: string): Promise => { - const temp = join(dirname(file.absolute), `.${basename(file.absolute)}.${randomUUID().slice(0, 8)}.tmp`) - const { mode } = await stat(file.absolute) - await writeFile(temp, content, { flag: "wx" }) +export const replaceLocal = async ( + absolute: string, + hash: string | undefined, + content: string | Uint8Array | ((temp: string) => Promise), + { tag = randomUUID().slice(0, 8), sha256 }: { tag?: string; sha256?: string } = {}, +): Promise<{ sha256: string; md5: string } | undefined> => { + const temp = localTemp(absolute, tag) + const mode = hash === undefined ? undefined : (await stat(absolute)).mode try { + if (typeof content === "function") await content(temp) + else await writeFile(temp, content, { flag: "wx" }) // A mode passed to the write goes through the umask; set it after, so a group-writable file stays one. - await chmod(temp, mode & 0o777) - if ((await hashFile(file.absolute)) !== file.hash) return undefined - await rename(temp, file.absolute) + if (mode !== undefined) await chmod(temp, mode & 0o777) + const written = await hashFile(temp) + if (sha256 !== undefined && written.sha256 !== sha256) + throw new CliError("invalid_response", "the downloaded file has other content than the listing said", { + reason: "download.mismatch", + }) + if (hash === undefined) { + // A link fails when the name exists, where a rename would replace a file created during the sync. + try { + await link(temp, absolute) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") return undefined + throw error + } + return written + } + if ((await hashFile(absolute)).sha256 !== hash) return undefined + await rename(temp, absolute) + return written } finally { await rm(temp, { force: true }) } - return hashText(content) } -/** Files under `root` the target pushes. Hidden files, editor leftovers and symbolic links are never followed. */ +/** + * Files under `root` the target pushes. Hidden files, editor leftovers and symbolic links are never followed. A file + * whose size and mtime match what `known` returns keeps that hash instead of being read again. + */ export const scanLocal = async ( root: string, kindOf: (path: string) => string | undefined, maxFiles = 10_000, + known?: (path: string) => { size: number; mtimeMs: number; hash: string } | undefined, ): Promise<{ files: LocalDocument[]; unsupported: number }> => { const files: LocalDocument[] = [] let unsupported = 0 @@ -123,7 +160,11 @@ export const scanLocal = async ( reason: "push.too-many-files", }) const absolute = join(root, path) - files.push({ path, absolute, kind, hash: await hashFile(absolute) }) + const { size, mtimeMs } = await stat(absolute) + const before = known?.(path) + const hash = + before && before.size === size && before.mtimeMs === mtimeMs ? before.hash : (await hashFile(absolute)).sha256 + files.push({ path, absolute, kind, hash, size, mtimeMs }) } } } @@ -298,12 +339,12 @@ export const pushDocuments = async (options: PushOptions): Promise [] as string[]) +vi.mock("node:fs/promises", async (original) => { + const fs = await original() + return { + ...fs, + readFile: (path: never, ...rest: never[]) => { + reads.push(String(path)) + return fs.readFile(path, ...rest) + }, + } +}) + +const FOLDER = "/Notes" +const TODAY = Date.parse("2026-11-18T12:00:00.000Z") +const hashes = (text: string) => ({ + sha256: createHash("sha256").update(text).digest("hex"), + md5: createHash("md5").update(text).digest("hex"), +}) +const temp = (prefix: string) => mkdtempSync(join(tmpdir(), prefix)) +const put = (root: string, path: string, content: string | Uint8Array) => { + mkdirSync(dirname(join(root, path)), { recursive: true }) + writeFileSync(join(root, path), content) +} +/** Every file under `root` but hidden ones, with its text. */ +const tree = async (root: string): Promise> => { + const found: Record = {} + for (const entry of await readdir(root, { recursive: true, withFileTypes: true })) { + const path = posix.relative(root, join(entry.parentPath, entry.name)) + if (entry.isFile() && !path.split("/").some((part) => part.startsWith("."))) + found[path] = readFileSync(join(root, path), "utf8") + } + return found +} + +/** A remote folder kept on disk (content) with ids and a revision in memory, strict about `expect`. */ +const fakeDisk = () => { + const dir = temp("file-mirror-remote-") + const ids = new Map() + const trash: { path: string; content: string }[] = [] + const calls: { op: string; path: string; expect?: Expect }[] = [] + let next = 0 + let revision = 0 + let gone = false + let complete = true + let rootId = "root-1" + const relative = (path: string) => posix.relative(FOLDER, path) + const touch = (path: string) => { + if (!ids.has(path)) ids.set(path, `res-${++next}`) + revision++ + } + const describe = async (path: string): Promise => { + const { sha256, md5 } = await hashFile(join(dir, path)) + return { + path, + id: ids.get(path) as string, + size: statSync(join(dir, path)).size, + sha256, + md5, + modifiedAt: "2026-11-18T10:00:00.000Z", + } + } + const check = async (path: string, expect: Expect) => { + const here = existsSync(join(dir, path)) ? await describe(path) : undefined + const ok = "absent" in expect ? !here : here?.sha256 === expect.sha256 && (!expect.md5 || here?.md5 === expect.md5) + if (!ok) throw new CliError("conflict", "the remote file is not as expected") + } + const target: FileTarget = { + identity: async () => "alpha", + list: async (root) => { + if (gone || root !== FOLDER) return undefined + const files: RemoteFile[] = [] + for (const path of [...ids.keys()].sort()) files.push(await describe(path)) + return { rootId, files, complete } + }, + download: async (path, into) => { + calls.push({ op: "download", path: relative(path) }) + await pipeline(createReadStream(join(dir, relative(path))), createWriteStream(into, { flags: "wx" })) + }, + upload: async (path, from, expect) => { + const at = relative(path) + calls.push({ op: "upload", path: at, expect }) + await check(at, expect) + mkdirSync(dirname(join(dir, at)), { recursive: true }) + await pipeline(createReadStream(from), createWriteStream(join(dir, at))) + touch(at) + return describe(at) + }, + trash: async (path, expect) => { + const at = relative(path) + calls.push({ op: "trash", path: at, expect }) + await check(at, expect) + trash.push({ path: at, content: readFileSync(join(dir, at), "utf8") }) + await unlink(join(dir, at)) + ids.delete(at) + revision++ + }, + ensureFolder: async (path) => { + calls.push({ op: "folder", path: relative(path) }) + mkdirSync(join(dir, relative(path)), { recursive: true }) + }, + revision: async () => String(revision), + } + return { + target, + calls, + trash, + /** Another client writes the file; an overwrite keeps its id. */ + write: (path: string, content: string | Uint8Array) => { + put(dir, path, content) + touch(path) + }, + /** Another client uploads a local file, streamed. */ + writeFrom: async (path: string, from: string) => { + await pipeline(createReadStream(from), createWriteStream(join(dir, path))) + touch(path) + }, + remove: (path: string) => { + rmSync(join(dir, path)) + ids.delete(path) + revision++ + }, + move: (from: string, to: string) => { + put(dir, to, readFileSync(join(dir, from))) + rmSync(join(dir, from)) + ids.set(to, ids.get(from) as string) + ids.delete(from) + revision++ + }, + files: () => tree(dir), + idOf: (path: string) => ids.get(path), + goAway: () => { + gone = true + }, + /** The next listings skip a page, as a provider's capped or failed page would. */ + listPartly: () => { + complete = false + }, + replaceRoot: () => { + rootId = "root-2" + }, + writes: () => calls.filter((call) => call.op !== "folder"), + } +} + +class Crash extends Error {} + +const setup = (local: Record = {}, remote: Record = {}) => { + const root = temp("file-mirror-local-") + for (const [path, content] of Object.entries(local)) put(root, path, content) + const disk = fakeDisk() + for (const [path, content] of Object.entries(remote)) disk.write(path, content) + const stateDir = temp("file-mirror-state-") + const mirror: Mirror = { id: mirrorId(root, FOLDER), path: root, folder: FOLDER } + const port = fileSyncState(stateDir, mirror, "alpha") + const versions = fileVersions(join(stateDir, "versions", mirror.id), () => TODAY) + let crashed = false + let crashAt: { action: string; side: string; when: "before" | "after" } | undefined + const attempts: { action: string; side: string; path: string }[] = [] + let ops = 0 + const options = (extra: Partial = {}): FileMirrorOptions => ({ + root, + folder: FOLDER, + target: disk.target, + // A crashed process saves nothing more. + state: { read: port.read, save: (state) => (crashed ? undefined : port.save(state)) }, + versions, + principal: "alpha", + remote: "disk", + guard: async (attempt, act) => { + attempts.push({ action: attempt.action, side: attempt.side, path: attempt.path }) + const hit = crashAt && crashAt.action === attempt.action && crashAt.side === attempt.side ? crashAt : undefined + if (hit?.when === "before") { + crashed = true + throw new Crash() + } + const done = await act() + if (hit) { + crashed = true + throw new Crash() + } + return done + }, + operationId: () => `op${String(++ops).padStart(6, "0")}`, + errorOf: (error) => { + if (error instanceof Crash) throw error + return isCliError(error) ? error : new CliError("provider_error", String(error)) + }, + assertHeld: () => {}, + now: () => TODAY, + ...extra, + }) + const sync = (keep?: Keep, extra: Partial = {}) => + syncFileMirror(options({ ...(keep ? { keep } : {}), ...extra })) + return { + root, + disk, + mirror, + port, + versions, + stateDir, + attempts, + sync, + options, + local: () => tree(root), + crashOn: (action: string, side: string, when: "before" | "after") => { + crashAt = { action, side, when } + }, + recover: () => { + crashAt = undefined + crashed = false + }, + } +} + +describe("syncFileMirror, first sync", () => { + it("adopts files that are the same on both sides and moves nothing", async () => { + const files = { "a.md": "one", "sub/b.txt": "two" } + const { sync, disk, local } = setup(files, files) + const result = await sync() + expect(result).toMatchObject({ uploaded: [], downloaded: [], adopted: ["a.md", "sub/b.txt"], conflicts: [] }) + expect(disk.writes()).toEqual([]) + expect(await sync()).toMatchObject({ adopted: [], unchanged: 2 }) + expect(await local()).toEqual(files) + }) + + it("reports a file with different content as a conflict and overwrites neither side", async () => { + const { sync, disk, local } = setup({ "a.md": "mine" }, { "a.md": "theirs" }) + const result = await sync() + expect(result.conflicts).toEqual([ + expect.objectContaining({ path: "a.md", reason: "created on both sides with different content", localSize: 4 }), + ]) + expect(disk.writes()).toEqual([]) + expect(await local()).toEqual({ "a.md": "mine" }) + expect(await disk.files()).toEqual({ "a.md": "theirs" }) + }) + + it("copies a file found on one side only to the other", async () => { + const { sync, disk, local } = setup({ "up/a.md": "local" }, { "down/b.md": "remote" }) + const result = await sync() + expect(result).toMatchObject({ uploaded: ["up/a.md"], downloaded: ["down/b.md"] }) + expect(disk.calls.find((call) => call.op === "upload")?.expect).toEqual({ absent: true }) + expect(disk.calls.filter((call) => call.op === "folder").map((call) => call.path)).toEqual(["up"]) + expect(await local()).toEqual(await disk.files()) + }) +}) + +describe("syncFileMirror, the three-way table", () => { + const synced = async (files: Record) => { + const context = setup(files, files) + await context.sync() + context.disk.calls.length = 0 + return context + } + + it("uploads a local change, expecting the remote as the listing saw it", async () => { + const context = await synced({ "a.md": "one", "b.md": "two" }) + put(context.root, "a.md", "one, edited") + const result = await context.sync() + expect(result).toMatchObject({ uploaded: ["a.md"], unchanged: 1 }) + expect(context.disk.calls[0]?.expect).toEqual(hashes("one")) + expect(await context.disk.files()).toMatchObject({ "a.md": "one, edited" }) + }) + + it("downloads a remote change", async () => { + const context = await synced({ "a.md": "one" }) + context.disk.write("a.md", "one, from the disk") + expect(await context.sync()).toMatchObject({ downloaded: ["a.md"], uploaded: [] }) + expect(await context.local()).toEqual({ "a.md": "one, from the disk" }) + }) + + it("adopts the same change made on both sides and reports different ones as a conflict", async () => { + const context = await synced({ "same.md": "x", "torn.md": "y" }) + put(context.root, "same.md", "x2") + context.disk.write("same.md", "x2") + put(context.root, "torn.md", "y local") + context.disk.write("torn.md", "y remote") + const result = await context.sync() + expect(result.adopted).toEqual(["same.md"]) + expect(result.conflicts).toEqual([ + expect.objectContaining({ path: "torn.md", reason: "changed on both sides", remoteSize: 8, localSize: 7 }), + ]) + expect(context.disk.writes()).toEqual([]) + expect(await context.local()).toMatchObject({ "torn.md": "y local" }) + }) + + it("sends a local deletion to the remote trash with the expected md5", async () => { + const context = await synced({ "a.md": "1", "b.md": "2", "c.md": "3", "d.md": "4" }) + rmSync(join(context.root, "a.md")) + const result = await context.sync() + expect(result.trashed).toEqual([{ path: "a.md", side: "remote" }]) + expect(context.disk.calls[0]).toEqual({ op: "trash", path: "a.md", expect: hashes("1") }) + expect(context.disk.trash).toEqual([{ path: "a.md", content: "1" }]) + }) + + it("moves a remote deletion into the local trash folder, never deleting it", async () => { + const context = await synced({ "a.md": "1", "b.md": "2", "c.md": "3", "d.md": "4" }) + context.disk.remove("a.md") + expect((await context.sync()).trashed).toEqual([{ path: "a.md", side: "local" }]) + expect(await context.local()).not.toHaveProperty("a.md") + expect(readFileSync(join(context.root, ".wirecat-trash", "2026-11-18", "a.md"), "utf8")).toBe("1") + }) + + it("forgets a file deleted on both sides", async () => { + const context = await synced({ "a.md": "1", "b.md": "2", "c.md": "3", "d.md": "4" }) + rmSync(join(context.root, "a.md")) + context.disk.remove("a.md") + expect(await context.sync()).toMatchObject({ trashed: [], conflicts: [] }) + expect(Object.keys(context.port.read().entries)).toEqual(["b.md", "c.md", "d.md"]) + }) + + it("reports a deletion against a change as a conflict, settled only by --keep", async () => { + const context = await synced({ "gone-here.md": "1", "gone-there.md": "2", "c.md": "3", "d.md": "4" }) + rmSync(join(context.root, "gone-here.md")) + context.disk.write("gone-here.md", "1, changed") + put(context.root, "gone-there.md", "2, changed") + context.disk.remove("gone-there.md") + const result = await context.sync() + expect(result.conflicts.map((conflict) => [conflict.path, conflict.reason])).toEqual([ + ["gone-here.md", "deleted here, changed on the remote"], + ["gone-there.md", "changed here, deleted on the remote"], + ]) + expect(context.disk.writes()).toEqual([]) + + const kept = await context.sync("remote") + expect(kept).toMatchObject({ downloaded: ["gone-here.md"], trashed: [{ path: "gone-there.md", side: "local" }] }) + expect(await context.local()).toEqual({ "gone-here.md": "1, changed", "c.md": "3", "d.md": "4" }) + }) + + it("--keep local keeps the changed local file against a remote deletion, and deletes against a remote change", async () => { + const context = await synced({ "gone-here.md": "1", "gone-there.md": "2", "c.md": "3", "d.md": "4" }) + rmSync(join(context.root, "gone-here.md")) + context.disk.write("gone-here.md", "1, changed") + put(context.root, "gone-there.md", "2, changed") + context.disk.remove("gone-there.md") + const result = await context.sync("local", { allowDeletions: 1 }) + expect(result).toMatchObject({ uploaded: ["gone-there.md"], trashed: [{ path: "gone-here.md", side: "remote" }] }) + expect(await context.disk.files()).toEqual({ "gone-there.md": "2, changed", "c.md": "3", "d.md": "4" }) + }) +}) + +describe("syncFileMirror, --keep on a file changed on both sides", () => { + const torn = async () => { + const context = setup({ "notes/a.md": "base" }, { "notes/a.md": "base" }) + await context.sync() + put(context.root, "notes/a.md", "local") + context.disk.write("notes/a.md", "remote") + context.disk.calls.length = 0 + return context + } + + it("local overwrites the remote, expecting what the listing saw", async () => { + const context = await torn() + expect(await context.sync("local")).toMatchObject({ uploaded: ["notes/a.md"], conflicts: [] }) + expect(await context.disk.files()).toEqual({ "notes/a.md": "local" }) + }) + + it("remote overwrites the local file", async () => { + const context = await torn() + expect(await context.sync("remote")).toMatchObject({ downloaded: ["notes/a.md"], conflicts: [] }) + expect(await context.local()).toEqual({ "notes/a.md": "remote" }) + }) + + it("both keeps the remote version as a dated copy on both sides, with nothing left to sync", async () => { + const context = await torn() + const result = await context.sync("both") + const copy = "notes/a (conflict 2026-11-18 disk).md" + expect(result).toMatchObject({ downloaded: [copy], uploaded: ["notes/a.md", copy], conflicts: [] }) + const both = { "notes/a.md": "local", [copy]: "remote" } + expect(await context.local()).toEqual(both) + expect(await context.disk.files()).toEqual(both) + context.disk.calls.length = 0 + expect(await context.sync()).toMatchObject({ unchanged: 2, adopted: [], conflicts: [] }) + expect(context.disk.writes()).toEqual([]) + }) +}) + +describe("syncFileMirror, deletion guard", () => { + it("stops before any write when a run would delete more than a quarter of the mirror", async () => { + const files = { "a.md": "1", "b.md": "2", "c.md": "3", "d.md": "4" } + const context = setup(files, files) + await context.sync() + rmSync(join(context.root, "a.md")) + rmSync(join(context.root, "b.md")) + put(context.root, "new.md", "n") + expect(await context.sync(undefined, { dryRun: true })).toMatchObject({ tooManyDeletions: 2 }) + await expect(context.sync()).rejects.toMatchObject({ + code: "confirmation_required", + details: { reason: "mirror.too-many-deletions", deletions: 2 }, + }) + expect(context.disk.writes()).toEqual([]) + expect(await context.sync(undefined, { allowDeletions: 2 })).toMatchObject({ uploaded: ["new.md"] }) + expect(context.disk.trash.map((item) => item.path)).toEqual(["a.md", "b.md"]) + }) + + it("stops at more than 50 deletions whatever the share", async () => { + const files = Object.fromEntries(Array.from({ length: 220 }, (_, n) => [`f${n}.md`, `${n}`])) + const context = setup(files, files) + await context.sync() + for (let n = 0; n < 51; n++) rmSync(join(context.root, `f${n}.md`)) + await expect(context.sync()).rejects.toMatchObject({ details: { deletions: 51 } }) + }) +}) + +describe("syncFileMirror, renames", () => { + it("renames the local file when the remote file moved, found by its id", async () => { + const files = { "a.md": "1", "b.md": "2" } + const context = setup(files, files) + await context.sync() + const id = context.disk.idOf("a.md") + context.disk.move("a.md", "archive/a.md") + context.disk.calls.length = 0 + const result = await context.sync() + expect(result).toMatchObject({ renamed: [{ from: "a.md", to: "archive/a.md" }], trashed: [], downloaded: [] }) + expect(context.disk.calls).toEqual([]) + expect(await context.local()).toEqual({ "archive/a.md": "1", "b.md": "2" }) + expect(context.port.read().entries["archive/a.md"]?.id).toBe(id) + expect(await context.sync()).toMatchObject({ unchanged: 2 }) + }) + + it("treats a local rename as a deletion and a new file", async () => { + const files = { "a.md": "1", "b.md": "2", "c.md": "3", "d.md": "4" } + const context = setup(files, files) + await context.sync() + put(context.root, "renamed.md", "1") + rmSync(join(context.root, "a.md")) + const result = await context.sync() + expect(result).toMatchObject({ uploaded: ["renamed.md"], trashed: [{ path: "a.md", side: "remote" }] }) + expect(await context.disk.files()).toEqual(await context.local()) + }) +}) + +describe("syncFileMirror, crash and resume", () => { + const files = { "a.md": "a", "b.md": "b", "c.md": "c", "d.md": "d" } + const rest = { "b.md": "b", "c.md": "c", "d.md": "d" } + const cases: [string, string, (context: ReturnType) => void, Record][] = [ + ["create", "remote", (context) => put(context.root, "new.md", "new"), { ...files, "new.md": "new" }], + ["update", "remote", (context) => put(context.root, "a.md", "a2"), { ...files, "a.md": "a2" }], + ["create", "local", (context) => context.disk.write("new.md", "new"), { ...files, "new.md": "new" }], + ["update", "local", (context) => context.disk.write("a.md", "a2"), { ...files, "a.md": "a2" }], + ["delete", "remote", (context) => rmSync(join(context.root, "a.md")), rest], + ["delete", "local", (context) => context.disk.remove("a.md"), rest], + ["move", "local", (context) => context.disk.move("a.md", "moved/a.md"), { ...rest, "moved/a.md": "a" }], + ] + for (const [action, side, change, after] of cases) + for (const when of ["before", "after"] as const) + it(`settles a crash ${when} a ${side} ${action} with no loss and no duplicate`, async () => { + const context = setup(files, files) + await context.sync() + change(context) + context.crashOn(action, side, when) + await expect(context.sync()).rejects.toBeInstanceOf(Crash) + context.recover() + const resumed = await context.sync() + expect(resumed).toMatchObject({ conflicts: [], errors: [] }) + expect(await context.local()).toEqual(after) + expect(await context.disk.files()).toEqual(after) + expect(context.disk.trash.length).toBe(action === "delete" && side === "remote" ? 1 : 0) + const trashed = join(context.root, ".wirecat-trash", "2026-11-18") + expect(existsSync(trashed) ? await readdir(trashed) : []).toEqual( + action === "delete" && side === "local" ? ["a.md"] : [], + ) + expect(await context.sync()).toMatchObject({ unchanged: Object.keys(after).length, adopted: [] }) + }) + + it("removes a download's temporary file left by a crash", async () => { + const context = setup({}, { "a.md": "a" }) + const state = context.port.read() + state.pending["a.md"] = { op: "download", operationId: "-4911" } + context.port.save(state) + const left = localTemp(join(context.root, "a.md"), createHash("sha256").update("-4911").digest("hex").slice(0, 8)) + writeFileSync(left, "half") + await context.sync() + expect(existsSync(left)).toBe(false) + expect(context.port.read().pending).toEqual({}) + expect(await context.local()).toEqual({ "a.md": "a" }) + }) + + it("does not retry a write whose outcome is unknown, and adopts it at the next listing", async () => { + const context = setup({ "a.md": "a" }) + const upload = context.disk.target.upload + let cut = true + const target: FileTarget = { + ...context.disk.target, + upload: async (...args) => { + await upload(...args) + if (cut) { + cut = false + throw new CliError("outcome_unknown", "the connection dropped after sending") + } + throw new Error("must not upload again") + }, + } + const first = await context.sync(undefined, { target }) + expect(first.errors).toEqual([expect.objectContaining({ path: "a.md", code: "outcome_unknown" })]) + expect(context.port.read().pending["a.md"]).toMatchObject({ op: "upload" }) + const second = await context.sync(undefined, { target }) + expect(second).toMatchObject({ adopted: ["a.md"], errors: [] }) + expect(context.port.read().pending).toEqual({}) + }) + + it("reports a remote file changed after the listing as a conflict and keeps both versions", async () => { + const context = setup({ "a.md": "a" }, { "a.md": "a" }) + await context.sync() + put(context.root, "a.md", "local") + const list = context.disk.target.list + const target: FileTarget = { + ...context.disk.target, + list: async (...args) => { + const listed = await list(...args) + context.disk.write("a.md", "sneaked in") + return listed + }, + } + const result = await context.sync(undefined, { target }) + expect(result.conflicts).toEqual([ + expect.objectContaining({ path: "a.md", reason: "the disk file changed during the sync" }), + ]) + expect(await context.disk.files()).toEqual({ "a.md": "sneaked in" }) + expect(await context.local()).toEqual({ "a.md": "local" }) + }) + + it("leaves a local file saved after the listing alone", async () => { + const context = setup({ "a.md": "a" }, { "a.md": "a" }) + await context.sync() + context.disk.write("a.md", "remote") + const download = context.disk.target.download + const target: FileTarget = { + ...context.disk.target, + download: async (...args) => { + put(context.root, "a.md", "saved meanwhile") + await download(...args) + }, + } + const result = await context.sync(undefined, { target }) + expect(result.conflicts).toEqual([ + expect.objectContaining({ path: "a.md", reason: "the local file changed during the sync" }), + ]) + expect(await context.local()).toEqual({ "a.md": "saved meanwhile" }) + }) +}) + +describe("syncFileMirror, read-back checks", () => { + it("never places a download whose content differs from the listing", async () => { + const context = setup({}, { "a.md": "listed" }) + const target: FileTarget = { + ...context.disk.target, + download: async (_path, into) => writeFileSync(into, "something else", { flag: "wx" }), + } + const result = await context.sync(undefined, { target }) + expect(result.errors).toEqual([expect.objectContaining({ path: "a.md", code: "invalid_response" })]) + expect(await context.local()).toEqual({}) + }) + + it("reports an upload that reads back with other content as a conflict and records nothing", async () => { + const context = setup({ "a.md": "local" }) + const upload = context.disk.target.upload + const target: FileTarget = { + ...context.disk.target, + upload: async (...args) => ({ ...(await upload(...args)), sha256: "other" }), + } + const result = await context.sync(undefined, { target }) + expect(result.conflicts).toEqual([ + expect.objectContaining({ path: "a.md", reason: "the local file changed during the upload" }), + ]) + expect(context.port.read().entries).toEqual({}) + }) +}) + +describe("syncFileMirror, content", () => { + it("syncs binary and empty files byte for byte", async () => { + const binary = Uint8Array.from({ length: 256 }, (_, n) => n) + const context = setup() + put(context.root, "blob.bin", binary) + put(context.root, "empty.txt", "") + context.disk.write("remote.bin", Uint8Array.from([0, 255, 0, 10, 13])) + context.disk.write("remote-empty", "") + const result = await context.sync() + expect(result.uploaded).toEqual(["blob.bin", "empty.txt"]) + expect(result.downloaded).toEqual(["remote-empty", "remote.bin"]) + expect(readFileSync(join(context.root, "remote.bin"))).toEqual(Buffer.from([0, 255, 0, 10, 13])) + expect(readFileSync(join(context.root, "remote-empty")).length).toBe(0) + expect(await context.sync()).toMatchObject({ unchanged: 4 }) + }) + + it("streams a large file both ways without reading it whole", async () => { + const context = setup() + const chunk = Buffer.alloc(1 << 20, 7) + const big = join(context.root, "big.bin") + await pipeline( + Readable.from( + (function* () { + for (let n = 0; n < 50; n++) yield chunk + })(), + ), + createWriteStream(big), + ) + reads.length = 0 + expect(await context.sync()).toMatchObject({ uploaded: ["big.bin"] }) + await context.disk.writeFrom("copy.bin", big) + expect(await context.sync()).toMatchObject({ downloaded: ["copy.bin"] }) + expect((await hashFile(join(context.root, "copy.bin"))).sha256).toBe((await hashFile(big)).sha256) + expect(reads.filter((path) => path.endsWith(".bin"))).toEqual([]) + }) +}) + +describe("syncFileMirror, refusals and dry run", () => { + it("writes nothing on either side on a dry run, and still shows the plan", async () => { + const context = setup({ "a.md": "1", "up.md": "u" }, { "a.md": "1", "down.md": "d" }) + const result = await context.sync(undefined, { dryRun: true }) + expect(result.plan).toEqual([ + { kind: "adopt", path: "a.md" }, + { kind: "download", path: "down.md" }, + { kind: "upload", path: "up.md" }, + ]) + expect(context.disk.calls).toEqual([]) + expect(context.attempts).toEqual([]) + expect(await context.local()).toEqual({ "a.md": "1", "up.md": "u" }) + expect(existsSync(join(context.stateDir, `${context.mirror.id}.json`))).toBe(false) + }) + + it("refuses another account", async () => { + const context = setup() + const target = { ...context.disk.target, identity: async () => "beta" } + await expect(context.sync(undefined, { target })).rejects.toMatchObject({ + details: { reason: "profile.wrong-account" }, + }) + }) + + it("skips hidden remote files and leaves a file without a hash for the next run", async () => { + const context = setup() + context.disk.write(".hidden", "h") + context.disk.write("a.md", "1") + const list = context.disk.target.list + const target: FileTarget = { + ...context.disk.target, + list: async (...args) => { + const listed = await list(...args) + return listed && { ...listed, files: listed.files.map(({ sha256: _, ...file }) => file) } + }, + } + const result = await context.sync(undefined, { target }) + expect(result.errors).toEqual([expect.objectContaining({ path: "a.md" })]) + expect(await context.local()).toEqual({}) + expect((await context.sync()).downloaded).toEqual(["a.md"]) + }) +}) + +describe("fileMirrorStatus and fileRemoteChanges", () => { + it("lists local changes and deletions without asking the remote", async () => { + const context = setup({ "a.md": "1", "b.md": "2" }) + await context.sync() + put(context.root, "a.md", "1, edited") + put(context.root, "c.md", "3") + rmSync(join(context.root, "b.md")) + const [item] = await fileMirrorStatus([context.mirror], () => context.port) + expect(item).toMatchObject({ upload: ["a.md", "c.md"], delete: ["b.md"], unchanged: 0, remote_checked: false }) + }) + + it("does not read a file again whose size and mtime match the last sync", async () => { + const context = setup({ "a.md": "1" }) + await context.sync() + const state = context.port.read() + state.entries["a.md"] = { ...(state.entries["a.md"] as FileEntry), sha256: "kept" } + context.port.save(state) + expect((await fileMirrorStatus([context.mirror], () => context.port))[0]).toMatchObject({ + upload: [], + unchanged: 1, + }) + }) + + it("lists again only when the remote revision moved", async () => { + const { disk } = setup() + const changes = fileRemoteChanges(disk.target) + const signal = new AbortController().signal + expect(await changes(signal)).toBe("all") + expect(await changes(signal)).toEqual([]) + disk.write("a.md", "1") + expect(await changes(signal)).toBe("all") + expect(await changes(signal)).toEqual([]) + const { revision: _, ...plain } = disk.target + expect(await fileRemoteChanges(plain)(signal)).toBe("all") + }) +}) + +describe("syncFileMirrors", () => { + it("reports conflicts with the remote named and one partial issue, and releases the lock", async () => { + const context = setup({ "a.md": "mine" }, { "a.md": "theirs" }) + const lock = { assertHeld: vi.fn(), release: vi.fn() } + const { root: _, folder: __, state: ___, assertHeld: ____, ...rest } = context.options() + const result = await syncFileMirrors({ + ...rest, + mirrors: [context.mirror], + stateOf: () => context.port, + versionsOf: () => context.versions, + lock: async () => lock, + }) + expect(result.items[0]?.conflicts[0]).toEqual( + expect.objectContaining({ path: "a.md", local_size: 4, disk_size: 6, disk_changed_at: expect.any(String) }), + ) + expect(result.items[0]?.conflicts[0]).not.toHaveProperty("url") + expect(result.issues).toEqual([ + { code: "provider_error", reason: "mirror.partial", source: `mirror ${context.mirror.id}` }, + ]) + expect(lock.release).toHaveBeenCalledOnce() + }) +}) + +describe("syncFileMirror, a deletion is proven or nothing is deleted", () => { + const files = { "a.md": "1", "b.md": "2", "c.md": "3", "d.md": "4" } + const synced = async () => { + const context = setup(files, files) + await context.sync() + context.disk.calls.length = 0 + return context + } + const nothingDeleted = async (context: Awaited>) => { + expect(context.disk.trash).toEqual([]) + expect(existsSync(join(context.root, ".wirecat-trash"))).toBe(false) + } + + it("does not delete locally a file missing from an incomplete remote listing, and reports it", async () => { + const context = await synced() + context.disk.remove("a.md") + context.disk.listPartly() + put(context.root, "b.md", "2, edited") + const result = await context.sync() + expect(result.unknown).toEqual([ + { path: "a.md", side: "remote", reason: "missing from an incomplete disk listing" }, + ]) + expect(result.uploaded).toEqual(["b.md"]) + await nothingDeleted(context) + expect(await context.local()).toMatchObject({ "a.md": "1" }) + }) + + it("deletes nothing when the remote listing fails", async () => { + const context = await synced() + rmSync(join(context.root, "a.md")) + const target: FileTarget = { + ...context.disk.target, + list: async () => { + throw new CliError("provider_unavailable", "page 2 failed") + }, + } + await expect(context.sync(undefined, { target })).rejects.toMatchObject({ code: "provider_unavailable" }) + await nothingDeleted(context) + }) + + it("deletes nothing on either side when the local folder is gone, empty or another folder", async () => { + for (const change of ["gone", "empty", "replaced"] as const) { + const context = await synced() + context.disk.remove("a.md") + if (change === "gone") rmSync(context.root, { recursive: true }) + else if (change === "empty") for (const name of Object.keys(files)) rmSync(join(context.root, name)) + else { + // Moved aside, not deleted, so the new folder cannot reuse the old inode number. + renameSync(context.root, `${context.root}-old`) + mkdirSync(context.root) + for (const name of ["b.md", "c.md", "d.md"]) put(context.root, name, files[name as "b.md"]) + } + const result = await context.sync() + expect(result.unknown).toHaveLength(4) + expect(result.unknown[0]?.side).toBe("local") + expect(context.disk.writes()).toEqual([]) + expect(await context.disk.files()).toEqual({ "b.md": "2", "c.md": "3", "d.md": "4" }) + } + }) + + it("deletes nothing on either side when the remote folder is gone or replaced", async () => { + for (const change of ["gone", "replaced"] as const) { + const context = await synced() + rmSync(join(context.root, "a.md")) + if (change === "gone") context.disk.goAway() + else context.disk.replaceRoot() + const result = await context.sync() + expect(result.unknown.map((item) => [item.path, item.side])).toEqual( + Object.keys(files).map((path) => [path, "remote"]), + ) + expect(context.disk.writes()).toEqual([]) + expect(await context.local()).toEqual({ "b.md": "2", "c.md": "3", "d.md": "4" }) + } + }) + + it("never takes a file the last sync did not see as deleted, even from an incomplete listing", async () => { + const context = await synced() + put(context.root, "new.md", "n") + context.disk.listPartly() + expect(await context.sync()).toMatchObject({ uploaded: ["new.md"], unknown: [], trashed: [] }) + }) + + it("counts the run as partial", async () => { + const context = await synced() + context.disk.remove("a.md") + context.disk.listPartly() + const { root: _, folder: __, state: ___, assertHeld: ____, versions: _____, ...rest } = context.options() + const result = await syncFileMirrors({ + ...rest, + mirrors: [context.mirror], + stateOf: () => context.port, + versionsOf: () => context.versions, + lock: async () => ({ assertHeld: () => {}, release: () => {} }), + }) + expect(result.issues).toEqual([expect.objectContaining({ reason: "mirror.partial" })]) + }) +}) + +describe("syncFileMirror, local versions", () => { + const synced = async () => { + const files = { "a.md": "1", "b.md": "2", "c.md": "3", "d.md": "4" } + const context = setup(files, files) + await context.sync() + return context + } + + it("keeps the local content before a pull overwrites it, and logs the change", async () => { + const context = await synced() + context.disk.write("a.md", "1, from the disk") + await context.sync() + const [event] = context.versions.list("a.md") + expect(event).toMatchObject({ + event: "changed", + direction: "remote-to-local", + old: hashes("1").sha256, + new: hashes("1, from the disk").sha256, + available: true, + }) + expect(await restoreFileVersion(context.root, "a.md", hashes("1").sha256, context.versions)).toBe(true) + expect(await context.local()).toMatchObject({ "a.md": "1" }) + expect(context.versions.list("a.md").at(-1)).toMatchObject({ + direction: "restored", + old: hashes("1, from the disk").sha256, + }) + expect(await context.sync()).toMatchObject({ uploaded: ["a.md"] }) + }) + + it("keeps a local file before --keep disk overwrites it and before a crossed deletion", async () => { + const context = await synced() + put(context.root, "a.md", "mine") + context.disk.write("a.md", "theirs") + context.disk.remove("b.md") + await context.sync("remote") + expect(context.versions.list("a.md")[0]).toMatchObject({ old: hashes("mine").sha256, available: true }) + expect(context.versions.list("b.md")[0]).toMatchObject({ + event: "deleted", + old: hashes("2").sha256, + available: true, + }) + }) + + it("logs the remote version an upload replaced", async () => { + const context = await synced() + put(context.root, "a.md", "1, edited") + rmSync(join(context.root, "b.md")) + await context.sync() + expect(context.versions.list("a.md")).toEqual([ + expect.objectContaining({ event: "changed", direction: "local-to-remote", old: hashes("1").sha256 }), + ]) + expect(context.versions.list("b.md")).toEqual([ + expect.objectContaining({ event: "deleted", direction: "local-to-remote", old: hashes("2").sha256 }), + ]) + }) + + it("keeps versions for 30 days, and the last version of a deleted file for good", async () => { + const dir = temp("file-mirror-versions-") + let clock = TODAY + const versions = fileVersions(dir, () => clock) + const file = join(temp("file-mirror-versions-src-"), "x") + for (const text of ["old", "gone"]) { + writeFileSync(file, text) + await versions.keep(file) + } + versions.log({ path: "x", event: "changed", direction: "remote-to-local", old: hashes("old").sha256 }) + versions.log({ path: "y", event: "deleted", direction: "remote-to-local", old: hashes("gone").sha256 }) + clock += 31 * 24 * 60 * 60 * 1000 + await versions.prune() + expect(versions.list("x")[0]?.available).toBe(false) + expect(versions.list("y")[0]?.available).toBe(true) + }) + + it("writes no version and no log on a dry run", async () => { + const context = await synced() + context.disk.write("a.md", "changed") + await context.sync(undefined, { dryRun: true }) + expect(context.versions.list("a.md")).toEqual([]) + }) +}) diff --git a/src/services/files/mirror.ts b/src/services/files/mirror.ts new file mode 100644 index 000000000..8b1491fae --- /dev/null +++ b/src/services/files/mirror.ts @@ -0,0 +1,647 @@ +import { createHash } from "node:crypto" +import { link, mkdir, rm, stat, unlink } from "node:fs/promises" +import { dirname, join, posix } from "node:path" +import { CliError } from "@wirecat/cli-base" +import * as v from "valibot" +import type { + Expect, + FileEntry, + FileMirrorState, + FileMirrorStatePort, + FileStepKind, + FileTarget, + RemoteFile, +} from "../../documents/file-mirror-model.js" +import type { Keep, LocalDocument } from "../../documents/push-model.js" +import { eachMirror, jsonState, type Mirror } from "../documents/mirrors.js" +import { + hashFile, + localTemp, + type MirrorAction, + type PushConflict, + replaceLocal, + scanLocal, + skippedName, +} from "../documents/push.js" +import type { FileVersions } from "./versions.js" + +type Local = LocalDocument & { size: number; mtimeMs: number } +type Side = "local" | "remote" + +export interface FileMirrorOptions { + /** Absolute path of the local folder. */ + root: string + /** The remote folder, as the target addresses it. */ + folder: string + target: FileTarget + state: FileMirrorStatePort + /** Keeps every local file's content before the sync overwrites or deletes it, and logs each change. */ + versions: FileVersions + principal: string + /** Settles conflicts; without it they are reported and left. */ + keep?: Keep + dryRun?: boolean + /** Lets a run delete this many files although the deletion guard would stop it. */ + allowDeletions?: number + /** How the app names the remote side; it ends a conflict copy's name. */ + remote: string + /** Runs one write behind the app's permission checks; `side` says which side it changes. */ + guard( + attempt: { action: MirrorAction; side: Side; path: string; operationId: string }, + act: () => Promise, + ): Promise + operationId(): string + errorOf(error: unknown): CliError + assertHeld(): void + signal?: AbortSignal + maxFiles?: number + now?(): number +} + +export interface FileStep { + kind: FileStepKind + path: string + /** A rename's new path, or a kept copy's name. */ + to?: string +} + +export interface FileMirrorResult { + uploaded: string[] + downloaded: string[] + trashed: { path: string; side: Side }[] + renamed: { from: string; to: string }[] + adopted: string[] + unchanged: number + conflicts: PushConflict[] + errors: { path: string; code: string; message: string }[] + /** + * Files of the last sync missing from a side that cannot prove a deletion: its listing was incomplete, or its root + * is gone or is another folder. Nothing is deleted for them. + */ + unknown: { path: string; side: Side; reason: string }[] + /** Every step but the unchanged files, on a dry run. */ + plan?: FileStep[] + /** On a dry run, the deletions that would stop the real run without `allowDeletions`. */ + tooManyDeletions?: number + stopped?: true +} + +interface Step extends FileStep { + local?: Local + remote?: RemoteFile + base?: FileEntry + reason?: string +} + +/** A run that would delete more than this many files, or this share of the mirror, stops before writing. */ +const MAX_DELETIONS = 50 +const MAX_DELETED_SHARE = 0.25 +const TRASH = ".wirecat-trash" +const SAVE_EVERY_MS = 1_000 + +/** A temporary file's tag from any operation id, safe in a file name. */ +const tagOf = (operationId: string) => createHash("sha256").update(operationId).digest("hex").slice(0, 8) + +const sameFile = (local: Local, remote: RemoteFile) => local.hash === remote.sha256 + +/** + * The three-way comparison of one path: the local file now, the remote file now and the last sync. A side is + * "same" when its content still has the last sync's hash. + */ +const decide = ( + path: string, + sure: Record, + local?: Local, + remote?: RemoteFile, + base?: FileEntry, + keep?: Keep, +): Step | undefined => { + const step = (kind: FileStepKind, reason?: string): Step => ({ + kind, + path, + ...(local ? { local } : {}), + ...(remote ? { remote } : {}), + ...(base ? { base } : {}), + ...(reason ? { reason } : {}), + }) + // Only a file the last sync saw, missing from a complete listing of the same root, counts as deleted. + if (base && !local && !sure.local) return step("unknown", "local") + if (base && !remote && !sure.remote) return step("unknown", "remote") + if (!local && !remote) return base ? step("forget") : undefined + if (local && remote && sameFile(local, remote)) + return base?.sha256 === local.hash && base.id === remote.id ? undefined : step("adopt") + if (!base) { + if (!remote) return step("upload") + if (!local) return step("download") + return settle("created on both sides with different content", "upload", "download", "keep-both") + } + const localSame = local?.hash === base.sha256 + const remoteSame = remote?.sha256 === base.sha256 + if (localSame) return step(remote ? "download" : "trash-local") + if (remoteSame) return step(local ? "upload" : "trash-remote") + if (!local) return settle("deleted here, changed on the remote", "trash-remote", "download", "download") + if (!remote) return settle("changed here, deleted on the remote", "upload", "trash-local", "upload") + return settle("changed on both sides", "upload", "download", "keep-both") + + function settle(reason: string, ...[ifLocal, ifRemote, ifBoth]: [FileStepKind, FileStepKind, FileStepKind]) { + if (keep === "local") return step(ifLocal) + if (keep === "remote") return step(ifRemote) + if (keep === "both") return step(ifBoth) + return step("conflict", reason) + } +} + +/** `name (conflict 2026-11-18 disk).ext`, numbered when that name is taken. */ +const copyName = (path: string, remote: string, date: string, taken: (path: string) => boolean) => { + const extension = posix.extname(path) + const stem = path.slice(0, path.length - extension.length) + for (let n = 1; ; n++) { + const candidate = `${stem} (conflict ${date} ${remote}${n > 1 ? ` ${n}` : ""})${extension}` + if (!taken(candidate)) return candidate + } +} + +/** Moves a file into `/.wirecat-trash//` while it still has `hash`; false when it changed. */ +const trashLocal = async (root: string, path: string, hash: string, date: string) => { + const from = join(root, path) + if ((await hashFile(from)).sha256 !== hash) return false + const into = join(root, TRASH, date, path) + await mkdir(dirname(into), { recursive: true }) + for (let n = 1; ; n++) { + try { + await link(from, n > 1 ? `${into}.${n}` : into) + break + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error + } + } + await unlink(from) + return true +} + +/** Renames a local file while it still has `hash` and nothing holds the new name; false otherwise. */ +const renameLocal = async (root: string, from: string, to: string, hash: string) => { + const source = join(root, from) + if ((await hashFile(source)).sha256 !== hash) return false + await mkdir(dirname(join(root, to)), { recursive: true }) + try { + await link(source, join(root, to)) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") return false + throw error + } + await unlink(source) + return true +} + +/** Every local file, hashed only when its size or mtime moved since the last sync. */ +const scan = async (root: string, entries: Record, maxFiles?: number) => { + const { files } = await scanLocal( + root, + () => "file", + maxFiles, + (path) => { + const entry = entries[path] + return entry && { size: entry.localSize, mtimeMs: entry.localMtimeMs, hash: entry.sha256 } + }, + ) + return files as Local[] +} + +/** Remote files the mirror covers: hidden files and editor leftovers are skipped, as they are locally. */ +const mirrored = (files: RemoteFile[]) => files.filter((file) => !file.path.split("/").some(skippedName)) + +/** + * Syncs a local folder and a remote folder of plain files both ways, whole files at a time. Each path is compared + * three ways (local now, remote now, last sync): a change on one side is copied to the other, the same content on + * both is adopted, and different changes on both are a conflict unless `keep` settles it. Deletions go to the + * remote's trash and to `/.wirecat-trash/`. A remote rename, found by the file's id, renames the local file. + * Every write is conditional on what the listing saw, so a crash at any point is settled by the next run. + */ +export const syncFileMirror = async (options: FileMirrorOptions): Promise => { + const { root, folder, target, principal, keep, signal, versions } = options + const now = options.now ?? Date.now + if (signal?.aborted) throw new CliError("cancelled", "sync was cancelled") + if ((await target.identity()) !== principal) + throw new CliError("authentication_error", "signed-in account does not match this profile", { + reason: "profile.wrong-account", + }) + const state = options.state.read() + if (state.principal !== principal || state.root !== root || state.target !== folder) + throw new CliError("configuration_error", "mirror state belongs to another account or mirror") + const synced = Object.keys(state.entries).length > 0 + const listed = await target.list(folder, signal) + const localRoot = await stat(root).catch(() => undefined) + const local = new Map( + (localRoot?.isDirectory() ? await scan(root, state.entries, options.maxFiles) : []).map((file) => [ + file.path, + file, + ]), + ) + // A folder that is gone, replaced, or empty where the state had files (an unmounted drive) is not this mirror. + const rootProblem = !listed + ? ["remote", `the ${options.remote} folder is gone`] + : state.rootId !== undefined && listed.rootId !== state.rootId + ? ["remote", `the ${options.remote} folder was replaced by another one`] + : !localRoot?.isDirectory() + ? ["local", "the local folder is gone"] + : state.localRoot && (state.localRoot.dev !== localRoot.dev || state.localRoot.ino !== localRoot.ino) + ? ["local", "the local folder was replaced by another one"] + : synced && !local.size + ? ["local", "the local folder is empty; it may be an unmounted drive"] + : undefined + const remote = new Map(rootProblem ? [] : mirrored(listed?.files ?? []).map((file) => [file.path, file])) + const sure = { + local: rootProblem?.[0] !== "local", + remote: rootProblem?.[0] !== "remote" && listed?.complete === true, + } + const base = { ...state.entries } + + const steps: Step[] = [] + const byId = new Map([...remote.values()].map((file) => [file.id, file])) + for (const [path, entry] of Object.entries(state.entries)) { + const moved = remote.has(path) ? undefined : byId.get(entry.id) + const file = local.get(path) + if (!moved || base[moved.path] || local.has(moved.path) || file?.hash !== entry.sha256) continue + steps.push({ kind: "rename-local", path, to: moved.path, local: file, base: entry }) + base[moved.path] = entry + delete base[path] + local.delete(path) + local.set(moved.path, { ...file, path: moved.path, absolute: join(root, moved.path) }) + } + + let unchanged = 0 + const touched: string[] = [] + const paths = [...new Set([...local.keys(), ...remote.keys(), ...Object.keys(base)])].sort() + const result: FileMirrorResult = { + uploaded: [], + downloaded: [], + trashed: [], + renamed: [], + adopted: [], + unchanged: 0, + conflicts: [], + errors: [], + unknown: [], + } + if (rootProblem) { + const [side, reason] = rootProblem as [Side, string] + result.unknown = Object.keys(state.entries).map((path) => ({ path, side, reason })) + return result + } + for (const path of paths) { + const file = remote.get(path) + if (file && file.sha256 === undefined) { + result.errors.push({ path, code: "provider_error", message: `the ${options.remote} has no hash for it yet` }) + continue + } + const step = decide(path, sure, local.get(path), file, base[path], keep) + if (step) steps.push(step) + else unchanged++ + const entry = base[path] + const here = local.get(path) + // Saved without a write, so the next scan does not read a file only touched since the last sync. + if (!step && entry && here && (entry.localMtimeMs !== here.mtimeMs || entry.localSize !== here.size)) + touched.push(path) + } + result.unchanged = unchanged + const conflict = (step: Step, reason: string) => + result.conflicts.push({ + path: step.path, + reason, + ...(step.local ? { localChangedAt: new Date(step.local.mtimeMs).toISOString(), localSize: step.local.size } : {}), + ...(step.remote ? { remoteChangedAt: step.remote.modifiedAt, remoteSize: step.remote.size } : {}), + }) + for (const step of steps) { + if (step.kind === "conflict") conflict(step, step.reason ?? "") + if (step.kind === "unknown") + result.unknown.push({ + path: step.path, + side: step.reason as Side, + reason: `missing from an incomplete ${options.remote} listing`, + }) + } + + const deletions = steps.filter((step) => step.kind === "trash-local" || step.kind === "trash-remote").length + const tooMany = deletions > MAX_DELETIONS || deletions > Object.keys(state.entries).length * MAX_DELETED_SHARE + if (options.dryRun) { + result.plan = steps.map(({ kind, path, to }) => ({ kind, path, ...(to ? { to } : {}) })) + if (tooMany) result.tooManyDeletions = deletions + return result + } + if (tooMany && (options.allowDeletions ?? 0) < deletions) + throw new CliError( + "confirmation_required", + `this sync would delete ${deletions} files; check them with --dry-run and allow them explicitly`, + { reason: "mirror.too-many-deletions", deletions }, + ) + + state.rootId ??= listed?.rootId + if (localRoot) state.localRoot ??= { dev: localRoot.dev, ino: localRoot.ino } + for (const [path, pending] of Object.entries(state.pending)) { + if (pending.op === "download") await rm(localTemp(join(root, path), tagOf(pending.operationId)), { force: true }) + delete state.pending[path] + } + let savedAt = now() + const save = (force = false) => { + if (!force && now() - savedAt < SAVE_EVERY_MS) return + options.state.save(state) + savedAt = now() + } + for (const path of touched) { + const here = local.get(path) + const entry = state.entries[path] + if (here && entry) state.entries[path] = { ...entry, localMtimeMs: here.mtimeMs, localSize: here.size } + } + save(true) + + const date = new Date(now()).toISOString().slice(0, 10) + const remotePath = (path: string) => posix.join(folder, path) + const folders = new Set([...remote.keys()].map((path) => posix.dirname(path))) + for (const path of [...folders]) for (let up = path; up !== "."; up = posix.dirname(up)) folders.add(up) + let lastOperation = "" + const write = (action: MirrorAction, side: Side, path: string, act: () => Promise, operationId?: string) => { + options.assertHeld() + lastOperation = operationId ?? options.operationId() + return options.guard({ action, side, path, operationId: lastOperation }, act) + } + const record = (path: string, file: RemoteFile, placed: { size: number; mtimeMs: number }, sha256: string) => { + state.entries[path] = { + id: file.id, + size: file.size, + sha256, + ...(file.md5 ? { md5: file.md5 } : {}), + remoteModified: file.modifiedAt, + localMtimeMs: placed.mtimeMs, + localSize: placed.size, + syncedAt: new Date(now()).toISOString(), + } + save() + } + + const upload = async (path: string, from: Local, expect: Expect, created: boolean) => { + for (const up of ancestors(path)) + if (!folders.has(up)) { + await write("folder", "remote", up, () => target.ensureFolder(remotePath(up), signal)) + folders.add(up) + } + const sent = await write(created ? "create" : "update", "remote", path, () => + target.upload(remotePath(path), from.absolute, expect, signal), + ) + if (sent.sha256 !== undefined && sent.sha256 !== from.hash) + throw new CliError("conflict", "the local file changed during the upload", { reason: "upload.mismatch" }) + record(path, sent, from, from.hash) + versions.log({ + path, + event: created ? "created" : "changed", + direction: "local-to-remote", + ...("sha256" in expect ? { old: expect.sha256 } : {}), + new: from.hash, + size: from.size, + }) + result.uploaded.push(path) + } + /** False when the local file changed (or appeared) during the sync, so nothing was written. */ + const download = async (path: string, file: RemoteFile, over?: Local) => { + const operationId = options.operationId() + state.pending[path] = { op: "download", operationId } + save(true) + const absolute = join(root, path) + await mkdir(dirname(absolute), { recursive: true }) + const old = over ? await versions.keep(absolute) : undefined + const written = await write( + over ? "update" : "create", + "local", + path, + () => + replaceLocal(absolute, over?.hash, (temp) => target.download(remotePath(file.path), temp, signal), { + tag: tagOf(operationId), + sha256: file.sha256 as string, + }), + operationId, + ) + delete state.pending[path] + if (!written) return false + record(path, file, await stat(absolute), written.sha256) + versions.log({ + path, + event: old ? "changed" : "created", + direction: "remote-to-local", + ...(old ? { old } : {}), + new: written.sha256, + size: file.size, + }) + result.downloaded.push(path) + return true + } + const expectOf = (file?: RemoteFile): Expect => + file ? { sha256: file.sha256 as string, ...(file.md5 ? { md5: file.md5 } : {}) } : { absent: true } + const changedHere = (step: Step) => conflict(step, "the local file changed during the sync") + + const run = async (step: Step) => { + const { path, local: here, remote: there, base: last } = step + switch (step.kind) { + case "conflict": + return + case "adopt": + if (here && there) record(path, there, here, here.hash) + result.adopted.push(path) + return + case "forget": + delete state.entries[path] + save() + return + case "upload": + if (here) await upload(path, here, expectOf(there), !there) + return + case "download": + if (there && !(await download(path, there, here))) changedHere(step) + return + case "trash-remote": + if (!there) return + await write("delete", "remote", path, () => + target.trash( + remotePath(path), + { sha256: there.sha256 as string, ...(there.md5 ? { md5: there.md5 } : {}) }, + signal, + ), + ) + delete state.entries[path] + save() + versions.log({ + path, + event: "deleted", + direction: "local-to-remote", + old: there.sha256 as string, + size: there.size, + }) + result.trashed.push({ path, side: "remote" }) + return + case "trash-local": { + if (!here) return + const old = await versions.keep(here.absolute) + if (!(await write("delete", "local", path, () => trashLocal(root, path, here.hash, date)))) + return changedHere(step) + delete state.entries[path] + save() + versions.log({ path, event: "deleted", direction: "remote-to-local", old, size: here.size }) + result.trashed.push({ path, side: "local" }) + return + } + case "rename-local": { + const to = step.to as string + if (!last || !(await write("move", "local", to, () => renameLocal(root, path, to, last.sha256)))) + return changedHere(step) + const placed = await stat(join(root, to)) + state.entries[to] = { ...last, localMtimeMs: placed.mtimeMs, localSize: placed.size } + delete state.entries[path] + save() + versions.log({ path, event: "renamed", direction: "remote-to-local", to, new: last.sha256, size: last.size }) + result.renamed.push({ from: path, to }) + return + } + case "keep-both": { + if (!here || !there) return + // The remote version lands beside the local one first, so the overwrite below loses nothing. + const copy = copyName(path, options.remote, date, (name) => local.has(name) || remote.has(name)) + if (!(await download(copy, there))) return changedHere(step) + await upload(path, here, expectOf(there), false) + const { size, mtimeMs } = await stat(join(root, copy)) + const copied = { ...here, path: copy, absolute: join(root, copy), hash: there.sha256 as string, size, mtimeMs } + await upload(copy, copied, { absent: true }, true) + return + } + } + } + + try { + for (const step of steps) { + if (signal?.aborted) { + result.stopped = true + break + } + try { + await run(step) + } catch (error) { + const failure = options.errorOf(error) + if (failure.code === "cancelled") { + result.stopped = true + break + } + if (failure.code === "conflict") + conflict( + step, + failure.details?.reason === "upload.mismatch" + ? failure.message + : `the ${options.remote} file changed during the sync`, + ) + else result.errors.push({ path: step.path, code: failure.code, message: failure.message }) + // Not retried: the next listing shows whether it landed, and adopts it when it did. + if (failure.code === "outcome_unknown") state.pending[step.path] = { op: step.kind, operationId: lastOperation } + } + } + } finally { + save(true) + } + await versions.prune() + return result +} + +const ancestors = (path: string) => { + const found: string[] = [] + for (let up = posix.dirname(path); up !== "."; up = posix.dirname(up)) found.unshift(up) + return found +} + +const entrySchema = v.object({ + id: v.string(), + size: v.number(), + sha256: v.string(), + md5: v.optional(v.string()), + remoteModified: v.string(), + localMtimeMs: v.number(), + localSize: v.number(), + syncedAt: v.string(), +}) +const stateSchema = v.object({ + version: v.literal(1), + principal: v.string(), + root: v.string(), + target: v.string(), + rootId: v.optional(v.string()), + localRoot: v.optional(v.object({ dev: v.number(), ino: v.number() })), + entries: v.record(v.string(), entrySchema), + pending: v.record(v.string(), v.object({ op: v.string(), operationId: v.string() })), +}) + +/** A file mirror's state in `/.json`, refused rather than reset when invalid. */ +export const fileSyncState = (dir: string, mirror: Mirror, principal: string): FileMirrorStatePort => + jsonState(join(dir, `${mirror.id}.json`), stateSchema, () => ({ + version: 1, + principal, + root: mirror.path, + target: mirror.folder, + entries: {}, + pending: {}, + })) + +/** What each mirror's next sync would send from local changes alone; the remote is not asked. */ +export const fileMirrorStatus = async (mirrors: Mirror[], stateOf: (mirror: Mirror) => FileMirrorStatePort) => { + const items = [] + for (const mirror of mirrors) { + const { entries } = stateOf(mirror).read() + const files = await scan(mirror.path, entries) + const seen = new Set(files.map((file) => file.path)) + items.push({ + id: mirror.id, + path: mirror.path, + folder: mirror.folder, + upload: files.filter((file) => entries[file.path]?.sha256 !== file.hash).map((file) => file.path), + delete: Object.keys(entries).filter((path) => !seen.has(path)), + unchanged: files.length - files.filter((file) => entries[file.path]?.sha256 !== file.hash).length, + remote_checked: false, + }) + } + return items +} + +/** + * A watch's `remoteChanges` for a file target: every mirror when the remote's revision moved since the last poll (or + * at the first poll), none when it did not, and every mirror at each poll when the target has no revision. + */ +export const fileRemoteChanges = (target: FileTarget) => { + let last: string | undefined + return async (signal: AbortSignal): Promise => { + if (!target.revision) return "all" + const revision = await target.revision(signal) + const moved = revision !== last + last = revision + return moved ? "all" : [] + } +} + +export interface FileMirrorSyncOptions + extends Pick< + FileMirrorOptions, + "target" | "principal" | "keep" | "dryRun" | "allowDeletions" | "guard" | "operationId" | "errorOf" | "signal" + > { + mirrors: Mirror[] + stateOf(mirror: Mirror): FileMirrorStatePort + versionsOf(mirror: Mirror): FileVersions + lock(): Promise<{ assertHeld(): void; release(): Promise | void }> + /** How the app names the remote side in reports and conflict copies. */ + remote: string +} + +/** Syncs each file mirror under one lock, with one `mirror.partial` issue per mirror left with conflicts or errors. */ +export const syncFileMirrors = (options: FileMirrorSyncOptions) => + eachMirror(options.mirrors, options.lock, options.remote, (mirror, assertHeld) => + syncFileMirror({ + ...options, + root: mirror.path, + folder: mirror.folder, + state: options.stateOf(mirror), + versions: options.versionsOf(mirror), + assertHeld, + }), + ) diff --git a/src/services/files/versions.ts b/src/services/files/versions.ts new file mode 100644 index 000000000..488ea38e0 --- /dev/null +++ b/src/services/files/versions.ts @@ -0,0 +1,149 @@ +import { createHash, randomUUID } from "node:crypto" +import { appendFileSync, createReadStream, createWriteStream, existsSync, mkdirSync, readFileSync } from "node:fs" +import { readdir, rename, rm } from "node:fs/promises" +import { join } from "node:path" +import { Transform } from "node:stream" +import { pipeline } from "node:stream/promises" +import { replaceLocal } from "../documents/push.js" + +/** Which way a sync moved a change. */ +export type VersionDirection = "local-to-remote" | "remote-to-local" | "restored" + +/** One line of a mirror's version log: what a sync did to a path, with the content's hash before and after. */ +export interface VersionEvent { + at: string + path: string + event: "created" | "changed" | "deleted" | "renamed" + direction: VersionDirection + /** The content the change replaced: the local file kept before a pull or deletion, the remote version an upload replaced. */ + old?: string + new?: string + size?: number + /** A rename's new path. */ + to?: string +} + +/** A version of a path, and whether its bytes are still in the store. */ +export interface FileVersion extends VersionEvent { + available: boolean +} + +export interface FileVersions { + /** Copies the file into the store before the sync overwrites or deletes it; returns the hash of what it copied. */ + keep(absolute: string): Promise + log(event: Omit): void + /** Every event of `path`, oldest first, including renames to or from it. */ + list(path: string): FileVersion[] + /** + * Writes a kept version to `into`, replacing the file only while it has `over`, or creating it when `over` is + * undefined. False when the store no longer has the version or the file was not as `over` said. + */ + write(sha256: string, into: string, over?: string): Promise + /** Applies the retention rule; see `fileVersions`. */ + prune(): Promise +} + +const RETAIN_MS = 30 * 24 * 60 * 60 * 1000 + +/** + * A mirror's local version store: `/objects/` holds each kept content once, `/log.jsonl` is + * append-only. The store lives under the profile's state folder, never inside the mirrored folder. Retention, one + * rule: a content stays while an event newer than 30 days names it, and the last content of a path whose newest + * event is a deletion stays for good. + */ +export const fileVersions = (dir: string, now: () => number = Date.now): FileVersions => { + const objects = join(dir, "objects") + const logPath = join(dir, "log.jsonl") + const events = (): VersionEvent[] => { + try { + return readFileSync(logPath, "utf8") + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as VersionEvent) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return [] + throw error + } + } + const has = (sha256: string) => existsSync(join(objects, sha256)) + return { + keep: async (absolute) => { + mkdirSync(objects, { recursive: true, mode: 0o700 }) + const temp = join(objects, `.${randomUUID()}.tmp`) + const hash = createHash("sha256") + try { + // Hashed while copied: the file may change under the sync, and the store is keyed by what it really holds. + await pipeline( + createReadStream(absolute), + new Transform({ + transform(chunk, _encoding, done) { + hash.update(chunk) + done(null, chunk) + }, + }), + createWriteStream(temp, { flags: "wx", mode: 0o600 }), + ) + const sha256 = hash.digest("hex") + if (!has(sha256)) await rename(temp, join(objects, sha256)) + return sha256 + } finally { + await rm(temp, { force: true }) + } + }, + log: (event) => { + mkdirSync(dir, { recursive: true, mode: 0o700 }) + appendFileSync(logPath, `${JSON.stringify({ at: new Date(now()).toISOString(), ...event })}\n`, { mode: 0o600 }) + }, + list: (path) => { + const found = events().filter((event) => event.path === path || event.to === path) + return found.map((event) => ({ + ...event, + available: event.old !== undefined && existsSync(join(objects, event.old)), + })) + }, + write: async (sha256, into, over) => { + if (!has(sha256)) return false + const copy = (temp: string) => + pipeline(createReadStream(join(objects, sha256)), createWriteStream(temp, { flags: "wx" })) + return (await replaceLocal(into, over, copy, { sha256 })) !== undefined + }, + prune: async () => { + const all = events() + const keep = new Set() + const last = new Map() + for (const event of all) { + last.set(event.path, event) + if (Date.parse(event.at) > now() - RETAIN_MS) + for (const hash of [event.old, event.new]) if (hash) keep.add(hash) + } + for (const event of last.values()) if (event.event === "deleted" && event.old) keep.add(event.old) + let names: string[] + try { + names = await readdir(objects) + } catch { + return + } + for (const name of names) + if (!name.startsWith(".") && !keep.has(name)) await rm(join(objects, name), { force: true }) + }, + } +} + +/** + * Puts a kept version of `path` back into the mirrored folder; the file there now is kept first. The next sync sends + * it like any local change. False when the store no longer has the version or the file changed meanwhile. + */ +export const restoreFileVersion = async (root: string, path: string, sha256: string, versions: FileVersions) => { + const absolute = join(root, path) + const current = existsSync(absolute) ? await versions.keep(absolute) : undefined + if (current === sha256) return true + if (!(await versions.write(sha256, absolute, current))) return false + versions.log({ + path, + event: current ? "changed" : "created", + direction: "restored", + ...(current ? { old: current } : {}), + new: sha256, + }) + return true +}