From 93f32bb1b08ebef6a049fb384eec0465902ee9e4 Mon Sep 17 00:00:00 2001 From: Simon Nodel Date: Fri, 4 Sep 2026 22:02:27 -0700 Subject: [PATCH] feat(api): revalidate the collection cache against disk on read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The app serves a collection from an in-memory tree it patches as the UI mutates resources. CLI commands write straight to disk, so the cache went stale and a browser refresh did not help — it re-read the same cache. Only a restart did. Detect the change on read instead of being told about it. Before serving the tree, cache status or a search, the API takes a stat-only fingerprint of the translations folder (file and folder counts, total size, newest mtime) and compares it with the one recorded at index time. A mismatch drops the cache, so the request answers 202 and the next one returns fresh data. Watching the folder was the obvious alternative and is not portable: inotify never fires for Windows-side writes on a WSL /mnt/c mount, and the same holds for several network and container mounts. A CLI-side HTTP invalidate is not portable either — localhost is not shared across the WSL boundary — and would miss every change that does not come from the CLI, such as a git checkout or a hand edit. The scan is throttled to once every 2000 ms (LINGO_TRACKER_REVALIDATE_INTERVAL_MS), and the API's own writes re-take the fingerprint on the next tick, coalesced, so bulk endpoints do not trigger a redundant re-index. Closes #86 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RifJr7xyhZkaTWn4ssV1b4 --- .../cache/collection-cache.service.spec.ts | 109 +++++++++++++ .../src/app/cache/collection-cache.service.ts | 131 +++++++++++++++- .../resources/resources.controller.spec.ts | 1 + .../resources/resources.controller.ts | 9 +- docs/api.md | 3 + libs/core/src/lib/resource/index.ts | 1 + .../src/lib/resource/tree-fingerprint.spec.ts | 146 +++++++++++++++++ .../core/src/lib/resource/tree-fingerprint.ts | 148 ++++++++++++++++++ 8 files changed, 545 insertions(+), 3 deletions(-) create mode 100644 libs/core/src/lib/resource/tree-fingerprint.spec.ts create mode 100644 libs/core/src/lib/resource/tree-fingerprint.ts diff --git a/apps/api/src/app/cache/collection-cache.service.spec.ts b/apps/api/src/app/cache/collection-cache.service.spec.ts index 4c4fb67c..e11a7013 100644 --- a/apps/api/src/app/cache/collection-cache.service.spec.ts +++ b/apps/api/src/app/cache/collection-cache.service.spec.ts @@ -1,3 +1,6 @@ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; import { Test, type TestingModule } from '@nestjs/testing'; import { CollectionCacheService, CacheStatus } from './collection-cache.service'; import * as core from '@simoncodes-ca/core'; @@ -14,6 +17,10 @@ jest.mock('@simoncodes-ca/core', () => { const mockCore = core as jest.Mocked; +// Revalidation is throttled in production; tests that are not about the throttle need every +// call to actually scan. Set before the service is constructed, since it reads this once. +process.env.LINGO_TRACKER_REVALIDATE_INTERVAL_MS = '0'; + describe('CollectionCacheService', () => { let service: CollectionCacheService; @@ -459,4 +466,106 @@ describe('CollectionCacheService', () => { expect(service.getCacheStatus('Main')).toBe(CacheStatus.NOT_STARTED); }); }); + describe('revalidate', () => { + let tempDir: string; + + const writeEntries = (entries: Record): void => { + const folderPath = path.join(tempDir, 'common'); + fs.mkdirSync(folderPath, { recursive: true }); + fs.writeFileSync(path.join(folderPath, 'resource_entries.json'), JSON.stringify(entries, null, 2), 'utf8'); + fs.writeFileSync(path.join(folderPath, 'tracker_meta.json'), JSON.stringify({}, null, 2), 'utf8'); + }; + + const indexTempCollection = async (): Promise => { + const mockTree = createMockTree(); + mockCore.loadResourceTree.mockReturnValue(mockTree); + mockCore.extractResourcesRecursively.mockReturnValue(mockTree.resources); + await service.indexCollection('Main', tempDir, 1); + }; + + beforeEach(() => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lingo-cache-revalidate-')); + writeEntries({ ok: { source: 'OK' } }); + }); + + afterEach(() => { + fs.rmSync(tempDir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); + }); + + it('does nothing when no collection is cached', () => { + expect(service.revalidate('Main', tempDir)).toBe(false); + }); + + it('does nothing when the cached collection is a different one', async () => { + await indexTempCollection(); + + expect(service.revalidate('Admin', tempDir)).toBe(false); + expect(service.getCacheStatus('Main')).toBe(CacheStatus.READY); + }); + + it('keeps the cache when nothing changed on disk', async () => { + await indexTempCollection(); + + expect(service.revalidate('Main', tempDir)).toBe(false); + expect(service.getCacheStatus('Main')).toBe(CacheStatus.READY); + }); + + it('drops the cache when a resource file changed outside the process', async () => { + await indexTempCollection(); + + writeEntries({ ok: { source: 'OK' }, maybe: { source: 'Maybe' } }); + + expect(service.revalidate('Main', tempDir)).toBe(true); + expect(service.getCacheStatus('Main')).toBe(CacheStatus.NOT_STARTED); + }); + + it('drops the cache when a resource folder is removed outside the process', async () => { + await indexTempCollection(); + + fs.rmSync(path.join(tempDir, 'common'), { recursive: true, force: true }); + + expect(service.revalidate('Main', tempDir)).toBe(true); + expect(service.getCacheStatus('Main')).toBe(CacheStatus.NOT_STARTED); + }); + + it('does not read its own write as an outside change', async () => { + await indexTempCollection(); + + // What an API mutation does: write to disk, then patch the cached tree. + writeEntries({ ok: { source: 'OK' }, maybe: { source: 'Maybe' } }); + service.addResourceToCache('Main', createMockTree().resources[0], ''); + + expect(service.revalidate('Main', tempDir)).toBe(false); + expect(service.getCacheStatus('Main')).toBe(CacheStatus.READY); + }); + + it('detects an outside change made after its own write settled', async () => { + await indexTempCollection(); + + writeEntries({ ok: { source: 'OK' }, maybe: { source: 'Maybe' } }); + service.addResourceToCache('Main', createMockTree().resources[0], ''); + service.refreshFingerprint(); + + writeEntries({ ok: { source: 'OK' }, maybe: { source: 'Maybe' }, later: { source: 'Later' } }); + + expect(service.revalidate('Main', tempDir)).toBe(true); + }); + + it('scans at most once per revalidation interval', async () => { + process.env.LINGO_TRACKER_REVALIDATE_INTERVAL_MS = '60000'; + const throttledService = new CollectionCacheService(); + process.env.LINGO_TRACKER_REVALIDATE_INTERVAL_MS = '0'; + + const mockTree = createMockTree(); + mockCore.loadResourceTree.mockReturnValue(mockTree); + mockCore.extractResourcesRecursively.mockReturnValue(mockTree.resources); + await throttledService.indexCollection('Main', tempDir, 1); + + writeEntries({ ok: { source: 'OK' }, maybe: { source: 'Maybe' } }); + + // Indexing takes a fingerprint, so the change is inside the interval that follows. + expect(throttledService.revalidate('Main', tempDir)).toBe(false); + expect(throttledService.getCacheStatus('Main')).toBe(CacheStatus.READY); + }); + }); }); diff --git a/apps/api/src/app/cache/collection-cache.service.ts b/apps/api/src/app/cache/collection-cache.service.ts index 8c6f2829..5d4c3e38 100644 --- a/apps/api/src/app/cache/collection-cache.service.ts +++ b/apps/api/src/app/cache/collection-cache.service.ts @@ -1,7 +1,15 @@ import { Injectable, Logger } from '@nestjs/common'; +import type { FolderChild, ResourceTreeEntry, ResourceTreeNode, TreeFingerprint } from '@simoncodes-ca/core'; import * as core from '@simoncodes-ca/core'; -import { extractResourcesRecursively } from '@simoncodes-ca/core'; -import type { ResourceTreeNode, ResourceTreeEntry, FolderChild } from '@simoncodes-ca/core'; +import { computeTreeFingerprint, extractResourcesRecursively, treeFingerprintsMatch } from '@simoncodes-ca/core'; + +/** + * How long a disk fingerprint is trusted before it is recomputed, in milliseconds. + * + * The scan is stat-only and costs a few milliseconds on a typical collection, but it runs + * on read paths, so it is throttled rather than run per request. + */ +const DEFAULT_REVALIDATION_INTERVAL_MS = 2000; export enum CacheStatus { NOT_STARTED = 'not-started', @@ -18,12 +26,22 @@ export interface CachedCollection { error?: string; totalKeys: number; localeCount: number; + /** Folder the tree was indexed from, needed to re-scan it later */ + translationsFolder: string | null; + /** Disk state as of the last index or self-write, used to spot outside changes */ + fingerprint: TreeFingerprint | null; } @Injectable() export class CollectionCacheService { readonly #logger = new Logger(CollectionCacheService.name); #cachedCollection: CachedCollection | null = null; + #lastRevalidationAt = 0; + #pendingFingerprintRefresh: NodeJS.Timeout | null = null; + + readonly #revalidationIntervalMs = Number( + process.env.LINGO_TRACKER_REVALIDATE_INTERVAL_MS ?? DEFAULT_REVALIDATION_INTERVAL_MS, + ); getCacheStatus(collectionName: string): CacheStatus { if (!this.#cachedCollection || this.#cachedCollection.collectionName !== collectionName) { @@ -92,6 +110,8 @@ export class CollectionCacheService { error, totalKeys: status === CacheStatus.READY && tree ? extractResourcesRecursively(tree).length : 0, localeCount: status === CacheStatus.READY ? (localeCount ?? 0) : 0, + translationsFolder: null, + fingerprint: null, }; } else { this.#cachedCollection.status = status; @@ -112,6 +132,8 @@ export class CollectionCacheService { } clearCache(): void { + this.#cancelPendingFingerprintRefresh(); + if (this.#cachedCollection) { this.#logger.log(`Clearing cache for collection: ${this.#cachedCollection.collectionName}`); this.#cachedCollection = null; @@ -155,6 +177,7 @@ export class CollectionCacheService { const existingChild = parentNode.children.find((c) => c.name === folderName); if (existingChild) { this.#logger.log(`Folder "${folderName}" already exists in cache at path "${parentPath || 'root'}"`); + this.#scheduleFingerprintRefresh(); return true; } @@ -175,6 +198,7 @@ export class CollectionCacheService { parentNode.children.sort((a, b) => a.name.localeCompare(b.name)); this.#logger.log(`Added folder "${folderName}" to cache at path "${parentPath || 'root'}"`); + this.#scheduleFingerprintRefresh(); return true; } @@ -227,6 +251,7 @@ export class CollectionCacheService { this.#logger.log(`Added resource "${resourceEntry.key}" to cache at path "${folderPath || 'root'}"`); } + this.#scheduleFingerprintRefresh(); return true; } @@ -279,6 +304,7 @@ export class CollectionCacheService { } this.#logger.log(`Removed folder "${folderPath}" from cache`); + this.#scheduleFingerprintRefresh(); return true; } @@ -329,6 +355,7 @@ export class CollectionCacheService { this.#cachedCollection.totalKeys--; this.#logger.log(`Removed resource "${resourceKey}" from cache at path "${folderPath || 'root'}"`); + this.#scheduleFingerprintRefresh(); return true; } @@ -418,9 +445,99 @@ export class CollectionCacheService { destParent.children.sort((a, b) => a.name.localeCompare(b.name)); this.#logger.log(`Moved folder "${sourceFolderPath}" to "${destinationFolderPath || 'root'}" in cache`); + this.#scheduleFingerprintRefresh(); + return true; + } + + /** + * Drops the cache when the translations folder has changed underneath it. + * + * The app caches a collection's whole tree in memory, so a CLI command, a `git checkout` + * or a hand edit would otherwise stay invisible until a restart — a browser refresh does + * not help, because it re-reads the same cache. Filesystem watching cannot fix this + * portably: inotify never fires for Windows-side writes on a WSL `/mnt/c` mount, and the + * same holds for several network and container mounts. So the check happens on read, + * against a stat-only fingerprint, throttled so it costs almost nothing. + * + * @param collectionName - The collection being read + * @param translationsFolder - The collection's translations folder + * @param cwd - Directory `translationsFolder` is resolved against + * @returns true when the cache was dropped and needs re-indexing + */ + revalidate(collectionName: string, translationsFolder: string, cwd?: string): boolean { + const cached = this.#cachedCollection; + + if (!cached || cached.collectionName !== collectionName || cached.status !== CacheStatus.READY) { + return false; + } + + const now = Date.now(); + if (now - this.#lastRevalidationAt < this.#revalidationIntervalMs) { + return false; + } + this.#lastRevalidationAt = now; + + const fingerprint = computeTreeFingerprint({ translationsFolder, cwd }); + + // A write of our own is still waiting for its deferred baseline refresh. Adopt the + // fingerprint now instead of reading our own change as somebody else's. + if (this.#pendingFingerprintRefresh !== null) { + this.#cancelPendingFingerprintRefresh(); + cached.fingerprint = fingerprint; + return false; + } + + if (treeFingerprintsMatch(cached.fingerprint, fingerprint)) { + return false; + } + + this.#logger.log(`Translations folder changed on disk for collection ${collectionName}, dropping cache`); + this.clearCache(); return true; } + /** + * Re-takes the disk fingerprint so the cache's own writes do not later read as external + * changes. Safe to call when no collection is cached. + */ + refreshFingerprint(): void { + this.#cancelPendingFingerprintRefresh(); + + const cached = this.#cachedCollection; + if (!cached?.translationsFolder) { + return; + } + + cached.fingerprint = computeTreeFingerprint({ translationsFolder: cached.translationsFolder }); + } + + /** + * Queues a fingerprint refresh for the end of the current tick. + * + * Bulk endpoints mutate the cache once per resource in a synchronous loop, so deferring + * collapses a whole batch into a single scan. + */ + #scheduleFingerprintRefresh(): void { + if (this.#pendingFingerprintRefresh !== null || !this.#cachedCollection?.translationsFolder) { + return; + } + + this.#pendingFingerprintRefresh = setTimeout(() => { + this.#pendingFingerprintRefresh = null; + this.refreshFingerprint(); + }, 0); + + // A pending refresh must never hold the process open on its own. + this.#pendingFingerprintRefresh.unref?.(); + } + + #cancelPendingFingerprintRefresh(): void { + if (this.#pendingFingerprintRefresh !== null) { + clearTimeout(this.#pendingFingerprintRefresh); + this.#pendingFingerprintRefresh = null; + } + } + async indexCollection(collectionName: string, translationsFolder: string, localeCount?: number): Promise { const currentStatus = this.getCacheStatus(collectionName); @@ -436,6 +553,10 @@ export class CollectionCacheService { this.#logger.log(`Starting indexing for collection: ${collectionName}`); try { + // Taken before the load: a write that lands mid-load then disagrees with this + // fingerprint, which costs one extra re-index but never loses the change. + const fingerprint = computeTreeFingerprint({ translationsFolder }); + const tree = core.loadResourceTree({ translationsFolder, path: '', @@ -456,6 +577,12 @@ export class CollectionCacheService { } this.setCacheStatus(indexingCollectionName, CacheStatus.READY, tree, undefined, localeCount); + + if (this.#cachedCollection) { + this.#cachedCollection.translationsFolder = translationsFolder; + this.#cachedCollection.fingerprint = fingerprint; + } + this.#lastRevalidationAt = Date.now(); } catch (error) { const duration = Date.now() - startTime; const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred'; diff --git a/apps/api/src/app/collections/resources/resources.controller.spec.ts b/apps/api/src/app/collections/resources/resources.controller.spec.ts index d2bd844b..a1a3b168 100644 --- a/apps/api/src/app/collections/resources/resources.controller.spec.ts +++ b/apps/api/src/app/collections/resources/resources.controller.spec.ts @@ -116,6 +116,7 @@ describe('ResourcesController', () => { getCacheStats: jest.fn(), indexCollection: jest.fn(), clearCache: jest.fn(), + revalidate: jest.fn().mockReturnValue(false), addResourceToCache: jest.fn().mockReturnValue(true), removeResourceFromCache: jest.fn().mockReturnValue(true), }; diff --git a/apps/api/src/app/collections/resources/resources.controller.ts b/apps/api/src/app/collections/resources/resources.controller.ts index bfdf1612..7eac8622 100644 --- a/apps/api/src/app/collections/resources/resources.controller.ts +++ b/apps/api/src/app/collections/resources/resources.controller.ts @@ -505,6 +505,10 @@ export class ResourcesController { const collection = config.collections[decodedCollectionName]; const translationsFolder = collection.translationsFolder; + // Pick up changes made outside this process (CLI commands, git checkouts, hand edits) + // before trusting the cache. + this.#cacheService.revalidate(decodedCollectionName, translationsFolder); + // Check cache status const cacheStatus = this.#cacheService.getCacheStatus(decodedCollectionName); @@ -631,10 +635,12 @@ export class ResourcesController { throw new NotFoundException(`Collection "${decodedCollectionName}" not found`); } + const collection = config.collections[decodedCollectionName]; + this.#cacheService.revalidate(decodedCollectionName, collection.translationsFolder); + const cacheStatus = this.#cacheService.getCacheStatus(decodedCollectionName); // If cache is not started, trigger indexing asynchronously - const collection = config.collections[decodedCollectionName]; if (cacheStatus === CacheStatus.NOT_STARTED) { const translationsFolder = collection.translationsFolder; const locales = collection.locales ?? config.locales ?? []; @@ -717,6 +723,7 @@ export class ResourcesController { const maxResults = Math.min(dto.maxResults || 100, 500); // Try to use cached tree for faster search + this.#cacheService.revalidate(decodedCollectionName, translationsFolder); const cachedTree = this.#cacheService.getCache(decodedCollectionName); let searchResults: SearchResult[]; diff --git a/docs/api.md b/docs/api.md index 9d3ec9e9..d8198050 100644 --- a/docs/api.md +++ b/docs/api.md @@ -11,6 +11,7 @@ The LingoTracker API provides REST endpoints for managing translation resources, - **Base URL**: `http://localhost:3030/api` - **Default Port**: `3030` (configurable via `LINGO_TRACKER_PORT` environment variable) +- **Cache revalidation interval**: `2000` ms (configurable via `LINGO_TRACKER_REVALIDATE_INTERVAL_MS` environment variable) - **Content-Type**: `application/json` - **Response Format**: JSON - **CORS**: Enabled with wildcard origin (`*`) in development @@ -1007,6 +1008,8 @@ interface MoveResourceResponseDto { Returns the resource tree for a collection, optionally scoped to a sub-path. The tree is built from an in-memory cache — the first request triggers background indexing and returns a `202 Accepted` with a status response until the cache is ready. +The cache is revalidated on read. Before the tree is served, the API takes a stat-only fingerprint of the collection's translations folder and compares it with the one recorded at index time; if they differ, the cache is dropped and re-indexed, so the request answers `202 Accepted` and the next one returns fresh data. This is what makes changes written outside the running app — a `lingo-tracker` CLI command, a `git checkout`, a hand edit — visible on a browser refresh instead of only after a restart. The scan is throttled to once every 2000 ms, configurable via the `LINGO_TRACKER_REVALIDATE_INTERVAL_MS` environment variable. + **Endpoint**: `GET /api/collections/:collectionName/resources/tree` **Path Parameters**: diff --git a/libs/core/src/lib/resource/index.ts b/libs/core/src/lib/resource/index.ts index b3e2804d..1ca96300 100644 --- a/libs/core/src/lib/resource/index.ts +++ b/libs/core/src/lib/resource/index.ts @@ -4,3 +4,4 @@ export * from './load-resource-tree'; export * from './load-full-resource-tree'; export * from './extract-subtree'; export * from './search'; +export * from './tree-fingerprint'; diff --git a/libs/core/src/lib/resource/tree-fingerprint.spec.ts b/libs/core/src/lib/resource/tree-fingerprint.spec.ts new file mode 100644 index 00000000..ca52b5e8 --- /dev/null +++ b/libs/core/src/lib/resource/tree-fingerprint.spec.ts @@ -0,0 +1,146 @@ +/** + * Real-filesystem tests for computeTreeFingerprint. + * + * The whole point of the fingerprint is what `stat` reports, so these run + * against the real filesystem rather than a mocked `fs`. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { computeTreeFingerprint, treeFingerprintsMatch } from './tree-fingerprint'; + +describe('computeTreeFingerprint', () => { + let tempDir: string; + + const writeResourceFolder = (relativePath: string, entries: Record): string => { + const folderPath = path.join(tempDir, relativePath); + fs.mkdirSync(folderPath, { recursive: true }); + fs.writeFileSync(path.join(folderPath, 'resource_entries.json'), JSON.stringify(entries, null, 2), 'utf8'); + fs.writeFileSync(path.join(folderPath, 'tracker_meta.json'), JSON.stringify({}, null, 2), 'utf8'); + return folderPath; + }; + + beforeEach(() => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lingo-fingerprint-test-')); + }); + + afterEach(() => { + fs.rmSync(tempDir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); + }); + + it('returns an all-zero fingerprint for a folder that does not exist', () => { + const fingerprint = computeTreeFingerprint({ translationsFolder: path.join(tempDir, 'missing') }); + + expect(fingerprint).toEqual({ fileCount: 0, folderCount: 0, totalSize: 0, maxMtimeMs: 0 }); + }); + + it('counts resource files across nested folders', () => { + writeResourceFolder('common/button', { ok: { source: 'OK' } }); + writeResourceFolder('common/dialog', { close: { source: 'Close' } }); + + const fingerprint = computeTreeFingerprint({ translationsFolder: tempDir }); + + // Two resource files per folder, three folders plus the root. + expect(fingerprint.fileCount).toBe(4); + expect(fingerprint.folderCount).toBe(4); + expect(fingerprint.totalSize).toBeGreaterThan(0); + }); + + it('resolves a relative translations folder against cwd', () => { + writeResourceFolder('translations/common', { ok: { source: 'OK' } }); + + const fingerprint = computeTreeFingerprint({ translationsFolder: 'translations', cwd: tempDir }); + + expect(fingerprint.fileCount).toBe(2); + }); + + it('ignores files that are not resource files', () => { + const folderPath = writeResourceFolder('common', { ok: { source: 'OK' } }); + fs.writeFileSync(path.join(folderPath, 'README.md'), 'not a resource file', 'utf8'); + + const fingerprint = computeTreeFingerprint({ translationsFolder: tempDir }); + + expect(fingerprint.fileCount).toBe(2); + }); + + it('is stable when nothing changes', () => { + writeResourceFolder('common', { ok: { source: 'OK' } }); + + const first = computeTreeFingerprint({ translationsFolder: tempDir }); + const second = computeTreeFingerprint({ translationsFolder: tempDir }); + + expect(treeFingerprintsMatch(first, second)).toBe(true); + }); + + it('changes when a resource file changes size, even within the same mtime tick', () => { + const folderPath = writeResourceFolder('common', { ok: { source: 'OK' } }); + const before = computeTreeFingerprint({ translationsFolder: tempDir }); + + // Deliberately not waiting for the clock to tick: coarse mtime granularity + // is exactly the case size and counts exist to cover. + fs.writeFileSync( + path.join(folderPath, 'resource_entries.json'), + JSON.stringify({ ok: { source: 'OK' }, cancel: { source: 'Cancel' } }, null, 2), + 'utf8', + ); + + const after = computeTreeFingerprint({ translationsFolder: tempDir }); + + expect(treeFingerprintsMatch(before, after)).toBe(false); + }); + + it('changes when a resource folder is added', () => { + writeResourceFolder('common', { ok: { source: 'OK' } }); + const before = computeTreeFingerprint({ translationsFolder: tempDir }); + + writeResourceFolder('common/nested', { close: { source: 'Close' } }); + const after = computeTreeFingerprint({ translationsFolder: tempDir }); + + expect(treeFingerprintsMatch(before, after)).toBe(false); + }); + + it('changes when a resource folder is removed', () => { + writeResourceFolder('common', { ok: { source: 'OK' } }); + const removedFolder = writeResourceFolder('other', { close: { source: 'Close' } }); + const before = computeTreeFingerprint({ translationsFolder: tempDir }); + + fs.rmSync(removedFolder, { recursive: true, force: true }); + const after = computeTreeFingerprint({ translationsFolder: tempDir }); + + expect(treeFingerprintsMatch(before, after)).toBe(false); + }); + + it('detects an empty folder being added, even though it holds no resource files', () => { + writeResourceFolder('common', { ok: { source: 'OK' } }); + const before = computeTreeFingerprint({ translationsFolder: tempDir }); + + fs.mkdirSync(path.join(tempDir, 'empty'), { recursive: true }); + const after = computeTreeFingerprint({ translationsFolder: tempDir }); + + expect(treeFingerprintsMatch(before, after)).toBe(false); + }); +}); + +describe('treeFingerprintsMatch', () => { + const fingerprint = { fileCount: 2, folderCount: 1, totalSize: 100, maxMtimeMs: 1000 }; + + it('treats a missing fingerprint as a mismatch', () => { + expect(treeFingerprintsMatch(null, fingerprint)).toBe(false); + expect(treeFingerprintsMatch(fingerprint, null)).toBe(false); + expect(treeFingerprintsMatch(null, null)).toBe(false); + }); + + it('matches identical fingerprints', () => { + expect(treeFingerprintsMatch(fingerprint, { ...fingerprint })).toBe(true); + }); + + it.each([ + ['fileCount', { fileCount: 3 }], + ['folderCount', { folderCount: 2 }], + ['totalSize', { totalSize: 101 }], + ['maxMtimeMs', { maxMtimeMs: 1001 }], + ])('does not match when %s differs', (_field, override) => { + expect(treeFingerprintsMatch(fingerprint, { ...fingerprint, ...override })).toBe(false); + }); +}); diff --git a/libs/core/src/lib/resource/tree-fingerprint.ts b/libs/core/src/lib/resource/tree-fingerprint.ts new file mode 100644 index 00000000..b0469070 --- /dev/null +++ b/libs/core/src/lib/resource/tree-fingerprint.ts @@ -0,0 +1,148 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { RESOURCE_ENTRIES_FILENAME, TRACKER_META_FILENAME } from '../../constants'; + +/** + * A cheap, stat-only summary of a translations tree on disk. + * + * Used to detect that something outside the running process (a CLI command, a + * `git checkout`, a hand edit) has changed the tree, without watching the + * filesystem. Watching is not an option here: inotify never fires for + * Windows-side writes on a WSL `/mnt/c` mount, and the same is true of several + * network and container mounts. + * + * The fingerprint deliberately reads no file contents — only directory entries + * and `stat` results — so it stays cheap enough to run on a request. + */ +export interface TreeFingerprint { + /** Number of `resource_entries.json` / `tracker_meta.json` files found */ + readonly fileCount: number; + /** Number of directories walked, including the root */ + readonly folderCount: number; + /** Sum of the sizes of the counted files, in bytes */ + readonly totalSize: number; + /** Newest mtime across the counted files, in milliseconds */ + readonly maxMtimeMs: number; +} + +export interface ComputeTreeFingerprintOptions { + /** Root translations folder path (absolute, or relative to `cwd`) */ + readonly translationsFolder: string; + /** Current working directory used to resolve `translationsFolder` */ + readonly cwd?: string; +} + +const EMPTY_FINGERPRINT: TreeFingerprint = { + fileCount: 0, + folderCount: 0, + totalSize: 0, + maxMtimeMs: 0, +}; + +/** + * Walks a translations folder and summarises its resource files. + * + * A missing root folder yields an all-zero fingerprint rather than throwing, so + * that a not-yet-created folder and a deleted one compare equal. + * + * @param options - Which folder to walk, and what to resolve it against + * @returns A fingerprint that changes whenever a resource file is added, + * removed, resized or touched + */ +export function computeTreeFingerprint(options: ComputeTreeFingerprintOptions): TreeFingerprint { + const { translationsFolder, cwd = process.cwd() } = options; + const rootPath = path.resolve(cwd, translationsFolder); + + if (!fs.existsSync(rootPath)) { + return EMPTY_FINGERPRINT; + } + + let fileCount = 0; + let folderCount = 0; + let totalSize = 0; + let maxMtimeMs = 0; + + const visitedPaths = new Set(); + const stack: string[] = [rootPath]; + + while (stack.length > 0) { + const folderPath = stack.pop(); + if (!folderPath) { + continue; + } + + // Symlinked folders can point back up the tree; realpath is what makes the + // visited set able to catch that. + let realPath: string; + try { + realPath = fs.realpathSync(folderPath); + } catch { + continue; + } + + if (visitedPaths.has(realPath)) { + continue; + } + visitedPaths.add(realPath); + + let dirEntries: fs.Dirent[]; + try { + dirEntries = fs.readdirSync(folderPath, { withFileTypes: true }); + } catch { + // A folder that disappeared mid-walk, or one we cannot read, simply does + // not contribute. The next fingerprint will disagree with this one, which + // is the correct outcome. + continue; + } + + folderCount++; + + for (const dirEntry of dirEntries) { + const entryPath = path.join(folderPath, dirEntry.name); + + if (dirEntry.isDirectory()) { + stack.push(entryPath); + continue; + } + + if (dirEntry.name !== RESOURCE_ENTRIES_FILENAME && dirEntry.name !== TRACKER_META_FILENAME) { + continue; + } + + try { + const stats = fs.statSync(entryPath); + fileCount++; + totalSize += stats.size; + maxMtimeMs = Math.max(maxMtimeMs, stats.mtimeMs); + } catch { + // Same reasoning as the unreadable-folder case above. + } + } + } + + return { fileCount, folderCount, totalSize, maxMtimeMs }; +} + +/** + * Compares two fingerprints. + * + * Size and counts are compared alongside mtime because mtime granularity is as + * coarse as one to two seconds on drvfs and FAT, so a same-second edit can + * leave mtime untouched. + * + * @param first - A fingerprint, or null when none has been taken yet + * @param second - The fingerprint to compare it against + * @returns true when both describe the same on-disk state + */ +export function treeFingerprintsMatch(first: TreeFingerprint | null, second: TreeFingerprint | null): boolean { + if (!first || !second) { + return false; + } + + return ( + first.fileCount === second.fileCount && + first.folderCount === second.folderCount && + first.totalSize === second.totalSize && + first.maxMtimeMs === second.maxMtimeMs + ); +}