From 52e7f80ea0d0ce57f0d018d41e8eb0a8e09ef618 Mon Sep 17 00:00:00 2001 From: Vladimir Haltakov Date: Sat, 13 Jun 2026 17:51:29 +0200 Subject: [PATCH 1/4] perf: generate thumbnails in parallel processGalleryThumbnails processed media files strictly one at a time, which dominates runtime on large galleries. Thumbnailing is IO/CPU bound and Sharp and ffmpeg release the JS thread, so the work parallelizes well. Add a small dependency-free mapWithConcurrency worker-pool helper (order-preserving) and use it to process each section's media with a pool sized to the available cores (cpus - 1). Per-file error isolation is unchanged. On a multi-core machine a large gallery now builds several times faster. Co-Authored-By: Claude Fable 5 --- gallery/src/modules/thumbnails/index.ts | 14 ++++-- gallery/src/utils/concurrency.ts | 34 ++++++++++++++ gallery/tests/concurrency.test.ts | 60 +++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 gallery/src/utils/concurrency.ts create mode 100644 gallery/tests/concurrency.test.ts diff --git a/gallery/src/modules/thumbnails/index.ts b/gallery/src/modules/thumbnails/index.ts index e0ce0bb..152c707 100644 --- a/gallery/src/modules/thumbnails/index.ts +++ b/gallery/src/modules/thumbnails/index.ts @@ -1,4 +1,5 @@ import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; import { @@ -12,6 +13,7 @@ import { getFileMtime } from './utils'; import { findGalleries, handleFileProcessingError } from '../../utils'; import { generateBlurHash } from '../../utils/blurhash'; +import { mapWithConcurrency } from '../../utils/concurrency'; import { getImageDescription } from '../../utils/descriptions'; import { parseGalleryJson } from '../../utils/gallery'; import { createImageThumbnails, loadImageWithMetadata, type ThumbnailSizeDimension } from '../../utils/image'; @@ -298,12 +300,16 @@ export async function processGalleryThumbnails( // If the mediaBasePath is not set, use the gallery directory const mediaBasePath = galleryData.mediaBasePath ?? path.join(galleryDir); - // Process all sections and their images + // Process media files in parallel. Thumbnailing is IO/CPU bound (Sharp and ffmpeg release the JS + // thread), so a worker pool sized to the available cores processes a large gallery several times + // faster than the previous one-at-a-time loop while keeping per-file error isolation. + const concurrency = Math.max(2, os.cpus().length - 1); + let processedCount = 0; for (const section of galleryData.sections) { - for (const [index, mediaFile] of section.images.entries()) { - section.images[index] = await processMediaFile(mediaFile, mediaBasePath, thumbnailsPath, thumbnailConfig, ui); - } + section.images = await mapWithConcurrency(section.images, concurrency, (mediaFile) => + processMediaFile(mediaFile, mediaBasePath, thumbnailsPath, thumbnailConfig, ui), + ); processedCount += section.images.length; } diff --git a/gallery/src/utils/concurrency.ts b/gallery/src/utils/concurrency.ts new file mode 100644 index 0000000..ecdfef2 --- /dev/null +++ b/gallery/src/utils/concurrency.ts @@ -0,0 +1,34 @@ +/** + * Maps over items with a bounded number of concurrent workers, preserving input order in the results. + * + * Used to parallelize IO/CPU-bound work (such as thumbnail generation) without spawning unbounded + * concurrency that would exhaust memory or oversubscribe the CPU. A fixed pool of workers pulls items + * from a shared cursor; because JavaScript runs the cursor read/increment without interleaving, no + * locking is required. + * + * @param items - The items to process + * @param concurrency - Maximum number of mapper invocations running at the same time + * @param mapper - Async function applied to each item; receives the item and its index + * @returns Promise resolving to the mapped results in the same order as the input + */ +export async function mapWithConcurrency( + items: readonly T[], + concurrency: number, + mapper: (item: T, index: number) => Promise, +): Promise { + const results: R[] = []; + let nextIndex = 0; + + const worker = async (): Promise => { + while (nextIndex < items.length) { + const currentIndex = nextIndex; + nextIndex += 1; + results[currentIndex] = await mapper(items[currentIndex], currentIndex); + } + }; + + const workerCount = Math.max(1, Math.min(concurrency, items.length)); + await Promise.all(Array.from({ length: workerCount }, () => worker())); + + return results; +} diff --git a/gallery/tests/concurrency.test.ts b/gallery/tests/concurrency.test.ts new file mode 100644 index 0000000..def5ad9 --- /dev/null +++ b/gallery/tests/concurrency.test.ts @@ -0,0 +1,60 @@ +import { mapWithConcurrency } from '../src/utils/concurrency'; + +const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +describe('mapWithConcurrency', () => { + test('should preserve input order in the results', async () => { + const items = [10, 5, 1, 8, 3]; + + // Items with shorter delays resolve first, but results must stay in input order + const results = await mapWithConcurrency(items, 3, async (value) => { + await delay(value); + return value * 2; + }); + + expect(results).toEqual([20, 10, 2, 16, 6]); + }); + + test('should pass the index to the mapper', async () => { + const items = ['a', 'b', 'c']; + + const results = await mapWithConcurrency(items, 2, async (value, index) => `${value}${index}`); + + expect(results).toEqual(['a0', 'b1', 'c2']); + }); + + test('should never run more than the requested number of workers at once', async () => { + const concurrency = 3; + let active = 0; + let maxActive = 0; + + await mapWithConcurrency( + Array.from({ length: 20 }, (_, index) => index), + concurrency, + async () => { + active += 1; + maxActive = Math.max(maxActive, active); + await delay(2); + active -= 1; + }, + ); + + expect(maxActive).toBeLessThanOrEqual(concurrency); + }); + + test('should process every item exactly once', async () => { + const items = Array.from({ length: 50 }, (_, index) => index); + const seen: number[] = []; + + await mapWithConcurrency(items, 8, async (value) => { + seen.push(value); + }); + + expect(seen.sort((a, b) => a - b)).toEqual(items); + }); + + test('should handle an empty input', async () => { + const results = await mapWithConcurrency([], 4, async (value) => value); + expect(results).toEqual([]); + }); +}); From 3f6d175b36bdde09a6bb8fd094c2201ed7c8204b Mon Sep 17 00:00:00 2001 From: Vladimir Haltakov Date: Sat, 13 Jun 2026 18:05:49 +0200 Subject: [PATCH 2/4] perf: make thumbnail format, quality and effort configurable Thumbnails were always encoded as AVIF with Sharp's default settings. AVIF gives small files but is slow to encode, which dominates build time on large galleries. Expose format (avif/webp/jpeg), quality and encoder effort through the existing 4-level config hierarchy (CLI > gallery.json > theme > defaults) so users can trade size for speed (e.g. webp with low effort) on big collections. - Extend ThumbnailConfigSchema and the gallery.json schema with the new optional fields so they validate and round-trip. - resizeImage/createImageThumbnails/createVideoThumbnails pass the encode options through to Sharp, clamping effort for webp and dropping it for jpeg. - Derive the thumbnail file extension from the format, and thread the resolved format into getSubgalleryThumbnailPath so sub-gallery thumbnail links match. - Add --thumbnail-format/--thumbnail-quality/--thumbnail-effort flags to the build and thumbnails commands; build persists them to gallery.json like the existing size/edge flags. Defaults are unchanged (avif, Sharp defaults), so existing galleries produce identical output. Co-Authored-By: Claude Fable 5 --- common/src/gallery/schemas.ts | 3 + common/src/theme/config.ts | 40 ++++++++++- common/src/theme/index.ts | 4 +- common/src/theme/paths.ts | 11 ++- common/src/theme/resolver.ts | 25 ++++--- common/src/theme/types.ts | 4 +- gallery/jest.config.cjs | 2 + gallery/src/index.ts | 8 ++- gallery/src/modules/build/index.ts | 19 ++--- gallery/src/modules/build/types/index.ts | 8 +++ gallery/src/modules/thumbnails/index.ts | 47 ++++++++---- gallery/src/modules/thumbnails/types/index.ts | 8 +++ gallery/src/utils/image.ts | 56 +++++++++++++-- gallery/src/utils/index.ts | 52 ++++++++++++++ gallery/src/utils/video.ts | 8 ++- gallery/tests/thumbnail-config.test.ts | 72 +++++++++++++++++++ 16 files changed, 317 insertions(+), 50 deletions(-) create mode 100644 gallery/tests/thumbnail-config.test.ts diff --git a/common/src/gallery/schemas.ts b/common/src/gallery/schemas.ts index 8ee3612..f865de3 100644 --- a/common/src/gallery/schemas.ts +++ b/common/src/gallery/schemas.ts @@ -125,6 +125,9 @@ export const GalleryDataSchema = z.object({ .object({ size: z.number().optional(), edge: z.enum(['auto', 'width', 'height']).optional(), + format: z.enum(['avif', 'webp', 'jpeg']).optional(), + quality: z.number().optional(), + effort: z.number().optional(), }) .optional(), metadata: GalleryMetadataSchema, diff --git a/common/src/theme/config.ts b/common/src/theme/config.ts index 34e7cf7..26e86b6 100644 --- a/common/src/theme/config.ts +++ b/common/src/theme/config.ts @@ -1,9 +1,18 @@ import { z } from 'zod'; +/** Supported output formats for generated thumbnails */ +export const THUMBNAIL_FORMATS = ['avif', 'webp', 'jpeg'] as const; + +/** TypeScript type for a thumbnail output format */ +export type ThumbnailFormat = (typeof THUMBNAIL_FORMATS)[number]; + /** Zod schema for thumbnail configuration */ export const ThumbnailConfigSchema = z.object({ size: z.number().min(50).max(4000).optional(), edge: z.enum(['auto', 'width', 'height']).optional(), + format: z.enum(THUMBNAIL_FORMATS).optional(), + quality: z.number().min(1).max(100).optional(), + effort: z.number().min(0).max(9).optional(), }); /** Zod schema for theme configuration file (themeConfig.json) */ @@ -17,12 +26,31 @@ export type ThumbnailConfig = z.infer; /** TypeScript type for theme configuration */ export type ThemeConfig = z.infer; +/** Fully resolved thumbnail configuration with size, edge and format always present */ +export interface ResolvedThumbnailConfig { + size: number; + edge: 'auto' | 'width' | 'height'; + format: ThumbnailFormat; + quality?: number; + effort?: number; +} + /** Default thumbnail configuration values */ -export const DEFAULT_THUMBNAIL_CONFIG: Required = { +export const DEFAULT_THUMBNAIL_CONFIG: ResolvedThumbnailConfig = { size: 300, edge: 'auto', + format: 'avif', }; +/** + * Returns the file extension (without a leading dot) used for thumbnails of the given format. + * @param format - The thumbnail output format + * @returns The file extension to use for generated thumbnails + */ +export function getThumbnailExtension(format: ThumbnailFormat): string { + return format === 'jpeg' ? 'jpg' : format; +} + /** * Extracts thumbnail config from gallery data. * @param gallery - The gallery data object @@ -32,6 +60,9 @@ export function extractThumbnailConfigFromGallery(gallery: { thumbnails?: Thumbn return { size: gallery.thumbnails?.size, edge: gallery.thumbnails?.edge, + format: gallery.thumbnails?.format, + quality: gallery.thumbnails?.quality, + effort: gallery.thumbnails?.effort, }; } @@ -45,15 +76,18 @@ export function extractThumbnailConfigFromGallery(gallery: { thumbnails?: Thumbn * @param cliConfig - Config from CLI flags (optional) * @param galleryConfig - Config from gallery.json (optional) * @param themeConfig - Config from themeConfig.json (optional) - * @returns Merged thumbnail configuration with all values resolved + * @returns Merged thumbnail configuration with all required values resolved */ export function mergeThumbnailConfig( cliConfig?: ThumbnailConfig, galleryConfig?: ThumbnailConfig, themeConfig?: ThumbnailConfig, -): Required { +): ResolvedThumbnailConfig { return { size: cliConfig?.size ?? galleryConfig?.size ?? themeConfig?.size ?? DEFAULT_THUMBNAIL_CONFIG.size, edge: cliConfig?.edge ?? galleryConfig?.edge ?? themeConfig?.edge ?? DEFAULT_THUMBNAIL_CONFIG.edge, + format: cliConfig?.format ?? galleryConfig?.format ?? themeConfig?.format ?? DEFAULT_THUMBNAIL_CONFIG.format, + quality: cliConfig?.quality ?? galleryConfig?.quality ?? themeConfig?.quality, + effort: cliConfig?.effort ?? galleryConfig?.effort ?? themeConfig?.effort, }; } diff --git a/common/src/theme/index.ts b/common/src/theme/index.ts index 5473588..e247844 100644 --- a/common/src/theme/index.ts +++ b/common/src/theme/index.ts @@ -2,11 +2,13 @@ export type { ResolvedGalleryData, ResolvedHero, ResolvedImage, ResolvedSection, ResolvedSubGallery } from './types'; // Theme config -export type { ThemeConfig, ThumbnailConfig } from './config'; +export type { ResolvedThumbnailConfig, ThemeConfig, ThumbnailConfig, ThumbnailFormat } from './config'; export { DEFAULT_THUMBNAIL_CONFIG, extractThumbnailConfigFromGallery, + getThumbnailExtension, mergeThumbnailConfig, + THUMBNAIL_FORMATS, ThemeConfigSchema, ThumbnailConfigSchema, } from './config'; diff --git a/common/src/theme/paths.ts b/common/src/theme/paths.ts index d81c9a1..b4f90ad 100644 --- a/common/src/theme/paths.ts +++ b/common/src/theme/paths.ts @@ -1,5 +1,7 @@ import path from 'node:path'; +import { getThumbnailExtension, type ThumbnailFormat } from './config'; + /** * Normalizes resource paths to be relative to the gallery root directory. * @@ -56,11 +58,16 @@ export function getPhotoPath(filename: string, mediaBaseUrl?: string, url?: stri * * @param headerImageFilename - The filename of the subgallery header image * @param resolvedSubgalleryPath - The resolved subgallery path relative to the gallery root + * @param format - The thumbnail output format used to derive the file extension (defaults to 'avif') * @returns The normalized path relative to the gallery root directory */ -export function getSubgalleryThumbnailPath(headerImageFilename: string, resolvedSubgalleryPath?: string): string { +export function getSubgalleryThumbnailPath( + headerImageFilename: string, + resolvedSubgalleryPath?: string, + format: ThumbnailFormat = 'avif', +): string { const basename = path.basename(headerImageFilename, path.extname(headerImageFilename)); - const thumbnailFilename = `${basename}.avif`; + const thumbnailFilename = `${basename}.${getThumbnailExtension(format)}`; const subgalleryFolder = resolvedSubgalleryPath || path.basename(path.dirname(headerImageFilename)); return path.join(subgalleryFolder, 'gallery', 'images', thumbnailFilename); diff --git a/common/src/theme/resolver.ts b/common/src/theme/resolver.ts index 27a4237..1f0cf1b 100644 --- a/common/src/theme/resolver.ts +++ b/common/src/theme/resolver.ts @@ -1,6 +1,11 @@ import path from 'node:path'; -import { extractThumbnailConfigFromGallery, mergeThumbnailConfig, type ThumbnailConfig } from './config'; +import { + extractThumbnailConfigFromGallery, + mergeThumbnailConfig, + type ThumbnailConfig, + type ThumbnailFormat, +} from './config'; import { LANDSCAPE_SIZES, PORTRAIT_SIZES } from './constants'; import { renderMarkdown } from './markdown'; import { buildHeroSrcset, getPhotoPath, getRelativePath, getSubgalleryThumbnailPath, getThumbnailPath } from './paths'; @@ -60,13 +65,17 @@ async function resolveSection( /** * Resolve a sub-gallery with computed thumbnail path and optional resolved path. */ -function resolveSubGallery(subGallery: SubGallery, galleryJsonPath?: string): ResolvedSubGallery { +function resolveSubGallery( + subGallery: SubGallery, + galleryJsonPath?: string, + thumbnailFormat?: ThumbnailFormat, +): ResolvedSubGallery { const resolvedPath = galleryJsonPath ? getRelativePath(subGallery.path, galleryJsonPath) : undefined; return { title: subGallery.title, headerImage: subGallery.headerImage, path: subGallery.path, - thumbnailPath: getSubgalleryThumbnailPath(subGallery.headerImage, resolvedPath), + thumbnailPath: getSubgalleryThumbnailPath(subGallery.headerImage, resolvedPath, thumbnailFormat), resolvedPath, }; } @@ -180,17 +189,17 @@ export async function resolveGalleryData( gallery.sections.map((section) => resolveSection(section, mediaBaseUrl, thumbsBaseUrl)), ); + // Merge thumbnail config: CLI > gallery.json > themeConfig > defaults + const galleryThumbnailConfig = extractThumbnailConfigFromGallery(gallery); + const thumbnails = mergeThumbnailConfig(cliConfig, galleryThumbnailConfig, themeConfig); + const resolvedSubGalleries = subGalleries?.galleries?.length ? { title: subGalleries.title, - galleries: subGalleries.galleries.map((sg) => resolveSubGallery(sg, galleryJsonPath)), + galleries: subGalleries.galleries.map((sg) => resolveSubGallery(sg, galleryJsonPath, thumbnails.format)), } : undefined; - // Merge thumbnail config: CLI > gallery.json > themeConfig > defaults - const galleryThumbnailConfig = extractThumbnailConfigFromGallery(gallery); - const thumbnails = mergeThumbnailConfig(cliConfig, galleryThumbnailConfig, themeConfig); - return { title: gallery.title, url: gallery.url, diff --git a/common/src/theme/types.ts b/common/src/theme/types.ts index 9a842f8..150fad2 100644 --- a/common/src/theme/types.ts +++ b/common/src/theme/types.ts @@ -1,5 +1,5 @@ import type { GalleryMetadata, HeaderImageVariants } from '../gallery'; -import type { ThumbnailConfig } from './config'; +import type { ResolvedThumbnailConfig } from './config'; /** Resolved hero data with all paths computed and markdown parsed */ export interface ResolvedHero { @@ -73,7 +73,7 @@ export interface ResolvedGalleryData { * Thumbnail configuration with dimension and edge settings. * Themes should use this for display sizing (e.g., row-height for modern theme). */ - thumbnails?: Required; + thumbnails?: ResolvedThumbnailConfig; /** Custom CSS variable overrides from gallery.json, passed through for theme injection. */ customStyles?: Record; } diff --git a/gallery/jest.config.cjs b/gallery/jest.config.cjs index 62fb1e5..90c2568 100644 --- a/gallery/jest.config.cjs +++ b/gallery/jest.config.cjs @@ -11,6 +11,8 @@ module.exports = { '^@simple-photo-gallery/common$': '/../common/src/gallery.ts', '^@simple-photo-gallery/common/theme$': '/../common/src/theme/index.ts', '^@simple-photo-gallery/common/theme/config$': '/../common/src/theme/config.ts', + // Use the CJS-compatible UMD build of marked, since Jest cannot parse its ESM build + '^marked$': '/../node_modules/marked/lib/marked.umd.js', }, transform: { '^.+\\.tsx?$': [ diff --git a/gallery/src/index.ts b/gallery/src/index.ts index 38b9195..d579e29 100644 --- a/gallery/src/index.ts +++ b/gallery/src/index.ts @@ -12,7 +12,7 @@ import { init } from './modules/init'; import { telemetry } from './modules/telemetry'; import { TelemetryService } from './modules/telemetry/service'; import { thumbnails } from './modules/thumbnails'; -import { parseTelemetryOption } from './utils'; +import { parseTelemetryOption, parseThumbnailFormat } from './utils'; import { checkForUpdates, displayUpdateNotification, waitForUpdateCheck } from './utils/version'; import packageJson from '../package.json' with { type: 'json' }; @@ -174,6 +174,9 @@ program .option('-r, --recursive', 'Scan subdirectories recursively', false) .option('--thumbnail-size ', 'Override thumbnail size in pixels', Number.parseInt) .option('--thumbnail-edge ', 'Override how thumbnail size is applied: auto, width, or height') + .option('--thumbnail-format ', 'Override thumbnail output format: avif, webp, or jpeg', parseThumbnailFormat) + .option('--thumbnail-quality ', 'Override thumbnail output quality (1-100)', Number.parseInt) + .option('--thumbnail-effort ', 'Override thumbnail encoder effort (higher is slower but smaller)', Number.parseInt) .action(withCommandContext((options, ui) => thumbnails(options, ui))); program @@ -191,6 +194,9 @@ program ) .option('--thumbnail-size ', 'Override thumbnail size in pixels', Number.parseInt) .option('--thumbnail-edge ', 'Override how thumbnail size is applied: auto, width, or height') + .option('--thumbnail-format ', 'Override thumbnail output format: avif, webp, or jpeg', parseThumbnailFormat) + .option('--thumbnail-quality ', 'Override thumbnail output quality (1-100)', Number.parseInt) + .option('--thumbnail-effort ', 'Override thumbnail encoder effort (higher is slower but smaller)', Number.parseInt) .action(withCommandContext((options, ui) => build(options, ui))); program diff --git a/gallery/src/modules/build/index.ts b/gallery/src/modules/build/index.ts index cc87a9d..f9db6f3 100644 --- a/gallery/src/modules/build/index.ts +++ b/gallery/src/modules/build/index.ts @@ -13,7 +13,7 @@ import { hasOldHeaderImages, } from './utils'; -import { findGalleries } from '../../utils'; +import { buildCliThumbnailConfig, findGalleries } from '../../utils'; import { parseGalleryJson } from '../../utils/gallery'; import { scanDirectory } from '../init'; import { processGalleryThumbnails } from '../thumbnails'; @@ -225,17 +225,13 @@ async function buildGallery( // If thumbnail settings are provided via CLI, update the gallery.json file if needed if (cliThumbnailConfig) { - const needsUpdate = - (cliThumbnailConfig.size !== undefined && galleryData.thumbnails?.size !== cliThumbnailConfig.size) || - (cliThumbnailConfig.edge !== undefined && galleryData.thumbnails?.edge !== cliThumbnailConfig.edge); + const currentThumbnails = galleryData.thumbnails ?? {}; + const cliKeys = Object.keys(cliThumbnailConfig) as (keyof typeof cliThumbnailConfig)[]; + const needsUpdate = cliKeys.some((key) => currentThumbnails[key] !== cliThumbnailConfig[key]); if (needsUpdate) { ui.debug('Updating gallery.json with thumbnail settings'); - galleryData.thumbnails = { - ...galleryData.thumbnails, - ...(cliThumbnailConfig.size !== undefined && { size: cliThumbnailConfig.size }), - ...(cliThumbnailConfig.edge !== undefined && { edge: cliThumbnailConfig.edge }), - }; + galleryData.thumbnails = { ...currentThumbnails, ...cliThumbnailConfig }; fs.writeFileSync(galleryJsonPath, JSON.stringify(galleryData, null, 2)); } } @@ -335,10 +331,7 @@ export async function build(options: BuildOptions, ui: ConsolaInstance): Promise } // Create CLI thumbnail config from options (only include values that were provided) - const cliThumbnailConfig: ThumbnailConfig | undefined = - options.thumbnailSize !== undefined || options.thumbnailEdge !== undefined - ? { size: options.thumbnailSize, edge: options.thumbnailEdge } - : undefined; + const cliThumbnailConfig = buildCliThumbnailConfig(options); // Process each gallery directory let totalGalleries = 0; diff --git a/gallery/src/modules/build/types/index.ts b/gallery/src/modules/build/types/index.ts index cb3f291..876b0e9 100644 --- a/gallery/src/modules/build/types/index.ts +++ b/gallery/src/modules/build/types/index.ts @@ -1,3 +1,5 @@ +import type { ThumbnailFormat } from '@simple-photo-gallery/common/theme'; + /** Options for building gallery HTML output */ export interface BuildOptions { /** Path to the directory containing the gallery */ @@ -18,4 +20,10 @@ export interface BuildOptions { thumbnailSize?: number; /** Override how thumbnail size should be applied: 'auto', 'width', or 'height' */ thumbnailEdge?: 'auto' | 'width' | 'height'; + /** Override thumbnail output format */ + thumbnailFormat?: ThumbnailFormat; + /** Override thumbnail output quality (1-100) */ + thumbnailQuality?: number; + /** Override thumbnail encoder effort */ + thumbnailEffort?: number; } diff --git a/gallery/src/modules/thumbnails/index.ts b/gallery/src/modules/thumbnails/index.ts index 152c707..c28437a 100644 --- a/gallery/src/modules/thumbnails/index.ts +++ b/gallery/src/modules/thumbnails/index.ts @@ -4,6 +4,7 @@ import path from 'node:path'; import { extractThumbnailConfigFromGallery, + getThumbnailExtension, mergeThumbnailConfig, loadThemeConfig, } from '@simple-photo-gallery/common/theme'; @@ -11,19 +12,24 @@ import { LogLevels, type ConsolaInstance } from 'consola'; import { getFileMtime } from './utils'; -import { findGalleries, handleFileProcessingError } from '../../utils'; +import { buildCliThumbnailConfig, findGalleries, handleFileProcessingError } from '../../utils'; import { generateBlurHash } from '../../utils/blurhash'; import { mapWithConcurrency } from '../../utils/concurrency'; import { getImageDescription } from '../../utils/descriptions'; import { parseGalleryJson } from '../../utils/gallery'; -import { createImageThumbnails, loadImageWithMetadata, type ThumbnailSizeDimension } from '../../utils/image'; +import { + createImageThumbnails, + loadImageWithMetadata, + type ImageEncodeOptions, + type ThumbnailSizeDimension, +} from '../../utils/image'; import { getVideoDimensions, createVideoThumbnails } from '../../utils/video'; import { resolveThemeDir } from '../build'; import type { ThumbnailOptions } from './types'; import type { CommandResultSummary } from '../telemetry/types'; import type { MediaFile } from '@simple-photo-gallery/common'; -import type { ThumbnailConfig } from '@simple-photo-gallery/common/theme'; +import type { ResolvedThumbnailConfig, ThumbnailConfig } from '@simple-photo-gallery/common/theme'; /** * Processes an image file to create thumbnail and extract metadata @@ -32,6 +38,7 @@ import type { ThumbnailConfig } from '@simple-photo-gallery/common/theme'; * @param thumbnailPathRetina - Path where retina thumbnail should be saved * @param thumbnailSize - Target size for thumbnail * @param thumbnailSizeDimension - How to apply size: 'auto', 'width', or 'height' + * @param encodeOptions - Encoding options (format, quality, effort) * @param lastMediaTimestamp - Optional timestamp to check if processing can be skipped * @returns Promise resolving to updated MediaFile or undefined if skipped */ @@ -41,6 +48,7 @@ export async function processImage( thumbnailPathRetina: string, thumbnailSize: number, thumbnailSizeDimension: ThumbnailSizeDimension = 'auto', + encodeOptions: ImageEncodeOptions = {}, lastMediaTimestamp?: Date, ): Promise { // Get the last media timestamp @@ -75,6 +83,7 @@ export async function processImage( thumbnailPathRetina, thumbnailSize, thumbnailSizeDimension, + encodeOptions, ); // Generate BlurHash from the thumbnail @@ -106,6 +115,7 @@ export async function processImage( * @param thumbnailSize - Target size for thumbnail * @param thumbnailSizeDimension - How to apply size: 'auto', 'width', or 'height' * @param verbose - Whether to enable verbose output + * @param encodeOptions - Encoding options (format, quality, effort) * @param lastMediaTimestamp - Optional timestamp to check if processing can be skipped * @returns Promise resolving to updated MediaFile or undefined if skipped */ @@ -116,6 +126,7 @@ async function processVideo( thumbnailSize: number, thumbnailSizeDimension: ThumbnailSizeDimension = 'auto', verbose: boolean, + encodeOptions: ImageEncodeOptions = {}, lastMediaTimestamp?: Date, ): Promise { // Get the last media timestamp @@ -138,6 +149,7 @@ async function processVideo( thumbnailSize, thumbnailSizeDimension, verbose, + encodeOptions, ); // Generate BlurHash from the thumbnail @@ -165,7 +177,7 @@ async function processVideo( * @param mediaFile - Media file to process * @param mediaBasePath - Base path for the media files * @param thumbnailsPath - Directory where thumbnails are stored - * @param thumbnailConfig - Thumbnail configuration (dimension and edge) + * @param thumbnailConfig - Resolved thumbnail configuration (size, edge, format, quality, effort) * @param ui - ConsolaInstance for logging * @returns Promise resolving to updated MediaFile */ @@ -173,7 +185,7 @@ async function processMediaFile( mediaFile: MediaFile, mediaBasePath: string, thumbnailsPath: string, - thumbnailConfig: Required, + thumbnailConfig: ResolvedThumbnailConfig, ui: ConsolaInstance, ): Promise { try { @@ -182,9 +194,15 @@ async function processMediaFile( const fileName = mediaFile.filename; const fileNameWithoutExt = path.parse(fileName).name; - const thumbnailFileName = `${fileNameWithoutExt}.avif`; - const thumbnailPath = path.join(thumbnailsPath, thumbnailFileName); - const thumbnailPathRetina = thumbnailPath.replace('.avif', '@2x.avif'); + const extension = getThumbnailExtension(thumbnailConfig.format); + const thumbnailPath = path.join(thumbnailsPath, `${fileNameWithoutExt}.${extension}`); + const thumbnailPathRetina = path.join(thumbnailsPath, `${fileNameWithoutExt}@2x.${extension}`); + + const encodeOptions: ImageEncodeOptions = { + format: thumbnailConfig.format, + quality: thumbnailConfig.quality, + effort: thumbnailConfig.effort, + }; const lastMediaTimestamp = mediaFile.lastMediaTimestamp ? new Date(mediaFile.lastMediaTimestamp) : undefined; const verbose = ui.level === LogLevels.debug; @@ -198,6 +216,7 @@ async function processMediaFile( thumbnailPathRetina, thumbnailConfig.size, thumbnailConfig.edge, + encodeOptions, lastMediaTimestamp, ) : processVideo( @@ -207,6 +226,7 @@ async function processMediaFile( thumbnailConfig.size, thumbnailConfig.edge, verbose, + encodeOptions, lastMediaTimestamp, )); @@ -295,7 +315,11 @@ export async function processGalleryThumbnails( // Merge with 4-level hierarchy: CLI > gallery.json > theme > defaults const thumbnailConfig = mergeThumbnailConfig(cliThumbnailConfig, galleryThumbnailConfig, themeConfig); - ui.debug(`Thumbnail config: size=${thumbnailConfig.size}, edge=${thumbnailConfig.edge}`); + ui.debug( + `Thumbnail config: size=${thumbnailConfig.size}, edge=${thumbnailConfig.edge}, format=${thumbnailConfig.format}` + + `${thumbnailConfig.quality === undefined ? '' : `, quality=${thumbnailConfig.quality}`}` + + `${thumbnailConfig.effort === undefined ? '' : `, effort=${thumbnailConfig.effort}`}`, + ); // If the mediaBasePath is not set, use the gallery directory const mediaBasePath = galleryData.mediaBasePath ?? path.join(galleryDir); @@ -341,10 +365,7 @@ export async function thumbnails(options: ThumbnailOptions, ui: ConsolaInstance) } // Create CLI thumbnail config from options (only include values that were provided) - const cliThumbnailConfig: ThumbnailConfig | undefined = - options.thumbnailSize !== undefined || options.thumbnailEdge !== undefined - ? { size: options.thumbnailSize, edge: options.thumbnailEdge } - : undefined; + const cliThumbnailConfig = buildCliThumbnailConfig(options); // Process each gallery directory let totalGalleries = 0; diff --git a/gallery/src/modules/thumbnails/types/index.ts b/gallery/src/modules/thumbnails/types/index.ts index c97196d..af830d3 100644 --- a/gallery/src/modules/thumbnails/types/index.ts +++ b/gallery/src/modules/thumbnails/types/index.ts @@ -1,3 +1,5 @@ +import type { ThumbnailFormat } from '@simple-photo-gallery/common/theme'; + /** Options for generating thumbnails */ export interface ThumbnailOptions { /** Path to the directory containing the gallery */ @@ -8,4 +10,10 @@ export interface ThumbnailOptions { thumbnailSize?: number; /** Override how thumbnail size should be applied: 'auto', 'width', or 'height' */ thumbnailEdge?: 'auto' | 'width' | 'height'; + /** Override thumbnail output format */ + thumbnailFormat?: ThumbnailFormat; + /** Override thumbnail output quality (1-100) */ + thumbnailQuality?: number; + /** Override thumbnail encoder effort */ + thumbnailEffort?: number; } diff --git a/gallery/src/utils/image.ts b/gallery/src/utils/image.ts index 57bed32..3fc9ce5 100644 --- a/gallery/src/utils/image.ts +++ b/gallery/src/utils/image.ts @@ -37,22 +37,68 @@ export async function loadImageWithMetadata(imagePath: string): Promise { + const { format = 'avif', quality, effort } = options; + // Resize the image without enlarging it - await image.resize(width, height, { withoutEnlargement: true }).toFormat(format).toFile(outputPath); + await image + .resize(width, height, { withoutEnlargement: true }) + .toFormat(format, buildFormatOptions(format, quality, effort)) + .toFile(outputPath); } /** @@ -90,6 +136,7 @@ export type ThumbnailSizeDimension = 'auto' | 'width' | 'height'; * @param outputPathRetina - Path where retina thumbnail should be saved * @param size - Target size for the thumbnail * @param sizeDimension - How to apply the size: 'auto' (longer edge), 'width', or 'height' + * @param encodeOptions - Encoding options (format, quality, effort) * @returns Promise resolving to thumbnail dimensions */ export async function createImageThumbnails( @@ -99,6 +146,7 @@ export async function createImageThumbnails( outputPathRetina: string, size: number, sizeDimension: ThumbnailSizeDimension = 'auto', + encodeOptions: ImageEncodeOptions = {}, ): Promise { // Get the original dimensions const originalWidth = metadata.width || 0; @@ -134,8 +182,8 @@ export async function createImageThumbnails( } // Resize the image and create the thumbnails - await resizeImage(image, outputPath, width, height); - await resizeImage(image, outputPathRetina, width * 2, height * 2); + await resizeImage(image, outputPath, width, height, encodeOptions); + await resizeImage(image, outputPathRetina, width * 2, height * 2, encodeOptions); // Return the dimensions of the thumbnail return { width, height }; diff --git a/gallery/src/utils/index.ts b/gallery/src/utils/index.ts index abc4ffa..0ae7816 100644 --- a/gallery/src/utils/index.ts +++ b/gallery/src/utils/index.ts @@ -1,6 +1,9 @@ import fs from 'node:fs'; import path from 'node:path'; +import { THUMBNAIL_FORMATS } from '@simple-photo-gallery/common/theme'; + +import type { ThumbnailConfig, ThumbnailFormat } from '@simple-photo-gallery/common/theme'; import type { ConsolaInstance } from 'consola'; /** @@ -73,3 +76,52 @@ export function parseTelemetryOption(value: string): '0' | '1' { return value; } + +/** + * Parses and validates the thumbnail format CLI option + * @param value - The value to parse + * @returns The parsed thumbnail format + */ +export function parseThumbnailFormat(value: string): ThumbnailFormat { + if (!(THUMBNAIL_FORMATS as readonly string[]).includes(value)) { + throw new Error(`Thumbnail format must be one of: ${THUMBNAIL_FORMATS.join(', ')}.`); + } + + return value as ThumbnailFormat; +} + +/** Subset of CLI options that map to thumbnail configuration overrides */ +export interface CliThumbnailOptions { + thumbnailSize?: number; + thumbnailEdge?: 'auto' | 'width' | 'height'; + thumbnailFormat?: ThumbnailFormat; + thumbnailQuality?: number; + thumbnailEffort?: number; +} + +/** + * Builds a ThumbnailConfig from CLI options, including only the values that were provided. + * @param options - CLI options carrying thumbnail overrides + * @returns A ThumbnailConfig with the provided overrides, or undefined when none were given + */ +export function buildCliThumbnailConfig(options: CliThumbnailOptions): ThumbnailConfig | undefined { + const config: ThumbnailConfig = {}; + + if (options.thumbnailSize !== undefined) { + config.size = options.thumbnailSize; + } + if (options.thumbnailEdge !== undefined) { + config.edge = options.thumbnailEdge; + } + if (options.thumbnailFormat !== undefined) { + config.format = options.thumbnailFormat; + } + if (options.thumbnailQuality !== undefined) { + config.quality = options.thumbnailQuality; + } + if (options.thumbnailEffort !== undefined) { + config.effort = options.thumbnailEffort; + } + + return Object.keys(config).length > 0 ? config : undefined; +} diff --git a/gallery/src/utils/video.ts b/gallery/src/utils/video.ts index fd724b0..2fe9f10 100644 --- a/gallery/src/utils/video.ts +++ b/gallery/src/utils/video.ts @@ -4,7 +4,7 @@ import { promises as fs } from 'node:fs'; import ffprobe from 'node-ffprobe'; import sharp from 'sharp'; -import { resizeImage, type ThumbnailSizeDimension } from './image'; +import { resizeImage, type ImageEncodeOptions, type ThumbnailSizeDimension } from './image'; import type { Dimensions } from '../types'; import type { Buffer } from 'node:buffer'; @@ -44,6 +44,7 @@ export async function getVideoDimensions(filePath: string): Promise * @param size - Target size for thumbnail * @param sizeDimension - How to apply size: 'auto' (longer edge), 'width', or 'height' * @param verbose - Whether to enable verbose ffmpeg output + * @param encodeOptions - Encoding options (format, quality, effort) * @returns Promise resolving to thumbnail dimensions */ export async function createVideoThumbnails( @@ -54,6 +55,7 @@ export async function createVideoThumbnails( size: number, sizeDimension: ThumbnailSizeDimension = 'auto', verbose: boolean = false, + encodeOptions: ImageEncodeOptions = {}, ): Promise { // Calculate dimensions maintaining aspect ratio based on sizeDimension const aspectRatio = videoDimensions.width / videoDimensions.height; @@ -106,8 +108,8 @@ export async function createVideoThumbnails( try { // Process the extracted frame with sharp const frameImage = sharp(tempFramePath); - await resizeImage(frameImage, outputPath, width, height); - await resizeImage(frameImage, outputPathRetina, width * 2, height * 2); + await resizeImage(frameImage, outputPath, width, height, encodeOptions); + await resizeImage(frameImage, outputPathRetina, width * 2, height * 2, encodeOptions); // Clean up temporary file try { diff --git a/gallery/tests/thumbnail-config.test.ts b/gallery/tests/thumbnail-config.test.ts new file mode 100644 index 0000000..e353799 --- /dev/null +++ b/gallery/tests/thumbnail-config.test.ts @@ -0,0 +1,72 @@ +import { getThumbnailExtension, mergeThumbnailConfig } from '../../common/src/theme/config'; +import { buildCliThumbnailConfig, parseThumbnailFormat } from '../src/utils'; + +describe('getThumbnailExtension', () => { + test('should return the format name for avif and webp', () => { + expect(getThumbnailExtension('avif')).toBe('avif'); + expect(getThumbnailExtension('webp')).toBe('webp'); + }); + + test('should map jpeg to the jpg extension', () => { + expect(getThumbnailExtension('jpeg')).toBe('jpg'); + }); +}); + +describe('mergeThumbnailConfig with format, quality and effort', () => { + test('should default to avif format with no quality or effort', () => { + const result = mergeThumbnailConfig(); + expect(result.format).toBe('avif'); + expect(result.quality).toBeUndefined(); + expect(result.effort).toBeUndefined(); + }); + + test('should resolve format with CLI > gallery > theme precedence', () => { + const result = mergeThumbnailConfig({ format: 'webp' }, { format: 'jpeg' }, { format: 'avif' }); + expect(result.format).toBe('webp'); + }); + + test('should fall back to gallery format when CLI does not set it', () => { + const result = mergeThumbnailConfig({ size: 200 }, { format: 'jpeg' }, { format: 'avif' }); + expect(result.format).toBe('jpeg'); + }); + + test('should resolve quality and effort across the hierarchy', () => { + const result = mergeThumbnailConfig({ effort: 2 }, { quality: 70 }, { quality: 50, effort: 9 }); + expect(result.quality).toBe(70); + expect(result.effort).toBe(2); + }); +}); + +describe('parseThumbnailFormat', () => { + test('should accept the supported formats', () => { + expect(parseThumbnailFormat('avif')).toBe('avif'); + expect(parseThumbnailFormat('webp')).toBe('webp'); + expect(parseThumbnailFormat('jpeg')).toBe('jpeg'); + }); + + test('should throw for an unsupported format', () => { + expect(() => parseThumbnailFormat('gif')).toThrow('Thumbnail format must be one of'); + }); +}); + +describe('buildCliThumbnailConfig', () => { + test('should return undefined when no thumbnail options are provided', () => { + expect(buildCliThumbnailConfig({})).toBeUndefined(); + }); + + test('should include only the provided options', () => { + const config = buildCliThumbnailConfig({ thumbnailFormat: 'webp', thumbnailEffort: 3 }); + expect(config).toEqual({ format: 'webp', effort: 3 }); + }); + + test('should map all thumbnail options to config keys', () => { + const config = buildCliThumbnailConfig({ + thumbnailSize: 400, + thumbnailEdge: 'width', + thumbnailFormat: 'jpeg', + thumbnailQuality: 80, + thumbnailEffort: 5, + }); + expect(config).toEqual({ size: 400, edge: 'width', format: 'jpeg', quality: 80, effort: 5 }); + }); +}); From 427bf0953059e829180847514583808b12529f62 Mon Sep 17 00:00:00 2001 From: Vladimir Haltakov Date: Sat, 13 Jun 2026 18:11:16 +0200 Subject: [PATCH 3/4] perf: read each original once when generating thumbnails For every image, the pipeline read the source from disk twice (once per thumbnail size, because Sharp re-reads a file-path input for each output) and then re-read and decoded the just-written thumbnail a third time to compute the BlurHash. Read the original into a Buffer once and reuse it for metadata, both thumbnail encodes and the BlurHash, and read EXIF from the same buffer. The BlurHash is now computed from the in-memory original instead of the written thumbnail, which is visually equivalent. The video frame is likewise read once and decoded from memory for both outputs. Co-Authored-By: Claude Fable 5 --- gallery/src/modules/thumbnails/index.ts | 12 ++++++++---- gallery/src/utils/blurhash.ts | 14 +++++++++----- gallery/src/utils/descriptions.ts | 9 ++++++--- gallery/src/utils/image.ts | 16 ++++++++++------ gallery/src/utils/video.ts | 6 ++++-- 5 files changed, 37 insertions(+), 20 deletions(-) diff --git a/gallery/src/modules/thumbnails/index.ts b/gallery/src/modules/thumbnails/index.ts index c28437a..b342de5 100644 --- a/gallery/src/modules/thumbnails/index.ts +++ b/gallery/src/modules/thumbnails/index.ts @@ -59,8 +59,12 @@ export async function processImage( return undefined; } + // Read the original once and reuse the buffer for metadata, thumbnails, blurhash and EXIF so the + // file (potentially tens of MB) is read from disk a single time instead of once per consumer. + const inputBuffer = await fs.promises.readFile(imagePath); + // Load the image and get metadata first to check orientation - const { image, metadata } = await loadImageWithMetadata(imagePath); + const { image, metadata } = await loadImageWithMetadata(inputBuffer); // Get the image dimensions const imageDimensions = { @@ -73,7 +77,7 @@ export async function processImage( } // Get the image description - const description = await getImageDescription(imagePath); + const description = await getImageDescription(inputBuffer); // Create the thumbnails const thumbnailDimensions = await createImageThumbnails( @@ -86,8 +90,8 @@ export async function processImage( encodeOptions, ); - // Generate BlurHash from the thumbnail - const blurHash = await generateBlurHash(thumbnailPath); + // Generate BlurHash from the in-memory original, avoiding a re-read and decode of the written thumbnail + const blurHash = await generateBlurHash(inputBuffer); // Return the updated media file return { diff --git a/gallery/src/utils/blurhash.ts b/gallery/src/utils/blurhash.ts index 078d148..0a02039 100644 --- a/gallery/src/utils/blurhash.ts +++ b/gallery/src/utils/blurhash.ts @@ -1,16 +1,20 @@ import { encode } from 'blurhash'; -import { loadImage } from './image'; +import { loadImage, type ImageSource } from './image'; /** - * Generates a BlurHash from an image file or Sharp instance - * @param imagePath - Path to image file or Sharp instance + * Generates a BlurHash from an image file path or its contents as a Buffer + * @param source - Path to the image file or its contents as a Buffer * @param componentX - Number of x components (default: 4) * @param componentY - Number of y components (default: 3) * @returns Promise resolving to BlurHash string */ -export async function generateBlurHash(imagePath: string, componentX: number = 4, componentY: number = 3): Promise { - const image = await loadImage(imagePath); +export async function generateBlurHash( + source: ImageSource, + componentX: number = 4, + componentY: number = 3, +): Promise { + const image = await loadImage(source); // Resize to small size for BlurHash computation to improve performance // BlurHash doesn't need high resolution diff --git a/gallery/src/utils/descriptions.ts b/gallery/src/utils/descriptions.ts index fff1f5d..1df9d7f 100644 --- a/gallery/src/utils/descriptions.ts +++ b/gallery/src/utils/descriptions.ts @@ -1,13 +1,16 @@ +import { Buffer } from 'node:buffer'; + import ExifReader from 'exifreader'; /** * Extracts description from image EXIF data - * @param image - Image path or File object + * @param image - Image path, file contents as a Buffer, or File object * @returns Promise resolving to image description or undefined if not found */ -export async function getImageDescription(image: string | File): Promise { +export async function getImageDescription(image: string | Buffer | File): Promise { try { - const tags = await ExifReader.load(image); + // ExifReader.load returns Tags synchronously for a Buffer and a Promise for a path or File + const tags = Buffer.isBuffer(image) ? ExifReader.load(image) : await ExifReader.load(image); // Description if (tags.description?.description) return tags.description.description; diff --git a/gallery/src/utils/image.ts b/gallery/src/utils/image.ts index 3fc9ce5..cc56ac8 100644 --- a/gallery/src/utils/image.ts +++ b/gallery/src/utils/image.ts @@ -1,24 +1,28 @@ import sharp from 'sharp'; import type { Dimensions, ImageWithMetadata } from '../types'; +import type { Buffer } from 'node:buffer'; import type { FormatEnum, Metadata, Sharp } from 'sharp'; +/** An image source accepted by Sharp: a file path or the image file contents in a Buffer */ +export type ImageSource = string | Buffer; + /** * Loads an image and auto-rotates it based on EXIF orientation. - * @param imagePath - Path to the image file + * @param image - Path to the image file or its contents as a Buffer * @returns Promise resolving to Sharp image instance */ -export async function loadImage(imagePath: string): Promise { - return sharp(imagePath).rotate(); +export async function loadImage(image: ImageSource): Promise { + return sharp(image).rotate(); } /** * Loads an image and its metadata, auto-rotating it based on EXIF orientation and swapping dimensions if needed. - * @param imagePath - Path to the image file + * @param source - Path to the image file or its contents as a Buffer * @returns Promise resolving to ImageWithMetadata object containing Sharp image instance and metadata */ -export async function loadImageWithMetadata(imagePath: string): Promise { - const image = sharp(imagePath); +export async function loadImageWithMetadata(source: ImageSource): Promise { + const image = sharp(source); const metadata = await image.metadata(); // Auto-rotate based on EXIF orientation diff --git a/gallery/src/utils/video.ts b/gallery/src/utils/video.ts index 2fe9f10..5bf4c3b 100644 --- a/gallery/src/utils/video.ts +++ b/gallery/src/utils/video.ts @@ -106,8 +106,10 @@ export async function createVideoThumbnails( ffmpeg.on('close', async (code: number) => { if (code === 0) { try { - // Process the extracted frame with sharp - const frameImage = sharp(tempFramePath); + // Read the extracted frame once and decode both thumbnails from memory instead of re-reading + // the temporary file from disk for each output + const frameBuffer = await fs.readFile(tempFramePath); + const frameImage = sharp(frameBuffer); await resizeImage(frameImage, outputPath, width, height, encodeOptions); await resizeImage(frameImage, outputPathRetina, width * 2, height * 2, encodeOptions); From feb48c0ab2e715e745f17c36050555604ceca6fc Mon Sep 17 00:00:00 2001 From: Vladimir Haltakov Date: Sat, 13 Jun 2026 18:15:33 +0200 Subject: [PATCH 4/4] perf: persist thumbnail progress incrementally and on interruption gallery.json (which records each file's lastMediaTimestamp) was only written once, after every thumbnail in the gallery finished. If a long run was interrupted - Ctrl-C, a crash, an OOM - all of that bookkeeping was lost and the next run regenerated every thumbnail from scratch even though the images were already on disk. Write gallery.json every 16 processed files and again on SIGINT/SIGTERM, using an atomic temp-file-and-rename so an interruption can never leave a truncated gallery.json. Combined with the existing mtime skip check, an interrupted run now resumes where it left off. Co-Authored-By: Claude Fable 5 --- gallery/src/modules/thumbnails/index.ts | 56 ++++++++++-- gallery/tests/thumbnails-progress.test.ts | 104 ++++++++++++++++++++++ 2 files changed, 152 insertions(+), 8 deletions(-) create mode 100644 gallery/tests/thumbnails-progress.test.ts diff --git a/gallery/src/modules/thumbnails/index.ts b/gallery/src/modules/thumbnails/index.ts index b342de5..d2ce2b8 100644 --- a/gallery/src/modules/thumbnails/index.ts +++ b/gallery/src/modules/thumbnails/index.ts @@ -1,6 +1,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import process from 'node:process'; import { extractThumbnailConfigFromGallery, @@ -328,22 +329,61 @@ export async function processGalleryThumbnails( // If the mediaBasePath is not set, use the gallery directory const mediaBasePath = galleryData.mediaBasePath ?? path.join(galleryDir); + // Persist gallery.json atomically (write to a temp file then rename) so an interruption can never + // leave a truncated gallery.json behind. + const writeGalleryData = (): void => { + const tempPath = `${galleryJsonPath}.tmp`; + fs.writeFileSync(tempPath, JSON.stringify(galleryData, null, 2)); + fs.renameSync(tempPath, galleryJsonPath); + }; + + // On interruption (e.g. Ctrl-C) flush the progress made so far before exiting. Each completed file + // records its lastMediaTimestamp, so a subsequent run skips everything already finished instead of + // starting the whole gallery over. + const flushAndExit = (exitCode: number) => (): void => { + writeGalleryData(); + // eslint-disable-next-line unicorn/no-process-exit + process.exit(exitCode); + }; + const onSigint = flushAndExit(130); + const onSigterm = flushAndExit(143); + process.once('SIGINT', onSigint); + process.once('SIGTERM', onSigterm); + // Process media files in parallel. Thumbnailing is IO/CPU bound (Sharp and ffmpeg release the JS // thread), so a worker pool sized to the available cores processes a large gallery several times // faster than the previous one-at-a-time loop while keeping per-file error isolation. const concurrency = Math.max(2, os.cpus().length - 1); + // Flush progress to disk every PROGRESS_WRITE_INTERVAL files so a long run that is interrupted can + // resume instead of redoing all the thumbnails already on disk. + const PROGRESS_WRITE_INTERVAL = 16; let processedCount = 0; - for (const section of galleryData.sections) { - section.images = await mapWithConcurrency(section.images, concurrency, (mediaFile) => - processMediaFile(mediaFile, mediaBasePath, thumbnailsPath, thumbnailConfig, ui), - ); - - processedCount += section.images.length; + let processedSinceWrite = 0; + + try { + for (const section of galleryData.sections) { + await mapWithConcurrency(section.images, concurrency, async (mediaFile, index) => { + const updated = await processMediaFile(mediaFile, mediaBasePath, thumbnailsPath, thumbnailConfig, ui); + section.images[index] = updated; + + processedCount += 1; + processedSinceWrite += 1; + if (processedSinceWrite >= PROGRESS_WRITE_INTERVAL) { + processedSinceWrite = 0; + writeGalleryData(); + } + + return updated; + }); + } + } finally { + process.removeListener('SIGINT', onSigint); + process.removeListener('SIGTERM', onSigterm); } - // Write updated gallery.json - fs.writeFileSync(galleryJsonPath, JSON.stringify(galleryData, null, 2)); + // Write the final gallery.json with all processed files + writeGalleryData(); ui.success(`Created thumbnails for ${processedCount} media files`); diff --git a/gallery/tests/thumbnails-progress.test.ts b/gallery/tests/thumbnails-progress.test.ts new file mode 100644 index 0000000..c83cef6 --- /dev/null +++ b/gallery/tests/thumbnails-progress.test.ts @@ -0,0 +1,104 @@ +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import process from 'node:process'; + +// resolveThemeDir uses import.meta, which ts-jest cannot evaluate; the test galleries set no theme so it +// is never called, but the module is still imported by thumbnails/index.ts, so mock it. +jest.mock('../src/modules/build', () => ({ + resolveThemeDir: jest.fn().mockResolvedValue('/tmp/theme'), +})); + +import { processGalleryThumbnails } from '../src/modules/thumbnails'; + +import type { ConsolaInstance } from 'consola'; + +const fixtureImages = path.resolve(process.cwd(), 'tests', 'fixtures', 'single'); + +const createMockUI = (): ConsolaInstance => + ({ + info: jest.fn(), + start: jest.fn(), + success: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + box: jest.fn(), + warn: jest.fn(), + }) as unknown as ConsolaInstance; + +function setupGallery(rootDir: string, filenames: string[]): void { + mkdirSync(path.join(rootDir, 'gallery'), { recursive: true }); + + for (const name of filenames) { + copyFileSync(path.join(fixtureImages, name), path.join(rootDir, name)); + } + + const galleryData = { + title: 'Test', + description: 'Test gallery', + headerImage: filenames[0], + metadata: {}, + sections: [{ images: filenames.map((filename) => ({ type: 'image', filename, width: 0, height: 0 })) }], + subGalleries: { title: 'Sub', galleries: [] }, + }; + + writeFileSync(path.join(rootDir, 'gallery', 'gallery.json'), JSON.stringify(galleryData, null, 2)); +} + +function readGalleryJson(rootDir: string): { sections: { images: Record[] }[] } { + return JSON.parse(readFileSync(path.join(rootDir, 'gallery', 'gallery.json'), 'utf8')); +} + +describe('processGalleryThumbnails progress persistence', () => { + let tempDir: string; + let filenames: string[]; + + beforeEach(() => { + tempDir = mkdtempSync(path.join(os.tmpdir(), 'spg-progress-')); + filenames = readdirSync(fixtureImages) + .filter((file) => file.toLowerCase().endsWith('.jpg')) + .slice(0, 3); + setupGallery(tempDir, filenames); + }); + + afterEach(() => { + if (existsSync(tempDir)) { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + test('persists thumbnails and timestamps and leaves no temporary file behind', async () => { + const count = await processGalleryThumbnails(tempDir, createMockUI()); + + expect(count).toBe(filenames.length); + + const data = readGalleryJson(tempDir); + for (const image of data.sections[0].images) { + const thumbnail = image.thumbnail as { path: string; blurHash?: string } | undefined; + expect(thumbnail).toBeDefined(); + expect(thumbnail!.blurHash).toBeDefined(); + expect(image.lastMediaTimestamp).toBeDefined(); + expect(existsSync(path.join(tempDir, 'gallery', 'images', thumbnail!.path))).toBe(true); + } + + // The atomic write should not leave the temporary file behind + expect(existsSync(path.join(tempDir, 'gallery', 'gallery.json.tmp'))).toBe(false); + }); + + test('a second run resumes without regenerating existing thumbnails', async () => { + await processGalleryThumbnails(tempDir, createMockUI()); + + const imagesDir = path.join(tempDir, 'gallery', 'images'); + const firstRun = readdirSync(imagesDir).sort(); + const firstTimestamps = readGalleryJson(tempDir).sections[0].images.map((image) => image.lastMediaTimestamp); + + await processGalleryThumbnails(tempDir, createMockUI()); + + const secondRun = readdirSync(imagesDir).sort(); + const secondTimestamps = readGalleryJson(tempDir).sections[0].images.map((image) => image.lastMediaTimestamp); + + // Already-processed files are skipped, so the thumbnails and their recorded timestamps are unchanged + expect(secondRun).toEqual(firstRun); + expect(secondTimestamps).toEqual(firstTimestamps); + }); +});