diff --git a/common/src/gallery/schemas.ts b/common/src/gallery/schemas.ts index 8ee3612..f865de3 100644 --- a/common/src/gallery/schemas.ts +++ b/common/src/gallery/schemas.ts @@ -125,6 +125,9 @@ export const GalleryDataSchema = z.object({ .object({ size: z.number().optional(), edge: z.enum(['auto', 'width', 'height']).optional(), + format: z.enum(['avif', 'webp', 'jpeg']).optional(), + quality: z.number().optional(), + effort: z.number().optional(), }) .optional(), metadata: GalleryMetadataSchema, diff --git a/common/src/theme/config.ts b/common/src/theme/config.ts index 34e7cf7..26e86b6 100644 --- a/common/src/theme/config.ts +++ b/common/src/theme/config.ts @@ -1,9 +1,18 @@ import { z } from 'zod'; +/** Supported output formats for generated thumbnails */ +export const THUMBNAIL_FORMATS = ['avif', 'webp', 'jpeg'] as const; + +/** TypeScript type for a thumbnail output format */ +export type ThumbnailFormat = (typeof THUMBNAIL_FORMATS)[number]; + /** Zod schema for thumbnail configuration */ export const ThumbnailConfigSchema = z.object({ size: z.number().min(50).max(4000).optional(), edge: z.enum(['auto', 'width', 'height']).optional(), + format: z.enum(THUMBNAIL_FORMATS).optional(), + quality: z.number().min(1).max(100).optional(), + effort: z.number().min(0).max(9).optional(), }); /** Zod schema for theme configuration file (themeConfig.json) */ @@ -17,12 +26,31 @@ export type ThumbnailConfig = z.infer; /** TypeScript type for theme configuration */ export type ThemeConfig = z.infer; +/** Fully resolved thumbnail configuration with size, edge and format always present */ +export interface ResolvedThumbnailConfig { + size: number; + edge: 'auto' | 'width' | 'height'; + format: ThumbnailFormat; + quality?: number; + effort?: number; +} + /** Default thumbnail configuration values */ -export const DEFAULT_THUMBNAIL_CONFIG: Required = { +export const DEFAULT_THUMBNAIL_CONFIG: ResolvedThumbnailConfig = { size: 300, edge: 'auto', + format: 'avif', }; +/** + * Returns the file extension (without a leading dot) used for thumbnails of the given format. + * @param format - The thumbnail output format + * @returns The file extension to use for generated thumbnails + */ +export function getThumbnailExtension(format: ThumbnailFormat): string { + return format === 'jpeg' ? 'jpg' : format; +} + /** * Extracts thumbnail config from gallery data. * @param gallery - The gallery data object @@ -32,6 +60,9 @@ export function extractThumbnailConfigFromGallery(gallery: { thumbnails?: Thumbn return { size: gallery.thumbnails?.size, edge: gallery.thumbnails?.edge, + format: gallery.thumbnails?.format, + quality: gallery.thumbnails?.quality, + effort: gallery.thumbnails?.effort, }; } @@ -45,15 +76,18 @@ export function extractThumbnailConfigFromGallery(gallery: { thumbnails?: Thumbn * @param cliConfig - Config from CLI flags (optional) * @param galleryConfig - Config from gallery.json (optional) * @param themeConfig - Config from themeConfig.json (optional) - * @returns Merged thumbnail configuration with all values resolved + * @returns Merged thumbnail configuration with all required values resolved */ export function mergeThumbnailConfig( cliConfig?: ThumbnailConfig, galleryConfig?: ThumbnailConfig, themeConfig?: ThumbnailConfig, -): Required { +): ResolvedThumbnailConfig { return { size: cliConfig?.size ?? galleryConfig?.size ?? themeConfig?.size ?? DEFAULT_THUMBNAIL_CONFIG.size, edge: cliConfig?.edge ?? galleryConfig?.edge ?? themeConfig?.edge ?? DEFAULT_THUMBNAIL_CONFIG.edge, + format: cliConfig?.format ?? galleryConfig?.format ?? themeConfig?.format ?? DEFAULT_THUMBNAIL_CONFIG.format, + quality: cliConfig?.quality ?? galleryConfig?.quality ?? themeConfig?.quality, + effort: cliConfig?.effort ?? galleryConfig?.effort ?? themeConfig?.effort, }; } diff --git a/common/src/theme/index.ts b/common/src/theme/index.ts index 5473588..e247844 100644 --- a/common/src/theme/index.ts +++ b/common/src/theme/index.ts @@ -2,11 +2,13 @@ export type { ResolvedGalleryData, ResolvedHero, ResolvedImage, ResolvedSection, ResolvedSubGallery } from './types'; // Theme config -export type { ThemeConfig, ThumbnailConfig } from './config'; +export type { ResolvedThumbnailConfig, ThemeConfig, ThumbnailConfig, ThumbnailFormat } from './config'; export { DEFAULT_THUMBNAIL_CONFIG, extractThumbnailConfigFromGallery, + getThumbnailExtension, mergeThumbnailConfig, + THUMBNAIL_FORMATS, ThemeConfigSchema, ThumbnailConfigSchema, } from './config'; diff --git a/common/src/theme/paths.ts b/common/src/theme/paths.ts index d81c9a1..b4f90ad 100644 --- a/common/src/theme/paths.ts +++ b/common/src/theme/paths.ts @@ -1,5 +1,7 @@ import path from 'node:path'; +import { getThumbnailExtension, type ThumbnailFormat } from './config'; + /** * Normalizes resource paths to be relative to the gallery root directory. * @@ -56,11 +58,16 @@ export function getPhotoPath(filename: string, mediaBaseUrl?: string, url?: stri * * @param headerImageFilename - The filename of the subgallery header image * @param resolvedSubgalleryPath - The resolved subgallery path relative to the gallery root + * @param format - The thumbnail output format used to derive the file extension (defaults to 'avif') * @returns The normalized path relative to the gallery root directory */ -export function getSubgalleryThumbnailPath(headerImageFilename: string, resolvedSubgalleryPath?: string): string { +export function getSubgalleryThumbnailPath( + headerImageFilename: string, + resolvedSubgalleryPath?: string, + format: ThumbnailFormat = 'avif', +): string { const basename = path.basename(headerImageFilename, path.extname(headerImageFilename)); - const thumbnailFilename = `${basename}.avif`; + const thumbnailFilename = `${basename}.${getThumbnailExtension(format)}`; const subgalleryFolder = resolvedSubgalleryPath || path.basename(path.dirname(headerImageFilename)); return path.join(subgalleryFolder, 'gallery', 'images', thumbnailFilename); diff --git a/common/src/theme/resolver.ts b/common/src/theme/resolver.ts index 27a4237..1f0cf1b 100644 --- a/common/src/theme/resolver.ts +++ b/common/src/theme/resolver.ts @@ -1,6 +1,11 @@ import path from 'node:path'; -import { extractThumbnailConfigFromGallery, mergeThumbnailConfig, type ThumbnailConfig } from './config'; +import { + extractThumbnailConfigFromGallery, + mergeThumbnailConfig, + type ThumbnailConfig, + type ThumbnailFormat, +} from './config'; import { LANDSCAPE_SIZES, PORTRAIT_SIZES } from './constants'; import { renderMarkdown } from './markdown'; import { buildHeroSrcset, getPhotoPath, getRelativePath, getSubgalleryThumbnailPath, getThumbnailPath } from './paths'; @@ -60,13 +65,17 @@ async function resolveSection( /** * Resolve a sub-gallery with computed thumbnail path and optional resolved path. */ -function resolveSubGallery(subGallery: SubGallery, galleryJsonPath?: string): ResolvedSubGallery { +function resolveSubGallery( + subGallery: SubGallery, + galleryJsonPath?: string, + thumbnailFormat?: ThumbnailFormat, +): ResolvedSubGallery { const resolvedPath = galleryJsonPath ? getRelativePath(subGallery.path, galleryJsonPath) : undefined; return { title: subGallery.title, headerImage: subGallery.headerImage, path: subGallery.path, - thumbnailPath: getSubgalleryThumbnailPath(subGallery.headerImage, resolvedPath), + thumbnailPath: getSubgalleryThumbnailPath(subGallery.headerImage, resolvedPath, thumbnailFormat), resolvedPath, }; } @@ -180,17 +189,17 @@ export async function resolveGalleryData( gallery.sections.map((section) => resolveSection(section, mediaBaseUrl, thumbsBaseUrl)), ); + // Merge thumbnail config: CLI > gallery.json > themeConfig > defaults + const galleryThumbnailConfig = extractThumbnailConfigFromGallery(gallery); + const thumbnails = mergeThumbnailConfig(cliConfig, galleryThumbnailConfig, themeConfig); + const resolvedSubGalleries = subGalleries?.galleries?.length ? { title: subGalleries.title, - galleries: subGalleries.galleries.map((sg) => resolveSubGallery(sg, galleryJsonPath)), + galleries: subGalleries.galleries.map((sg) => resolveSubGallery(sg, galleryJsonPath, thumbnails.format)), } : undefined; - // Merge thumbnail config: CLI > gallery.json > themeConfig > defaults - const galleryThumbnailConfig = extractThumbnailConfigFromGallery(gallery); - const thumbnails = mergeThumbnailConfig(cliConfig, galleryThumbnailConfig, themeConfig); - return { title: gallery.title, url: gallery.url, diff --git a/common/src/theme/types.ts b/common/src/theme/types.ts index 9a842f8..150fad2 100644 --- a/common/src/theme/types.ts +++ b/common/src/theme/types.ts @@ -1,5 +1,5 @@ import type { GalleryMetadata, HeaderImageVariants } from '../gallery'; -import type { ThumbnailConfig } from './config'; +import type { ResolvedThumbnailConfig } from './config'; /** Resolved hero data with all paths computed and markdown parsed */ export interface ResolvedHero { @@ -73,7 +73,7 @@ export interface ResolvedGalleryData { * Thumbnail configuration with dimension and edge settings. * Themes should use this for display sizing (e.g., row-height for modern theme). */ - thumbnails?: Required; + thumbnails?: ResolvedThumbnailConfig; /** Custom CSS variable overrides from gallery.json, passed through for theme injection. */ customStyles?: Record; } diff --git a/gallery/jest.config.cjs b/gallery/jest.config.cjs index 62fb1e5..90c2568 100644 --- a/gallery/jest.config.cjs +++ b/gallery/jest.config.cjs @@ -11,6 +11,8 @@ module.exports = { '^@simple-photo-gallery/common$': '/../common/src/gallery.ts', '^@simple-photo-gallery/common/theme$': '/../common/src/theme/index.ts', '^@simple-photo-gallery/common/theme/config$': '/../common/src/theme/config.ts', + // Use the CJS-compatible UMD build of marked, since Jest cannot parse its ESM build + '^marked$': '/../node_modules/marked/lib/marked.umd.js', }, transform: { '^.+\\.tsx?$': [ diff --git a/gallery/src/index.ts b/gallery/src/index.ts index 38b9195..d579e29 100644 --- a/gallery/src/index.ts +++ b/gallery/src/index.ts @@ -12,7 +12,7 @@ import { init } from './modules/init'; import { telemetry } from './modules/telemetry'; import { TelemetryService } from './modules/telemetry/service'; import { thumbnails } from './modules/thumbnails'; -import { parseTelemetryOption } from './utils'; +import { parseTelemetryOption, parseThumbnailFormat } from './utils'; import { checkForUpdates, displayUpdateNotification, waitForUpdateCheck } from './utils/version'; import packageJson from '../package.json' with { type: 'json' }; @@ -174,6 +174,9 @@ program .option('-r, --recursive', 'Scan subdirectories recursively', false) .option('--thumbnail-size ', 'Override thumbnail size in pixels', Number.parseInt) .option('--thumbnail-edge ', 'Override how thumbnail size is applied: auto, width, or height') + .option('--thumbnail-format ', 'Override thumbnail output format: avif, webp, or jpeg', parseThumbnailFormat) + .option('--thumbnail-quality ', 'Override thumbnail output quality (1-100)', Number.parseInt) + .option('--thumbnail-effort ', 'Override thumbnail encoder effort (higher is slower but smaller)', Number.parseInt) .action(withCommandContext((options, ui) => thumbnails(options, ui))); program @@ -191,6 +194,9 @@ program ) .option('--thumbnail-size ', 'Override thumbnail size in pixels', Number.parseInt) .option('--thumbnail-edge ', 'Override how thumbnail size is applied: auto, width, or height') + .option('--thumbnail-format ', 'Override thumbnail output format: avif, webp, or jpeg', parseThumbnailFormat) + .option('--thumbnail-quality ', 'Override thumbnail output quality (1-100)', Number.parseInt) + .option('--thumbnail-effort ', 'Override thumbnail encoder effort (higher is slower but smaller)', Number.parseInt) .action(withCommandContext((options, ui) => build(options, ui))); program diff --git a/gallery/src/modules/build/index.ts b/gallery/src/modules/build/index.ts index cc87a9d..f9db6f3 100644 --- a/gallery/src/modules/build/index.ts +++ b/gallery/src/modules/build/index.ts @@ -13,7 +13,7 @@ import { hasOldHeaderImages, } from './utils'; -import { findGalleries } from '../../utils'; +import { buildCliThumbnailConfig, findGalleries } from '../../utils'; import { parseGalleryJson } from '../../utils/gallery'; import { scanDirectory } from '../init'; import { processGalleryThumbnails } from '../thumbnails'; @@ -225,17 +225,13 @@ async function buildGallery( // If thumbnail settings are provided via CLI, update the gallery.json file if needed if (cliThumbnailConfig) { - const needsUpdate = - (cliThumbnailConfig.size !== undefined && galleryData.thumbnails?.size !== cliThumbnailConfig.size) || - (cliThumbnailConfig.edge !== undefined && galleryData.thumbnails?.edge !== cliThumbnailConfig.edge); + const currentThumbnails = galleryData.thumbnails ?? {}; + const cliKeys = Object.keys(cliThumbnailConfig) as (keyof typeof cliThumbnailConfig)[]; + const needsUpdate = cliKeys.some((key) => currentThumbnails[key] !== cliThumbnailConfig[key]); if (needsUpdate) { ui.debug('Updating gallery.json with thumbnail settings'); - galleryData.thumbnails = { - ...galleryData.thumbnails, - ...(cliThumbnailConfig.size !== undefined && { size: cliThumbnailConfig.size }), - ...(cliThumbnailConfig.edge !== undefined && { edge: cliThumbnailConfig.edge }), - }; + galleryData.thumbnails = { ...currentThumbnails, ...cliThumbnailConfig }; fs.writeFileSync(galleryJsonPath, JSON.stringify(galleryData, null, 2)); } } @@ -335,10 +331,7 @@ export async function build(options: BuildOptions, ui: ConsolaInstance): Promise } // Create CLI thumbnail config from options (only include values that were provided) - const cliThumbnailConfig: ThumbnailConfig | undefined = - options.thumbnailSize !== undefined || options.thumbnailEdge !== undefined - ? { size: options.thumbnailSize, edge: options.thumbnailEdge } - : undefined; + const cliThumbnailConfig = buildCliThumbnailConfig(options); // Process each gallery directory let totalGalleries = 0; diff --git a/gallery/src/modules/build/types/index.ts b/gallery/src/modules/build/types/index.ts index cb3f291..876b0e9 100644 --- a/gallery/src/modules/build/types/index.ts +++ b/gallery/src/modules/build/types/index.ts @@ -1,3 +1,5 @@ +import type { ThumbnailFormat } from '@simple-photo-gallery/common/theme'; + /** Options for building gallery HTML output */ export interface BuildOptions { /** Path to the directory containing the gallery */ @@ -18,4 +20,10 @@ export interface BuildOptions { thumbnailSize?: number; /** Override how thumbnail size should be applied: 'auto', 'width', or 'height' */ thumbnailEdge?: 'auto' | 'width' | 'height'; + /** Override thumbnail output format */ + thumbnailFormat?: ThumbnailFormat; + /** Override thumbnail output quality (1-100) */ + thumbnailQuality?: number; + /** Override thumbnail encoder effort */ + thumbnailEffort?: number; } diff --git a/gallery/src/modules/thumbnails/index.ts b/gallery/src/modules/thumbnails/index.ts index e0ce0bb..d2ce2b8 100644 --- a/gallery/src/modules/thumbnails/index.ts +++ b/gallery/src/modules/thumbnails/index.ts @@ -1,8 +1,11 @@ import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; +import process from 'node:process'; import { extractThumbnailConfigFromGallery, + getThumbnailExtension, mergeThumbnailConfig, loadThemeConfig, } from '@simple-photo-gallery/common/theme'; @@ -10,18 +13,24 @@ import { LogLevels, type ConsolaInstance } from 'consola'; import { getFileMtime } from './utils'; -import { findGalleries, handleFileProcessingError } from '../../utils'; +import { buildCliThumbnailConfig, findGalleries, handleFileProcessingError } from '../../utils'; import { generateBlurHash } from '../../utils/blurhash'; +import { mapWithConcurrency } from '../../utils/concurrency'; import { getImageDescription } from '../../utils/descriptions'; import { parseGalleryJson } from '../../utils/gallery'; -import { createImageThumbnails, loadImageWithMetadata, type ThumbnailSizeDimension } from '../../utils/image'; +import { + createImageThumbnails, + loadImageWithMetadata, + type ImageEncodeOptions, + type ThumbnailSizeDimension, +} from '../../utils/image'; import { getVideoDimensions, createVideoThumbnails } from '../../utils/video'; import { resolveThemeDir } from '../build'; import type { ThumbnailOptions } from './types'; import type { CommandResultSummary } from '../telemetry/types'; import type { MediaFile } from '@simple-photo-gallery/common'; -import type { ThumbnailConfig } from '@simple-photo-gallery/common/theme'; +import type { ResolvedThumbnailConfig, ThumbnailConfig } from '@simple-photo-gallery/common/theme'; /** * Processes an image file to create thumbnail and extract metadata @@ -30,6 +39,7 @@ import type { ThumbnailConfig } from '@simple-photo-gallery/common/theme'; * @param thumbnailPathRetina - Path where retina thumbnail should be saved * @param thumbnailSize - Target size for thumbnail * @param thumbnailSizeDimension - How to apply size: 'auto', 'width', or 'height' + * @param encodeOptions - Encoding options (format, quality, effort) * @param lastMediaTimestamp - Optional timestamp to check if processing can be skipped * @returns Promise resolving to updated MediaFile or undefined if skipped */ @@ -39,6 +49,7 @@ export async function processImage( thumbnailPathRetina: string, thumbnailSize: number, thumbnailSizeDimension: ThumbnailSizeDimension = 'auto', + encodeOptions: ImageEncodeOptions = {}, lastMediaTimestamp?: Date, ): Promise { // Get the last media timestamp @@ -49,8 +60,12 @@ export async function processImage( return undefined; } + // Read the original once and reuse the buffer for metadata, thumbnails, blurhash and EXIF so the + // file (potentially tens of MB) is read from disk a single time instead of once per consumer. + const inputBuffer = await fs.promises.readFile(imagePath); + // Load the image and get metadata first to check orientation - const { image, metadata } = await loadImageWithMetadata(imagePath); + const { image, metadata } = await loadImageWithMetadata(inputBuffer); // Get the image dimensions const imageDimensions = { @@ -63,7 +78,7 @@ export async function processImage( } // Get the image description - const description = await getImageDescription(imagePath); + const description = await getImageDescription(inputBuffer); // Create the thumbnails const thumbnailDimensions = await createImageThumbnails( @@ -73,10 +88,11 @@ export async function processImage( thumbnailPathRetina, thumbnailSize, thumbnailSizeDimension, + encodeOptions, ); - // Generate BlurHash from the thumbnail - const blurHash = await generateBlurHash(thumbnailPath); + // Generate BlurHash from the in-memory original, avoiding a re-read and decode of the written thumbnail + const blurHash = await generateBlurHash(inputBuffer); // Return the updated media file return { @@ -104,6 +120,7 @@ export async function processImage( * @param thumbnailSize - Target size for thumbnail * @param thumbnailSizeDimension - How to apply size: 'auto', 'width', or 'height' * @param verbose - Whether to enable verbose output + * @param encodeOptions - Encoding options (format, quality, effort) * @param lastMediaTimestamp - Optional timestamp to check if processing can be skipped * @returns Promise resolving to updated MediaFile or undefined if skipped */ @@ -114,6 +131,7 @@ async function processVideo( thumbnailSize: number, thumbnailSizeDimension: ThumbnailSizeDimension = 'auto', verbose: boolean, + encodeOptions: ImageEncodeOptions = {}, lastMediaTimestamp?: Date, ): Promise { // Get the last media timestamp @@ -136,6 +154,7 @@ async function processVideo( thumbnailSize, thumbnailSizeDimension, verbose, + encodeOptions, ); // Generate BlurHash from the thumbnail @@ -163,7 +182,7 @@ async function processVideo( * @param mediaFile - Media file to process * @param mediaBasePath - Base path for the media files * @param thumbnailsPath - Directory where thumbnails are stored - * @param thumbnailConfig - Thumbnail configuration (dimension and edge) + * @param thumbnailConfig - Resolved thumbnail configuration (size, edge, format, quality, effort) * @param ui - ConsolaInstance for logging * @returns Promise resolving to updated MediaFile */ @@ -171,7 +190,7 @@ async function processMediaFile( mediaFile: MediaFile, mediaBasePath: string, thumbnailsPath: string, - thumbnailConfig: Required, + thumbnailConfig: ResolvedThumbnailConfig, ui: ConsolaInstance, ): Promise { try { @@ -180,9 +199,15 @@ async function processMediaFile( const fileName = mediaFile.filename; const fileNameWithoutExt = path.parse(fileName).name; - const thumbnailFileName = `${fileNameWithoutExt}.avif`; - const thumbnailPath = path.join(thumbnailsPath, thumbnailFileName); - const thumbnailPathRetina = thumbnailPath.replace('.avif', '@2x.avif'); + const extension = getThumbnailExtension(thumbnailConfig.format); + const thumbnailPath = path.join(thumbnailsPath, `${fileNameWithoutExt}.${extension}`); + const thumbnailPathRetina = path.join(thumbnailsPath, `${fileNameWithoutExt}@2x.${extension}`); + + const encodeOptions: ImageEncodeOptions = { + format: thumbnailConfig.format, + quality: thumbnailConfig.quality, + effort: thumbnailConfig.effort, + }; const lastMediaTimestamp = mediaFile.lastMediaTimestamp ? new Date(mediaFile.lastMediaTimestamp) : undefined; const verbose = ui.level === LogLevels.debug; @@ -196,6 +221,7 @@ async function processMediaFile( thumbnailPathRetina, thumbnailConfig.size, thumbnailConfig.edge, + encodeOptions, lastMediaTimestamp, ) : processVideo( @@ -205,6 +231,7 @@ async function processMediaFile( thumbnailConfig.size, thumbnailConfig.edge, verbose, + encodeOptions, lastMediaTimestamp, )); @@ -293,23 +320,70 @@ export async function processGalleryThumbnails( // Merge with 4-level hierarchy: CLI > gallery.json > theme > defaults const thumbnailConfig = mergeThumbnailConfig(cliThumbnailConfig, galleryThumbnailConfig, themeConfig); - ui.debug(`Thumbnail config: size=${thumbnailConfig.size}, edge=${thumbnailConfig.edge}`); + ui.debug( + `Thumbnail config: size=${thumbnailConfig.size}, edge=${thumbnailConfig.edge}, format=${thumbnailConfig.format}` + + `${thumbnailConfig.quality === undefined ? '' : `, quality=${thumbnailConfig.quality}`}` + + `${thumbnailConfig.effort === undefined ? '' : `, effort=${thumbnailConfig.effort}`}`, + ); // If the mediaBasePath is not set, use the gallery directory const mediaBasePath = galleryData.mediaBasePath ?? path.join(galleryDir); - // Process all sections and their images + // Persist gallery.json atomically (write to a temp file then rename) so an interruption can never + // leave a truncated gallery.json behind. + const writeGalleryData = (): void => { + const tempPath = `${galleryJsonPath}.tmp`; + fs.writeFileSync(tempPath, JSON.stringify(galleryData, null, 2)); + fs.renameSync(tempPath, galleryJsonPath); + }; + + // On interruption (e.g. Ctrl-C) flush the progress made so far before exiting. Each completed file + // records its lastMediaTimestamp, so a subsequent run skips everything already finished instead of + // starting the whole gallery over. + const flushAndExit = (exitCode: number) => (): void => { + writeGalleryData(); + // eslint-disable-next-line unicorn/no-process-exit + process.exit(exitCode); + }; + const onSigint = flushAndExit(130); + const onSigterm = flushAndExit(143); + process.once('SIGINT', onSigint); + process.once('SIGTERM', onSigterm); + + // Process media files in parallel. Thumbnailing is IO/CPU bound (Sharp and ffmpeg release the JS + // thread), so a worker pool sized to the available cores processes a large gallery several times + // faster than the previous one-at-a-time loop while keeping per-file error isolation. + const concurrency = Math.max(2, os.cpus().length - 1); + + // Flush progress to disk every PROGRESS_WRITE_INTERVAL files so a long run that is interrupted can + // resume instead of redoing all the thumbnails already on disk. + const PROGRESS_WRITE_INTERVAL = 16; let processedCount = 0; - for (const section of galleryData.sections) { - for (const [index, mediaFile] of section.images.entries()) { - section.images[index] = await processMediaFile(mediaFile, mediaBasePath, thumbnailsPath, thumbnailConfig, ui); + let processedSinceWrite = 0; + + try { + for (const section of galleryData.sections) { + await mapWithConcurrency(section.images, concurrency, async (mediaFile, index) => { + const updated = await processMediaFile(mediaFile, mediaBasePath, thumbnailsPath, thumbnailConfig, ui); + section.images[index] = updated; + + processedCount += 1; + processedSinceWrite += 1; + if (processedSinceWrite >= PROGRESS_WRITE_INTERVAL) { + processedSinceWrite = 0; + writeGalleryData(); + } + + return updated; + }); } - - processedCount += section.images.length; + } finally { + process.removeListener('SIGINT', onSigint); + process.removeListener('SIGTERM', onSigterm); } - // Write updated gallery.json - fs.writeFileSync(galleryJsonPath, JSON.stringify(galleryData, null, 2)); + // Write the final gallery.json with all processed files + writeGalleryData(); ui.success(`Created thumbnails for ${processedCount} media files`); @@ -335,10 +409,7 @@ export async function thumbnails(options: ThumbnailOptions, ui: ConsolaInstance) } // Create CLI thumbnail config from options (only include values that were provided) - const cliThumbnailConfig: ThumbnailConfig | undefined = - options.thumbnailSize !== undefined || options.thumbnailEdge !== undefined - ? { size: options.thumbnailSize, edge: options.thumbnailEdge } - : undefined; + const cliThumbnailConfig = buildCliThumbnailConfig(options); // Process each gallery directory let totalGalleries = 0; diff --git a/gallery/src/modules/thumbnails/types/index.ts b/gallery/src/modules/thumbnails/types/index.ts index c97196d..af830d3 100644 --- a/gallery/src/modules/thumbnails/types/index.ts +++ b/gallery/src/modules/thumbnails/types/index.ts @@ -1,3 +1,5 @@ +import type { ThumbnailFormat } from '@simple-photo-gallery/common/theme'; + /** Options for generating thumbnails */ export interface ThumbnailOptions { /** Path to the directory containing the gallery */ @@ -8,4 +10,10 @@ export interface ThumbnailOptions { thumbnailSize?: number; /** Override how thumbnail size should be applied: 'auto', 'width', or 'height' */ thumbnailEdge?: 'auto' | 'width' | 'height'; + /** Override thumbnail output format */ + thumbnailFormat?: ThumbnailFormat; + /** Override thumbnail output quality (1-100) */ + thumbnailQuality?: number; + /** Override thumbnail encoder effort */ + thumbnailEffort?: number; } diff --git a/gallery/src/utils/blurhash.ts b/gallery/src/utils/blurhash.ts index 078d148..0a02039 100644 --- a/gallery/src/utils/blurhash.ts +++ b/gallery/src/utils/blurhash.ts @@ -1,16 +1,20 @@ import { encode } from 'blurhash'; -import { loadImage } from './image'; +import { loadImage, type ImageSource } from './image'; /** - * Generates a BlurHash from an image file or Sharp instance - * @param imagePath - Path to image file or Sharp instance + * Generates a BlurHash from an image file path or its contents as a Buffer + * @param source - Path to the image file or its contents as a Buffer * @param componentX - Number of x components (default: 4) * @param componentY - Number of y components (default: 3) * @returns Promise resolving to BlurHash string */ -export async function generateBlurHash(imagePath: string, componentX: number = 4, componentY: number = 3): Promise { - const image = await loadImage(imagePath); +export async function generateBlurHash( + source: ImageSource, + componentX: number = 4, + componentY: number = 3, +): Promise { + const image = await loadImage(source); // Resize to small size for BlurHash computation to improve performance // BlurHash doesn't need high resolution diff --git a/gallery/src/utils/concurrency.ts b/gallery/src/utils/concurrency.ts new file mode 100644 index 0000000..ecdfef2 --- /dev/null +++ b/gallery/src/utils/concurrency.ts @@ -0,0 +1,34 @@ +/** + * Maps over items with a bounded number of concurrent workers, preserving input order in the results. + * + * Used to parallelize IO/CPU-bound work (such as thumbnail generation) without spawning unbounded + * concurrency that would exhaust memory or oversubscribe the CPU. A fixed pool of workers pulls items + * from a shared cursor; because JavaScript runs the cursor read/increment without interleaving, no + * locking is required. + * + * @param items - The items to process + * @param concurrency - Maximum number of mapper invocations running at the same time + * @param mapper - Async function applied to each item; receives the item and its index + * @returns Promise resolving to the mapped results in the same order as the input + */ +export async function mapWithConcurrency( + items: readonly T[], + concurrency: number, + mapper: (item: T, index: number) => Promise, +): Promise { + const results: R[] = []; + let nextIndex = 0; + + const worker = async (): Promise => { + while (nextIndex < items.length) { + const currentIndex = nextIndex; + nextIndex += 1; + results[currentIndex] = await mapper(items[currentIndex], currentIndex); + } + }; + + const workerCount = Math.max(1, Math.min(concurrency, items.length)); + await Promise.all(Array.from({ length: workerCount }, () => worker())); + + return results; +} diff --git a/gallery/src/utils/descriptions.ts b/gallery/src/utils/descriptions.ts index fff1f5d..1df9d7f 100644 --- a/gallery/src/utils/descriptions.ts +++ b/gallery/src/utils/descriptions.ts @@ -1,13 +1,16 @@ +import { Buffer } from 'node:buffer'; + import ExifReader from 'exifreader'; /** * Extracts description from image EXIF data - * @param image - Image path or File object + * @param image - Image path, file contents as a Buffer, or File object * @returns Promise resolving to image description or undefined if not found */ -export async function getImageDescription(image: string | File): Promise { +export async function getImageDescription(image: string | Buffer | File): Promise { try { - const tags = await ExifReader.load(image); + // ExifReader.load returns Tags synchronously for a Buffer and a Promise for a path or File + const tags = Buffer.isBuffer(image) ? ExifReader.load(image) : await ExifReader.load(image); // Description if (tags.description?.description) return tags.description.description; diff --git a/gallery/src/utils/image.ts b/gallery/src/utils/image.ts index 57bed32..cc56ac8 100644 --- a/gallery/src/utils/image.ts +++ b/gallery/src/utils/image.ts @@ -1,24 +1,28 @@ import sharp from 'sharp'; import type { Dimensions, ImageWithMetadata } from '../types'; +import type { Buffer } from 'node:buffer'; import type { FormatEnum, Metadata, Sharp } from 'sharp'; +/** An image source accepted by Sharp: a file path or the image file contents in a Buffer */ +export type ImageSource = string | Buffer; + /** * Loads an image and auto-rotates it based on EXIF orientation. - * @param imagePath - Path to the image file + * @param image - Path to the image file or its contents as a Buffer * @returns Promise resolving to Sharp image instance */ -export async function loadImage(imagePath: string): Promise { - return sharp(imagePath).rotate(); +export async function loadImage(image: ImageSource): Promise { + return sharp(image).rotate(); } /** * Loads an image and its metadata, auto-rotating it based on EXIF orientation and swapping dimensions if needed. - * @param imagePath - Path to the image file + * @param source - Path to the image file or its contents as a Buffer * @returns Promise resolving to ImageWithMetadata object containing Sharp image instance and metadata */ -export async function loadImageWithMetadata(imagePath: string): Promise { - const image = sharp(imagePath); +export async function loadImageWithMetadata(source: ImageSource): Promise { + const image = sharp(source); const metadata = await image.metadata(); // Auto-rotate based on EXIF orientation @@ -37,22 +41,68 @@ export async function loadImageWithMetadata(imagePath: string): Promise { + const { format = 'avif', quality, effort } = options; + // Resize the image without enlarging it - await image.resize(width, height, { withoutEnlargement: true }).toFormat(format).toFile(outputPath); + await image + .resize(width, height, { withoutEnlargement: true }) + .toFormat(format, buildFormatOptions(format, quality, effort)) + .toFile(outputPath); } /** @@ -90,6 +140,7 @@ export type ThumbnailSizeDimension = 'auto' | 'width' | 'height'; * @param outputPathRetina - Path where retina thumbnail should be saved * @param size - Target size for the thumbnail * @param sizeDimension - How to apply the size: 'auto' (longer edge), 'width', or 'height' + * @param encodeOptions - Encoding options (format, quality, effort) * @returns Promise resolving to thumbnail dimensions */ export async function createImageThumbnails( @@ -99,6 +150,7 @@ export async function createImageThumbnails( outputPathRetina: string, size: number, sizeDimension: ThumbnailSizeDimension = 'auto', + encodeOptions: ImageEncodeOptions = {}, ): Promise { // Get the original dimensions const originalWidth = metadata.width || 0; @@ -134,8 +186,8 @@ export async function createImageThumbnails( } // Resize the image and create the thumbnails - await resizeImage(image, outputPath, width, height); - await resizeImage(image, outputPathRetina, width * 2, height * 2); + await resizeImage(image, outputPath, width, height, encodeOptions); + await resizeImage(image, outputPathRetina, width * 2, height * 2, encodeOptions); // Return the dimensions of the thumbnail return { width, height }; diff --git a/gallery/src/utils/index.ts b/gallery/src/utils/index.ts index abc4ffa..0ae7816 100644 --- a/gallery/src/utils/index.ts +++ b/gallery/src/utils/index.ts @@ -1,6 +1,9 @@ import fs from 'node:fs'; import path from 'node:path'; +import { THUMBNAIL_FORMATS } from '@simple-photo-gallery/common/theme'; + +import type { ThumbnailConfig, ThumbnailFormat } from '@simple-photo-gallery/common/theme'; import type { ConsolaInstance } from 'consola'; /** @@ -73,3 +76,52 @@ export function parseTelemetryOption(value: string): '0' | '1' { return value; } + +/** + * Parses and validates the thumbnail format CLI option + * @param value - The value to parse + * @returns The parsed thumbnail format + */ +export function parseThumbnailFormat(value: string): ThumbnailFormat { + if (!(THUMBNAIL_FORMATS as readonly string[]).includes(value)) { + throw new Error(`Thumbnail format must be one of: ${THUMBNAIL_FORMATS.join(', ')}.`); + } + + return value as ThumbnailFormat; +} + +/** Subset of CLI options that map to thumbnail configuration overrides */ +export interface CliThumbnailOptions { + thumbnailSize?: number; + thumbnailEdge?: 'auto' | 'width' | 'height'; + thumbnailFormat?: ThumbnailFormat; + thumbnailQuality?: number; + thumbnailEffort?: number; +} + +/** + * Builds a ThumbnailConfig from CLI options, including only the values that were provided. + * @param options - CLI options carrying thumbnail overrides + * @returns A ThumbnailConfig with the provided overrides, or undefined when none were given + */ +export function buildCliThumbnailConfig(options: CliThumbnailOptions): ThumbnailConfig | undefined { + const config: ThumbnailConfig = {}; + + if (options.thumbnailSize !== undefined) { + config.size = options.thumbnailSize; + } + if (options.thumbnailEdge !== undefined) { + config.edge = options.thumbnailEdge; + } + if (options.thumbnailFormat !== undefined) { + config.format = options.thumbnailFormat; + } + if (options.thumbnailQuality !== undefined) { + config.quality = options.thumbnailQuality; + } + if (options.thumbnailEffort !== undefined) { + config.effort = options.thumbnailEffort; + } + + return Object.keys(config).length > 0 ? config : undefined; +} diff --git a/gallery/src/utils/video.ts b/gallery/src/utils/video.ts index fd724b0..5bf4c3b 100644 --- a/gallery/src/utils/video.ts +++ b/gallery/src/utils/video.ts @@ -4,7 +4,7 @@ import { promises as fs } from 'node:fs'; import ffprobe from 'node-ffprobe'; import sharp from 'sharp'; -import { resizeImage, type ThumbnailSizeDimension } from './image'; +import { resizeImage, type ImageEncodeOptions, type ThumbnailSizeDimension } from './image'; import type { Dimensions } from '../types'; import type { Buffer } from 'node:buffer'; @@ -44,6 +44,7 @@ export async function getVideoDimensions(filePath: string): Promise * @param size - Target size for thumbnail * @param sizeDimension - How to apply size: 'auto' (longer edge), 'width', or 'height' * @param verbose - Whether to enable verbose ffmpeg output + * @param encodeOptions - Encoding options (format, quality, effort) * @returns Promise resolving to thumbnail dimensions */ export async function createVideoThumbnails( @@ -54,6 +55,7 @@ export async function createVideoThumbnails( size: number, sizeDimension: ThumbnailSizeDimension = 'auto', verbose: boolean = false, + encodeOptions: ImageEncodeOptions = {}, ): Promise { // Calculate dimensions maintaining aspect ratio based on sizeDimension const aspectRatio = videoDimensions.width / videoDimensions.height; @@ -104,10 +106,12 @@ export async function createVideoThumbnails( ffmpeg.on('close', async (code: number) => { if (code === 0) { try { - // Process the extracted frame with sharp - const frameImage = sharp(tempFramePath); - await resizeImage(frameImage, outputPath, width, height); - await resizeImage(frameImage, outputPathRetina, width * 2, height * 2); + // Read the extracted frame once and decode both thumbnails from memory instead of re-reading + // the temporary file from disk for each output + const frameBuffer = await fs.readFile(tempFramePath); + const frameImage = sharp(frameBuffer); + await resizeImage(frameImage, outputPath, width, height, encodeOptions); + await resizeImage(frameImage, outputPathRetina, width * 2, height * 2, encodeOptions); // Clean up temporary file try { diff --git a/gallery/tests/concurrency.test.ts b/gallery/tests/concurrency.test.ts new file mode 100644 index 0000000..def5ad9 --- /dev/null +++ b/gallery/tests/concurrency.test.ts @@ -0,0 +1,60 @@ +import { mapWithConcurrency } from '../src/utils/concurrency'; + +const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +describe('mapWithConcurrency', () => { + test('should preserve input order in the results', async () => { + const items = [10, 5, 1, 8, 3]; + + // Items with shorter delays resolve first, but results must stay in input order + const results = await mapWithConcurrency(items, 3, async (value) => { + await delay(value); + return value * 2; + }); + + expect(results).toEqual([20, 10, 2, 16, 6]); + }); + + test('should pass the index to the mapper', async () => { + const items = ['a', 'b', 'c']; + + const results = await mapWithConcurrency(items, 2, async (value, index) => `${value}${index}`); + + expect(results).toEqual(['a0', 'b1', 'c2']); + }); + + test('should never run more than the requested number of workers at once', async () => { + const concurrency = 3; + let active = 0; + let maxActive = 0; + + await mapWithConcurrency( + Array.from({ length: 20 }, (_, index) => index), + concurrency, + async () => { + active += 1; + maxActive = Math.max(maxActive, active); + await delay(2); + active -= 1; + }, + ); + + expect(maxActive).toBeLessThanOrEqual(concurrency); + }); + + test('should process every item exactly once', async () => { + const items = Array.from({ length: 50 }, (_, index) => index); + const seen: number[] = []; + + await mapWithConcurrency(items, 8, async (value) => { + seen.push(value); + }); + + expect(seen.sort((a, b) => a - b)).toEqual(items); + }); + + test('should handle an empty input', async () => { + const results = await mapWithConcurrency([], 4, async (value) => value); + expect(results).toEqual([]); + }); +}); diff --git a/gallery/tests/thumbnail-config.test.ts b/gallery/tests/thumbnail-config.test.ts new file mode 100644 index 0000000..e353799 --- /dev/null +++ b/gallery/tests/thumbnail-config.test.ts @@ -0,0 +1,72 @@ +import { getThumbnailExtension, mergeThumbnailConfig } from '../../common/src/theme/config'; +import { buildCliThumbnailConfig, parseThumbnailFormat } from '../src/utils'; + +describe('getThumbnailExtension', () => { + test('should return the format name for avif and webp', () => { + expect(getThumbnailExtension('avif')).toBe('avif'); + expect(getThumbnailExtension('webp')).toBe('webp'); + }); + + test('should map jpeg to the jpg extension', () => { + expect(getThumbnailExtension('jpeg')).toBe('jpg'); + }); +}); + +describe('mergeThumbnailConfig with format, quality and effort', () => { + test('should default to avif format with no quality or effort', () => { + const result = mergeThumbnailConfig(); + expect(result.format).toBe('avif'); + expect(result.quality).toBeUndefined(); + expect(result.effort).toBeUndefined(); + }); + + test('should resolve format with CLI > gallery > theme precedence', () => { + const result = mergeThumbnailConfig({ format: 'webp' }, { format: 'jpeg' }, { format: 'avif' }); + expect(result.format).toBe('webp'); + }); + + test('should fall back to gallery format when CLI does not set it', () => { + const result = mergeThumbnailConfig({ size: 200 }, { format: 'jpeg' }, { format: 'avif' }); + expect(result.format).toBe('jpeg'); + }); + + test('should resolve quality and effort across the hierarchy', () => { + const result = mergeThumbnailConfig({ effort: 2 }, { quality: 70 }, { quality: 50, effort: 9 }); + expect(result.quality).toBe(70); + expect(result.effort).toBe(2); + }); +}); + +describe('parseThumbnailFormat', () => { + test('should accept the supported formats', () => { + expect(parseThumbnailFormat('avif')).toBe('avif'); + expect(parseThumbnailFormat('webp')).toBe('webp'); + expect(parseThumbnailFormat('jpeg')).toBe('jpeg'); + }); + + test('should throw for an unsupported format', () => { + expect(() => parseThumbnailFormat('gif')).toThrow('Thumbnail format must be one of'); + }); +}); + +describe('buildCliThumbnailConfig', () => { + test('should return undefined when no thumbnail options are provided', () => { + expect(buildCliThumbnailConfig({})).toBeUndefined(); + }); + + test('should include only the provided options', () => { + const config = buildCliThumbnailConfig({ thumbnailFormat: 'webp', thumbnailEffort: 3 }); + expect(config).toEqual({ format: 'webp', effort: 3 }); + }); + + test('should map all thumbnail options to config keys', () => { + const config = buildCliThumbnailConfig({ + thumbnailSize: 400, + thumbnailEdge: 'width', + thumbnailFormat: 'jpeg', + thumbnailQuality: 80, + thumbnailEffort: 5, + }); + expect(config).toEqual({ size: 400, edge: 'width', format: 'jpeg', quality: 80, effort: 5 }); + }); +}); diff --git a/gallery/tests/thumbnails-progress.test.ts b/gallery/tests/thumbnails-progress.test.ts new file mode 100644 index 0000000..c83cef6 --- /dev/null +++ b/gallery/tests/thumbnails-progress.test.ts @@ -0,0 +1,104 @@ +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import process from 'node:process'; + +// resolveThemeDir uses import.meta, which ts-jest cannot evaluate; the test galleries set no theme so it +// is never called, but the module is still imported by thumbnails/index.ts, so mock it. +jest.mock('../src/modules/build', () => ({ + resolveThemeDir: jest.fn().mockResolvedValue('/tmp/theme'), +})); + +import { processGalleryThumbnails } from '../src/modules/thumbnails'; + +import type { ConsolaInstance } from 'consola'; + +const fixtureImages = path.resolve(process.cwd(), 'tests', 'fixtures', 'single'); + +const createMockUI = (): ConsolaInstance => + ({ + info: jest.fn(), + start: jest.fn(), + success: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + box: jest.fn(), + warn: jest.fn(), + }) as unknown as ConsolaInstance; + +function setupGallery(rootDir: string, filenames: string[]): void { + mkdirSync(path.join(rootDir, 'gallery'), { recursive: true }); + + for (const name of filenames) { + copyFileSync(path.join(fixtureImages, name), path.join(rootDir, name)); + } + + const galleryData = { + title: 'Test', + description: 'Test gallery', + headerImage: filenames[0], + metadata: {}, + sections: [{ images: filenames.map((filename) => ({ type: 'image', filename, width: 0, height: 0 })) }], + subGalleries: { title: 'Sub', galleries: [] }, + }; + + writeFileSync(path.join(rootDir, 'gallery', 'gallery.json'), JSON.stringify(galleryData, null, 2)); +} + +function readGalleryJson(rootDir: string): { sections: { images: Record[] }[] } { + return JSON.parse(readFileSync(path.join(rootDir, 'gallery', 'gallery.json'), 'utf8')); +} + +describe('processGalleryThumbnails progress persistence', () => { + let tempDir: string; + let filenames: string[]; + + beforeEach(() => { + tempDir = mkdtempSync(path.join(os.tmpdir(), 'spg-progress-')); + filenames = readdirSync(fixtureImages) + .filter((file) => file.toLowerCase().endsWith('.jpg')) + .slice(0, 3); + setupGallery(tempDir, filenames); + }); + + afterEach(() => { + if (existsSync(tempDir)) { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + test('persists thumbnails and timestamps and leaves no temporary file behind', async () => { + const count = await processGalleryThumbnails(tempDir, createMockUI()); + + expect(count).toBe(filenames.length); + + const data = readGalleryJson(tempDir); + for (const image of data.sections[0].images) { + const thumbnail = image.thumbnail as { path: string; blurHash?: string } | undefined; + expect(thumbnail).toBeDefined(); + expect(thumbnail!.blurHash).toBeDefined(); + expect(image.lastMediaTimestamp).toBeDefined(); + expect(existsSync(path.join(tempDir, 'gallery', 'images', thumbnail!.path))).toBe(true); + } + + // The atomic write should not leave the temporary file behind + expect(existsSync(path.join(tempDir, 'gallery', 'gallery.json.tmp'))).toBe(false); + }); + + test('a second run resumes without regenerating existing thumbnails', async () => { + await processGalleryThumbnails(tempDir, createMockUI()); + + const imagesDir = path.join(tempDir, 'gallery', 'images'); + const firstRun = readdirSync(imagesDir).sort(); + const firstTimestamps = readGalleryJson(tempDir).sections[0].images.map((image) => image.lastMediaTimestamp); + + await processGalleryThumbnails(tempDir, createMockUI()); + + const secondRun = readdirSync(imagesDir).sort(); + const secondTimestamps = readGalleryJson(tempDir).sections[0].images.map((image) => image.lastMediaTimestamp); + + // Already-processed files are skipped, so the thumbnails and their recorded timestamps are unchanged + expect(secondRun).toEqual(firstRun); + expect(secondTimestamps).toEqual(firstTimestamps); + }); +});