diff --git a/app/note/[id].tsx b/app/note/[id].tsx index de495e3..e3cb139 100644 --- a/app/note/[id].tsx +++ b/app/note/[id].tsx @@ -952,7 +952,9 @@ export default function NoteScreen() { Data { + guard url.isFileURL, maxPixelSize > 0, + let source = CGImageSourceCreateWithURL(url as CFURL, nil) else { + throw DecodeError.invalidSource + } + let options: [CFString: Any] = [ + kCGImageSourceCreateThumbnailFromImageAlways: true, + kCGImageSourceCreateThumbnailWithTransform: true, + kCGImageSourceThumbnailMaxPixelSize: maxPixelSize, + kCGImageSourceShouldCacheImmediately: true, + ] + guard let image = CGImageSourceCreateThumbnailAtIndex(source, 0, options as CFDictionary) else { + throw DecodeError.decodeFailed + } + let data = CFDataCreateMutable(nil, 0)! + guard let destination = CGImageDestinationCreateWithData( + data, UTType.png.identifier as CFString, 1, nil + ) else { + throw DecodeError.encodeFailed + } + CGImageDestinationAddImage(destination, image, nil) + guard CGImageDestinationFinalize(destination) else { throw DecodeError.encodeFailed } + return data as Data + } +} diff --git a/ios/OpenNotes/PDFUtilsModule.m b/ios/OpenNotes/PDFUtilsModule.m index 857f284..a365140 100644 --- a/ios/OpenNotes/PDFUtilsModule.m +++ b/ios/OpenNotes/PDFUtilsModule.m @@ -2,6 +2,10 @@ @interface RCT_EXTERN_MODULE(PDFUtilsModule, NSObject) +RCT_EXTERN_METHOD(readImageForThumbnail:(NSString *)fileUri + resolver:(RCTPromiseResolveBlock)resolver + rejecter:(RCTPromiseRejectBlock)rejecter) + RCT_EXTERN_METHOD(getPageCount:(NSString *)filePath resolver:(RCTPromiseResolveBlock)resolver rejecter:(RCTPromiseRejectBlock)rejecter) diff --git a/ios/OpenNotes/PDFUtilsModule.swift b/ios/OpenNotes/PDFUtilsModule.swift index 0d6b4dd..7246ae1 100644 --- a/ios/OpenNotes/PDFUtilsModule.swift +++ b/ios/OpenNotes/PDFUtilsModule.swift @@ -8,6 +8,26 @@ class PDFUtilsModule: NSObject { return false } + @objc + func readImageForThumbnail(_ fileUri: String, + resolver: @escaping RCTPromiseResolveBlock, + rejecter: @escaping RCTPromiseRejectBlock) { + DispatchQueue.global(qos: .userInitiated).async { + autoreleasepool { + guard let url = PDFUtilsModule.localFileURL(fileUri) else { + rejecter("E_INVALID_PATH", "Invalid image file URL", nil) + return + } + do { + let data = try ImageThumbnailEncoder.pngData(at: url) + resolver("data:image/png;base64," + data.base64EncodedString()) + } catch { + rejecter("E_IMAGE_DECODE", "Could not decode image for thumbnail", error) + } + } + } + } + /// A file URL for a file:// URI or a plain absolute path. private static func localFileURL(_ path: String) -> URL? { if path.hasPrefix("file://") { diff --git a/package.json b/package.json index 883fe2a..339d63e 100644 --- a/package.json +++ b/package.json @@ -19,8 +19,9 @@ "android": "expo run:android", "test:lifecycle": "node --experimental-strip-types --test scripts/lifecyclePolicy.test.mjs scripts/reviewPromptService.test.mjs", "test:catalog": "node --experimental-strip-types --test scripts/catalogStore.test.mjs", - "test": "node --experimental-strip-types --test scripts/lifecyclePolicy.test.mjs scripts/reviewPromptService.test.mjs scripts/catalogStore.test.mjs scripts/backupEngine.test.mjs scripts/exportService.test.mjs scripts/overlayTextInsertion.test.mjs scripts/appDocumentUri.test.mjs scripts/crashReport.test.mjs", + "test": "node --experimental-strip-types --test scripts/lifecyclePolicy.test.mjs scripts/reviewPromptService.test.mjs scripts/catalogStore.test.mjs scripts/backupEngine.test.mjs scripts/exportService.test.mjs scripts/overlayTextInsertion.test.mjs scripts/appDocumentUri.test.mjs scripts/crashReport.test.mjs scripts/noteThumbnails.test.mjs scripts/thumbnailRendering.test.mjs", "test:backup": "node --experimental-strip-types --test scripts/backupEngine.test.mjs", + "test:thumbnail-native": "xcrun swiftc ios/OpenNotes/ImageThumbnailEncoder.swift scripts/imageThumbnailSmoke.swift -o /tmp/opennotes-image-thumbnail-smoke && /tmp/opennotes-image-thumbnail-smoke", "typecheck": "tsc --noEmit -p tsconfig.json" }, "dependencies": { diff --git a/scripts/catalogStore.test.mjs b/scripts/catalogStore.test.mjs index aba65d0..317f0c2 100644 --- a/scripts/catalogStore.test.mjs +++ b/scripts/catalogStore.test.mjs @@ -10,6 +10,21 @@ import { parseCatalog, } from '../src/services/catalogStore.ts'; +test('catalog reload preserves composed thumbnail versions and still accepts old notes', () => { + const catalog = parseCatalog(JSON.stringify({ + version: 1, folders: [], notes: [ + { id: 'old', title: 'Old note', thumbnailUri: 'old-ink-preview' }, + { id: 'new', title: 'Photo note', thumbnailUri: 'composed-cover', thumbnailVersion: 1 }, + ], + })); + assert.equal(catalog.notes[0].thumbnailUri, 'old-ink-preview'); + assert.equal(catalog.notes[0].thumbnailVersion, undefined); + assert.equal(catalog.notes[1].thumbnailVersion, 1); + const restored = parseCatalog(JSON.stringify(catalog)); + assert.equal(restored.notes[1].thumbnailUri, 'composed-cover'); + assert.equal(restored.notes[1].thumbnailVersion, 1); +}); + function makeKv(initial = {}) { const map = new Map(Object.entries(initial)); return { diff --git a/scripts/imageThumbnailSmoke.swift b/scripts/imageThumbnailSmoke.swift new file mode 100644 index 0000000..b6b64ba --- /dev/null +++ b/scripts/imageThumbnailSmoke.swift @@ -0,0 +1,60 @@ +import Foundation +import CoreGraphics +import ImageIO +import UniformTypeIdentifiers + +@main +struct ImageThumbnailSmoke { + static func main() throws { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let context = CGContext(data: nil, width: 2000, height: 1000, + bitsPerComponent: 8, bytesPerRow: 0, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)! + context.setFillColor(CGColor(red: 0, green: 0.7, blue: 0.2, alpha: 1)) + context.fill(CGRect(x: 0, y: 0, width: 2000, height: 1000)) + let image = context.makeImage()! + var checked = 0 + for (name, type, orientation) in [ + ("photo.png", UTType.png, 1), + ("photo.jpg", UTType.jpeg, 6), + ("photo.heic", UTType.heic, 1), + ] { + let url = directory.appendingPathComponent(name) + let destination = CGImageDestinationCreateWithURL(url as CFURL, type.identifier as CFString, 1, nil)! + CGImageDestinationAddImage(destination, image, [kCGImagePropertyOrientation: orientation] as CFDictionary) + precondition(CGImageDestinationFinalize(destination), "Could not create fixture") + let before = try Data(contentsOf: url) + let thumbnail = try ImageThumbnailEncoder.pngData(at: url, maxPixelSize: 256) + let source = CGImageSourceCreateWithData(thumbnail as CFData, nil)! + precondition(CGImageSourceGetType(source) as String? == UTType.png.identifier) + let decoded = CGImageSourceCreateImageAtIndex(source, 0, nil)! + precondition(decoded.width == (orientation == 6 ? 128 : 256), "Wrong thumbnail width") + precondition(decoded.height == (orientation == 6 ? 256 : 128), "Wrong thumbnail height/orientation") + var pixel = [UInt8](repeating: 0, count: 4) + let sample = CGContext(data: &pixel, width: 1, height: 1, bitsPerComponent: 8, + bytesPerRow: 4, space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)! + sample.draw(decoded, in: CGRect(x: 0, y: 0, width: 1, height: 1)) + precondition(pixel[0] < 40 && pixel[1] > 140 && pixel[2] < 100, "Image colors were lost") + let after = try Data(contentsOf: url) + precondition(after == before, "Original photo was changed") + checked += 1 + } + for url in [directory.appendingPathComponent("missing.heic"), directory.appendingPathComponent("broken.png")] { + if url.lastPathComponent == "broken.png" { try Data("invalid".utf8).write(to: url) } + do { + _ = try ImageThumbnailEncoder.pngData(at: url) + preconditionFailure("Invalid input should be rejected") + } catch { checked += 1 } + } + do { + _ = try ImageThumbnailEncoder.pngData(at: URL(string: "https://example.com/image.heic")!) + preconditionFailure("Remote URLs must not be fetched by the native helper") + } catch { checked += 1 } + FileHandle.standardOutput.write(Data("Passed \(checked) native image thumbnail checks (PNG, JPEG orientation, HEIC, missing/corrupt assets).\n".utf8)) + } +} diff --git a/scripts/noteThumbnails.test.mjs b/scripts/noteThumbnails.test.mjs new file mode 100644 index 0000000..daed3ef --- /dev/null +++ b/scripts/noteThumbnails.test.mjs @@ -0,0 +1,167 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import test from 'node:test'; +import vm from 'node:vm'; +import ts from 'typescript'; + +const source = ts.transpileModule( + readFileSync(new URL('../src/services/notesRepo.ts', import.meta.url), 'utf8'), + { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 } }, +).outputText; + +const ink = 'data:image/png;base64,aW5r'; +const composite = 'data:image/png;base64,cGhvdG8='; +const photo = { id: 'photo', type: 'image', sourceUri: 'file:///photo.jpg', x: 20, y: 30, width: 100, height: 80 }; +const body = () => ({ version: '1.0', pages: [{ id: 'page-1', data: '{"pages":{}}', previewUri: ink, insertedElements: [photo] }] }); + +function harness({ writeOk = true, renderFails = false, render = async () => composite, readKind = 'ok' } = {}) { + let catalog = { notes: [{ id: 'note', createdAt: '2026-01-01', updatedAt: '2026-01-02', thumbnailUri: ink, backgroundType: 'plain' }] }; + let stored = body(); + let reads = 0; + const exports = {}; + vm.runInNewContext(source, { + exports, + require(name) { + if (name === './catalogEnv') return { catalogStore: { + getCatalog: async () => structuredClone(catalog), + mutate: async fn => { catalog = fn(catalog) ?? catalog; return catalog; }, + } }; + if (name === './noteBodyStorage') return { + writeBody: async (_id, data) => { if (writeOk) stored = structuredClone(data); return writeOk; }, + readBody: async () => { reads += 1; return { kind: readKind, data: structuredClone(stored) }; }, + }; + if (name === './thumbnailService') return { + THUMBNAIL_VERSION: 1, + createPageThumbnail: async (...args) => { + if (renderFails) throw new Error('Image decoder unavailable'); + return render(...args); + }, + }; + if (name === '../i18n') return { t: { common: { untitled: 'Untitled' } } }; + if (name === '../utils/id') return {}; + if (name === './catalogStore') return {}; + if (name === './pdfStorage' || name === './imageInsertStorage') return {}; + throw new Error('Unexpected import: ' + name); + }, + }); + return { ...exports, note: () => catalog.notes[0], stored: () => stored, reads: () => reads, + removeNote: () => { catalog.notes = []; } }; +} + +test('saving uses a composed cover without flattening overlays into the note body', async () => { + const h = harness(); + const original = body(); + const result = await h.saveNoteBody('note', original); + assert.equal(result.ok, true); + assert.equal(h.note().thumbnailUri, composite); + assert.equal(h.note().thumbnailVersion, 1); + assert.deepEqual(h.stored(), original); + assert.equal(h.stored().pages[0].previewUri, ink); +}); + +test('failed body writes leave the previous cover and metadata untouched', async () => { + const h = harness({ writeOk: false }); + const before = structuredClone(h.note()); + assert.equal((await h.saveNoteBody('note', body())).ok, false); + assert.deepEqual(h.note(), before); +}); + +test('thumbnail failure cannot fail a note save or discard its images', async () => { + const h = harness({ renderFails: true }); + assert.equal((await h.saveNoteBody('note', body())).ok, true); + assert.equal(h.stored().pages[0].insertedElements[0].sourceUri, photo.sourceUri); + assert.equal(h.note().thumbnailUri, ink); + assert.notEqual(h.note().thumbnailVersion, 1); +}); + +test('legacy thumbnail repair leaves the note body and modification date untouched', async () => { + const h = harness(); + const before = structuredClone(h.stored()); + assert.equal(await h.getNoteThumbnail(h.note()), composite); + assert.equal(h.note().updatedAt, '2026-01-02'); + assert.equal(h.note().thumbnailVersion, 1); + assert.deepEqual(h.stored(), before); +}); + +test('failed legacy thumbnail repair keeps the existing cover eligible for retry', async () => { + const h = harness({ renderFails: true }); + assert.equal(await h.getNoteThumbnail(h.note()), ink); + assert.notEqual(h.note().thumbnailVersion, 1); +}); + +test('image-only saves compose the first page even when no ink preview exists', async () => { + const h = harness({ render: async page => page.insertedElements?.length ? composite : null }); + const original = body(); + delete original.pages[0].previewUri; + assert.equal((await h.saveNoteBody('note', original)).ok, true); + assert.equal(h.note().thumbnailUri, composite); + assert.equal(h.stored().pages[0].previewUri, undefined); +}); + +test('saving an image-only edit uses the new overlay data instead of the previous body', async () => { + const h = harness({ render: async page => page.insertedElements[0].x === 300 ? composite : ink }); + const original = body(); + original.pages[0].insertedElements = [{ ...photo, x: 300 }]; + await h.saveNoteBody('note', original); + assert.equal(h.note().thumbnailUri, composite); +}); + +test('deleting all overlays restores the original ink-only cover', async () => { + const h = harness({ render: async page => page.insertedElements?.length ? composite : page.previewUri }); + await h.saveNoteBody('note', body()); + const original = body(); + original.pages[0].insertedElements = []; + await h.saveNoteBody('note', original); + assert.equal(h.note().thumbnailUri, ink); +}); + +for (const readKind of ['unreadable', 'missing']) { + test(`${readKind} legacy bodies retain their last thumbnail without being rewritten`, async () => { + const h = harness({ readKind }); + const before = structuredClone(h.note()); + assert.equal(await h.getNoteThumbnail(h.note()), ink); + assert.deepEqual(h.note(), before); + }); +} + +test('current thumbnail versions do not re-read or rewrite legacy bodies', async () => { + const h = harness(); + await h.updateMetadata('note', { thumbnailVersion: 1, thumbnailUri: composite }); + assert.equal(await h.getNoteThumbnail(h.note()), composite); + assert.equal(h.reads(), 0); +}); + +test('an old thumbnail repair cannot overwrite a newer note save', async () => { + let finishRender; + let startRender; + const started = new Promise(resolve => { startRender = resolve; }); + const h = harness({ render: () => new Promise(resolve => { finishRender = resolve; startRender(); }) }); + const repairing = h.getNoteThumbnail(structuredClone(h.note())); + await started; + await h.updateMetadata('note', { updatedAt: '2026-10-22', thumbnailUri: 'new-cover', thumbnailVersion: 1 }); + finishRender(composite); + assert.equal(await repairing, 'new-cover'); + assert.equal(h.note().thumbnailUri, 'new-cover'); +}); + +test('a repair completing after note deletion cannot resurrect the note', async () => { + let finishRender; + let startRender; + const started = new Promise(resolve => { startRender = resolve; }); + const h = harness({ render: () => new Promise(resolve => { finishRender = resolve; startRender(); }) }); + const repairing = h.getNoteThumbnail(h.note()); + await started; + h.removeNote(); + finishRender(composite); + assert.equal(await repairing, null); + assert.equal(h.note(), undefined); +}); + +test('duplicate card requests share a single repair of the same legacy cover', async () => { + const h = harness(); + const first = h.getNoteThumbnail(h.note()); + const second = h.getNoteThumbnail(h.note()); + assert.equal(await first, composite); + assert.equal(await second, composite); + assert.equal(h.reads(), 1); +}); diff --git a/scripts/thumbnailRendering.test.mjs b/scripts/thumbnailRendering.test.mjs new file mode 100644 index 0000000..9f0e5ff --- /dev/null +++ b/scripts/thumbnailRendering.test.mjs @@ -0,0 +1,270 @@ +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; +import { readFileSync } from 'node:fs'; +import test from 'node:test'; +import vm from 'node:vm'; +import ts from 'typescript'; +import { createPromiseQueue } from '../src/utils/promiseQueue.ts'; + +const require = createRequire(import.meta.url); +const initCanvasKit = require('canvaskit-wasm/bin/full/canvaskit.js'); +const kit = await initCanvasKit({ locateFile: file => require.resolve('canvaskit-wasm/bin/full/' + file) }); +const { JsiSkApi } = require('../node_modules/@shopify/react-native-skia/lib/commonjs/skia/web/JsiSkia.js'); +const Skia = JsiSkApi(kit); +const fonts = Skia.TypefaceFontProvider.Make(); +const fontData = Skia.Data.fromBytes(readFileSync(require.resolve( + '@shopify/react-native-skia/src/skia/__tests__/assets/Roboto-Regular.ttf', +))); +const font = Skia.Typeface.MakeFreeTypeFaceFromData(fontData); +fonts.registerFont(font, 'Roboto'); +const makeParagraph = Skia.ParagraphBuilder.Make.bind(Skia.ParagraphBuilder); +Skia.ParagraphBuilder.Make = style => { + const builder = makeParagraph(style, fonts); + const pushStyle = builder.pushStyle.bind(builder); + builder.pushStyle = textStyle => pushStyle({ ...textStyle, fontFamilies: ['Roboto'] }); + return builder; +}; +const source = ts.transpileModule( + readFileSync(new URL('../src/services/thumbnailService.ts', import.meta.url), 'utf8'), + { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 } }, +).outputText; + +function solid(color, width = 20, height = 20) { + const surface = Skia.Surface.Make(width, height); + surface.getCanvas().clear(Skia.Color(color)); + const image = surface.makeImageSnapshot(); + const uri = 'data:image/png;base64,' + image.encodeToBase64(); + image.dispose(); + surface.dispose(); + return uri; +} + +const white = solid('#ffffff'); +const red = solid('#ff0000'); +const blue = solid('#0000ff'); +const black = solid('#000000'); + +function harness({ nativeFails = false, nativeParagraphBuilder = false, nativeImage } = {}) { + const exports = {}; + const nativeCalls = []; + vm.runInNewContext(source, { exports, require(name) { + if (name === '../utils/promiseQueue') return { createPromiseQueue }; + if (name === './pdfUtilsModule') return { PDFUtilsModule: nativeImage ? { + readImageForThumbnail: nativeImage, + } : undefined }; + if (name === '@shopify/react-native-skia') return { Skia: nativeParagraphBuilder ? { + ...Skia, ParagraphBuilder: { Make: (...args) => { + const builder = Skia.ParagraphBuilder.Make(...args); + builder.dispose = undefined; + return builder; + } }, + } : Skia, ClipOp: { Intersect: 1 } }; + if (name === '@mathnotes/mobile-ink') return { batchExportPages: async (...args) => { + nativeCalls.push(args); + if (nativeFails) throw new Error('Native renderer unavailable'); + return [white]; + } }; + throw new Error('Unexpected import: ' + name); + } }); + return { ...exports, nativeCalls }; +} + +function page(elements = [], extra = {}) { + return { id: 'page-1', data: '{"pages":{}}', previewUri: white, insertedElements: elements, ...extra }; +} +function photo(sourceUri = red, extra = {}) { + return { id: 'photo', type: 'image', sourceUri, x: 100, y: 100, width: 200, height: 200, rotation: 0, ...extra }; +} +let lastRaster; +function pixel(uri, x, y) { + if (lastRaster?.uri !== uri) { + const data = Skia.Data.fromBase64(uri.split(',')[1]); + const image = Skia.Image.MakeImageFromEncoded(data); + lastRaster = { uri, width: image.width(), pixels: image.readPixels(0, 0, { + width: image.width(), height: image.height(), colorType: 4, alphaType: 1, + }) }; + image.dispose(); + data.dispose(); + } + const offset = (y * lastRaster.width + x) * 4; + return [...lastRaster.pixels.subarray(offset, offset + 4)]; +} + +test('legacy ink-only previews are returned byte-for-byte without rendering', async () => { + const h = harness(); + assert.equal(await h.createPageThumbnail(page()), white); + assert.equal(h.nativeCalls.length, 0); + assert.equal(await h.createPageThumbnail(undefined), null); +}); + +test('a photo is visible at its page coordinates over the background', async () => { + const result = await harness().createPageThumbnail(page([photo()])); + assert.deepEqual(pixel(result, 100, 100), [255, 0, 0, 255]); + assert.deepEqual(pixel(result, 10, 10), [255, 255, 255, 255]); +}); + +test('image-only notes acquire a background even without an ink preview', async () => { + const h = harness(); + const result = await h.createPageThumbnail(page([photo()], { previewUri: undefined }), 'grid'); + assert.deepEqual(pixel(result, 100, 100), [255, 0, 0, 255]); + assert.equal(h.nativeCalls[0][1][0], 'grid'); +}); + +test('a moved or deleted photo does not remain in the composed thumbnail', async () => { + const h = harness(); + const result = await h.createPageThumbnail(page([photo(red, { x: 400 })])); + assert.deepEqual(pixel(result, 100, 100), [255, 255, 255, 255]); + assert.deepEqual(pixel(result, 250, 100), [255, 0, 0, 255]); + assert.equal(await h.createPageThumbnail(page()), white); +}); + +test('overlapping images respect their z-order', async () => { + const result = await harness().createPageThumbnail(page([ + photo(blue, { id: 'front', zIndex: 5 }), photo(red, { id: 'back', zIndex: 1 }), + ])); + assert.deepEqual(pixel(result, 100, 100), [0, 0, 255, 255]); +}); + +test('legacy images without zIndex retain the editor sibling order beside explicit index one', async () => { + const result = await harness().createPageThumbnail(page([ + photo(red, { id: 'front', zIndex: 1 }), photo(blue, { id: 'legacy' }), + ])); + assert.deepEqual(pixel(result, 100, 100), [255, 0, 0, 255]); +}); + +test('rotation is applied around the image center', async () => { + const result = await harness().createPageThumbnail(page([photo(red, { width: 200, height: 100, rotation: 90 })])); + assert.deepEqual(pixel(result, 100, 110), [255, 0, 0, 255]); + assert.deepEqual(pixel(result, 60, 75), [255, 255, 255, 255]); +}); + +test('crop rectangles select the requested portion of an image', async () => { + const surface = Skia.Surface.Make(20, 20); + const canvas = surface.getCanvas(); + canvas.clear(Skia.Color('#ff0000')); + const paint = Skia.Paint(); + paint.setColor(Skia.Color('#0000ff')); + canvas.drawRect(Skia.XYWHRect(10, 0, 10, 20), paint); + const image = surface.makeImageSnapshot(); + const uri = 'data:image/png;base64,' + image.encodeToBase64(); + const result = await harness().createPageThumbnail(page([photo(uri, { + width: 100, height: 200, cropRect: { x: 0.5, y: 0, width: 0.5, height: 1 }, + })])); + assert.deepEqual(pixel(result, 75, 100), [0, 0, 255, 255]); + image.dispose(); surface.dispose(); paint.dispose(); +}); + +test('handwriting remains below inserted images', async () => { + const result = await harness().createPageThumbnail(page([photo()], { previewUri: black })); + assert.deepEqual(pixel(result, 10, 10), [0, 0, 0, 255]); + assert.deepEqual(pixel(result, 100, 100), [255, 0, 0, 255]); +}); + +test('rendering preserves the original notebook data including unknown legacy fields', async () => { + const original = page([photo()], { legacyField: { original: true } }); + const snapshot = structuredClone(original); + await harness().createPageThumbnail(original); + assert.deepEqual(original, snapshot); +}); + +test('unreadable image assets report failure instead of caching an incomplete cover', async () => { + await assert.rejects(harness().createPageThumbnail(page([photo('data:image/png;base64,invalid')]))); +}); + +test('legacy HEIC files use the native decoder when Skia cannot read them', async () => { + const h = harness({ nativeImage: async uri => { + assert.equal(uri, 'file:///legacy/photo.heic'); + return blue; + } }); + const result = await h.createPageThumbnail(page([photo('file:///legacy/photo.heic')])); + assert.deepEqual(pixel(result, 100, 100), [0, 0, 255, 255]); +}); + +test('a failed native image fallback does not cache an incomplete thumbnail', async () => { + const h = harness({ nativeImage: async () => { throw new Error('Image missing'); } }); + await assert.rejects(h.createPageThumbnail(page([photo('file:///missing/photo.heic')])), /Image missing/); +}); + +test('PDF thumbnails without a preview request the correct background page', async () => { + const h = harness(); + await h.createPageThumbnail(page([photo()], { previewUri: undefined, pdfPageNumber: 3 }), 'pdf', 'file:///lecture.pdf', 1); + assert.equal(h.nativeCalls[0][5], 'file:///lecture.pdf'); + assert.equal(h.nativeCalls[0][6][0], 2); +}); + +test('typed text is included and clipped to its original text box', async () => { + const result = await harness().createPageThumbnail(page([], { textBoxes: [{ + id: 'caption', content: 'Visible text\nSecond line\nHidden line', color: '#000000', + x: 100, y: 100, width: 300, height: 70, fontSize: 40, + }] })); + let darkPixels = 0; + for (let y = 55; y < 85; y += 2) { + for (let x = 55; x < 190; x += 2) { + if (pixel(result, x, y)[0] < 128) darkPixels += 1; + } + } + assert.ok(darkPixels > 30, 'Expected visible text glyphs inside the text box'); + assert.deepEqual(pixel(result, 60, 95), [255, 255, 255, 255]); +}); + +test('native paragraph builders without an explicit dispose method still produce a complete cover', async () => { + const result = await harness({ nativeParagraphBuilder: true }).createPageThumbnail(page([photo()], { + textBoxes: [{ id: 'caption', content: 'Caption', color: '#123456', + x: 100, y: 400, width: 300, height: 100, fontSize: 40 }], + })); + assert.deepEqual(pixel(result, 100, 100), [255, 0, 0, 255]); +}); + +test('legacy renderedImageUri assets are supported without sourceUri', async () => { + const result = await harness().createPageThumbnail(page([photo(undefined, { + sourceUri: undefined, renderedImageUri: blue, + })])); + assert.deepEqual(pixel(result, 100, 100), [0, 0, 255, 255]); +}); + +test('a failed background render does not manufacture a blank completed preview', async () => { + await assert.rejects(harness({ nativeFails: true }).createPageThumbnail( + page([photo()], { previewUri: undefined }), + ), /Native renderer unavailable/); +}); + +test('malformed numeric bounds cannot poison the graphics context', async () => { + const result = await harness().createPageThumbnail(page([photo(red, { + x: NaN, y: Infinity, width: NaN, height: undefined, rotation: Infinity, + })])); + assert.deepEqual(pixel(result, 20, 20), [255, 0, 0, 255]); +}); + +test('ordinary image overlays cover text just as they do in the editor', async () => { + const result = await harness().createPageThumbnail(page([photo()], { textBoxes: [{ + id: 'behind-photo', content: 'Hidden text', color: '#000000', + x: 100, y: 100, width: 200, height: 200, fontSize: 40, + }] })); + for (let y = 55; y < 85; y += 3) { + for (let x = 55; x < 145; x += 3) { + assert.deepEqual(pixel(result, x, y), [255, 0, 0, 255]); + } + } +}); + +test('text is visible above images explicitly sent behind it', async () => { + const result = await harness().createPageThumbnail(page([photo(red, { zIndex: -1 })], { textBoxes: [{ + id: 'above-photo', content: 'Visible text', color: '#000000', + x: 100, y: 100, width: 200, height: 200, fontSize: 40, + }] })); + let darkPixels = 0; + for (let y = 55; y < 85; y += 3) { + for (let x = 55; x < 145; x += 3) { + if (pixel(result, x, y)[0] < 128) darkPixels += 1; + } + } + assert.ok(darkPixels > 10); +}); + +test('cancelled queued sidebar previews skip native capture instead of delaying later saves', async () => { + const h = harness(); + assert.equal(await h.createPageThumbnail(page([photo()], { previewUri: undefined }), 'plain', null, 0, () => true), null); + assert.equal(h.nativeCalls.length, 0); + const saved = await h.createPageThumbnail(page([photo()])); + assert.deepEqual(pixel(saved, 100, 100), [255, 0, 0, 255]); +}); diff --git a/src/components/editor/PageSidebar.tsx b/src/components/editor/PageSidebar.tsx index 70b01a2..807788c 100644 --- a/src/components/editor/PageSidebar.tsx +++ b/src/components/editor/PageSidebar.tsx @@ -1,6 +1,5 @@ import React, { useCallback, useEffect, useRef, useState } from 'react'; import { - Image, Keyboard, Pressable, ScrollView, @@ -15,6 +14,7 @@ import { useTheme } from '../../hooks/useTheme'; import { radius, spacing } from '../../theme/spacing'; import { typography } from '../../theme/typography'; import { t } from '../../i18n'; +import { PageThumbnailImage } from './PageThumbnailImage'; const SIDEBAR_WIDTH = 136; const THUMBNAIL_WIDTH = 78; @@ -28,6 +28,8 @@ export interface PageSidebarProps { bottomInset: number; onClose: () => void; onJumpToPage: (pageIndex: number) => void; + backgroundType: string; + pdfUri?: string | null; } function clampPageNumber(value: number, pageCount: number): number { @@ -43,6 +45,8 @@ export function PageSidebar({ bottomInset, onClose, onJumpToPage, + backgroundType, + pdfUri, }: PageSidebarProps) { const theme = useTheme(); const scrollRef = useRef(null); @@ -168,19 +172,15 @@ export function PageSidebar({ active && { shadowColor: theme.colors.accent }, ]} > - {page.previewUri ? ( - - ) : ( + {!page.previewUri ? ( <> - )} + ) : null} + ; +}) { + const renderPage = useMemo(() => ({ + id: page.id, title: '', rotation: 0, data: page.data, previewUri: page.previewUri, + insertedElements: page.insertedElements, textBoxes: page.textBoxes, + pageType: page.pageType, pdfPageNumber: page.pdfPageNumber, + }), [page.id, page.data, page.previewUri, page.insertedElements, page.textBoxes, + page.pageType, page.pdfPageNumber]); + const [cover, setCover] = useState<{ page: NotebookPage; uri: string | null } | null>(null); + const uri = cover?.page === renderPage ? cover.uri : renderPage.previewUri; + useEffect(() => { + let active = true; + void createPageThumbnail(renderPage, backgroundType, pdfUri, pageIndex, () => !active) + .then(next => { if (active) setCover({ page: renderPage, uri: next }); }) + .catch(() => { if (active) setCover({ page: renderPage, uri: renderPage.previewUri ?? null }); }); + return () => { active = false; }; + }, [renderPage, backgroundType, pdfUri, pageIndex]); + return uri ? : null; +} diff --git a/src/components/library/NoteCard.tsx b/src/components/library/NoteCard.tsx index c9acb9c..1811fae 100644 --- a/src/components/library/NoteCard.tsx +++ b/src/components/library/NoteCard.tsx @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useEffect, useState } from 'react'; import { Image, Pressable, StyleSheet, Text, View, useWindowDimensions } from 'react-native'; import { Ionicons } from '@expo/vector-icons'; import { useTheme } from '../../hooks/useTheme'; @@ -7,6 +7,7 @@ import { radius, spacing } from '../../theme/spacing'; import { typography } from '../../theme/typography'; import { formatRelative } from '../../utils/relativeTime'; import type { NoteMetadata } from '../../types/note'; +import { getNoteThumbnail } from '../../services/notesRepo'; export interface NoteCardProps { note: NoteMetadata; @@ -18,6 +19,15 @@ export interface NoteCardProps { export function NoteCard({ note, onPress, onLongPress, onMenuPress }: NoteCardProps) { const theme = useTheme(); const tileWidth = useLibraryTileWidth(); + const [cover, setCover] = useState<{ note: NoteMetadata; uri: string | null } | null>(null); + const thumbnailUri = cover?.note === note ? cover.uri : note.thumbnailUri; + useEffect(() => { + let active = true; + void getNoteThumbnail(note).then(uri => { + if (active) setCover({ note, uri }); + }); + return () => { active = false; }; + }, [note]); return ( - {note.thumbnailUri ? ( + {thumbnailUri ? ( diff --git a/src/services/catalogStore.ts b/src/services/catalogStore.ts index b1ab9d5..81e1107 100644 --- a/src/services/catalogStore.ts +++ b/src/services/catalogStore.ts @@ -123,6 +123,7 @@ function normalizeNote(value: unknown): NoteMetadata | null { backgroundType, pdfUri: isNonEmptyString(v.pdfUri) ? v.pdfUri : null, thumbnailUri: isNonEmptyString(v.thumbnailUri) ? v.thumbnailUri : null, + ...(v.thumbnailVersion === 1 ? { thumbnailVersion: 1 } : {}), }; } diff --git a/src/services/notesRepo.ts b/src/services/notesRepo.ts index bf9c467..53e1048 100644 --- a/src/services/notesRepo.ts +++ b/src/services/notesRepo.ts @@ -7,6 +7,7 @@ import { pruneTombstones } from './catalogStore'; import { deleteBody, readBody, writeBody, type BodyReadResult } from './noteBodyStorage'; import { deletePdfForNote } from './pdfStorage'; import { deleteImagesForNote } from './imageInsertStorage'; +import { createPageThumbnail, THUMBNAIL_VERSION } from './thumbnailService'; export async function listAllMetadata(): Promise { const catalog = await catalogStore.getCatalog(); @@ -74,6 +75,38 @@ export async function readNoteBody(id: string): Promise { return readBody(id); } +const pendingThumbnails = new Map>(); + +/** Repairs legacy covers without rewriting bodies or changing library order. */ +export function getNoteThumbnail(note: NoteMetadata): Promise { + if (note.thumbnailVersion === THUMBNAIL_VERSION) return Promise.resolve(note.thumbnailUri); + const pending = pendingThumbnails.get(note.id); + if (pending) return pending; + const repair = (async () => { + try { + const body = await readBody(note.id); + if (body.kind !== 'ok') return note.thumbnailUri; + const uri = await createPageThumbnail(body.data.pages[0], note.backgroundType, note.pdfUri); + let currentUri = note.thumbnailUri; + await catalogStore.mutate(catalog => { + const current = catalog.notes.find(candidate => candidate.id === note.id); + currentUri = current?.thumbnailUri ?? null; + if (!current || current.updatedAt !== note.updatedAt || current.thumbnailVersion === THUMBNAIL_VERSION) return null; + currentUri = uri; + return { ...catalog, notes: catalog.notes.map(candidate => candidate.id === note.id + ? { ...candidate, thumbnailUri: uri, thumbnailVersion: THUMBNAIL_VERSION } : candidate) }; + }); + return currentUri; + } catch { + return note.thumbnailUri; + } finally { + pendingThumbnails.delete(note.id); + } + })(); + pendingThumbnails.set(note.id, repair); + return repair; +} + export async function saveNoteBody( id: string, data: SerializedNotebookData, @@ -84,9 +117,18 @@ export async function saveNoteBody( // pointing at the last version that actually persisted. return { ok: false, metadata: null }; } - const previewUri = data.pages[0]?.previewUri ?? null; + let previewUri = data.pages[0]?.previewUri ?? null; + let thumbnailVersion: number | undefined; + try { + const current = await getNote(id); + previewUri = await createPageThumbnail(data.pages[0], current?.backgroundType, current?.pdfUri); + thumbnailVersion = THUMBNAIL_VERSION; + } catch { + // Preview generation is disposable; it must never block a durable note save. + } const meta = await updateMetadata(id, { thumbnailUri: previewUri, + thumbnailVersion, updatedAt: new Date().toISOString(), }); return { ok: true, metadata: meta }; diff --git a/src/services/pdfUtilsModule.ts b/src/services/pdfUtilsModule.ts index bebd548..12ff948 100644 --- a/src/services/pdfUtilsModule.ts +++ b/src/services/pdfUtilsModule.ts @@ -5,6 +5,7 @@ export type PDFUtilsModuleType = { getPageCount?: (filePath: string) => Promise; copySecurityScopedFileToTmp?: (sourceUrl: string) => Promise; mergePdfFiles?: (fileUris: string[]) => Promise; + readImageForThumbnail?: (fileUri: string) => Promise; }; export const PDFUtilsModule = NativeModules.PDFUtilsModule as PDFUtilsModuleType | undefined; diff --git a/src/services/thumbnailService.ts b/src/services/thumbnailService.ts new file mode 100644 index 0000000..11824d5 --- /dev/null +++ b/src/services/thumbnailService.ts @@ -0,0 +1,159 @@ +import { ClipOp, Skia, type SkImage } from '@shopify/react-native-skia'; +import { batchExportPages, type InsertedElement, type NotebookPage } from '@mathnotes/mobile-ink'; +import { createPromiseQueue } from '../utils/promiseQueue'; +import { PDFUtilsModule } from './pdfUtilsModule'; + +export const THUMBNAIL_VERSION = 1; +const PAGE_WIDTH = 820; +const PAGE_HEIGHT = 1061; +const SCALE = 0.5; +const renderQueue = createPromiseQueue(); + +function finite(value: number | undefined, fallback: number): number { + return typeof value === 'number' && Number.isFinite(value) ? value : fallback; +} + +async function decodeImage(uri: string): Promise { + try { + return await readSkiaImage(uri); + } catch (error) { + if (!uri.startsWith('file://') || !PDFUtilsModule?.readImageForThumbnail) throw error; + // ImageIO can decode legacy HEIC photos that Skia's codecs do not support. + return readSkiaImage(await PDFUtilsModule.readImageForThumbnail(uri)); + } +} + +async function readSkiaImage(uri: string): Promise { + const data = uri.startsWith('data:image/') + ? Skia.Data.fromBase64(uri.slice(uri.indexOf(',') + 1)) + : await Skia.Data.fromURI(uri); + try { + const image = Skia.Image.MakeImageFromEncoded(data); + if (!image) throw new Error('Thumbnail image could not be decoded'); + return image; + } finally { + data.dispose(); + } +} + +/** Composes a separate cover; engine previews must stay free of host overlays. */ +export function createPageThumbnail( + page: NotebookPage | undefined, + backgroundType = 'plain', + pdfUri?: string | null, + pageIndex = 0, + shouldCancel?: () => boolean, +): Promise { + return renderQueue.enqueue(async () => shouldCancel?.() + ? null : composePageThumbnail(page, backgroundType, pdfUri, pageIndex)); +} + +async function composePageThumbnail( + page: NotebookPage | undefined, + backgroundType: string, + pdfUri: string | null | undefined, + pageIndex: number, +): Promise { + if (!page) return null; + const elements = [...(page.insertedElements ?? [])] + .filter(element => element.type === 'image' || element.sourceUri || element.renderedImageUri) + .sort((a, b) => finite(a.zIndex, 1) - finite(b.zIndex, 1) + || finite(a.zIndex, 0) - finite(b.zIndex, 0)); + const textBoxes = (page.textBoxes ?? []).filter(box => Boolean(box.content)); + if (!elements.length && !textBoxes.length) return page.previewUri ?? null; + + let raster = page.previewUri; + if (!raster) { + [raster] = await batchExportPages( + [page.data ?? '{"pages":{}}'], [page.pageType ?? backgroundType], PAGE_WIDTH, PAGE_HEIGHT, SCALE, + pdfUri ?? undefined, [page.pdfPageNumber ? page.pdfPageNumber - 1 : pageIndex], + ); + if (!raster) throw new Error('Thumbnail background could not be rendered'); + } + + const surface = Skia.Surface.Make(PAGE_WIDTH * SCALE, Math.ceil(PAGE_HEIGHT * SCALE)); + if (!surface) throw new Error('Thumbnail surface could not be allocated'); + const paint = Skia.Paint(); + try { + const canvas = surface.getCanvas(); + canvas.clear(Skia.Color('white')); + canvas.scale(SCALE, SCALE); + const base = await decodeImage(raster); + try { + canvas.drawImageRect(base, Skia.XYWHRect(0, 0, base.width(), base.height()), + Skia.XYWHRect(0, 0, PAGE_WIDTH, PAGE_HEIGHT), paint); + } finally { + base.dispose(); + } + + const drawElement = async (element: InsertedElement) => { + const uri = element.sourceUri ?? element.renderedImageUri; + if (!uri) throw new Error('Thumbnail image has no source'); + const image = await decodeImage(uri); + try { + const x = finite(element.x, 0); + const y = finite(element.y, 0); + const width = Math.max(1, finite(element.width, 240)); + const height = Math.max(1, finite(element.height, 240)); + const crop = element.cropRect; + const fullWidth = width / Math.max(0.001, finite(crop?.width, 1)); + const fullHeight = height / Math.max(0.001, finite(crop?.height, 1)); + // Match ImageInsertOverlay's resizeMode="cover" before applying its crop. + const coverScale = Math.max(fullWidth / image.width(), fullHeight / image.height()); + const sourceWidth = fullWidth / coverScale; + const sourceHeight = fullHeight / coverScale; + canvas.save(); + canvas.rotate(finite(element.rotation, 0), x + width / 2, y + height / 2); + canvas.clipRect(Skia.XYWHRect(x, y, width, height), ClipOp.Intersect, true); + canvas.drawImageRect(image, + Skia.XYWHRect((image.width() - sourceWidth) / 2, (image.height() - sourceHeight) / 2, sourceWidth, sourceHeight), + Skia.XYWHRect(x - finite(crop?.x, 0) * fullWidth, y - finite(crop?.y, 0) * fullHeight, fullWidth, fullHeight), paint); + canvas.restore(); + } finally { + image.dispose(); + } + }; + + // Text siblings have zIndex 0; image siblings default to zIndex 1. + for (const element of elements.filter(element => finite(element.zIndex, 1) <= 0)) { + await drawElement(element); + } + + for (const box of textBoxes) { + const builder = Skia.ParagraphBuilder.Make(); + builder.pushStyle({ + color: Skia.Color(box.color || '#000000'), fontSize: Math.max(1, finite(box.fontSize, 18)), + heightMultiplier: 1.25, + }); + const paragraph = builder.addText(box.content).build(); + try { + const x = finite(box.x, 0); + const y = finite(box.y, 0); + const width = Math.max(1, finite(box.width, 200)); + const height = Math.max(1, finite(box.height, 100)); + paragraph.layout(Math.max(1, width - 16)); + canvas.save(); + canvas.clipRect(Skia.XYWHRect(x, y, width, height), ClipOp.Intersect, true); + paragraph.paint(canvas, x + 8, y + 8); + canvas.restore(); + } finally { + paragraph.dispose(); + // Skia 2.2's native builder is garbage-collected and omits this method. + builder.dispose?.(); + } + } + for (const element of elements.filter(element => finite(element.zIndex, 1) > 0)) { + await drawElement(element); + } + surface.flush(); + const image = surface.makeImageSnapshot(); + try { + return 'data:image/png;base64,' + image.encodeToBase64(); + } finally { + image.dispose(); + } + } finally { + paint.dispose(); + surface.dispose(); + } +} diff --git a/src/types/note.ts b/src/types/note.ts index e29755c..9df568a 100644 --- a/src/types/note.ts +++ b/src/types/note.ts @@ -31,4 +31,5 @@ export interface NoteMetadata { backgroundType: BackgroundType; pdfUri: string | null; thumbnailUri: string | null; + thumbnailVersion?: number; }