diff --git a/packages/app-expo/assets/reader/reader.html b/packages/app-expo/assets/reader/reader.html index 6967e9048..3e6d24bf5 100644 --- a/packages/app-expo/assets/reader/reader.html +++ b/packages/app-expo/assets/reader/reader.html @@ -262,6 +262,7 @@ let bookTextMetricsTimer = null; let bookmarkPullGestureActive = false; let pullBookmarkResetTimer = null; + let extractionSessions = null; let refreshAnnotationsTimer = null; let activeFootnoteTipKey = null; let bookmarkPullStateMeta = { @@ -1418,6 +1419,9 @@ case 'extractBookChapters': await handleExtractBookChapters(msg); break; + case 'cancelExtraction': + if (msg.requestId) getExtractionSessions().cancel(msg.requestId); + break; case 'getChapterParagraphs': handleGetChapterParagraphs(); break; @@ -1435,9 +1439,61 @@ } // ─── Book loading ─── + const SUPPORTED_BOOK_FORMATS = new Set(['epub', 'pdf', 'txt', 'umd', 'mobi', 'azw', 'azw3']); + const BOOK_MIME_TYPES = { + epub: 'application/epub+zip', + pdf: 'application/pdf', + txt: 'text/plain', + umd: 'application/epub+zip', + mobi: 'application/x-mobipocket-ebook', + azw: 'application/vnd.amazon.ebook', + azw3: 'application/vnd.amazon.ebook', + }; + const BOOK_FORMATS_BY_MIME = { + 'application/epub+zip': 'epub', + 'application/pdf': 'pdf', + 'text/plain': 'txt', + 'application/x-mobipocket-ebook': 'mobi', + 'application/vnd.amazon.ebook': 'azw3', + }; + + function resolveBookFormat(msg) { + const storedFormat = String(msg.bookFormat || '').trim().toLowerCase(); + if (SUPPORTED_BOOK_FORMATS.has(storedFormat)) return storedFormat; + + const cleanFileName = String(msg.fileName || '').split(/[?#]/, 1)[0]; + const extension = cleanFileName.split('.').pop().toLowerCase(); + if (SUPPORTED_BOOK_FORMATS.has(extension)) return extension; + + const mimeType = String(msg.mimeType || '').split(';', 1)[0].trim().toLowerCase(); + return BOOK_FORMATS_BY_MIME[mimeType] || null; + } + + function getExtractionSessions() { + if (!extractionSessions) { + extractionSessions = new window.ReaderExtractionSessions(); + } + return extractionSessions; + } + + function getBookFileName(msg) { + const format = resolveBookFormat(msg); + const cleanFileName = String(msg.fileName || '').split(/[?#]/, 1)[0].split(/[\\/]/).pop(); + if (!format) return cleanFileName || 'book.epub'; + const baseName = cleanFileName?.replace(/\.[^.]*$/, '') || 'book'; + return `${baseName}.${format}`; + } + + function getBookMimeType(msg, fallback) { + const format = resolveBookFormat(msg); + return BOOK_MIME_TYPES[format] || msg.mimeType || fallback || 'application/octet-stream'; + } + async function openBook(msg) { const container = document.getElementById('reader-container'); const loading = document.getElementById('loading'); + const fileName = getBookFileName(msg); + const mimeType = getBookMimeType(msg); currentBookIsPdf = isPDFBookMessage(msg); pdfPageLightCache = {}; pdfDocIndexMap = new WeakMap(); @@ -1448,12 +1504,13 @@ } try { + getExtractionSessions().throwIfCancelled(msg.requestId); let hasSignalledLoaded = false; const markLoaded = () => { if (loading) loading.classList.add('hidden'); if (!hasSignalledLoaded) { hasSignalledLoaded = true; - postToRN('loaded', {}); + postToRN('loaded', { requestId: msg.requestId }); } }; @@ -1462,22 +1519,22 @@ const binary = atob(msg.base64); const bytes = new Uint8Array(binary.length); for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); - file = new File([bytes], msg.fileName || 'book.epub', { - type: msg.mimeType || 'application/epub+zip' + file = new File([bytes], fileName, { + type: mimeType }); } else if (msg.uri) { postToRN('debug', { message: `[ReaderFetch] open ${JSON.stringify({ uri: msg.uri, - fileName: msg.fileName || '', - mimeType: msg.mimeType || '' + fileName, + mimeType })}` }); // Try Range-based lazy loading for ZIP-based formats (EPUB, CBZ, FBZ) // This avoids loading the entire file into memory — only reads // the ZIP central directory (~few KB) then fetches entries on demand. - const isZipFormat = /\.(epub|cbz|fb2\.zip|fbz)$/i.test(msg.fileName || ''); + const isZipFormat = /\.(epub|cbz|fb2\.zip|fbz)$/i.test(fileName); let lazyBook = null; if (isZipFormat) { @@ -1530,14 +1587,14 @@ // Try Range-based lazy loading for PDF // pdf.js natively supports Range requests via url + disableAutoFetch - if (!lazyBook && /\.pdf$/i.test(msg.fileName || '')) { + if (!lazyBook && /\.pdf$/i.test(fileName)) { try { const headRes = await fetch(msg.uri, { method: 'HEAD' }); const acceptRanges = headRes.headers.get('Accept-Ranges'); const contentLength = parseInt(headRes.headers.get('Content-Length'), 10); if (acceptRanges === 'bytes' && contentLength > 0 && window._makePDFFromURL) { - lazyBook = await window._makePDFFromURL(msg.uri, msg.fileName || 'book.pdf'); + lazyBook = await window._makePDFFromURL(msg.uri, fileName); } } catch (pdfLazyErr) { console.warn('[Reader] PDF lazy loading failed, falling back to full fetch:', pdfLazyErr); @@ -1550,8 +1607,8 @@ const isLocalFileStatus = res.status === 0 && /^file:\/\//i.test(msg.uri); if (!res.ok && !isLocalFileStatus) throw new Error(`Failed to fetch: ${res.status}`); const blob = await res.blob(); - file = new File([blob], msg.fileName || 'book.epub', { - type: msg.mimeType || blob.type || 'application/octet-stream' + file = new File([blob], fileName, { + type: getBookMimeType(msg, blob.type) }); } else { // Skip makeBook below — we already have the book object @@ -1561,7 +1618,11 @@ throw new Error('No book data provided'); } - const book = (file && file.sections) ? file : await makeBook(file); + const loadBook = async () => (file && file.sections) ? file : await makeBook(file); + const book = msg.requestId + ? await getExtractionSessions().open(msg.requestId, loadBook) + : await loadBook(); + getExtractionSessions().throwIfCancelled(msg.requestId); currentBook = book; attachBookTransformHandler(book); @@ -1802,7 +1863,8 @@ console.error('[WebView] Error in openBook:', err); const message = `${String(err)}${msg?.uri ? ` (${msg.uri})` : ''}`; loading.innerHTML = '
' + escapeHtml(message) + '
'; - postToRN('error', { message }); + postToRN('error', { message, requestId: msg.requestId }); + getExtractionSessions().release(msg.requestId); } } @@ -4296,8 +4358,7 @@ // ─── Chapter Extraction for Vectorization ─── function isPDFBookMessage(msg) { - const mimeType = (msg.mimeType || '').split(';')[0].trim().toLowerCase(); - return mimeType === 'application/pdf' || /\.pdf$/i.test(msg.fileName || ''); + return resolveBookFormat(msg) === 'pdf'; } async function createBookFileFromMessage(msg) { @@ -4305,8 +4366,8 @@ const binary = atob(msg.base64); const bytes = new Uint8Array(binary.length); for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); - return new File([bytes], msg.fileName || 'book.epub', { - type: msg.mimeType || 'application/epub+zip' + return new File([bytes], getBookFileName(msg), { + type: getBookMimeType(msg) }); } @@ -4315,8 +4376,8 @@ const isLocalFileStatus = res.status === 0 && /^file:\/\//i.test(msg.uri); if (!res.ok && !isLocalFileStatus) throw new Error(`Failed to fetch: ${res.status}`); const blob = await res.blob(); - return new File([blob], msg.fileName || 'book.epub', { - type: msg.mimeType || blob.type || 'application/octet-stream' + return new File([blob], getBookFileName(msg), { + type: getBookMimeType(msg, blob.type) }); } @@ -4324,7 +4385,9 @@ } async function handleExtractBookChapters(msg) { + const requestId = msg.requestId; try { + getExtractionSessions().throwIfCancelled(requestId); if (isPDFBookMessage(msg)) { if (typeof window._extractPDFChapters !== 'function') { throw new Error('PDF extraction is not available in this reader build'); @@ -4344,6 +4407,7 @@ postToRN('debug', { message: `[PDFExtract] page ${JSON.stringify(detail)}` }); } }); + getExtractionSessions().throwIfCancelled(requestId); postToRN('debug', { message: `[PDFExtract] done ${JSON.stringify({ @@ -4353,27 +4417,42 @@ })}` }); - postToRN('chaptersExtracted', { chapters }); + postToRN('chaptersExtracted', { requestId, chapters }); return; } - currentBook = await makeBook(await createBookFileFromMessage(msg)); - await handleExtractChapters(); + const requestBook = await getExtractionSessions().open( + requestId, + async () => makeBook(await createBookFileFromMessage(msg)) + ); + currentBook = requestBook; + getExtractionSessions().throwIfCancelled(requestId); + await handleExtractChapters(requestId); } catch (err) { console.error('[WebView] Error extracting book chapters:', err); - postToRN('chaptersExtracted', { error: String(err) }); + postToRN('chaptersExtracted', { requestId, error: String(err) }); + } finally { + getExtractionSessions().release(requestId); } } - async function handleExtractChapters() { - if (!currentBook) { - postToRN('chaptersExtracted', { error: 'No book loaded' }); + async function handleExtractChapters(requestId) { + let extractionBook; + try { + extractionBook = requestId ? getExtractionSessions().getBook(requestId) : currentBook; + } catch (err) { + postToRN('chaptersExtracted', { requestId, error: String(err) }); + return; + } + if (!extractionBook) { + postToRN('chaptersExtracted', { requestId, error: 'No book loaded' }); return; } try { - const sections = currentBook.sections || []; - const toc = currentBook.toc || []; + getExtractionSessions().throwIfCancelled(requestId); + const sections = extractionBook.sections || []; + const toc = extractionBook.toc || []; // Build map of href to title mapping const tocMap = new Map(); @@ -4398,6 +4477,7 @@ let skippedNoCreateDocument = 0; for (let i = 0; i < sections.length; i++) { + getExtractionSessions().throwIfCancelled(requestId); const section = sections[i]; if (!section.createDocument) { skippedNoCreateDocument += 1; @@ -4406,6 +4486,7 @@ try { const doc = await section.createDocument(); + getExtractionSessions().throwIfCancelled(requestId); if (!doc.body) continue; const title = tocMap.get(i) || tocMap.get(section.href || "") || `Section ${i + 1}`; @@ -4432,6 +4513,8 @@ } } + getExtractionSessions().throwIfCancelled(requestId); + console.log('[WebView] Chapter extraction summary:', { sections: sections.length, chapters: chapters.length, @@ -4439,10 +4522,12 @@ skippedNoCreateDocument }); - postToRN('chaptersExtracted', { chapters }); + postToRN('chaptersExtracted', { requestId, chapters }); } catch (err) { console.error('[WebView] Error extracting chapters:', err); - postToRN('chaptersExtracted', { error: String(err) }); + postToRN('chaptersExtracted', { requestId, error: String(err) }); + } finally { + getExtractionSessions().release(requestId); } } @@ -5025,8 +5110,8 @@ diff --git a/packages/app-expo/assets/reader/reader.template.html b/packages/app-expo/assets/reader/reader.template.html index d3658a0b3..78a8f3aab 100644 --- a/packages/app-expo/assets/reader/reader.template.html +++ b/packages/app-expo/assets/reader/reader.template.html @@ -215,6 +215,7 @@ let bookTextMetricsTimer = null; let bookmarkPullGestureActive = false; let pullBookmarkResetTimer = null; + let extractionSessions = null; let refreshAnnotationsTimer = null; let activeFootnoteTipKey = null; let bookmarkPullStateMeta = { @@ -1371,6 +1372,9 @@ case 'extractBookChapters': await handleExtractBookChapters(msg); break; + case 'cancelExtraction': + if (msg.requestId) getExtractionSessions().cancel(msg.requestId); + break; case 'getChapterParagraphs': handleGetChapterParagraphs(); break; @@ -1388,9 +1392,61 @@ } // ─── Book loading ─── + const SUPPORTED_BOOK_FORMATS = new Set(['epub', 'pdf', 'txt', 'umd', 'mobi', 'azw', 'azw3']); + const BOOK_MIME_TYPES = { + epub: 'application/epub+zip', + pdf: 'application/pdf', + txt: 'text/plain', + umd: 'application/epub+zip', + mobi: 'application/x-mobipocket-ebook', + azw: 'application/vnd.amazon.ebook', + azw3: 'application/vnd.amazon.ebook', + }; + const BOOK_FORMATS_BY_MIME = { + 'application/epub+zip': 'epub', + 'application/pdf': 'pdf', + 'text/plain': 'txt', + 'application/x-mobipocket-ebook': 'mobi', + 'application/vnd.amazon.ebook': 'azw3', + }; + + function resolveBookFormat(msg) { + const storedFormat = String(msg.bookFormat || '').trim().toLowerCase(); + if (SUPPORTED_BOOK_FORMATS.has(storedFormat)) return storedFormat; + + const cleanFileName = String(msg.fileName || '').split(/[?#]/, 1)[0]; + const extension = cleanFileName.split('.').pop().toLowerCase(); + if (SUPPORTED_BOOK_FORMATS.has(extension)) return extension; + + const mimeType = String(msg.mimeType || '').split(';', 1)[0].trim().toLowerCase(); + return BOOK_FORMATS_BY_MIME[mimeType] || null; + } + + function getExtractionSessions() { + if (!extractionSessions) { + extractionSessions = new window.ReaderExtractionSessions(); + } + return extractionSessions; + } + + function getBookFileName(msg) { + const format = resolveBookFormat(msg); + const cleanFileName = String(msg.fileName || '').split(/[?#]/, 1)[0].split(/[\\/]/).pop(); + if (!format) return cleanFileName || 'book.epub'; + const baseName = cleanFileName?.replace(/\.[^.]*$/, '') || 'book'; + return `${baseName}.${format}`; + } + + function getBookMimeType(msg, fallback) { + const format = resolveBookFormat(msg); + return BOOK_MIME_TYPES[format] || msg.mimeType || fallback || 'application/octet-stream'; + } + async function openBook(msg) { const container = document.getElementById('reader-container'); const loading = document.getElementById('loading'); + const fileName = getBookFileName(msg); + const mimeType = getBookMimeType(msg); currentBookIsPdf = isPDFBookMessage(msg); pdfPageLightCache = {}; pdfDocIndexMap = new WeakMap(); @@ -1401,12 +1457,13 @@ } try { + getExtractionSessions().throwIfCancelled(msg.requestId); let hasSignalledLoaded = false; const markLoaded = () => { if (loading) loading.classList.add('hidden'); if (!hasSignalledLoaded) { hasSignalledLoaded = true; - postToRN('loaded', {}); + postToRN('loaded', { requestId: msg.requestId }); } }; @@ -1415,22 +1472,22 @@ const binary = atob(msg.base64); const bytes = new Uint8Array(binary.length); for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); - file = new File([bytes], msg.fileName || 'book.epub', { - type: msg.mimeType || 'application/epub+zip' + file = new File([bytes], fileName, { + type: mimeType }); } else if (msg.uri) { postToRN('debug', { message: `[ReaderFetch] open ${JSON.stringify({ uri: msg.uri, - fileName: msg.fileName || '', - mimeType: msg.mimeType || '' + fileName, + mimeType })}` }); // Try Range-based lazy loading for ZIP-based formats (EPUB, CBZ, FBZ) // This avoids loading the entire file into memory — only reads // the ZIP central directory (~few KB) then fetches entries on demand. - const isZipFormat = /\.(epub|cbz|fb2\.zip|fbz)$/i.test(msg.fileName || ''); + const isZipFormat = /\.(epub|cbz|fb2\.zip|fbz)$/i.test(fileName); let lazyBook = null; if (isZipFormat) { @@ -1483,14 +1540,14 @@ // Try Range-based lazy loading for PDF // pdf.js natively supports Range requests via url + disableAutoFetch - if (!lazyBook && /\.pdf$/i.test(msg.fileName || '')) { + if (!lazyBook && /\.pdf$/i.test(fileName)) { try { const headRes = await fetch(msg.uri, { method: 'HEAD' }); const acceptRanges = headRes.headers.get('Accept-Ranges'); const contentLength = parseInt(headRes.headers.get('Content-Length'), 10); if (acceptRanges === 'bytes' && contentLength > 0 && window._makePDFFromURL) { - lazyBook = await window._makePDFFromURL(msg.uri, msg.fileName || 'book.pdf'); + lazyBook = await window._makePDFFromURL(msg.uri, fileName); } } catch (pdfLazyErr) { console.warn('[Reader] PDF lazy loading failed, falling back to full fetch:', pdfLazyErr); @@ -1503,8 +1560,8 @@ const isLocalFileStatus = res.status === 0 && /^file:\/\//i.test(msg.uri); if (!res.ok && !isLocalFileStatus) throw new Error(`Failed to fetch: ${res.status}`); const blob = await res.blob(); - file = new File([blob], msg.fileName || 'book.epub', { - type: msg.mimeType || blob.type || 'application/octet-stream' + file = new File([blob], fileName, { + type: getBookMimeType(msg, blob.type) }); } else { // Skip makeBook below — we already have the book object @@ -1514,7 +1571,11 @@ throw new Error('No book data provided'); } - const book = (file && file.sections) ? file : await makeBook(file); + const loadBook = async () => (file && file.sections) ? file : await makeBook(file); + const book = msg.requestId + ? await getExtractionSessions().open(msg.requestId, loadBook) + : await loadBook(); + getExtractionSessions().throwIfCancelled(msg.requestId); currentBook = book; attachBookTransformHandler(book); @@ -1755,7 +1816,8 @@ console.error('[WebView] Error in openBook:', err); const message = `${String(err)}${msg?.uri ? ` (${msg.uri})` : ''}`; loading.innerHTML = '
' + escapeHtml(message) + '
'; - postToRN('error', { message }); + postToRN('error', { message, requestId: msg.requestId }); + getExtractionSessions().release(msg.requestId); } } @@ -4249,8 +4311,7 @@ // ─── Chapter Extraction for Vectorization ─── function isPDFBookMessage(msg) { - const mimeType = (msg.mimeType || '').split(';')[0].trim().toLowerCase(); - return mimeType === 'application/pdf' || /\.pdf$/i.test(msg.fileName || ''); + return resolveBookFormat(msg) === 'pdf'; } async function createBookFileFromMessage(msg) { @@ -4258,8 +4319,8 @@ const binary = atob(msg.base64); const bytes = new Uint8Array(binary.length); for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); - return new File([bytes], msg.fileName || 'book.epub', { - type: msg.mimeType || 'application/epub+zip' + return new File([bytes], getBookFileName(msg), { + type: getBookMimeType(msg) }); } @@ -4268,8 +4329,8 @@ const isLocalFileStatus = res.status === 0 && /^file:\/\//i.test(msg.uri); if (!res.ok && !isLocalFileStatus) throw new Error(`Failed to fetch: ${res.status}`); const blob = await res.blob(); - return new File([blob], msg.fileName || 'book.epub', { - type: msg.mimeType || blob.type || 'application/octet-stream' + return new File([blob], getBookFileName(msg), { + type: getBookMimeType(msg, blob.type) }); } @@ -4277,7 +4338,9 @@ } async function handleExtractBookChapters(msg) { + const requestId = msg.requestId; try { + getExtractionSessions().throwIfCancelled(requestId); if (isPDFBookMessage(msg)) { if (typeof window._extractPDFChapters !== 'function') { throw new Error('PDF extraction is not available in this reader build'); @@ -4297,6 +4360,7 @@ postToRN('debug', { message: `[PDFExtract] page ${JSON.stringify(detail)}` }); } }); + getExtractionSessions().throwIfCancelled(requestId); postToRN('debug', { message: `[PDFExtract] done ${JSON.stringify({ @@ -4306,27 +4370,42 @@ })}` }); - postToRN('chaptersExtracted', { chapters }); + postToRN('chaptersExtracted', { requestId, chapters }); return; } - currentBook = await makeBook(await createBookFileFromMessage(msg)); - await handleExtractChapters(); + const requestBook = await getExtractionSessions().open( + requestId, + async () => makeBook(await createBookFileFromMessage(msg)) + ); + currentBook = requestBook; + getExtractionSessions().throwIfCancelled(requestId); + await handleExtractChapters(requestId); } catch (err) { console.error('[WebView] Error extracting book chapters:', err); - postToRN('chaptersExtracted', { error: String(err) }); + postToRN('chaptersExtracted', { requestId, error: String(err) }); + } finally { + getExtractionSessions().release(requestId); } } - async function handleExtractChapters() { - if (!currentBook) { - postToRN('chaptersExtracted', { error: 'No book loaded' }); + async function handleExtractChapters(requestId) { + let extractionBook; + try { + extractionBook = requestId ? getExtractionSessions().getBook(requestId) : currentBook; + } catch (err) { + postToRN('chaptersExtracted', { requestId, error: String(err) }); + return; + } + if (!extractionBook) { + postToRN('chaptersExtracted', { requestId, error: 'No book loaded' }); return; } try { - const sections = currentBook.sections || []; - const toc = currentBook.toc || []; + getExtractionSessions().throwIfCancelled(requestId); + const sections = extractionBook.sections || []; + const toc = extractionBook.toc || []; // Build map of href to title mapping const tocMap = new Map(); @@ -4351,6 +4430,7 @@ let skippedNoCreateDocument = 0; for (let i = 0; i < sections.length; i++) { + getExtractionSessions().throwIfCancelled(requestId); const section = sections[i]; if (!section.createDocument) { skippedNoCreateDocument += 1; @@ -4359,6 +4439,7 @@ try { const doc = await section.createDocument(); + getExtractionSessions().throwIfCancelled(requestId); if (!doc.body) continue; const title = tocMap.get(i) || tocMap.get(section.href || "") || `Section ${i + 1}`; @@ -4385,6 +4466,8 @@ } } + getExtractionSessions().throwIfCancelled(requestId); + console.log('[WebView] Chapter extraction summary:', { sections: sections.length, chapters: chapters.length, @@ -4392,10 +4475,12 @@ skippedNoCreateDocument }); - postToRN('chaptersExtracted', { chapters }); + postToRN('chaptersExtracted', { requestId, chapters }); } catch (err) { console.error('[WebView] Error extracting chapters:', err); - postToRN('chaptersExtracted', { error: String(err) }); + postToRN('chaptersExtracted', { requestId, error: String(err) }); + } finally { + getExtractionSessions().release(requestId); } } diff --git a/packages/app-expo/scripts/build-reader.js b/packages/app-expo/scripts/build-reader.js index ec5c53b56..d4aeb34c6 100644 --- a/packages/app-expo/scripts/build-reader.js +++ b/packages/app-expo/scripts/build-reader.js @@ -12,6 +12,7 @@ const FOLIATE_DIR = path.resolve(__dirname, "../../foliate-js"); const ASSETS_DIR = path.resolve(__dirname, "../assets/reader"); const TEMPLATE = path.resolve(ASSETS_DIR, "reader.template.html"); const OUTPUT = path.resolve(ASSETS_DIR, "reader.html"); +const EXTRACTION_SESSIONS = path.resolve(__dirname, "../src/lib/rag/reader-extraction-sessions.ts"); const JUSTIFIED_TEXT = path.resolve(ASSETS_DIR, "justified-text.js"); async function buildReader() { @@ -23,6 +24,7 @@ async function buildReader() { import { configure, ZipReader, BlobReader, TextWriter, BlobWriter } from "${FOLIATE_DIR.replace(/\\/g, "/")}/vendor/zip.js"; import { EPUB } from "${FOLIATE_DIR.replace(/\\/g, "/")}/epub.js"; import { extractPDFChapters, makePDFFromURL } from "${FOLIATE_DIR.replace(/\\/g, "/")}/pdf.js"; + import { ReaderExtractionSessions } from "${EXTRACTION_SESSIONS.replace(/\\/g, "/")}"; window.makeBook = makeBook; window.Overlayer = Overlayer; @@ -33,6 +35,7 @@ async function buildReader() { window._EPUB = EPUB; window._makePDFFromURL = makePDFFromURL; window._extractPDFChapters = extractPDFChapters; + window.ReaderExtractionSessions = ReaderExtractionSessions; if (!customElements.get('foliate-view')) { customElements.define('foliate-view', View); diff --git a/packages/app-expo/src/components/library/BookCard.tsx b/packages/app-expo/src/components/library/BookCard.tsx index 746f0a141..e0a6a6ed4 100644 --- a/packages/app-expo/src/components/library/BookCard.tsx +++ b/packages/app-expo/src/components/library/BookCard.tsx @@ -1,4 +1,5 @@ import { CheckIcon, ClockIcon, Loader2Icon, MoreVerticalIcon } from "@/components/ui/Icon"; +import { isVectorizationCancellable } from "@/lib/rag/vectorization-cancel-state"; import { useColors } from "@/styles/theme"; import { getPlatformService } from "@readany/core/services"; /** @@ -54,6 +55,7 @@ interface BookCardProps { onShowDetails?: (book: Book) => void; onManageTags?: (book: Book) => void; onVectorize?: (book: Book) => void; + onCancelVectorize?: (bookId: string) => void; isVectorizing?: boolean; isQueued?: boolean; vectorProgress?: { status: string; processedChunks: number; totalChunks: number } | null; @@ -72,6 +74,7 @@ export const BookCard = memo(function BookCard({ onShowDetails, onManageTags, onVectorize, + onCancelVectorize, isVectorizing, isQueued, vectorProgress, @@ -124,6 +127,10 @@ export const BookCard = memo(function BookCard({ ? Math.round((vectorProgress.processedChunks / vectorProgress.totalChunks) * 100) : 0 : 0; + const canCancelVectorization = isVectorizationCancellable( + Boolean(isVectorizing), + vectorProgress?.status, + ); const measureAnchor = useCallback(async () => { const measureNode = (node: View | null, fallbackToBottomRight = false) => @@ -287,31 +294,74 @@ export const BookCard = memo(function BookCard({ )} {/* Vectorization progress overlay */} - {isVectorizing && ( + {isVectorizing && canCancelVectorization && ( + { + event.stopPropagation(); + onCancelVectorize?.(book.id); + }} + > + {vectorProgress?.status !== "cancelled" && } + + {vectorProgress?.status === "cancelling" + ? t("home.vec_cancelling", "Cancelling…") + : vectorProgress?.status === "cancelled" + ? t("home.vec_cancelled", "Cancelled") + : vectorProgress?.status === "chunking" + ? `${vecPct}%` + : vectorProgress?.status === "embedding" + ? `${vecPct}%` + : vectorProgress?.status === "indexing" + ? t("home.vec_indexing") + : vectorProgress?.status === "completed" + ? "✓" + : vectorProgress?.status === "error" + ? "✗" + : t("home.vec_processing")} + + {vectorProgress?.status !== "cancelling" && + vectorProgress?.status !== "cancelled" && ( + {t("home.vec_cancel", "Cancel")} + )} + + )} + {isVectorizing && !canCancelVectorization && ( - + {vectorProgress?.status !== "cancelled" && } - {vectorProgress?.status === "chunking" - ? `${vecPct}%` - : vectorProgress?.status === "embedding" - ? `${vecPct}%` - : vectorProgress?.status === "indexing" - ? t("home.vec_indexing") - : vectorProgress?.status === "completed" - ? "✓" - : vectorProgress?.status === "error" - ? "✗" - : t("home.vec_processing")} + {vectorProgress?.status === "cancelling" + ? t("home.vec_cancelling", "Cancelling…") + : vectorProgress?.status === "cancelled" + ? t("home.vec_cancelled", "Cancelled") + : vectorProgress?.status === "completed" + ? "✓" + : vectorProgress?.status === "error" + ? "✕" + : t("home.vec_processing")} )} {/* Queued overlay */} {isQueued && !isVectorizing && ( - + { + event.stopPropagation(); + onCancelVectorize?.(book.id); + }} + > {t("home.vec_queued", "排队中")} - + {t("home.vec_cancel", "Cancel")} + )} {/* Remote status overlay (on-demand download) */} diff --git a/packages/app-expo/src/components/rag/ExtractorWebView.tsx b/packages/app-expo/src/components/rag/ExtractorWebView.tsx index 2609a7b01..7d4280193 100644 --- a/packages/app-expo/src/components/rag/ExtractorWebView.tsx +++ b/packages/app-expo/src/components/rag/ExtractorWebView.tsx @@ -1,59 +1,64 @@ import type { ChapterData } from "@readany/core/rag"; +import type { Book } from "@readany/core/types"; import { Asset } from "expo-asset"; import { forwardRef, useCallback, useEffect, useImperativeHandle, useRef, useState } from "react"; import { StyleSheet, View } from "react-native"; -import { WebView } from "react-native-webview"; +import { WebView, type WebViewMessageEvent } from "react-native-webview"; +import { toBookExtractionError } from "../../lib/rag/extractor-error"; +import { createExtractorCommand } from "../../lib/rag/extractor-format"; +import { ExtractorRequestBoundary } from "../../lib/rag/extractor-request-boundary"; const READER_HTML_ASSET = Asset.fromModule(require("../../../assets/reader/reader.html")); const EXTRACTION_TIMEOUT_MS = 45_000; -const EXTRACTOR_EXTENSIONS_BY_MIME: Record = { - "application/epub+zip": "epub", - "application/pdf": "pdf", - "application/x-mobipocket-ebook": "mobi", - "application/vnd.amazon.ebook": "azw3", - "application/vnd.comicbook+zip": "cbz", - "application/x-fictionbook+xml": "fb2", - "application/x-zip-compressed-fb2": "fbz", - "text/plain": "txt", -}; - -function getExtractorFileName(mimeType: string) { - const normalized = mimeType.split(";")[0]?.trim().toLowerCase() || "application/epub+zip"; - return `book.${EXTRACTOR_EXTENSIONS_BY_MIME[normalized] || "epub"}`; -} - -function isPDFMimeType(mimeType: string) { - return mimeType.split(";")[0]?.trim().toLowerCase() === "application/pdf"; -} - export interface ExtractorRef { - extractChapters: (base64BookData: string, mimeType?: string) => Promise; + extractChapters: ( + base64BookData: string, + mimeType?: string, + bookFormat?: Book["format"], + fileName?: string, + signal?: AbortSignal, + ) => Promise; } -interface PendingExtraction { - resolve: (chapters: ChapterData[]) => void; - reject: (err: Error) => void; - timeoutId: ReturnType; +function getAbortError(signal: AbortSignal): Error { + const reason = signal.reason; + if (reason instanceof Error) { + if (reason.name !== "AbortError") reason.name = "AbortError"; + return reason; + } + const error = new Error("Vectorization cancelled"); + error.name = "AbortError"; + return error; } export const ExtractorWebView = forwardRef((_, ref) => { const webViewRef = useRef(null); const [htmlUri, setHtmlUri] = useState(null); const [ready, setReady] = useState(false); - - // Pending extraction requests - const pendingRequests = useRef([]); + const [requestBoundary] = useState( + () => + new ExtractorRequestBoundary({ + timeoutMs: EXTRACTION_TIMEOUT_MS, + sendCancel: (requestId) => { + webViewRef.current?.injectJavaScript(` + window.postMessage(${JSON.stringify( + JSON.stringify({ type: "cancelExtraction", requestId }), + )}, "*"); + true; + `); + }, + onCancelError: (requestId, error) => { + console.warn(`[ExtractorWebView] Failed to cancel request ${requestId}:`, error); + }, + }), + ); useEffect(() => { return () => { - for (const pending of pendingRequests.current) { - clearTimeout(pending.timeoutId); - pending.reject(new Error("Extractor WebView unmounted")); - } - pendingRequests.current = []; + requestBoundary.rejectAll(); }; - }, []); + }, [requestBoundary]); useEffect(() => { const loadAsset = async () => { @@ -69,75 +74,102 @@ export const ExtractorWebView = forwardRef((_, ref) => { loadAsset(); }, []); - // biome-ignore lint/suspicious/noExplicitAny: Required for React Native WebView events - const handleMessage = useCallback((event: any) => { - try { - const msg = JSON.parse(event.nativeEvent.data); - if (msg.type === "ready") { - setReady(true); - } else if (msg.type === "loaded") { - // Trigger extraction once the book is fully loaded - webViewRef.current?.injectJavaScript(` + const handleMessage = useCallback( + (event: WebViewMessageEvent) => { + try { + const msg = JSON.parse(event.nativeEvent.data); + if (msg.type === "ready") { + setReady(true); + } else if (msg.type === "loaded") { + if (!requestBoundary.has(msg.requestId)) return; + // Trigger extraction once the book is fully loaded + webViewRef.current?.injectJavaScript(` if (window.handleExtractChapters) { - window.handleExtractChapters(); + window.handleExtractChapters(${JSON.stringify(msg.requestId)}); } else { - window.ReactNativeWebView.postMessage(JSON.stringify({ type: 'chaptersExtracted', error: 'Extraction not supported' })); + window.ReactNativeWebView.postMessage(JSON.stringify({ type: 'chaptersExtracted', requestId: ${JSON.stringify(msg.requestId)}, error: 'Extraction not supported' })); } true; `); - } else if (msg.type === "chaptersExtracted") { - const pending = pendingRequests.current.shift(); - if (!pending) return; - - clearTimeout(pending.timeoutId); - if (msg.error) { - pending.reject(new Error(msg.error)); - } else if (msg.chapters) { - pending.resolve(msg.chapters); - } - } else if (msg.type === "debug") { - console.log("[ExtractorWebView]", msg.message); - } else if (msg.type === "error") { - console.error("[ExtractorWebView] WebView error:", msg.message); - const pending = pendingRequests.current.shift(); - if (pending) { - clearTimeout(pending.timeoutId); - pending.reject(new Error(msg.message)); + } else if (msg.type === "chaptersExtracted") { + const classificationFormat = requestBoundary.getContext(msg.requestId); + if (msg.error) { + requestBoundary.reject( + msg.requestId, + toBookExtractionError(new Error(String(msg.error)), classificationFormat), + ); + } else if (msg.chapters) { + requestBoundary.resolve(msg.requestId, msg.chapters); + } + } else if (msg.type === "debug") { + console.log("[ExtractorWebView]", msg.message); + } else if (msg.type === "error") { + if (!requestBoundary.has(msg.requestId)) return; + console.error("[ExtractorWebView] WebView error:", msg.message); + const classificationFormat = requestBoundary.getContext(msg.requestId); + requestBoundary.reject( + msg.requestId, + toBookExtractionError(new Error(String(msg.message)), classificationFormat), + ); } + } catch (err) { + console.warn("[ExtractorWebView] Failed to parse message:", err); } - } catch (err) { - console.warn("[ExtractorWebView] Failed to parse message:", err); - } - }, []); + }, + [requestBoundary], + ); useImperativeHandle(ref, () => ({ - extractChapters: (base64BookData: string, mimeType = "application/epub+zip") => { + extractChapters: ( + base64BookData: string, + mimeType = "application/epub+zip", + bookFormat?: Book["format"], + fileName?: string, + signal?: AbortSignal, + ) => { + const requestId = `extract-${Date.now()}-${Math.random().toString(36).slice(2)}`; + const baseCommand = createExtractorCommand({ + base64BookData, + mimeType, + bookFormat, + fileName, + }); + const command = { ...baseCommand, requestId }; + const classificationFormat = command.bookFormat ?? undefined; return new Promise((resolve, reject) => { + if (signal?.aborted) return reject(getAbortError(signal)); if (!ready || !webViewRef.current) { - return reject(new Error("Extractor WebView not ready")); + return reject( + toBookExtractionError(new Error("Extractor WebView not ready"), classificationFormat), + ); } - const timeoutId = setTimeout(() => { - const index = pendingRequests.current.findIndex((pending) => pending.reject === reject); - if (index >= 0) pendingRequests.current.splice(index, 1); - reject(new Error("Timed out extracting book content")); - }, EXTRACTION_TIMEOUT_MS); - - pendingRequests.current.push({ resolve, reject, timeoutId }); + requestBoundary.add({ + requestId, + resolve, + reject, + context: classificationFormat, + signal, + abortError: () => getAbortError(signal as AbortSignal), + timeoutError: () => + toBookExtractionError( + new Error("Timed out extracting book content"), + classificationFormat, + ), + disposeError: () => + toBookExtractionError(new Error("Extractor WebView unmounted"), classificationFormat), + }); // Command the webview to open the book first. // It will reply with "loaded" when it finishes rendering. - const cmd = { - type: isPDFMimeType(mimeType) ? "extractBookChapters" : "openBook", - base64: base64BookData, - mimeType, - fileName: getExtractorFileName(mimeType), - }; - - webViewRef.current.injectJavaScript(` - window.postMessage(${JSON.stringify(JSON.stringify(cmd))}, "*"); - true; - `); + try { + webViewRef.current.injectJavaScript(` + window.postMessage(${JSON.stringify(JSON.stringify(command))}, "*"); + true; + `); + } catch (error) { + requestBoundary.reject(requestId, toBookExtractionError(error, classificationFormat)); + } }); }, })); diff --git a/packages/app-expo/src/lib/rag/__fixtures__/README.md b/packages/app-expo/src/lib/rag/__fixtures__/README.md new file mode 100644 index 000000000..92f46bd7f --- /dev/null +++ b/packages/app-expo/src/lib/rag/__fixtures__/README.md @@ -0,0 +1,18 @@ +# MOBI-family integration fixtures + +These are byte-for-byte Project Gutenberg downloads of ebook 11, *Alice's Adventures in Wonderland* by Lewis Carroll. They were retrieved on 2026-08-16 and are used only to prove ReadAny's real foliate MOBI-family extraction path. + +| Fixture | Official acquisition URL | Resolved Project Gutenberg file | Bytes | SHA-256 | +| --- | --- | --- | ---: | --- | +| `gutenberg-11.mobi` | | | 240,898 | `4cc3901c405178935a0d4b25ac03bdafc776e0ec3ce81b24482844f2a47ecd13` | +| `gutenberg-11.azw3` | | | 256,060 | `fffee390f393ecf004f65c7fcd2cbefb3ee2652ff6f3fa8daa09c8a9a5644df0` | + +## Byte identity + +Both files have the Palm Database type/creator bytes `BOOKMOBI` and PalmDOC encryption value `0` (unencrypted). The older-Kindle download declares MOBI version 6. The KF8 download declares MOBI version 8 and is stored here with the `.azw3` extension so the integration test covers ReadAny's AZW3 input path without converting or modifying the source bytes. + +Project Gutenberg currently publishes the KF8 acquisition with a `.mobi` filename. The Library of Congress format description records that Amazon registered `application/vnd.amazon.mobi8-ebook` for the MOBI version that uses the `.azw3` extension: . That byte-level version evidence, rather than a fabricated conversion, is why the untouched KF8 download is the AZW3 fixture. + +## Rights and repository suitability + +Project Gutenberg's ebook page identifies this title as public domain in the USA: . Each fixture includes the Project Gutenberg License and its distribution terms in the ebook text. The two fixtures total 496,958 bytes (about 485 KiB), small enough for deterministic upstream integration coverage while retaining real MOBI v6 and KF8 containers, metadata, sections, compression, and images. diff --git a/packages/app-expo/src/lib/rag/__fixtures__/gutenberg-11.azw3 b/packages/app-expo/src/lib/rag/__fixtures__/gutenberg-11.azw3 new file mode 100644 index 000000000..36f5e4e81 Binary files /dev/null and b/packages/app-expo/src/lib/rag/__fixtures__/gutenberg-11.azw3 differ diff --git a/packages/app-expo/src/lib/rag/__fixtures__/gutenberg-11.mobi b/packages/app-expo/src/lib/rag/__fixtures__/gutenberg-11.mobi new file mode 100644 index 000000000..562b9ab8d Binary files /dev/null and b/packages/app-expo/src/lib/rag/__fixtures__/gutenberg-11.mobi differ diff --git a/packages/app-expo/src/lib/rag/auto-vectorize-book.ts b/packages/app-expo/src/lib/rag/auto-vectorize-book.ts index f798a8abc..f19265504 100644 --- a/packages/app-expo/src/lib/rag/auto-vectorize-book.ts +++ b/packages/app-expo/src/lib/rag/auto-vectorize-book.ts @@ -2,19 +2,7 @@ import { getPlatformService } from "@readany/core/services"; import type { Book } from "@readany/core/types"; import * as FileSystem from "expo-file-system/legacy"; import { queueBook as queueAutoVectorize } from "./auto-vectorize-service"; - -const MIME_TYPES: Record = { - epub: "application/epub+zip", - pdf: "application/pdf", - txt: "text/plain", - // Mobile UMD imports are converted and stored as EPUB before vectorization. - umd: "application/epub+zip", -}; - -export function getMobileVectorizeMimeType(format: string | undefined): string | null { - const normalized = String(format || "").toLowerCase(); - return MIME_TYPES[normalized] ?? null; -} +import { getMobileVectorizeCapability } from "./mobile-vectorize-capability"; function bytesToBase64(bytes: Uint8Array): string { const chunkSize = 0x8000; @@ -57,8 +45,9 @@ export async function inspectMobileBookForVectorize(book: Book): Promise<{ reason?: "unsupported-format" | "missing-file"; }> { const absPath = await resolveMobileBookPath(book.filePath); - const mimeType = getMobileVectorizeMimeType(book.format); - if (!mimeType) { + const capability = getMobileVectorizeCapability(book.format); + const { mimeType } = capability; + if (!capability.supported) { return { absPath, mimeType, size: null, canVectorize: false, reason: "unsupported-format" }; } diff --git a/packages/app-expo/src/lib/rag/auto-vectorize-service.test.ts b/packages/app-expo/src/lib/rag/auto-vectorize-service.test.ts new file mode 100644 index 000000000..2ff709047 --- /dev/null +++ b/packages/app-expo/src/lib/rag/auto-vectorize-service.test.ts @@ -0,0 +1,109 @@ +import { VectorizationCleanupError } from "@readany/core/rag"; +import type { Book } from "@readany/core/types"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +const vectorizeMocks = vi.hoisted(() => ({ + resetBookVectorization: vi.fn(), + triggerVectorizeBook: vi.fn(), +})); + +vi.mock("./vectorize-trigger", () => vectorizeMocks); + +import { isProcessing, queueBook, setCallback, setExtractorRef } from "./auto-vectorize-service"; + +const book: Book = { + id: "book-1", + filePath: "books/protected.mobi", + format: "mobi", + meta: { title: "Protected book", author: "Author" }, + addedAt: 1, + updatedAt: 1, + progress: 0, + isVectorized: true, + vectorizeProgress: 1, + tags: [], + syncStatus: "local", +}; + +afterEach(() => { + setExtractorRef(null); + setCallback(null); + vi.clearAllMocks(); +}); + +describe("automatic vectorization failure lifecycle", () => { + it("cleans the book before publishing a classified extraction error", async () => { + const events: string[] = []; + vectorizeMocks.resetBookVectorization.mockImplementation(async () => { + events.push("cleanup"); + }); + setExtractorRef({ + extractChapters: vi.fn().mockRejectedValue(new Error("Encrypted MOBI records")), + }); + + const errorPublished = new Promise((resolve) => { + setCallback((_bookId, progress) => { + if (progress.status === "error") { + events.push(`error:${progress.errorCategory}`); + resolve(); + } + }); + }); + + await queueBook(book, "base64", "application/x-mobipocket-ebook"); + await errorPublished; + + expect(events).toEqual(["cleanup", "error:drm-protected"]); + expect(vectorizeMocks.triggerVectorizeBook).not.toHaveBeenCalled(); + }); + + it("publishes the failure and releases the queue if cleanup itself rejects", async () => { + const cleanupError = new Error("cleanup failed"); + vectorizeMocks.resetBookVectorization.mockRejectedValueOnce(cleanupError); + setExtractorRef({ + extractChapters: vi.fn().mockRejectedValue(new Error("loader failed")), + }); + const callback = vi.fn(); + setCallback(callback); + + await queueBook(book, "base64", "application/x-mobipocket-ebook"); + + await vi.waitFor(() => { + expect(callback).toHaveBeenCalledWith("book-1", { + status: "error", + progress: 0, + error: expect.anything(), + errorCategory: "unknown", + cleanupError, + }); + expect(isProcessing()).toBe(false); + }); + }); + + it("surfaces core cleanup failure instead of discarding it", async () => { + const cleanupError = new Error("partial vectors remain"); + setExtractorRef({ + extractChapters: vi.fn().mockResolvedValue([{ index: 0, title: "Chapter", content: "text" }]), + }); + vectorizeMocks.triggerVectorizeBook.mockRejectedValueOnce( + new VectorizationCleanupError(new Error("cancelled"), cleanupError), + ); + const callback = vi.fn(); + setCallback(callback); + + await queueBook(book, "base64", "application/x-mobipocket-ebook"); + + await vi.waitFor(() => { + expect(callback).toHaveBeenCalledWith( + "book-1", + expect.objectContaining({ + status: "error", + error: expect.any(VectorizationCleanupError), + cleanupError, + }), + ); + expect(vectorizeMocks.resetBookVectorization).not.toHaveBeenCalled(); + expect(isProcessing()).toBe(false); + }); + }); +}); diff --git a/packages/app-expo/src/lib/rag/auto-vectorize-service.ts b/packages/app-expo/src/lib/rag/auto-vectorize-service.ts index 4d27ec229..220963ad8 100644 --- a/packages/app-expo/src/lib/rag/auto-vectorize-service.ts +++ b/packages/app-expo/src/lib/rag/auto-vectorize-service.ts @@ -1,10 +1,27 @@ import type { ChapterData } from "@readany/core/rag"; import type { Book } from "@readany/core/types"; - -export type AutoVectorizeCallback = (bookId: string, progress: { status: string; progress: number }) => void; +import type { BookExtractionErrorCategory } from "./extractor-error"; +import { runVectorizeQueueJob, throwIfQueueJobAborted } from "./vectorize-queue-job"; + +export type AutoVectorizeCallback = ( + bookId: string, + progress: { + status: string; + progress: number; + error?: unknown; + errorCategory?: BookExtractionErrorCategory; + cleanupError?: unknown; + }, +) => void; interface ExtractorRef { - extractChapters: (base64BookData: string, mimeType?: string) => Promise; + extractChapters: ( + base64BookData: string, + mimeType?: string, + bookFormat?: Book["format"], + fileName?: string, + signal?: AbortSignal, + ) => Promise; } interface QueueItem { @@ -15,7 +32,7 @@ interface QueueItem { let extractorRef: ExtractorRef | null = null; let callback: AutoVectorizeCallback | null = null; -let queue: QueueItem[] = []; +const queue: QueueItem[] = []; let processing = false; export function setExtractorRef(ref: ExtractorRef | null) { @@ -45,41 +62,71 @@ async function processQueue() { if (processing) return; processing = true; - const { triggerVectorizeBook } = await import("./vectorize-trigger"); - - while (queue.length > 0) { - const item = queue.shift(); - if (!item) break; - - const { book, base64Data, mimeType } = item; - - try { - callback?.(book.id, { status: "extracting", progress: 0 }); - - if (!extractorRef) { - console.warn("[AutoVectorize] Extractor not ready, skipping"); - continue; - } - - const chapters = await extractorRef.extractChapters(base64Data, mimeType); - if (!chapters || chapters.length === 0) { - console.warn(`[AutoVectorize] No chapters for ${book.meta.title}`); - continue; - } - - callback?.(book.id, { status: "vectorizing", progress: 0 }); - - await triggerVectorizeBook(book.id, book.filePath, chapters, (progress) => { - const pct = progress.totalChunks > 0 ? progress.processedChunks / progress.totalChunks : 0; - callback?.(book.id, { status: "vectorizing", progress: pct }); + try { + const { resetBookVectorization, triggerVectorizeBook } = await import("./vectorize-trigger"); + + while (queue.length > 0) { + const item = queue.shift(); + if (!item) break; + + const { book, base64Data, mimeType } = item; + + await runVectorizeQueueJob({ + format: book.format, + extract: async (signal) => { + throwIfQueueJobAborted(signal); + if (!extractorRef) throw new Error("Extractor WebView not ready"); + return extractorRef.extractChapters( + base64Data, + mimeType, + book.format, + book.filePath, + signal, + ); + }, + vectorize: async (chapters, onProgress, signal) => { + await triggerVectorizeBook( + book.id, + book.filePath, + chapters, + (progress) => { + const pct = + progress.totalChunks > 0 ? progress.processedChunks / progress.totalChunks : 0; + onProgress?.(pct); + }, + signal, + ); + }, + cleanup: () => resetBookVectorization(book.id), + onEvent: (event) => { + if (event.status === "extracting") { + callback?.(book.id, { status: "extracting", progress: 0 }); + } else if (event.status === "vectorizing") { + callback?.(book.id, { status: "vectorizing", progress: event.progress ?? 0 }); + } else if (event.status === "completed") { + callback?.(book.id, { status: "completed", progress: 1 }); + } else if (event.status === "cancelled") { + callback?.(book.id, { status: "cancelled", progress: 0 }); + } else { + console.error(`[AutoVectorize] Failed for ${book.meta.title}:`, event.error); + if (event.cleanupError) { + console.error( + `[AutoVectorize] Failed to clean up ${book.meta.title}:`, + event.cleanupError, + ); + } + callback?.(book.id, { + status: "error", + progress: 0, + error: event.error, + errorCategory: event.errorCategory, + cleanupError: event.cleanupError, + }); + } + }, }); - - callback?.(book.id, { status: "completed", progress: 1 }); - } catch (err) { - console.error(`[AutoVectorize] Failed for ${book.meta.title}:`, err); - callback?.(book.id, { status: "error", progress: 0 }); } + } finally { + processing = false; } - - processing = false; } diff --git a/packages/app-expo/src/lib/rag/extractor-error.test.ts b/packages/app-expo/src/lib/rag/extractor-error.test.ts new file mode 100644 index 000000000..155dca04c --- /dev/null +++ b/packages/app-expo/src/lib/rag/extractor-error.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "vitest"; +import { + BookExtractionError, + classifyBookExtractionError, + getBookExtractionErrorMessageKeys, + toBookExtractionError, +} from "./extractor-error"; + +describe("classifyBookExtractionError", () => { + it.each([ + ["mobi", "Encrypted MOBI records are not supported"], + ["azw", "DRM protected content"], + ["azw3", "encryption is not supported"], + ])("classifies narrow protection evidence for %s", (format, message) => { + expect(classifyBookExtractionError(new Error(message), format)).toBe("drm-protected"); + }); + + it.each(["epub", "pdf", "kfx", undefined])( + "does not classify protection wording for non-MOBI format %s", + (format) => { + expect(classifyBookExtractionError(new Error("encrypted content"), format)).toBe("unknown"); + }, + ); + + it.each([ + "invalid PDB record offset", + "truncated MOBI header", + "invalid record structure", + "record offset is outside the file", + "Invalid HUFF record", + "Invalid CDIC record", + "Invalid INDX record", + "Invalid TAGX section", + "Invalid EXTH header", + "Missing MOBI header", + "Missing FDST record", + "Record index out of bounds", + "Offset is outside the bounds of the DataView", + ])("classifies malformed structure evidence: %s", (message) => { + expect(classifyBookExtractionError(new Error(message), "mobi")).toBe("malformed"); + }); + + it("does not guess DRM from a generic MOBI parser failure", () => { + expect(classifyBookExtractionError(new Error("loader failed"), "mobi")).toBe("unknown"); + }); + + it("classifies explicit unsupported-format failures", () => { + expect(classifyBookExtractionError(new Error("unsupported format: kfx"), "kfx")).toBe( + "unsupported-format", + ); + }); + + it("handles non-Error rejections without broadening classification", () => { + expect(classifyBookExtractionError("loader failed", "azw3")).toBe("unknown"); + }); +}); + +describe("BookExtractionError", () => { + it("carries bounded parser classification across the extraction boundary", () => { + const cause = new Error("Encrypted MOBI records are not supported"); + + const error = toBookExtractionError(cause, "mobi"); + + expect(error).toBeInstanceOf(BookExtractionError); + expect(error.message).toBe(cause.message); + expect(error.category).toBe("drm-protected"); + expect(error.cause).toBe(cause); + }); + + it("keeps extraction message keys distinct for every category", () => { + expect(getBookExtractionErrorMessageKeys("drm-protected")).toEqual({ + title: "vectorize.protectedBookTitle", + description: "vectorize.protectedBookDesc", + }); + expect(getBookExtractionErrorMessageKeys("malformed")).toEqual({ + title: "vectorize.malformedBookTitle", + description: "vectorize.malformedBookDesc", + }); + expect(getBookExtractionErrorMessageKeys("unsupported-format")).toEqual({ + title: "vectorize.unsupportedFormatTitle", + description: "vectorize.unsupportedFormatDesc", + }); + expect(getBookExtractionErrorMessageKeys("unknown")).toEqual({ + title: "vectorize.extractionFailedTitle", + description: "vectorize.extractionFailedDesc", + }); + }); +}); diff --git a/packages/app-expo/src/lib/rag/extractor-error.ts b/packages/app-expo/src/lib/rag/extractor-error.ts new file mode 100644 index 000000000..0d9e22ae7 --- /dev/null +++ b/packages/app-expo/src/lib/rag/extractor-error.ts @@ -0,0 +1,79 @@ +export type BookExtractionErrorCategory = + | "drm-protected" + | "malformed" + | "unsupported-format" + | "unknown"; + +const MOBI_FAMILY = new Set(["mobi", "azw", "azw3"]); +const PROTECTION_EVIDENCE = /\b(?:encrypt(?:ed|ion)?|drm|protected)\b/i; +const GENERIC_MALFORMED_EVIDENCE = + /\b(?:truncat(?:ed|ion)|invalid\s+(?:(?:pdb|mobi)\s+)?record(?:\s+(?:offset|structure|header|index))?|record\s+(?:offset|structure|header|index))\b/i; +const MOBI_PARSER_MALFORMED_EVIDENCE = + /^(?:Invalid (?:HUFF|CDIC|INDX) record|Invalid TAGX section|Invalid EXTH header|Missing MOBI header|Missing FDST record|Record index out of bounds|Offset is outside (?:the )?bounds of (?:the )?DataView)$/i; +const UNSUPPORTED_FORMAT_EVIDENCE = /\bunsupported\s+(?:book\s+)?format\b/i; + +const MESSAGE_KEYS: Record = { + "drm-protected": { + title: "vectorize.protectedBookTitle", + description: "vectorize.protectedBookDesc", + }, + malformed: { + title: "vectorize.malformedBookTitle", + description: "vectorize.malformedBookDesc", + }, + "unsupported-format": { + title: "vectorize.unsupportedFormatTitle", + description: "vectorize.unsupportedFormatDesc", + }, + unknown: { + title: "vectorize.extractionFailedTitle", + description: "vectorize.extractionFailedDesc", + }, +}; + +export function classifyBookExtractionError( + error: unknown, + format: string | undefined, +): BookExtractionErrorCategory { + const message = error instanceof Error ? error.message : String(error); + const parserMessage = message.replace(/^(?:Error|RangeError):\s*/i, ""); + const normalizedFormat = format?.trim().toLowerCase(); + + if ( + normalizedFormat && + MOBI_FAMILY.has(normalizedFormat) && + PROTECTION_EVIDENCE.test(parserMessage) + ) { + return "drm-protected"; + } + if ( + GENERIC_MALFORMED_EVIDENCE.test(parserMessage) || + MOBI_PARSER_MALFORMED_EVIDENCE.test(parserMessage) + ) + return "malformed"; + if (UNSUPPORTED_FORMAT_EVIDENCE.test(parserMessage)) return "unsupported-format"; + return "unknown"; +} + +export class BookExtractionError extends Error { + readonly category: BookExtractionErrorCategory; + override readonly cause: unknown; + + constructor(error: unknown, format: string | undefined) { + super(error instanceof Error ? error.message : String(error)); + this.name = "BookExtractionError"; + this.category = classifyBookExtractionError(error, format); + this.cause = error; + } +} + +export function toBookExtractionError( + error: unknown, + format: string | undefined, +): BookExtractionError { + return error instanceof BookExtractionError ? error : new BookExtractionError(error, format); +} + +export function getBookExtractionErrorMessageKeys(category: BookExtractionErrorCategory) { + return MESSAGE_KEYS[category]; +} diff --git a/packages/app-expo/src/lib/rag/extractor-format.test.ts b/packages/app-expo/src/lib/rag/extractor-format.test.ts new file mode 100644 index 000000000..7f67a1552 --- /dev/null +++ b/packages/app-expo/src/lib/rag/extractor-format.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "vitest"; +import { createExtractorCommand, resolveExtractorFormat } from "./extractor-format"; + +describe("resolveExtractorFormat", () => { + it.each(["epub", "pdf", "txt", "umd", "mobi", "azw", "azw3"])( + "prefers the supported stored %s format", + (bookFormat) => { + expect( + resolveExtractorFormat({ + bookFormat, + mimeType: "application/octet-stream", + fileName: "misleading.epub", + }), + ).toBe(bookFormat); + }, + ); + + it("uses a supported filename extension when the stored format is unavailable", () => { + expect( + resolveExtractorFormat({ + bookFormat: undefined, + mimeType: "application/vnd.amazon.ebook", + fileName: "x.AZW3", + }), + ).toBe("azw3"); + }); + + it.each([ + ["application/epub+zip", "epub"], + ["application/pdf", "pdf"], + ["text/plain; charset=utf-8", "txt"], + ["application/x-mobipocket-ebook", "mobi"], + ["application/vnd.amazon.ebook", "azw3"], + ])("falls back from %s to %s", (mimeType, format) => { + expect(resolveExtractorFormat({ mimeType })).toBe(format); + }); + + it("normalizes stored format and ignores query text after the filename extension", () => { + expect(resolveExtractorFormat({ bookFormat: "MOBI" })).toBe("mobi"); + expect(resolveExtractorFormat({ fileName: "download.AZW?token=1" })).toBe("azw"); + }); + + it("rejects KFX and unknown signals", () => { + expect( + resolveExtractorFormat({ + bookFormat: "kfx", + mimeType: "application/octet-stream", + fileName: "x.kfx", + }), + ).toBeNull(); + expect(resolveExtractorFormat({ bookFormat: "unknown" })).toBeNull(); + expect(resolveExtractorFormat({})).toBeNull(); + }); +}); + +describe("createExtractorCommand", () => { + it.each([ + [ + { bookFormat: "mobi", mimeType: "application/pdf", fileName: "stored.pdf" }, + { type: "openBook", bookFormat: "mobi", fileName: "stored.mobi" }, + ], + [ + { mimeType: "application/pdf", fileName: "filename.mobi" }, + { type: "openBook", bookFormat: "mobi", fileName: "filename.mobi" }, + ], + [ + { + bookFormat: "pdf", + mimeType: "application/x-mobipocket-ebook", + fileName: "stored.mobi", + }, + { type: "extractBookChapters", bookFormat: "pdf", fileName: "stored.pdf" }, + ], + [ + { mimeType: "application/x-mobipocket-ebook", fileName: "filename.pdf" }, + { type: "extractBookChapters", bookFormat: "pdf", fileName: "filename.pdf" }, + ], + ])("dispatches from resolved format for %#", (input, expected) => { + expect(createExtractorCommand({ base64BookData: "data", ...input })).toMatchObject(expected); + }); +}); + +describe("extractor pending-request classification", () => { + it.each([ + [{ mimeType: "application/octet-stream", fileName: "inferred.mobi" }, "mobi"], + [{ mimeType: "application/vnd.amazon.ebook" }, "azw3"], + ])("preserves the resolved %s format for error classification", (input, expected) => { + const command = createExtractorCommand({ base64BookData: "data", ...input }); + + expect(command.bookFormat).toBe(expected); + expect(command.bookFormat ?? undefined).toBe(expected); + }); +}); diff --git a/packages/app-expo/src/lib/rag/extractor-format.ts b/packages/app-expo/src/lib/rag/extractor-format.ts new file mode 100644 index 000000000..19a882a11 --- /dev/null +++ b/packages/app-expo/src/lib/rag/extractor-format.ts @@ -0,0 +1,84 @@ +import type { Book } from "@readany/core/types"; + +const EXTRACTOR_EXTENSIONS_BY_MIME: Record = { + "application/epub+zip": "epub", + "application/pdf": "pdf", + "application/x-mobipocket-ebook": "mobi", + "application/vnd.amazon.ebook": "azw3", + "application/vnd.comicbook+zip": "cbz", + "application/x-fictionbook+xml": "fb2", + "application/x-zip-compressed-fb2": "fbz", + "text/plain": "txt", +}; + +const SUPPORTED_FORMATS = new Set([ + "epub", + "pdf", + "txt", + "umd", + "mobi", + "azw", + "azw3", +]); + +const FORMAT_BY_MIME_TYPE: Partial> = { + "application/epub+zip": "epub", + "application/pdf": "pdf", + "application/vnd.amazon.ebook": "azw3", + "application/x-mobipocket-ebook": "mobi", + "text/plain": "txt", +}; + +function asSupportedFormat(value: string | undefined): Book["format"] | null { + const normalized = value?.trim().toLowerCase() as Book["format"] | undefined; + return normalized && SUPPORTED_FORMATS.has(normalized) ? normalized : null; +} + +export function resolveExtractorFormat(input: { + bookFormat?: string; + mimeType?: string; + fileName?: string; +}): Book["format"] | null { + const storedFormat = asSupportedFormat(input.bookFormat); + if (storedFormat) return storedFormat; + + const cleanFileName = input.fileName?.split(/[?#]/, 1)[0]; + const extension = cleanFileName?.split(".").pop(); + const fileFormat = asSupportedFormat(extension); + if (fileFormat) return fileFormat; + + const normalizedMimeType = input.mimeType?.split(";", 1)[0]?.trim().toLowerCase(); + return normalizedMimeType ? FORMAT_BY_MIME_TYPE[normalizedMimeType] || null : null; +} + +function getExtractorFileName( + mimeType: string, + bookFormat: Book["format"] | null, + fileName?: string, +) { + const cleanFileName = fileName?.split(/[?#]/, 1)[0]?.split(/[\\/]/).pop(); + if (bookFormat) { + const baseName = cleanFileName?.replace(/\.[^.]*$/, "") || "book"; + return `${baseName}.${bookFormat}`; + } + if (cleanFileName) return cleanFileName; + + const normalizedMimeType = mimeType.split(";")[0]?.trim().toLowerCase() || "application/epub+zip"; + return `book.${EXTRACTOR_EXTENSIONS_BY_MIME[normalizedMimeType] || "epub"}`; +} + +export function createExtractorCommand(input: { + base64BookData: string; + mimeType: string; + bookFormat?: string; + fileName?: string; +}) { + const resolvedFormat = resolveExtractorFormat(input); + return { + type: resolvedFormat === "pdf" ? "extractBookChapters" : "openBook", + base64: input.base64BookData, + mimeType: input.mimeType, + bookFormat: resolvedFormat, + fileName: getExtractorFileName(input.mimeType, resolvedFormat, input.fileName), + }; +} diff --git a/packages/app-expo/src/lib/rag/extractor-request-boundary.test.ts b/packages/app-expo/src/lib/rag/extractor-request-boundary.test.ts new file mode 100644 index 000000000..9f075c85b --- /dev/null +++ b/packages/app-expo/src/lib/rag/extractor-request-boundary.test.ts @@ -0,0 +1,104 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { ExtractorRequestBoundary } from "./extractor-request-boundary"; +import { ReaderExtractionSessions } from "./reader-extraction-sessions"; + +function deferred() { + let resolve: (value: T) => void = () => {}; + const promise = new Promise((promiseResolve) => { + resolve = promiseResolve; + }); + return { promise, resolve }; +} + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("ExtractorRequestBoundary timeout", () => { + it("cancels and releases timed-out A before isolated B completes", async () => { + vi.useFakeTimers(); + const order: string[] = []; + const sessions = new ReaderExtractionSessions<{ chapters: string[] }>(); + const boundary = new ExtractorRequestBoundary({ + timeoutMs: 45_000, + sendCancel: (requestId) => { + order.push(`cancel:${requestId}`); + sessions.cancel(requestId); + }, + }); + const parserA = deferred<{ chapters: string[] }>(); + const openA = sessions.open("A", () => parserA.promise); + const resultA = new Promise((resolve, reject) => { + boundary.add({ + requestId: "A", + resolve, + reject: (error) => { + order.push("reject:A"); + reject(error); + }, + timeoutError: () => new Error("Timed out extracting book content"), + }); + }); + + const timedOut = expect(resultA).rejects.toThrow("Timed out extracting book content"); + await vi.advanceTimersByTimeAsync(45_000); + await timedOut; + expect(order).toEqual(["cancel:A", "reject:A"]); + + const cancelledOpen = expect(openA).rejects.toMatchObject({ name: "AbortError" }); + parserA.resolve({ chapters: ["A chapter"] }); + await cancelledOpen; + expect(boundary.resolve("A", ["late A chapter"])).toBe(false); + + const openB = sessions.open("B", async () => ({ chapters: ["B chapter"] })); + const resultB = new Promise((resolve, reject) => { + boundary.add({ + requestId: "B", + resolve, + reject, + timeoutError: () => new Error("B timed out"), + }); + }); + const bookB = await openB; + expect(boundary.resolve("B", bookB.chapters)).toBe(true); + + await expect(resultB).resolves.toEqual(["B chapter"]); + expect(sessions.getBook("B").chapters).toEqual(["B chapter"]); + await vi.runAllTimersAsync(); + expect(order).toEqual(["cancel:A", "reject:A"]); + }); + + it("keeps explicit abort cancellation one-shot and ignores its late reply", async () => { + vi.useFakeTimers(); + const cancelled: string[] = []; + const controller = new AbortController(); + const boundary = new ExtractorRequestBoundary({ + timeoutMs: 45_000, + sendCancel: (requestId) => cancelled.push(requestId), + }); + const result = new Promise((resolve, reject) => { + boundary.add({ + requestId: "A", + resolve, + reject, + signal: controller.signal, + abortError: () => { + const error = new Error("cancelled"); + error.name = "AbortError"; + return error; + }, + timeoutError: () => new Error("timed out"), + }); + }); + const rejected = expect(result).rejects.toMatchObject({ name: "AbortError" }); + + controller.abort(); + controller.abort(); + + await rejected; + expect(cancelled).toEqual(["A"]); + expect(boundary.resolve("A", ["late chapter"])).toBe(false); + await vi.runAllTimersAsync(); + expect(cancelled).toEqual(["A"]); + }); +}); diff --git a/packages/app-expo/src/lib/rag/extractor-request-boundary.ts b/packages/app-expo/src/lib/rag/extractor-request-boundary.ts new file mode 100644 index 000000000..b54f3f3cf --- /dev/null +++ b/packages/app-expo/src/lib/rag/extractor-request-boundary.ts @@ -0,0 +1,108 @@ +interface ExtractorRequestBoundaryOptions { + timeoutMs: number; + sendCancel: (requestId: string) => void; + onCancelError?: (requestId: string, error: unknown) => void; +} + +interface AddExtractorRequest { + requestId: string; + resolve: (result: Result) => void; + reject: (error: Error) => void; + timeoutError: () => Error; + disposeError?: () => Error; + signal?: AbortSignal; + abortError?: () => Error; + context?: Context; +} + +interface PendingExtractorRequest extends AddExtractorRequest { + timeoutId: ReturnType; + abortHandler?: () => void; +} + +function defaultAbortError(): Error { + const error = new Error("Vectorization cancelled"); + error.name = "AbortError"; + return error; +} + +/** Owns RN pending requests and their cancellation notification to the reader. */ +export class ExtractorRequestBoundary { + private readonly requests = new Map>(); + + constructor(private readonly options: ExtractorRequestBoundaryOptions) {} + + add(request: AddExtractorRequest): void { + if (this.requests.has(request.requestId)) { + throw new Error(`Duplicate extractor request: ${request.requestId}`); + } + + const pending: PendingExtractorRequest = { + ...request, + timeoutId: setTimeout(() => { + this.cancel(request.requestId, request.timeoutError()); + }, this.options.timeoutMs), + }; + this.requests.set(request.requestId, pending); + + if (request.signal) { + pending.abortHandler = () => { + this.cancel(request.requestId, request.abortError?.() ?? defaultAbortError()); + }; + if (request.signal.aborted) pending.abortHandler(); + else request.signal.addEventListener("abort", pending.abortHandler, { once: true }); + } + } + + has(requestId: string): boolean { + return this.requests.has(requestId); + } + + getContext(requestId: string): Context | undefined { + return this.requests.get(requestId)?.context; + } + + resolve(requestId: string, result: Result): boolean { + const pending = this.take(requestId); + if (!pending) return false; + pending.resolve(result); + return true; + } + + reject(requestId: string, error: Error): boolean { + const pending = this.take(requestId); + if (!pending) return false; + pending.reject(error); + return true; + } + + cancel(requestId: string, error: Error): boolean { + const pending = this.take(requestId); + if (!pending) return false; + try { + this.options.sendCancel(requestId); + } catch (cancelError) { + this.options.onCancelError?.(requestId, cancelError); + } + pending.reject(error); + return true; + } + + rejectAll(): void { + for (const requestId of [...this.requests.keys()]) { + const pending = this.take(requestId); + if (pending) pending.reject(pending.disposeError?.() ?? new Error("Extractor disposed")); + } + } + + private take(requestId: string): PendingExtractorRequest | undefined { + const pending = this.requests.get(requestId); + if (!pending) return undefined; + this.requests.delete(requestId); + clearTimeout(pending.timeoutId); + if (pending.abortHandler) { + pending.signal?.removeEventListener("abort", pending.abortHandler); + } + return pending; + } +} diff --git a/packages/app-expo/src/lib/rag/mobi-extraction.integration.test.ts b/packages/app-expo/src/lib/rag/mobi-extraction.integration.test.ts new file mode 100644 index 000000000..811cacf04 --- /dev/null +++ b/packages/app-expo/src/lib/rag/mobi-extraction.integration.test.ts @@ -0,0 +1,166 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { MOBI } from "../../../../foliate-js/mobi.js"; +import { unzlibSync } from "../../../../foliate-js/vendor/fflate.js"; +import { BookExtractionError } from "./extractor-error"; +import { runVectorizeQueueJob } from "./vectorize-queue-job"; + +const FIXTURE_URL = new URL("./__fixtures__/", import.meta.url); + +function decodeHtmlEntities(value: string): string { + return value + .replace(/&#(x?[\da-f]+);/gi, (_match, digits: string) => { + const hexadecimal = digits[0]?.toLowerCase() === "x"; + return String.fromCodePoint( + Number.parseInt(hexadecimal ? digits.slice(1) : digits, hexadecimal ? 16 : 10), + ); + }) + .replace(/&(amp|apos|gt|lt|nbsp|quot);/gi, (_match, entity: string) => { + const values: Record = { + amp: "&", + apos: "'", + gt: ">", + lt: "<", + nbsp: " ", + quot: '"', + }; + return values[entity.toLowerCase()] ?? _match; + }); +} + +function htmlToText(value: string): string { + return decodeHtmlEntities( + value + .replace(//gi, " ") + .replace(//gi, " ") + .replace(/<[^>]+>/g, " "), + ); +} + +class TextOnlyDocument { + readonly documentElement: { textContent: string }; + + constructor(source: string) { + this.documentElement = { textContent: htmlToText(source) }; + } + + getElementsByTagName(): never[] { + return []; + } + + querySelectorAll(): never[] { + return []; + } +} + +class TextOnlyDomParser { + parseFromString(source: string): TextOnlyDocument { + return new TextOnlyDocument(source); + } +} + +function installParserTestDom() { + vi.stubGlobal("DOMParser", TextOnlyDomParser); + vi.stubGlobal("XMLSerializer", class {}); + vi.stubGlobal("CSS", { escape: (value: string) => value }); + vi.stubGlobal("document", { + createElement: () => { + let html = ""; + return { + set innerHTML(value: string) { + html = value; + }, + get value() { + return decodeHtmlEntities(html); + }, + }; + }, + }); +} + +async function extractFixtureSections(fixture: string): Promise { + const fixturePath = fileURLToPath(new URL(fixture, FIXTURE_URL).href); + const bytes = new Uint8Array(readFileSync(fixturePath)); + const book = await new MOBI({ unzlib: unzlibSync }).open( + new Blob([bytes], { type: "application/x-mobipocket-ebook" }), + ); + + const sections: string[] = []; + for (const section of book.sections) { + if (!section.createDocument) continue; + const document = await section.createDocument(); + const text = document.documentElement?.textContent?.replace(/\s+/g, " ").trim(); + if (text) sections.push(text); + } + return sections; +} + +function writeAscii(bytes: Uint8Array, offset: number, value: string) { + for (let index = 0; index < value.length; index++) { + bytes[offset + index] = value.charCodeAt(index); + } +} + +function createProtectedMobiFile(): Blob { + const bytes = new Uint8Array(512); + const view = new DataView(bytes.buffer); + const recordOffset = 86; + + writeAscii(bytes, 60, "BOOKMOBI"); + view.setUint16(76, 1); + view.setUint32(78, recordOffset); + view.setUint16(recordOffset, 1); + view.setUint16(recordOffset + 12, 1); + writeAscii(bytes, recordOffset + 16, "MOBI"); + view.setUint32(recordOffset + 20, 232); + view.setUint32(recordOffset + 28, 65001); + view.setUint32(recordOffset + 36, 6); + + return new Blob([bytes], { type: "application/x-mobipocket-ebook" }); +} + +describe("MOBI-family extraction fixtures", () => { + beforeAll(installParserTestDom); + afterAll(() => vi.unstubAllGlobals()); + + it.each(["gutenberg-11.mobi", "gutenberg-11.azw3"])( + "extracts ordered Alice text from %s with the foliate MOBI loader", + async (fixture) => { + const sections = await extractFixtureSections(fixture); + + expect(sections.length).toBeGreaterThan(10); + const firstChapter = sections.findIndex((text) => + text.includes("Alice was beginning to get very tired"), + ); + const secondChapter = sections.findIndex((text) => text.includes("Curiouser and curiouser")); + const queenChapter = sections.findIndex((text) => + text.includes("The Queen turned crimson with fury"), + ); + expect(firstChapter).toBeGreaterThanOrEqual(0); + expect(secondChapter).toBeGreaterThan(firstChapter); + expect(queenChapter).toBeGreaterThan(secondChapter); + }, + ); + + it("rejects a bounded protected header, cleans up, and never vectorizes", async () => { + const parser = new MOBI({ unzlib: unzlibSync }); + const cleanup = vi.fn().mockResolvedValue(undefined); + const vectorize = vi.fn(); + + const result = await runVectorizeQueueJob({ + format: "mobi", + extract: async () => (await parser.open(createProtectedMobiFile())) as never, + vectorize, + cleanup, + onEvent: vi.fn(), + }); + + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected protected extraction to fail"); + expect(result.error).toBeInstanceOf(BookExtractionError); + expect(result.error).toMatchObject({ category: "drm-protected" }); + expect(cleanup).toHaveBeenCalledOnce(); + expect(vectorize).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/app-expo/src/lib/rag/mobi-parser-boundary.test.ts b/packages/app-expo/src/lib/rag/mobi-parser-boundary.test.ts new file mode 100644 index 000000000..2ce301738 --- /dev/null +++ b/packages/app-expo/src/lib/rag/mobi-parser-boundary.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest"; +import { MOBI } from "../../../../foliate-js/mobi.js"; + +function writeAscii(bytes: Uint8Array, offset: number, value: string) { + for (let index = 0; index < value.length; index++) { + bytes[offset + index] = value.charCodeAt(index); + } +} + +function createProtectedMobiFile(): Blob { + const bytes = new Uint8Array(512); + const view = new DataView(bytes.buffer); + const recordOffset = 86; + + writeAscii(bytes, 60, "BOOKMOBI"); + view.setUint16(76, 1); + view.setUint32(78, recordOffset); + + view.setUint16(recordOffset, 1); + view.setUint16(recordOffset + 12, 1); + writeAscii(bytes, recordOffset + 16, "MOBI"); + view.setUint32(recordOffset + 20, 232); + view.setUint32(recordOffset + 28, 65001); + view.setUint32(recordOffset + 32, 1); + view.setUint32(recordOffset + 36, 6); + view.setUint32(recordOffset + 108, 1); + + return new Blob([bytes], { type: "application/x-mobipocket-ebook" }); +} + +function createProtectedComboMobiFile(): Blob { + const bytes = new Uint8Array(800); + const view = new DataView(bytes.buffer); + const mobi6Offset = 94; + const kf8Offset = 400; + + writeAscii(bytes, 60, "BOOKMOBI"); + view.setUint16(76, 2); + view.setUint32(78, mobi6Offset); + view.setUint32(86, kf8Offset); + + view.setUint16(mobi6Offset, 1); + writeAscii(bytes, mobi6Offset + 16, "MOBI"); + view.setUint32(mobi6Offset + 20, 232); + view.setUint32(mobi6Offset + 28, 65001); + view.setUint32(mobi6Offset + 32, 1); + view.setUint32(mobi6Offset + 36, 6); + view.setUint32(mobi6Offset + 108, 2); + view.setUint32(mobi6Offset + 128, 0b100_0000); + writeAscii(bytes, mobi6Offset + 248, "EXTH"); + view.setUint32(mobi6Offset + 252, 24); + view.setUint32(mobi6Offset + 256, 1); + view.setUint32(mobi6Offset + 260, 121); + view.setUint32(mobi6Offset + 264, 12); + view.setUint32(mobi6Offset + 268, 1); + + view.setUint16(kf8Offset, 1); + view.setUint16(kf8Offset + 12, 1); + writeAscii(bytes, kf8Offset + 16, "MOBI"); + view.setUint32(kf8Offset + 20, 232); + view.setUint32(kf8Offset + 28, 65001); + view.setUint32(kf8Offset + 32, 2); + view.setUint32(kf8Offset + 36, 8); + + return new Blob([bytes], { type: "application/x-mobipocket-ebook" }); +} + +describe("MOBI parser protection boundary", () => { + it("rejects encrypted PalmDOC records before text extraction", async () => { + const parser = new MOBI({ unzlib: (value: Uint8Array) => value }); + + await expect(parser.open(createProtectedMobiFile())).rejects.toThrow( + "Encrypted MOBI records are not supported", + ); + }); + + it("does not swallow encryption in a combo file's KF8 records", async () => { + const parser = new MOBI({ unzlib: (value: Uint8Array) => value }); + + await expect(parser.open(createProtectedComboMobiFile())).rejects.toThrow( + "Encrypted MOBI records are not supported", + ); + }); +}); diff --git a/packages/app-expo/src/lib/rag/mobile-vectorize-capability.test.ts b/packages/app-expo/src/lib/rag/mobile-vectorize-capability.test.ts new file mode 100644 index 000000000..b9c209f2d --- /dev/null +++ b/packages/app-expo/src/lib/rag/mobile-vectorize-capability.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; +import { + MOBILE_VECTORIZE_UNSUPPORTED_FORMAT_DESCRIPTION, + getMobileVectorizeCapability, +} from "./mobile-vectorize-capability"; + +describe("getMobileVectorizeCapability", () => { + it.each([ + ["epub", "application/epub+zip"], + ["pdf", "application/pdf"], + ["txt", "text/plain"], + ["umd", "application/epub+zip"], + ["mobi", "application/x-mobipocket-ebook"], + ["azw", "application/vnd.amazon.ebook"], + ["azw3", "application/vnd.amazon.ebook"], + ])("supports %s", (format, mimeType) => { + expect(getMobileVectorizeCapability(format)).toEqual({ supported: true, mimeType }); + }); + + it("normalizes case and rejects other formats", () => { + expect(getMobileVectorizeCapability("MOBI")).toEqual({ + supported: true, + mimeType: "application/x-mobipocket-ebook", + }); + expect(getMobileVectorizeCapability("kfx")).toEqual({ supported: false, mimeType: null }); + expect(getMobileVectorizeCapability("unknown")).toEqual({ supported: false, mimeType: null }); + expect(getMobileVectorizeCapability(undefined)).toEqual({ supported: false, mimeType: null }); + }); + + it("describes MOBI-family support as DRM-free", () => { + expect(MOBILE_VECTORIZE_UNSUPPORTED_FORMAT_DESCRIPTION).toBe( + "Mobile vectorization supports EPUB, PDF, TXT, UMD, and DRM-free MOBI, AZW, and AZW3 books.", + ); + }); +}); diff --git a/packages/app-expo/src/lib/rag/mobile-vectorize-capability.ts b/packages/app-expo/src/lib/rag/mobile-vectorize-capability.ts new file mode 100644 index 000000000..958a405ad --- /dev/null +++ b/packages/app-expo/src/lib/rag/mobile-vectorize-capability.ts @@ -0,0 +1,20 @@ +const MOBILE_VECTORIZE_MIME = { + epub: "application/epub+zip", + pdf: "application/pdf", + txt: "text/plain", + umd: "application/epub+zip", + mobi: "application/x-mobipocket-ebook", + azw: "application/vnd.amazon.ebook", + azw3: "application/vnd.amazon.ebook", +} as const; + +export const MOBILE_VECTORIZE_UNSUPPORTED_FORMAT_DESCRIPTION = + "Mobile vectorization supports EPUB, PDF, TXT, UMD, and DRM-free MOBI, AZW, and AZW3 books."; + +export function getMobileVectorizeCapability(format: string | undefined) { + const normalized = String(format || "").toLowerCase() as keyof typeof MOBILE_VECTORIZE_MIME; + const mimeType = MOBILE_VECTORIZE_MIME[normalized]; + return mimeType + ? { supported: true as const, mimeType } + : { supported: false as const, mimeType: null }; +} diff --git a/packages/app-expo/src/lib/rag/reader-extraction-sessions.test.ts b/packages/app-expo/src/lib/rag/reader-extraction-sessions.test.ts new file mode 100644 index 000000000..81355e8d1 --- /dev/null +++ b/packages/app-expo/src/lib/rag/reader-extraction-sessions.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; +import { ReaderExtractionSessions } from "./reader-extraction-sessions"; + +function deferred() { + let resolve: (value: T) => void = () => {}; + const promise = new Promise((promiseResolve) => { + resolve = promiseResolve; + }); + return { promise, resolve }; +} + +describe("ReaderExtractionSessions", () => { + it("keeps A and B paired with their own books when B opens first", async () => { + const sessions = new ReaderExtractionSessions<{ chapters: string[] }>(); + const a = deferred<{ chapters: string[] }>(); + const b = deferred<{ chapters: string[] }>(); + + const openA = sessions.open("A", () => a.promise); + const openB = sessions.open("B", () => b.promise); + b.resolve({ chapters: ["B chapter"] }); + await openB; + a.resolve({ chapters: ["A chapter"] }); + await openA; + + expect(sessions.getBook("A").chapters).toEqual(["A chapter"]); + expect(sessions.getBook("B").chapters).toEqual(["B chapter"]); + }); + + it("rejects cancelled A after its parser settles without contaminating B", async () => { + const sessions = new ReaderExtractionSessions<{ chapters: string[] }>(); + const a = deferred<{ chapters: string[] }>(); + const b = deferred<{ chapters: string[] }>(); + + const openA = sessions.open("A", () => a.promise); + sessions.cancel("A"); + const openB = sessions.open("B", () => b.promise); + b.resolve({ chapters: ["B chapter"] }); + await openB; + a.resolve({ chapters: ["A chapter"] }); + + await expect(openA).rejects.toMatchObject({ name: "AbortError" }); + expect(sessions.getBook("B").chapters).toEqual(["B chapter"]); + expect(() => sessions.getBook("A")).toThrow(/cancelled|not available/i); + }); +}); diff --git a/packages/app-expo/src/lib/rag/reader-extraction-sessions.ts b/packages/app-expo/src/lib/rag/reader-extraction-sessions.ts new file mode 100644 index 000000000..8e8ab312d --- /dev/null +++ b/packages/app-expo/src/lib/rag/reader-extraction-sessions.ts @@ -0,0 +1,50 @@ +function createAbortError(): Error { + const error = new Error("Vectorization cancelled"); + error.name = "AbortError"; + return error; +} + +/** Owns parser results by request so concurrent reader work cannot cross books. */ +export class ReaderExtractionSessions { + private readonly books = new Map(); + private readonly cancelled = new Set(); + + async open(requestId: string, load: () => Promise): Promise { + this.throwIfCancelled(requestId); + try { + const book = await load(); + this.throwIfCancelled(requestId); + this.books.set(requestId, book); + return book; + } catch (error) { + this.books.delete(requestId); + if (this.cancelled.has(requestId)) { + this.cancelled.delete(requestId); + throw createAbortError(); + } + throw error; + } + } + + cancel(requestId: string): void { + this.cancelled.add(requestId); + this.books.delete(requestId); + } + + getBook(requestId: string): Book { + this.throwIfCancelled(requestId); + const book = this.books.get(requestId); + if (!book) throw new Error(`Book for extraction request ${requestId} is not available`); + return book; + } + + throwIfCancelled(requestId?: string): void { + if (requestId && this.cancelled.has(requestId)) throw createAbortError(); + } + + release(requestId?: string): void { + if (!requestId) return; + this.books.delete(requestId); + this.cancelled.delete(requestId); + } +} diff --git a/packages/app-expo/src/lib/rag/vectorization-cancel-state.test.ts b/packages/app-expo/src/lib/rag/vectorization-cancel-state.test.ts new file mode 100644 index 000000000..d27d564ff --- /dev/null +++ b/packages/app-expo/src/lib/rag/vectorization-cancel-state.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest"; +import { isVectorizationCancellable } from "./vectorization-cancel-state"; + +describe("isVectorizationCancellable", () => { + it.each([undefined, "loading", "extracting", "chunking", "vectorizing", "embedding", "indexing"])( + "allows cancellation while active in %s", + (status) => expect(isVectorizationCancellable(true, status)).toBe(true), + ); + + it.each(["cancelling", "completed", "error", "cancelled"])( + "disables the terminal %s overlay", + (status) => expect(isVectorizationCancellable(true, status)).toBe(false), + ); + + it("does not expose a cancel action for an inactive card", () => { + expect(isVectorizationCancellable(false, "embedding")).toBe(false); + }); +}); diff --git a/packages/app-expo/src/lib/rag/vectorization-cancel-state.ts b/packages/app-expo/src/lib/rag/vectorization-cancel-state.ts new file mode 100644 index 000000000..4843881ae --- /dev/null +++ b/packages/app-expo/src/lib/rag/vectorization-cancel-state.ts @@ -0,0 +1,12 @@ +const CANCELLABLE_STATUSES = new Set([ + "loading", + "extracting", + "chunking", + "vectorizing", + "embedding", + "indexing", +]); + +export function isVectorizationCancellable(isActive: boolean, status?: string): boolean { + return isActive && (status === undefined || CANCELLABLE_STATUSES.has(status)); +} diff --git a/packages/app-expo/src/lib/rag/vectorize-queue-job.test.ts b/packages/app-expo/src/lib/rag/vectorize-queue-job.test.ts new file mode 100644 index 000000000..cf4ceb679 --- /dev/null +++ b/packages/app-expo/src/lib/rag/vectorize-queue-job.test.ts @@ -0,0 +1,205 @@ +import { VectorizationCleanupError } from "@readany/core/rag"; +import { describe, expect, it, vi } from "vitest"; +import { MOBI } from "../../../../foliate-js/mobi.js"; +import { BookExtractionError } from "./extractor-error"; +import { runVectorizeQueueJob } from "./vectorize-queue-job"; + +function writeAscii(bytes: Uint8Array, offset: number, value: string) { + for (let index = 0; index < value.length; index++) { + bytes[offset + index] = value.charCodeAt(index); + } +} + +function createProtectedMobiFile(): Blob { + const bytes = new Uint8Array(512); + const view = new DataView(bytes.buffer); + const recordOffset = 86; + + writeAscii(bytes, 60, "BOOKMOBI"); + view.setUint16(76, 1); + view.setUint32(78, recordOffset); + view.setUint16(recordOffset, 1); + view.setUint16(recordOffset + 12, 1); + writeAscii(bytes, recordOffset + 16, "MOBI"); + view.setUint32(recordOffset + 20, 232); + view.setUint32(recordOffset + 28, 65001); + view.setUint32(recordOffset + 36, 6); + + return new Blob([bytes], { type: "application/x-mobipocket-ebook" }); +} + +describe("runVectorizeQueueJob", () => { + it("routes a real protected MOBI parser failure through cleanup without vectorizing", async () => { + const order: string[] = []; + const vectorize = vi.fn(); + const parser = new MOBI({ unzlib: (value: Uint8Array) => value }); + + const result = await runVectorizeQueueJob({ + format: "mobi", + extract: async () => (await parser.open(createProtectedMobiFile())) as never, + vectorize, + cleanup: async () => { + order.push("cleanup"); + }, + onEvent: (event) => order.push(event.status), + }); + + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected protected extraction to fail"); + expect(result.error).toBeInstanceOf(BookExtractionError); + expect(result.error).toMatchObject({ category: "drm-protected" }); + expect(order).toEqual(["extracting", "cleanup", "error"]); + expect(vectorize).not.toHaveBeenCalled(); + }); + + it.each(["manual", "automatic"])( + "classifies raw parser failures at the shared %s queue boundary", + async () => { + const order: string[] = []; + + const result = await runVectorizeQueueJob({ + format: "azw3", + extract: async () => { + throw new Error("Invalid HUFF record"); + }, + vectorize: vi.fn(), + cleanup: async () => { + order.push("cleanup"); + }, + onEvent: (event) => order.push(event.status), + }); + + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected parser failure"); + expect(result.error).toMatchObject({ category: "malformed" }); + expect(order).toEqual(["extracting", "cleanup", "error"]); + }, + ); + + it("awaits cleanup before publishing cancellation during extraction", async () => { + const order: string[] = []; + const controller = new AbortController(); + let releaseCleanup: (() => void) | undefined; + const cleanupReleased = new Promise((resolve) => { + releaseCleanup = resolve; + }); + + const job = runVectorizeQueueJob({ + format: "mobi", + signal: controller.signal, + extract: (signal) => + new Promise((_, reject) => { + signal.addEventListener("abort", () => reject(signal.reason), { once: true }); + }), + vectorize: vi.fn(), + cleanup: async () => { + order.push("cleanup:start"); + await cleanupReleased; + order.push("cleanup:end"); + }, + onEvent: (event) => order.push(event.status), + }); + controller.abort(); + + await vi.waitFor(() => expect(order).toContain("cleanup:start")); + expect(order).not.toContain("cancelled"); + releaseCleanup?.(); + const result = await job; + + expect(result).toMatchObject({ ok: false, cancelled: true }); + expect(order).toEqual(["extracting", "cleanup:start", "cleanup:end", "cancelled"]); + expect(order).not.toContain("completed"); + }); + + it("cancels during vectorization without publishing completion", async () => { + const statuses: string[] = []; + const controller = new AbortController(); + const cleanup = vi.fn().mockResolvedValue(undefined); + + const result = await runVectorizeQueueJob({ + format: "mobi", + signal: controller.signal, + extract: async () => [{ index: 0, title: "Chapter", content: "text", segments: [] }], + vectorize: async (_chapters, onProgress, signal) => { + onProgress?.({ status: "cancelled" }); + controller.abort(); + signal.throwIfAborted(); + }, + cleanup, + onEvent: (event) => statuses.push(event.status), + }); + + expect(result).toMatchObject({ ok: false, cancelled: true }); + expect(cleanup).not.toHaveBeenCalled(); + expect(statuses).toEqual(["extracting", "vectorizing", "cancelled"]); + expect(statuses).not.toContain("completed"); + }); + + it("publishes an error when core cancellation cleanup rejects", async () => { + const statuses: string[] = []; + const controller = new AbortController(); + const cleanup = vi.fn(); + const cleanupError = new Error("failed to delete partial chunks"); + + const result = await runVectorizeQueueJob({ + signal: controller.signal, + extract: async () => [{ index: 0, title: "Chapter", content: "text", segments: [] }], + vectorize: async () => { + controller.abort(); + throw new VectorizationCleanupError(controller.signal.reason, cleanupError); + }, + cleanup, + onEvent: (event) => statuses.push(event.status), + }); + + expect(result).toMatchObject({ ok: false, cancelled: false, cleanupError }); + expect(cleanup).not.toHaveBeenCalled(); + expect(statuses).toEqual(["extracting", "vectorizing", "error"]); + expect(statuses).not.toContain("cancelled"); + expect(statuses).not.toContain("completed"); + }); + + it("publishes error only after a rejected cleanup attempt", async () => { + const order: string[] = []; + const cleanupError = new Error("cleanup failed"); + + const result = await runVectorizeQueueJob({ + format: "mobi", + extract: async () => { + throw new Error("loader failed"); + }, + vectorize: vi.fn(), + cleanup: async () => { + order.push("cleanup"); + throw cleanupError; + }, + onEvent: (event) => order.push(event.status), + }); + + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected extraction failure"); + expect(result.cleanupError).toBe(cleanupError); + expect(order).toEqual(["extracting", "cleanup", "error"]); + }); + + it("does not double-clean a final state-write rejection or publish completion", async () => { + const statuses: string[] = []; + const cleanup = vi.fn().mockResolvedValue(undefined); + + const result = await runVectorizeQueueJob({ + format: "mobi", + extract: async () => [{ index: 0, title: "Chapter", content: "text", segments: [] }], + vectorize: async (_chapters, onProgress) => { + onProgress?.({ status: "completed" }); + throw new Error("failed to persist vectorized state"); + }, + cleanup, + onEvent: (event) => statuses.push(event.status), + }); + + expect(result.ok).toBe(false); + expect(cleanup).not.toHaveBeenCalled(); + expect(statuses).toEqual(["extracting", "vectorizing", "error"]); + expect(statuses).not.toContain("completed"); + }); +}); diff --git a/packages/app-expo/src/lib/rag/vectorize-queue-job.ts b/packages/app-expo/src/lib/rag/vectorize-queue-job.ts new file mode 100644 index 000000000..5b4cb9913 --- /dev/null +++ b/packages/app-expo/src/lib/rag/vectorize-queue-job.ts @@ -0,0 +1,124 @@ +import { type ChapterData, VectorizationCleanupError } from "@readany/core/rag"; +import { + BookExtractionError, + type BookExtractionErrorCategory, + toBookExtractionError, +} from "./extractor-error"; + +export type VectorizeQueueJobEvent = + | { status: "extracting" } + | { status: "vectorizing"; progress?: Progress } + | { status: "completed" } + | { status: "cancelled" } + | { + status: "error"; + error: unknown; + errorCategory?: BookExtractionErrorCategory; + cleanupError?: unknown; + }; + +export type VectorizeQueueJobResult = + | { ok: true } + | { ok: false; error: unknown; cancelled?: boolean; cleanupError?: unknown }; + +interface VectorizeQueueJobOptions { + format?: string; + signal?: AbortSignal; + extract: (signal: AbortSignal) => Promise; + vectorize: ( + chapters: ChapterData[], + onProgress: ((progress: Progress) => void) | undefined, + signal: AbortSignal, + ) => Promise; + cleanup: () => Promise; + onEvent: (event: VectorizeQueueJobEvent) => void; +} + +function createFallbackSignal(): AbortSignal { + return new AbortController().signal; +} + +export function throwIfQueueJobAborted(signal: AbortSignal): void { + if (!signal.aborted) return; + const reason = signal.reason; + if (reason instanceof Error) { + if (reason.name !== "AbortError") reason.name = "AbortError"; + throw reason; + } + const error = new Error("Vectorization cancelled"); + error.name = "AbortError"; + throw error; +} + +function isAbortFailure(error: unknown, signal: AbortSignal): boolean { + return signal.aborted || (error instanceof Error && error.name === "AbortError"); +} + +function isTerminalProgress(progress: unknown): boolean { + return ( + typeof progress === "object" && + progress !== null && + "status" in progress && + (progress.status === "completed" || + progress.status === "error" || + progress.status === "cancelled") + ); +} + +export async function runVectorizeQueueJob( + options: VectorizeQueueJobOptions, +): Promise { + let phase: "extracting" | "vectorizing" = "extracting"; + const signal = options.signal ?? createFallbackSignal(); + options.onEvent({ status: "extracting" }); + + try { + throwIfQueueJobAborted(signal); + const chapters = await options.extract(signal); + throwIfQueueJobAborted(signal); + if (!chapters.length) { + throw toBookExtractionError(new Error("No chapters extracted from book"), options.format); + } + + phase = "vectorizing"; + options.onEvent({ status: "vectorizing" }); + await options.vectorize( + chapters, + (progress) => { + if (!isTerminalProgress(progress)) { + options.onEvent({ status: "vectorizing", progress }); + } + }, + signal, + ); + throwIfQueueJobAborted(signal); + + options.onEvent({ status: "completed" }); + return { ok: true }; + } catch (error) { + const failure = phase === "extracting" ? toBookExtractionError(error, options.format) : error; + let cleanupError = error instanceof VectorizationCleanupError ? error.cleanupError : undefined; + if (phase === "extracting") { + try { + await options.cleanup(); + } catch (errorDuringCleanup) { + cleanupError = errorDuringCleanup; + } + } + const cancelled = !cleanupError && isAbortFailure(error, signal); + + if (cancelled) { + options.onEvent({ status: "cancelled" }); + } else { + options.onEvent({ + status: "error", + error: failure, + errorCategory: failure instanceof BookExtractionError ? failure.category : undefined, + cleanupError, + }); + } + return cleanupError + ? { ok: false, error: failure, cancelled, cleanupError } + : { ok: false, error: failure, cancelled }; + } +} diff --git a/packages/app-expo/src/lib/rag/vectorize-trigger.test.ts b/packages/app-expo/src/lib/rag/vectorize-trigger.test.ts new file mode 100644 index 000000000..0615e9e4d --- /dev/null +++ b/packages/app-expo/src/lib/rag/vectorize-trigger.test.ts @@ -0,0 +1,64 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + coreTriggerVectorizeBook: vi.fn(), + updateBook: vi.fn(), + updateBookStrict: vi.fn(), +})); + +vi.mock("@readany/core/rag", () => ({ + resetBookVectorization: vi.fn(), + triggerVectorizeBook: mocks.coreTriggerVectorizeBook, +})); + +vi.mock("@/stores/library-store", () => ({ + useLibraryStore: { + getState: () => ({ + updateBook: mocks.updateBook, + updateBookStrict: mocks.updateBookStrict, + }), + }, +})); + +vi.mock("@/stores/vector-model-store", () => ({ + useVectorModelStore: { + getState: () => ({ + vectorModelEnabled: true, + vectorModelMode: "builtin", + selectedBuiltinModelId: "test-model", + getSelectedVectorModel: () => null, + }), + }, +})); + +import { triggerVectorizeBook } from "./vectorize-trigger"; + +describe("triggerVectorizeBook persistence boundary", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.updateBook.mockResolvedValue(undefined); + mocks.updateBookStrict.mockRejectedValue(new Error("database write failed")); + mocks.coreTriggerVectorizeBook.mockImplementation( + async (_bookId, _chapters, _config, callbacks) => { + await callbacks.onBookUpdate("book-1", { + isVectorized: true, + vectorizeProgress: 1, + }); + }, + ); + }); + + it("propagates a failed durable vectorized-state write", async () => { + await expect( + triggerVectorizeBook("book-1", "book.mobi", [ + { index: 0, title: "Chapter", content: "text", segments: [] }, + ]), + ).rejects.toThrow("database write failed"); + + expect(mocks.updateBookStrict).toHaveBeenCalledWith("book-1", { + isVectorized: true, + vectorizeProgress: 1, + }); + expect(mocks.updateBook).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/app-expo/src/lib/rag/vectorize-trigger.ts b/packages/app-expo/src/lib/rag/vectorize-trigger.ts index d9b4db8a0..964ec6072 100644 --- a/packages/app-expo/src/lib/rag/vectorize-trigger.ts +++ b/packages/app-expo/src/lib/rag/vectorize-trigger.ts @@ -1,6 +1,9 @@ import { useLibraryStore } from "@/stores/library-store"; import { useVectorModelStore } from "@/stores/vector-model-store"; -import { triggerVectorizeBook as coreTriggerVectorizeBook } from "@readany/core/rag"; +import { + resetBookVectorization as coreResetBookVectorization, + triggerVectorizeBook as coreTriggerVectorizeBook, +} from "@readany/core/rag"; import type { ChapterData, VectorizeStatusCallback, @@ -9,11 +12,19 @@ import type { export type { VectorizeStatusCallback }; +export async function resetBookVectorization(bookId: string): Promise { + await coreResetBookVectorization(bookId, { + onBookUpdate: useLibraryStore.getState().updateBookStrict, + onBookReset: useLibraryStore.getState().resetBookVectorizationState, + }); +} + export async function triggerVectorizeBook( bookId: string, _filePath: string, chapters: ChapterData[], onProgress?: VectorizeStatusCallback, + signal?: AbortSignal, ): Promise { const vmState = useVectorModelStore.getState(); @@ -35,9 +46,10 @@ export async function triggerVectorizeBook( // 2. Build callbacks for state updates const callbacks = { - onBookUpdate: useLibraryStore.getState().updateBook, + onBookUpdate: useLibraryStore.getState().updateBookStrict, + onBookReset: useLibraryStore.getState().resetBookVectorizationState, }; // 3. Delegate to core vectorization pipeline which does the chunking & embedding - await coreTriggerVectorizeBook(bookId, chapters, config, callbacks, onProgress); + await coreTriggerVectorizeBook(bookId, chapters, config, callbacks, onProgress, signal); } diff --git a/packages/app-expo/src/screens/LibraryScreen.tsx b/packages/app-expo/src/screens/LibraryScreen.tsx index d5f3a1fb6..5e464e1b4 100644 --- a/packages/app-expo/src/screens/LibraryScreen.tsx +++ b/packages/app-expo/src/screens/LibraryScreen.tsx @@ -23,6 +23,7 @@ import { SyncButton } from "@/components/ui/SyncButton"; import { useResponsiveLayout } from "@/hooks/use-responsive-layout"; import { openMobileBook } from "@/lib/library/open-mobile-book"; import { setCallback, setExtractorRef } from "@/lib/rag/auto-vectorize-service"; +import { getBookExtractionErrorMessageKeys } from "@/lib/rag/extractor-error"; import type { RootStackParamList } from "@/navigation/RootNavigator"; import { WebDavConnectSheet } from "@/screens/library/WebDavConnectSheet"; import { WebDavImportSourceSheet } from "@/screens/library/WebDavImportSourceSheet"; @@ -232,9 +233,8 @@ export function LibraryScreen() { onSuccess: () => {}, }); - const { vectorQueue, vectorizingBookId, vectorProgress, handleVectorize } = useVectorizationQueue( - { extractorRef, nav }, - ); + const { vectorQueue, vectorizingBookId, vectorProgress, handleVectorize, cancelVectorize } = + useVectorizationQueue({ extractorRef, nav }); const openSearch = useCallback(() => { setShowSearch(true); @@ -294,6 +294,8 @@ export function LibraryScreen() { return extractorRef.current.extractChapters( bytesToBase64(bytes), mimeTypes[String(book.format || "").toLowerCase()] || "application/epub+zip", + book.format, + filePath, ); }, }); @@ -301,13 +303,24 @@ export function LibraryScreen() { console.log( `[AutoVectorize] Book ${bookId}: ${progress.status} (${Math.round(progress.progress * 100)}%)`, ); + if (progress.status === "error") { + if (progress.errorCategory) { + const keys = getBookExtractionErrorMessageKeys(progress.errorCategory); + Alert.alert(t(keys.title), t(keys.description)); + } else { + Alert.alert( + t("vectorize.vectorizationFailedTitle"), + t("vectorize.vectorizationFailedDesc"), + ); + } + } }); return () => { setExtractorRef(null); setFallbackContentProvider(null); setCallback(null); }; - }, []); + }, [t]); useEffect(() => { return onLibraryChanged((deletedTags) => loadBooks(deletedTags)); @@ -754,6 +767,7 @@ export function LibraryScreen() { onShowDetails={handleShowDetails} onManageTags={handleManageTags} onVectorize={handleVectorize} + onCancelVectorize={cancelVectorize} isVectorizing={vectorizingBookId === item.book.id} isQueued={vectorQueue.some((b) => b.id === item.book.id)} vectorProgress={vectorizingBookId === item.book.id ? vectorProgress : null} @@ -774,6 +788,7 @@ export function LibraryScreen() { handleShowDetails, handleOpen, handleVectorize, + cancelVectorize, removeBook, s.gridItem, selectedBookIds, diff --git a/packages/app-expo/src/screens/library/useVectorizationQueue.ts b/packages/app-expo/src/screens/library/useVectorizationQueue.ts index fa280e992..5a21ab615 100644 --- a/packages/app-expo/src/screens/library/useVectorizationQueue.ts +++ b/packages/app-expo/src/screens/library/useVectorizationQueue.ts @@ -1,6 +1,9 @@ import type { ExtractorRef } from "@/components/rag/ExtractorWebView"; import { inspectMobileBookForVectorize } from "@/lib/rag/auto-vectorize-book"; -import { triggerVectorizeBook } from "@/lib/rag/vectorize-trigger"; +import { getBookExtractionErrorMessageKeys } from "@/lib/rag/extractor-error"; +import { MOBILE_VECTORIZE_UNSUPPORTED_FORMAT_DESCRIPTION } from "@/lib/rag/mobile-vectorize-capability"; +import { runVectorizeQueueJob, throwIfQueueJobAborted } from "@/lib/rag/vectorize-queue-job"; +import { resetBookVectorization, triggerVectorizeBook } from "@/lib/rag/vectorize-trigger"; import type { RootStackParamList } from "@/navigation/RootNavigator"; import { useVectorModelStore } from "@/stores/vector-model-store"; import type { NativeStackNavigationProp } from "@react-navigation/native-stack"; @@ -9,6 +12,7 @@ import * as FileSystem from "expo-file-system/legacy"; import { useCallback, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import { Alert } from "react-native"; +import { VectorizationQueue } from "./vectorization-queue"; type Nav = NativeStackNavigationProp; @@ -20,68 +24,107 @@ interface UseVectorizationQueueOptions { export function useVectorizationQueue({ extractorRef, nav }: UseVectorizationQueueOptions) { const { t } = useTranslation(); const [vectorQueue, setVectorQueue] = useState([]); - const vectorQueueRef = useRef([]); + const queueRef = useRef(new VectorizationQueue()); const [vectorizingBookId, setVectorizingBookId] = useState(null); const [vectorizingBookTitle, setVectorizingBookTitle] = useState(""); const [vectorProgress, setVectorProgress] = useState(null); const isProcessingRef = useRef(false); const processOneBook = useCallback( - async (book: Book) => { + async (book: Book, signal: AbortSignal) => { setVectorizingBookId(book.id); setVectorizingBookTitle(book.meta.title); - setVectorProgress({ - bookId: book.id, - status: "chunking", - processedChunks: 0, - totalChunks: 0, + return runVectorizeQueueJob({ + format: book.format, + signal, + extract: async (jobSignal) => { + throwIfQueueJobAborted(jobSignal); + if (!extractorRef.current) throw new Error("Extractor WebView not ready"); + + const info = await inspectMobileBookForVectorize(book); + if (!info.canVectorize || !info.mimeType) { + throw new Error(`Book cannot be vectorized on mobile: ${info.reason ?? "unknown"}`); + } + + const base64 = await FileSystem.readAsStringAsync(info.absPath, { + encoding: FileSystem.EncodingType.Base64, + }); + throwIfQueueJobAborted(jobSignal); + return extractorRef.current.extractChapters( + base64, + info.mimeType, + book.format, + info.absPath, + jobSignal, + ); + }, + vectorize: (chapters, onProgress, jobSignal) => + triggerVectorizeBook(book.id, book.filePath, chapters, onProgress, jobSignal), + cleanup: () => resetBookVectorization(book.id), + onEvent: (event) => { + if (event.status === "extracting") { + setVectorProgress({ + bookId: book.id, + status: "chunking", + processedChunks: 0, + totalChunks: 0, + }); + } else if (event.status === "vectorizing") { + if (event.progress) setVectorProgress({ ...event.progress }); + } else if (event.status === "completed") { + setVectorProgress({ + bookId: book.id, + status: "completed", + processedChunks: 1, + totalChunks: 1, + }); + } else if (event.status === "cancelled") { + setVectorProgress({ + bookId: book.id, + status: "cancelled", + processedChunks: 0, + totalChunks: 0, + }); + } else { + console.error( + `[useVectorizationQueue] Vectorization failed for "${book.meta.title}":`, + event.error, + ); + if (event.cleanupError) { + console.error( + `[useVectorizationQueue] Failed to clean up "${book.meta.title}":`, + event.cleanupError, + ); + } + setVectorProgress({ + bookId: book.id, + status: "error", + processedChunks: 0, + totalChunks: 0, + }); + + if (event.cleanupError) { + Alert.alert( + t("vectorize.cleanupFailedTitle", "Couldn't fully clean up indexing"), + t( + "vectorize.cleanupFailedDesc", + "The book remains marked not indexed, but some partial search data could not be removed. Retry indexing, then restart the app if this book still appears in search.", + ), + ); + } else if (event.errorCategory) { + const keys = getBookExtractionErrorMessageKeys(event.errorCategory); + Alert.alert(t(keys.title), t(keys.description)); + } else { + Alert.alert( + t("vectorize.vectorizationFailedTitle"), + t("vectorize.vectorizationFailedDesc"), + ); + } + } + }, }); - - try { - if (!extractorRef.current) { - throw new Error("Extractor WebView not ready"); - } - - const info = await inspectMobileBookForVectorize(book); - if (!info.canVectorize || !info.mimeType) { - throw new Error(`Book cannot be vectorized on mobile: ${info.reason ?? "unknown"}`); - } - - const base64 = await FileSystem.readAsStringAsync(info.absPath, { - encoding: FileSystem.EncodingType.Base64, - }); - - const chapters = await extractorRef.current.extractChapters(base64, info.mimeType); - if (!chapters || chapters.length === 0) { - throw new Error("No chapters extracted from book"); - } - - await triggerVectorizeBook(book.id, book.filePath, chapters, (progress) => { - setVectorProgress({ ...progress }); - }); - - setVectorProgress({ - bookId: book.id, - status: "completed", - processedChunks: 1, - totalChunks: 1, - }); - await new Promise((resolve) => setTimeout(resolve, 800)); - } catch (err) { - console.error( - `[useVectorizationQueue] Vectorization failed for "${book.meta.title}":`, - err, - ); - setVectorProgress({ - bookId: book.id, - status: "error", - processedChunks: 0, - totalChunks: 0, - }); - await new Promise((resolve) => setTimeout(resolve, 1500)); - } }, - [extractorRef], + [extractorRef, t], ); const processQueue = useCallback(async () => { @@ -89,12 +132,20 @@ export function useVectorizationQueue({ extractorRef, nav }: UseVectorizationQue isProcessingRef.current = true; try { - while (vectorQueueRef.current.length > 0) { - const [nextBook, ...remainingBooks] = vectorQueueRef.current; - if (!nextBook) break; - vectorQueueRef.current = remainingBooks; - setVectorQueue([...vectorQueueRef.current]); - await processOneBook(nextBook); + while (true) { + const nextJob = queueRef.current.startNext(); + if (!nextJob) break; + setVectorQueue(queueRef.current.snapshot()); + try { + const result = await processOneBook(nextJob.book, nextJob.signal); + queueRef.current.markTerminal(nextJob.book.id); + await new Promise((resolve) => + setTimeout(resolve, result.ok ? 800 : result.cancelled ? 500 : 1500), + ); + } finally { + queueRef.current.markTerminal(nextJob.book.id); + queueRef.current.finish(nextJob.book.id); + } } } finally { isProcessingRef.current = false; @@ -110,10 +161,7 @@ export function useVectorizationQueue({ extractorRef, nav }: UseVectorizationQue if (info.reason === "unsupported-format") { Alert.alert( t("vectorize.unsupportedFormatTitle", "Unsupported format"), - t( - "vectorize.unsupportedFormatDesc", - "Mobile vectorization currently supports EPUB, PDF, TXT, and UMD books.", - ), + t("vectorize.unsupportedFormatDesc", MOBILE_VECTORIZE_UNSUPPORTED_FORMAT_DESCRIPTION), ); return; } @@ -127,11 +175,8 @@ export function useVectorizationQueue({ extractorRef, nav }: UseVectorizationQue ); return; } - const alreadyQueued = vectorQueueRef.current.some((b) => b.id === book.id); - if (alreadyQueued || vectorizingBookId === book.id) return; - - vectorQueueRef.current = [...vectorQueueRef.current, book]; - setVectorQueue([...vectorQueueRef.current]); + if (!queueRef.current.enqueue(book)) return; + setVectorQueue(queueRef.current.snapshot()); if (!isProcessingRef.current) { processQueue(); @@ -158,14 +203,33 @@ export function useVectorizationQueue({ extractorRef, nav }: UseVectorizationQue ); }); }, - [nav, t, vectorizingBookId, processQueue], + [nav, t, processQueue], ); + const cancelVectorize = useCallback((bookId: string) => { + const outcome = queueRef.current.cancel(bookId); + if (outcome === "queued") { + setVectorQueue(queueRef.current.snapshot()); + } else if (outcome === "active") { + setVectorProgress((current) => + current + ? { ...current, status: "cancelling" } + : { + bookId, + status: "cancelling", + processedChunks: 0, + totalChunks: 0, + }, + ); + } + }, []); + return { vectorQueue, vectorizingBookId, vectorizingBookTitle, vectorProgress, handleVectorize, + cancelVectorize, }; } diff --git a/packages/app-expo/src/screens/library/vectorization-queue.test.ts b/packages/app-expo/src/screens/library/vectorization-queue.test.ts new file mode 100644 index 000000000..da6a70b5d --- /dev/null +++ b/packages/app-expo/src/screens/library/vectorization-queue.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; +import { VectorizationQueue } from "./vectorization-queue"; + +describe("VectorizationQueue", () => { + it("removes a queued book without disturbing the active job", () => { + const queue = new VectorizationQueue<{ id: string }>(); + queue.enqueue({ id: "active" }); + queue.enqueue({ id: "queued" }); + const active = queue.startNext(); + + expect(queue.cancel("queued")).toBe("queued"); + expect(active?.book.id).toBe("active"); + expect(active?.signal.aborted).toBe(false); + expect(queue.snapshot()).toEqual([]); + }); + + it("aborts the active job and keeps it active until cleanup finishes", () => { + const queue = new VectorizationQueue<{ id: string }>(); + queue.enqueue({ id: "active" }); + const active = queue.startNext(); + + expect(queue.cancel("active")).toBe("active"); + expect(active?.signal.aborted).toBe(true); + expect(queue.activeBookId).toBe("active"); + + queue.finish("active"); + expect(queue.activeBookId).toBeNull(); + }); + + it.each(["completed", "error", "cancelled"])( + "ignores a late cancel tap while %s is being presented", + () => { + const queue = new VectorizationQueue<{ id: string }>(); + queue.enqueue({ id: "active" }); + const active = queue.startNext(); + queue.markTerminal("active"); + + expect(queue.cancel("active")).toBe("not-cancellable"); + expect(active?.signal.aborted).toBe(false); + expect(queue.activeBookId).toBe("active"); + }, + ); + + it("makes a second tap inert after cancellation begins", () => { + const queue = new VectorizationQueue<{ id: string }>(); + queue.enqueue({ id: "active" }); + queue.startNext(); + + expect(queue.cancel("active")).toBe("active"); + expect(queue.cancel("active")).toBe("not-cancellable"); + }); +}); diff --git a/packages/app-expo/src/screens/library/vectorization-queue.ts b/packages/app-expo/src/screens/library/vectorization-queue.ts new file mode 100644 index 000000000..7d0faf874 --- /dev/null +++ b/packages/app-expo/src/screens/library/vectorization-queue.ts @@ -0,0 +1,50 @@ +export class VectorizationQueue { + private queuedBooks: Book[] = []; + private active: { book: Book; controller: AbortController; cancellable: boolean } | null = null; + + get activeBookId(): string | null { + return this.active?.book.id ?? null; + } + + snapshot(): Book[] { + return [...this.queuedBooks]; + } + + enqueue(book: Book): boolean { + if (this.active?.book.id === book.id || this.queuedBooks.some((item) => item.id === book.id)) { + return false; + } + this.queuedBooks.push(book); + return true; + } + + startNext(): { book: Book; signal: AbortSignal } | null { + if (this.active) return null; + const book = this.queuedBooks.shift(); + if (!book) return null; + const controller = new AbortController(); + this.active = { book, controller, cancellable: true }; + return { book, signal: controller.signal }; + } + + cancel(bookId: string): "active" | "queued" | "not-cancellable" | "not-found" { + if (this.active?.book.id === bookId) { + if (!this.active.cancellable) return "not-cancellable"; + this.active.cancellable = false; + this.active.controller.abort(); + return "active"; + } + const nextQueue = this.queuedBooks.filter((book) => book.id !== bookId); + if (nextQueue.length === this.queuedBooks.length) return "not-found"; + this.queuedBooks = nextQueue; + return "queued"; + } + + markTerminal(bookId: string): void { + if (this.active?.book.id === bookId) this.active.cancellable = false; + } + + finish(bookId: string): void { + if (this.active?.book.id === bookId) this.active = null; + } +} diff --git a/packages/app-expo/src/stores/library-store.ts b/packages/app-expo/src/stores/library-store.ts index d9014c14c..6c00832ca 100644 --- a/packages/app-expo/src/stores/library-store.ts +++ b/packages/app-expo/src/stores/library-store.ts @@ -4,6 +4,7 @@ import { extractBookMetadataFromFile, } from "@/lib/book/metadata-extractor"; import { queueBook as queueAutoVectorize } from "@/lib/rag/auto-vectorize-service"; +import { getMobileVectorizeCapability } from "@/lib/rag/mobile-vectorize-capability"; import { type ImportBooksResult, createEmptyImportBooksResult, @@ -75,6 +76,8 @@ export interface LibraryState { addBook: (book: Book) => Promise; removeBook: (bookId: string, options?: RemoveBookOptions) => Promise; updateBook: (bookId: string, updates: Partial) => Promise; + updateBookStrict: (bookId: string, updates: Partial) => Promise; + resetBookVectorizationState: (bookId: string) => Promise; setFilter: (filter: Partial) => void; setViewMode: (mode: LibraryViewMode) => void; setSortField: (field: SortField) => void; @@ -200,10 +203,6 @@ async function extractMobileImportMetadata(params: { }; } -function shouldAutoVectorizeMobile(format: Book["format"]): boolean { - return format === "epub" || format === "txt" || format === "umd"; -} - /** * Ensure raw bytes are UTF-8 encoded. Hermes (React Native) only supports * UTF-8 in TextDecoder — GBK/GB18030/Shift-JIS etc. are NOT supported. @@ -830,6 +829,33 @@ export const useLibraryStore = create((set, get) => ({ ); }, + updateBookStrict: async (bookId, updates) => { + await persistBookUpdate(bookId, updates); + set((state) => ({ + books: state.books.map((b) => (b.id === bookId ? { ...b, ...updates } : b)), + allTags: + updates.tags !== undefined + ? Array.from(new Set([...state.allTags, ...updates.tags])).sort() + : state.allTags, + })); + debouncedSave("library-books", get().books); + }, + + resetBookVectorizationState: async (bookId) => { + const reset = { isVectorized: false, vectorizeProgress: 0 } as const; + let persistenceError: unknown; + try { + await persistBookUpdate(bookId, reset); + } catch (error) { + persistenceError = error; + } + set((state) => ({ + books: state.books.map((book) => (book.id === bookId ? { ...book, ...reset } : book)), + })); + debouncedSave("library-books", get().books); + if (persistenceError) throw persistenceError; + }, + setFilter: (filter) => set((state) => ({ filter: { ...state.filter, ...filter } })), setViewMode: (mode) => set({ viewMode: mode }), setSortField: (field) => set((state) => ({ filter: { ...state.filter, sortField: field } })), @@ -995,14 +1021,15 @@ export const useLibraryStore = create((set, get) => ({ // successful import doesn't get reported as a failed import. try { const vmState = useVectorModelStore.getState(); + const vectorizeCapability = getMobileVectorizeCapability(book.format); if ( vmState.autoVectorizeOnImport && vmState.vectorModelEnabled && vmState.hasVectorCapability() && - shouldAutoVectorizeMobile("txt") + vectorizeCapability.supported ) { const base64 = bytesToBase64(conversion.epubBytes); - queueAutoVectorize(book, base64, "application/epub+zip"); + queueAutoVectorize(book, base64, vectorizeCapability.mimeType); } } catch (autoVectorizeErr) { console.warn( @@ -1116,14 +1143,15 @@ export const useLibraryStore = create((set, get) => ({ try { const vmState = useVectorModelStore.getState(); + const vectorizeCapability = getMobileVectorizeCapability(book.format); if ( vmState.autoVectorizeOnImport && vmState.vectorModelEnabled && vmState.hasVectorCapability() && - shouldAutoVectorizeMobile("umd") + vectorizeCapability.supported ) { const base64 = bytesToBase64(conversion.epubBytes); - queueAutoVectorize(book, base64, "application/epub+zip"); + queueAutoVectorize(book, base64, vectorizeCapability.mimeType); } } catch (autoVectorizeErr) { console.warn( @@ -1235,28 +1263,16 @@ export const useLibraryStore = create((set, get) => ({ // successful import doesn't get reported as a failed import. try { const vmState = useVectorModelStore.getState(); + const vectorizeCapability = getMobileVectorizeCapability(format); if ( vmState.autoVectorizeOnImport && vmState.vectorModelEnabled && vmState.hasVectorCapability() && - shouldAutoVectorizeMobile(format) + vectorizeCapability.supported ) { const sourceBytes = await platform.readFile(filePath); const base64 = bytesToBase64(sourceBytes); - const mimeTypes: Record = { - epub: "application/epub+zip", - pdf: "application/pdf", - mobi: "application/x-mobipocket-ebook", - azw: "application/vnd.amazon.ebook", - azw3: "application/vnd.amazon.ebook", - cbz: "application/vnd.comicbook+zip", - cbr: "application/vnd.comicbook+zip", - fb2: "application/x-fictionbook+xml", - fbz: "application/x-zip-compressed-fb2", - txt: "text/plain", - }; - const mimeType = mimeTypes[format] || "application/epub+zip"; - queueAutoVectorize(book, base64, mimeType); + queueAutoVectorize(book, base64, vectorizeCapability.mimeType); } else if (vmState.autoVectorizeOnImport && vmState.vectorModelEnabled) { console.warn( `[importBooks] Skip auto-vectorize for unsupported mobile import: ${fileName} (${fileSize} bytes, format=${format})`, diff --git a/packages/app-expo/src/stores/library-store.vectorization.test.ts b/packages/app-expo/src/stores/library-store.vectorization.test.ts new file mode 100644 index 000000000..e7cd66215 --- /dev/null +++ b/packages/app-expo/src/stores/library-store.vectorization.test.ts @@ -0,0 +1,133 @@ +import type { Book } from "@readany/core/types"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + updateBook: vi.fn(), + debouncedSave: vi.fn(), +})); + +vi.mock("@/lib/book/metadata-extractor", () => ({ + createRangeReadableFile: vi.fn(), + extractBookMetadata: vi.fn(), + extractBookMetadataFromFile: vi.fn(), +})); + +vi.mock("@/lib/rag/auto-vectorize-service", () => ({ queueBook: vi.fn() })); +vi.mock("@/lib/rag/mobile-vectorize-capability", () => ({ + getMobileVectorizeCapability: vi.fn(), +})); +vi.mock("./vector-model-store", () => ({ + useVectorModelStore: { getState: vi.fn() }, +})); + +vi.mock("@readany/core/db/database", () => ({ + updateBook: mocks.updateBook, +})); + +vi.mock("@readany/core/db/write-retry", () => ({ + runWithDbRetry: (operation: () => Promise) => operation(), +})); + +vi.mock("./persist", () => ({ + debouncedSave: mocks.debouncedSave, + loadFromFS: vi.fn(), +})); + +import { useLibraryStore } from "./library-store"; + +const book: Book = { + id: "book-1", + filePath: "books/book.mobi", + format: "mobi", + meta: { title: "Book", author: "Author" }, + addedAt: 1, + updatedAt: 1, + progress: 0, + isVectorized: false, + vectorizeProgress: 0, + tags: [], + syncStatus: "local", +}; + +describe("library store strict vectorization updates", () => { + beforeEach(() => { + vi.clearAllMocks(); + useLibraryStore.getState().setBooks([{ ...book }]); + }); + + afterEach(() => { + useLibraryStore.getState().setBooks([]); + }); + + it("keeps completion invisible until the database write resolves", async () => { + let resolveWrite: (() => void) | undefined; + mocks.updateBook.mockImplementation( + () => + new Promise((resolve) => { + resolveWrite = resolve; + }), + ); + const visibleVectorizedStates: boolean[] = []; + const unsubscribe = useLibraryStore.subscribe((state) => { + if (state.books[0]?.isVectorized) visibleVectorizedStates.push(true); + }); + + const update = useLibraryStore.getState().updateBookStrict("book-1", { + isVectorized: true, + vectorizeProgress: 1, + }); + + expect(useLibraryStore.getState().books[0]?.isVectorized).toBe(false); + expect(mocks.debouncedSave).not.toHaveBeenCalled(); + expect(visibleVectorizedStates).toEqual([]); + + resolveWrite?.(); + await update; + + expect(useLibraryStore.getState().books[0]?.isVectorized).toBe(true); + expect(visibleVectorizedStates).toEqual([true]); + expect(mocks.debouncedSave).toHaveBeenCalledOnce(); + unsubscribe(); + }); + + it("does not expose or cache completion when the database write rejects", async () => { + mocks.updateBook.mockRejectedValue(new Error("database write failed")); + const visibleVectorizedStates: boolean[] = []; + const unsubscribe = useLibraryStore.subscribe((state) => { + if (state.books[0]?.isVectorized) visibleVectorizedStates.push(true); + }); + + await expect( + useLibraryStore.getState().updateBookStrict("book-1", { + isVectorized: true, + vectorizeProgress: 1, + }), + ).rejects.toThrow("database write failed"); + + expect(useLibraryStore.getState().books[0]?.isVectorized).toBe(false); + expect(useLibraryStore.getState().books[0]?.vectorizeProgress).toBe(0); + expect(visibleVectorizedStates).toEqual([]); + expect(mocks.debouncedSave).not.toHaveBeenCalled(); + unsubscribe(); + }); + + it("clears visible and cached indexed state when the durable cleanup write rejects", async () => { + useLibraryStore.getState().setBooks([{ ...book, isVectorized: true, vectorizeProgress: 1 }]); + mocks.updateBook.mockRejectedValue(new Error("database write failed")); + + await expect(useLibraryStore.getState().resetBookVectorizationState("book-1")).rejects.toThrow( + "database write failed", + ); + + expect(useLibraryStore.getState().books[0]).toMatchObject({ + isVectorized: false, + vectorizeProgress: 0, + }); + expect(mocks.debouncedSave).toHaveBeenCalledWith( + "library-books", + expect.arrayContaining([ + expect.objectContaining({ id: "book-1", isVectorized: false, vectorizeProgress: 0 }), + ]), + ); + }); +}); diff --git a/packages/app/src/lib/rag/vectorize-trigger.test.ts b/packages/app/src/lib/rag/vectorize-trigger.test.ts new file mode 100644 index 000000000..2d559d3ab --- /dev/null +++ b/packages/app/src/lib/rag/vectorize-trigger.test.ts @@ -0,0 +1,67 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + coreTriggerVectorizeBook: vi.fn(), + updateBook: vi.fn(), + updateBookStrict: vi.fn(), +})); + +vi.mock("@readany/core/rag", () => ({ + triggerVectorizeBook: mocks.coreTriggerVectorizeBook, +})); +vi.mock("@/lib/storage/desktop-library-root", () => ({ + resolveDesktopDataPath: vi.fn(async (path: string) => path), +})); +vi.mock("@/lib/reader/document-loader", () => ({})); +vi.mock("./book-extractor", () => ({ + extractBookChapters: vi.fn(async () => [ + { index: 0, title: "Chapter", content: "text", segments: [] }, + ]), +})); +vi.mock("@/stores/library-store", () => ({ + useLibraryStore: { + getState: () => ({ + updateBook: mocks.updateBook, + updateBookStrict: mocks.updateBookStrict, + }), + }, +})); +vi.mock("@/stores/vector-model-store", () => ({ + useVectorModelStore: { + getState: () => ({ + vectorModelEnabled: true, + vectorModelMode: "builtin", + selectedBuiltinModelId: "test-model", + getSelectedVectorModel: () => null, + }), + }, +})); + +import { triggerVectorizeBook } from "./vectorize-trigger"; + +describe("desktop vectorization persistence adapter", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.updateBookStrict.mockRejectedValue(new Error("database write failed")); + mocks.coreTriggerVectorizeBook.mockImplementation( + async (_bookId, _chapters, _config, callbacks) => { + await callbacks.onBookUpdate("book-1", { + isVectorized: true, + vectorizeProgress: 1, + }); + }, + ); + }); + + it("propagates a rejected strict state write from the core boundary", async () => { + await expect(triggerVectorizeBook("book-1", "book.epub")).rejects.toThrow( + "database write failed", + ); + + expect(mocks.updateBookStrict).toHaveBeenCalledWith("book-1", { + isVectorized: true, + vectorizeProgress: 1, + }); + expect(mocks.updateBook).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/app/src/lib/rag/vectorize-trigger.ts b/packages/app/src/lib/rag/vectorize-trigger.ts index 331fa7c13..61d478118 100644 --- a/packages/app/src/lib/rag/vectorize-trigger.ts +++ b/packages/app/src/lib/rag/vectorize-trigger.ts @@ -49,7 +49,8 @@ export async function triggerVectorizeBook( // Build callbacks that write back to Zustand store const callbacks = { - onBookUpdate: useLibraryStore.getState().updateBook, + onBookUpdate: useLibraryStore.getState().updateBookStrict, + onBookReset: useLibraryStore.getState().resetBookVectorizationState, }; // Extract chapters from the book file (platform-specific: Tauri + foliate-js) diff --git a/packages/app/src/stores/library-store.ts b/packages/app/src/stores/library-store.ts index 73fb2b85f..f1df0a8b4 100644 --- a/packages/app/src/stores/library-store.ts +++ b/packages/app/src/stores/library-store.ts @@ -354,6 +354,8 @@ export interface LibraryState { addBook: (book: Book) => void; removeBook: (bookId: string, options?: RemoveBookOptions) => Promise; updateBook: (bookId: string, updates: Partial) => Promise; + updateBookStrict: (bookId: string, updates: Partial) => Promise; + resetBookVectorizationState: (bookId: string) => Promise; setFilter: (filter: Partial) => void; setViewMode: (mode: LibraryViewMode) => void; setSortField: (field: SortField) => void; @@ -854,6 +856,33 @@ export const useLibraryStore = create((set, get) => ({ .catch((err) => console.error("Failed to update book in database:", err)); }, + updateBookStrict: async (bookId, updates) => { + await db.updateBook(bookId, updates); + set((state) => ({ + books: state.books.map((book) => (book.id === bookId ? { ...book, ...updates } : book)), + allTags: + updates.tags !== undefined + ? Array.from(new Set([...state.allTags, ...updates.tags])).sort() + : state.allTags, + })); + debouncedSave("library-books", get().books); + }, + + resetBookVectorizationState: async (bookId) => { + const reset = { isVectorized: false, vectorizeProgress: 0 } as const; + let persistenceError: unknown; + try { + await db.updateBook(bookId, reset); + } catch (error) { + persistenceError = error; + } + set((state) => ({ + books: state.books.map((book) => (book.id === bookId ? { ...book, ...reset } : book)), + })); + debouncedSave("library-books", get().books); + if (persistenceError) throw persistenceError; + }, + setFilter: (filter) => set((state) => ({ filter: { ...state.filter, ...filter } })), setViewMode: (mode) => set({ viewMode: mode }), diff --git a/packages/app/src/stores/library-store.vectorization.test.ts b/packages/app/src/stores/library-store.vectorization.test.ts new file mode 100644 index 000000000..0866df7f0 --- /dev/null +++ b/packages/app/src/stores/library-store.vectorization.test.ts @@ -0,0 +1,61 @@ +import type { Book } from "@readany/core/types"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + updateBook: vi.fn(), + debouncedSave: vi.fn(), +})); + +vi.mock("@/lib/db/database", () => ({ updateBook: mocks.updateBook })); +vi.mock("@/lib/rag/vectorize-trigger", () => ({ triggerVectorizeBook: vi.fn() })); +vi.mock("@/lib/storage/desktop-library-root", () => ({ + getDesktopLibraryRoot: vi.fn(), + isDesktopManagedRelativePath: vi.fn(), + resolveDesktopDataPath: vi.fn(), +})); +vi.mock("@readany/core/stores/persist", () => ({ + debouncedSave: mocks.debouncedSave, + loadFromFS: vi.fn(), +})); +vi.mock("@readany/core/stores/vector-model-store", () => ({ + useVectorModelStore: { getState: vi.fn() }, +})); + +import { useLibraryStore } from "./library-store"; + +const book: Book = { + id: "book-1", + filePath: "books/book.epub", + format: "epub", + meta: { title: "Book", author: "Author" }, + addedAt: 1, + updatedAt: 1, + progress: 0, + isVectorized: false, + vectorizeProgress: 0, + tags: [], + syncStatus: "local", +}; + +describe("desktop library strict vectorization updates", () => { + beforeEach(() => { + vi.clearAllMocks(); + useLibraryStore.getState().setBooks([{ ...book }]); + }); + + afterEach(() => useLibraryStore.getState().setBooks([])); + + it("does not expose or cache completion when the database rejects", async () => { + mocks.updateBook.mockRejectedValue(new Error("database write failed")); + + await expect( + useLibraryStore.getState().updateBookStrict("book-1", { + isVectorized: true, + vectorizeProgress: 1, + }), + ).rejects.toThrow("database write failed"); + + expect(useLibraryStore.getState().books[0]?.isVectorized).toBe(false); + expect(mocks.debouncedSave).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/core/src/i18n/library-vectorize-locales.test.ts b/packages/core/src/i18n/library-vectorize-locales.test.ts new file mode 100644 index 000000000..5477849ba --- /dev/null +++ b/packages/core/src/i18n/library-vectorize-locales.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; +import en from "./locales/en/library.json"; +import es from "./locales/es/library.json"; +import fr from "./locales/fr/library.json"; +import ja from "./locales/ja/library.json"; +import ko from "./locales/ko/library.json"; +import zhTW from "./locales/zh-TW/library.json"; +import zh from "./locales/zh/library.json"; + +const locales = { en, es, fr, ja, ko, zh, "zh-TW": zhTW }; +const VECTORIZE_ERROR_KEYS = [ + "protectedBookTitle", + "protectedBookDesc", + "malformedBookTitle", + "malformedBookDesc", + "unsupportedFormatTitle", + "unsupportedFormatDesc", + "extractionFailedTitle", + "extractionFailedDesc", + "vectorizationFailedTitle", + "vectorizationFailedDesc", + "cleanupFailedTitle", + "cleanupFailedDesc", +] as const; + +describe("library vectorization error translations", () => { + it.each(Object.entries(locales))("defines every actionable message in %s", (_name, locale) => { + for (const key of VECTORIZE_ERROR_KEYS) { + expect(locale.vectorize[key]).toEqual(expect.any(String)); + expect(locale.vectorize[key].trim()).not.toBe(""); + } + }); + + it("states the DRM-free support boundary accurately in English", () => { + expect(en.vectorize.protectedBookDesc).toBe( + "ReadAny can vectorize DRM-free MOBI, AZW, and AZW3 books, but this file appears to be protected.", + ); + }); +}); diff --git a/packages/core/src/i18n/locales/en/library.json b/packages/core/src/i18n/locales/en/library.json index a6229cf6a..97b4664ab 100644 --- a/packages/core/src/i18n/locales/en/library.json +++ b/packages/core/src/i18n/locales/en/library.json @@ -161,6 +161,20 @@ "detailsManagedMetadataNote": "ReadAny maintains this metadata for library display, search, and sync naming. The original book file is not modified.", "bookNotFound": "Book not found" }, + "vectorize": { + "protectedBookTitle": "Protected book", + "protectedBookDesc": "ReadAny can vectorize DRM-free MOBI, AZW, and AZW3 books, but this file appears to be protected.", + "malformedBookTitle": "Damaged or incomplete book", + "malformedBookDesc": "ReadAny couldn't read this book because its file structure appears damaged or incomplete. Try downloading or exporting a fresh DRM-free copy.", + "unsupportedFormatTitle": "Unsupported format", + "unsupportedFormatDesc": "ReadAny can vectorize EPUB, PDF, TXT, UMD, and DRM-free MOBI, AZW, and AZW3 books. This file format is not supported.", + "extractionFailedTitle": "Couldn't read book", + "extractionFailedDesc": "ReadAny couldn't extract readable text from this book. Try downloading or exporting a fresh DRM-free copy.", + "vectorizationFailedTitle": "Couldn't vectorize book", + "vectorizationFailedDesc": "ReadAny couldn't finish vectorizing this book. Check your vector model settings and try again.", + "cleanupFailedTitle": "Couldn't fully clean up indexing", + "cleanupFailedDesc": "The book remains marked not indexed, but some partial search data could not be removed. Retry indexing, then restart the app if this book still appears in search." + }, "home": { "library": "Library", "addBook": "Add Book", @@ -184,6 +198,10 @@ "vec_indexing": "Indexing...", "vec_processing": "Processing...", "vec_queued": "Queued", + "vec_cancel": "Cancel", + "vec_cancelQueued": "Cancel queued vectorization", + "vec_cancelling": "Cancelling…", + "vec_cancelled": "Cancelled", "manageTags": "Manage Tags", "remote": "Download required", "downloading": "Downloading" diff --git a/packages/core/src/i18n/locales/es/library.json b/packages/core/src/i18n/locales/es/library.json index 5dc3c17f2..2f6a875b7 100644 --- a/packages/core/src/i18n/locales/es/library.json +++ b/packages/core/src/i18n/locales/es/library.json @@ -148,6 +148,20 @@ "detailsManagedMetadataNote": "ReadAny mantiene estos metadatos para mostrar, buscar y nombrar en la sincronización. El archivo original del libro no se modifica.", "bookNotFound": "Libro no encontrado" }, + "vectorize": { + "protectedBookTitle": "Libro protegido", + "protectedBookDesc": "ReadAny puede vectorizar libros MOBI, AZW y AZW3 sin DRM, pero este archivo parece estar protegido.", + "malformedBookTitle": "Libro dañado o incompleto", + "malformedBookDesc": "ReadAny no pudo leer este libro porque la estructura del archivo parece dañada o incompleta. Prueba a descargar o exportar una copia nueva sin DRM.", + "unsupportedFormatTitle": "Formato no compatible", + "unsupportedFormatDesc": "ReadAny puede vectorizar EPUB, PDF, TXT, UMD y libros MOBI, AZW y AZW3 sin DRM. Este formato de archivo no es compatible.", + "extractionFailedTitle": "No se pudo leer el libro", + "extractionFailedDesc": "ReadAny no pudo extraer texto legible de este libro. Prueba a descargar o exportar una copia nueva sin DRM.", + "vectorizationFailedTitle": "No se pudo vectorizar el libro", + "vectorizationFailedDesc": "ReadAny no pudo terminar de vectorizar este libro. Comprueba la configuración del modelo vectorial e inténtalo de nuevo.", + "cleanupFailedTitle": "No se pudo limpiar por completo la indexación", + "cleanupFailedDesc": "El libro sigue marcado como no indexado, pero no se pudieron eliminar algunos datos parciales de búsqueda. Vuelve a indexarlo y reinicia la aplicación si aún aparece en la búsqueda." + }, "home": { "library": "Biblioteca", "addBook": "Agregar libro", @@ -170,6 +184,10 @@ "vec_indexing": "Indexando...", "vec_processing": "Procesando...", "vec_queued": "En cola", + "vec_cancel": "Cancelar", + "vec_cancelQueued": "Cancelar vectorización en cola", + "vec_cancelling": "Cancelando…", + "vec_cancelled": "Cancelado", "manageTags": "Administrar etiquetas", "remote": "Requiere descarga", "downloading": "Descargando" diff --git a/packages/core/src/i18n/locales/fr/library.json b/packages/core/src/i18n/locales/fr/library.json index fe3c20224..9703d9b94 100644 --- a/packages/core/src/i18n/locales/fr/library.json +++ b/packages/core/src/i18n/locales/fr/library.json @@ -148,6 +148,20 @@ "detailsManagedMetadataNote": "ReadAny conserve ces métadonnées pour l'affichage, la recherche et le nommage de synchronisation. Le fichier original du livre n'est pas modifié.", "bookNotFound": "Livre introuvable" }, + "vectorize": { + "protectedBookTitle": "Livre protégé", + "protectedBookDesc": "ReadAny peut vectoriser les livres MOBI, AZW et AZW3 sans DRM, mais ce fichier semble être protégé.", + "malformedBookTitle": "Livre endommagé ou incomplet", + "malformedBookDesc": "ReadAny n'a pas pu lire ce livre, car la structure du fichier semble endommagée ou incomplète. Essayez de télécharger ou d'exporter une nouvelle copie sans DRM.", + "unsupportedFormatTitle": "Format non pris en charge", + "unsupportedFormatDesc": "ReadAny peut vectoriser les fichiers EPUB, PDF, TXT et UMD, ainsi que les livres MOBI, AZW et AZW3 sans DRM. Ce format de fichier n'est pas pris en charge.", + "extractionFailedTitle": "Impossible de lire le livre", + "extractionFailedDesc": "ReadAny n'a pas pu extraire de texte lisible de ce livre. Essayez de télécharger ou d'exporter une nouvelle copie sans DRM.", + "vectorizationFailedTitle": "Impossible de vectoriser le livre", + "vectorizationFailedDesc": "ReadAny n'a pas pu terminer la vectorisation de ce livre. Vérifiez les paramètres du modèle vectoriel et réessayez.", + "cleanupFailedTitle": "Nettoyage incomplet de l'indexation", + "cleanupFailedDesc": "Le livre reste indiqué comme non indexé, mais certaines données de recherche partielles n'ont pas pu être supprimées. Relancez l'indexation, puis redémarrez l'application si ce livre apparaît encore dans la recherche." + }, "home": { "library": "Bibliothèque", "addBook": "Ajouter un livre", @@ -170,6 +184,10 @@ "vec_indexing": "Indexation...", "vec_processing": "Traitement...", "vec_queued": "En file d'attente", + "vec_cancel": "Annuler", + "vec_cancelQueued": "Annuler la vectorisation en attente", + "vec_cancelling": "Annulation…", + "vec_cancelled": "Annulée", "manageTags": "Gérer les tags", "remote": "Téléchargement requis", "downloading": "Téléchargement" diff --git a/packages/core/src/i18n/locales/ja/library.json b/packages/core/src/i18n/locales/ja/library.json index efd242e63..42e0b1fe1 100644 --- a/packages/core/src/i18n/locales/ja/library.json +++ b/packages/core/src/i18n/locales/ja/library.json @@ -148,6 +148,20 @@ "detailsManagedMetadataNote": "ReadAny はこのメタデータをライブラリ表示、検索、同期時の命名に使用します。元の書籍ファイルは変更しません。", "bookNotFound": "書籍が見つかりません" }, + "vectorize": { + "protectedBookTitle": "保護された書籍", + "protectedBookDesc": "ReadAny は DRM フリーの MOBI、AZW、AZW3 書籍をベクトル化できますが、このファイルは保護されているようです。", + "malformedBookTitle": "破損または不完全な書籍", + "malformedBookDesc": "ファイル構造が破損しているか不完全なため、この書籍を読み取れませんでした。DRM フリーの新しいコピーをダウンロードまたは書き出してください。", + "unsupportedFormatTitle": "未対応の形式", + "unsupportedFormatDesc": "ReadAny は EPUB、PDF、TXT、UMD、および DRM フリーの MOBI、AZW、AZW3 書籍をベクトル化できます。このファイル形式には対応していません。", + "extractionFailedTitle": "書籍を読み取れませんでした", + "extractionFailedDesc": "この書籍から読み取り可能なテキストを抽出できませんでした。DRM フリーの新しいコピーをダウンロードまたは書き出してください。", + "vectorizationFailedTitle": "書籍をベクトル化できませんでした", + "vectorizationFailedDesc": "この書籍のベクトル化を完了できませんでした。ベクトルモデルの設定を確認して、もう一度お試しください。", + "cleanupFailedTitle": "インデックスを完全に削除できませんでした", + "cleanupFailedDesc": "書籍は未インデックスとして表示されますが、一部の検索データを削除できませんでした。再度インデックスを作成し、検索に残る場合はアプリを再起動してください。" + }, "home": { "library": "ライブラリ", "addBook": "本を追加", @@ -170,6 +184,10 @@ "vec_indexing": "インデックス作成中...", "vec_processing": "処理中...", "vec_queued": "キュー待ち", + "vec_cancel": "キャンセル", + "vec_cancelQueued": "キュー内のベクトル化をキャンセル", + "vec_cancelling": "キャンセル中…", + "vec_cancelled": "キャンセル済み", "manageTags": "タグを管理", "remote": "ダウンロードが必要", "downloading": "ダウンロード中" diff --git a/packages/core/src/i18n/locales/ko/library.json b/packages/core/src/i18n/locales/ko/library.json index 5c216bddb..a61d48f32 100644 --- a/packages/core/src/i18n/locales/ko/library.json +++ b/packages/core/src/i18n/locales/ko/library.json @@ -148,6 +148,20 @@ "detailsManagedMetadataNote": "ReadAny는 이 메타데이터를 서재 표시, 검색, 동기화 이름에 사용합니다. 원본 책 파일은 수정하지 않습니다.", "bookNotFound": "책을 찾을 수 없어요" }, + "vectorize": { + "protectedBookTitle": "보호된 책", + "protectedBookDesc": "ReadAny는 DRM이 없는 MOBI, AZW, AZW3 책을 벡터화할 수 있지만, 이 파일은 보호된 것으로 보입니다.", + "malformedBookTitle": "손상되었거나 불완전한 책", + "malformedBookDesc": "파일 구조가 손상되었거나 불완전하여 이 책을 읽지 못했습니다. DRM이 없는 새 사본을 다운로드하거나 내보낸 뒤 다시 시도하세요.", + "unsupportedFormatTitle": "지원하지 않는 형식", + "unsupportedFormatDesc": "ReadAny는 EPUB, PDF, TXT, UMD 및 DRM이 없는 MOBI, AZW, AZW3 책을 벡터화할 수 있습니다. 이 파일 형식은 지원하지 않습니다.", + "extractionFailedTitle": "책을 읽을 수 없음", + "extractionFailedDesc": "이 책에서 읽을 수 있는 텍스트를 추출하지 못했습니다. DRM이 없는 새 사본을 다운로드하거나 내보낸 뒤 다시 시도하세요.", + "vectorizationFailedTitle": "책을 벡터화할 수 없음", + "vectorizationFailedDesc": "이 책의 벡터화를 완료하지 못했습니다. 벡터 모델 설정을 확인하고 다시 시도하세요.", + "cleanupFailedTitle": "인덱싱을 완전히 정리하지 못함", + "cleanupFailedDesc": "책은 인덱싱되지 않은 상태로 표시되지만 일부 검색 데이터를 제거하지 못했습니다. 다시 인덱싱하고 검색에 계속 나타나면 앱을 다시 시작하세요." + }, "home": { "library": "서재", "addBook": "책 추가", @@ -170,6 +184,10 @@ "vec_indexing": "인덱싱 중...", "vec_processing": "처리 중...", "vec_queued": "대기 중", + "vec_cancel": "취소", + "vec_cancelQueued": "대기 중인 벡터화 취소", + "vec_cancelling": "취소 중…", + "vec_cancelled": "취소됨", "manageTags": "태그 관리", "remote": "다운로드 필요", "downloading": "다운로드 중" diff --git a/packages/core/src/i18n/locales/zh-TW/library.json b/packages/core/src/i18n/locales/zh-TW/library.json index 811eca7d6..2ab628124 100644 --- a/packages/core/src/i18n/locales/zh-TW/library.json +++ b/packages/core/src/i18n/locales/zh-TW/library.json @@ -148,6 +148,20 @@ "detailsManagedMetadataNote": "ReadAny 會維護這份元資訊,用於書架顯示、搜尋和同步命名,不會修改原始書籍檔案。", "bookNotFound": "書籍不存在" }, + "vectorize": { + "protectedBookTitle": "受保護的書籍", + "protectedBookDesc": "ReadAny 可以向量化不含 DRM 的 MOBI、AZW 和 AZW3 書籍,但此檔案似乎受到保護。", + "malformedBookTitle": "書籍檔案損壞或不完整", + "malformedBookDesc": "ReadAny 無法讀取此書,因為檔案結構似乎已損壞或不完整。請重新下載或匯出一份不含 DRM 的副本。", + "unsupportedFormatTitle": "不支援的格式", + "unsupportedFormatDesc": "ReadAny 可以向量化 EPUB、PDF、TXT、UMD,以及不含 DRM 的 MOBI、AZW 和 AZW3 書籍。此檔案格式不受支援。", + "extractionFailedTitle": "無法讀取書籍", + "extractionFailedDesc": "ReadAny 無法從此書中擷取可讀文字。請重新下載或匯出一份不含 DRM 的副本。", + "vectorizationFailedTitle": "無法向量化書籍", + "vectorizationFailedDesc": "ReadAny 無法完成此書的向量化。請檢查向量模型設定後再試一次。", + "cleanupFailedTitle": "無法完全清理索引", + "cleanupFailedDesc": "本書仍會顯示為未索引,但部分搜尋資料無法刪除。請重新建立索引;如果搜尋中仍出現本書,請重新啟動應用程式。" + }, "home": { "library": "書庫", "addBook": "新增書籍", @@ -170,6 +184,10 @@ "vec_indexing": "入庫中...", "vec_processing": "處理中...", "vec_queued": "排隊中", + "vec_cancel": "取消", + "vec_cancelQueued": "取消佇列中的向量化", + "vec_cancelling": "正在取消…", + "vec_cancelled": "已取消", "manageTags": "管理標籤", "remote": "需下載", "downloading": "下載中" diff --git a/packages/core/src/i18n/locales/zh/library.json b/packages/core/src/i18n/locales/zh/library.json index 440790e3c..90faf5d63 100644 --- a/packages/core/src/i18n/locales/zh/library.json +++ b/packages/core/src/i18n/locales/zh/library.json @@ -161,6 +161,20 @@ "detailsManagedMetadataNote": "ReadAny 会维护这份元信息,用于书架展示、搜索和同步命名,不会修改原始书籍文件。", "bookNotFound": "书籍不存在" }, + "vectorize": { + "protectedBookTitle": "受保护的书籍", + "protectedBookDesc": "ReadAny 可以向量化不含 DRM 的 MOBI、AZW 和 AZW3 书籍,但此文件似乎受到保护。", + "malformedBookTitle": "书籍文件损坏或不完整", + "malformedBookDesc": "ReadAny 无法读取此书,因为文件结构似乎已损坏或不完整。请重新下载或导出一份不含 DRM 的副本。", + "unsupportedFormatTitle": "不支持的格式", + "unsupportedFormatDesc": "ReadAny 可以向量化 EPUB、PDF、TXT、UMD,以及不含 DRM 的 MOBI、AZW 和 AZW3 书籍。此文件格式不受支持。", + "extractionFailedTitle": "无法读取书籍", + "extractionFailedDesc": "ReadAny 无法从此书中提取可读文本。请重新下载或导出一份不含 DRM 的副本。", + "vectorizationFailedTitle": "无法向量化书籍", + "vectorizationFailedDesc": "ReadAny 无法完成此书的向量化。请检查向量模型设置后重试。", + "cleanupFailedTitle": "无法完全清理索引", + "cleanupFailedDesc": "本书仍会显示为未索引,但部分搜索数据无法删除。请重新建立索引;如果搜索中仍出现本书,请重启应用。" + }, "home": { "library": "书库", "addBook": "添加书籍", @@ -184,6 +198,10 @@ "vec_indexing": "入库中...", "vec_processing": "处理中...", "vec_queued": "排队中", + "vec_cancel": "取消", + "vec_cancelQueued": "取消排队中的向量化", + "vec_cancelling": "正在取消…", + "vec_cancelled": "已取消", "manageTags": "管理标签", "remote": "需下载", "downloading": "下载中" diff --git a/packages/core/src/rag/index.ts b/packages/core/src/rag/index.ts index bde1b66bc..f4341b739 100644 --- a/packages/core/src/rag/index.ts +++ b/packages/core/src/rag/index.ts @@ -62,7 +62,11 @@ export type { export { vectorizeBook } from "./vectorize"; export type { VectorizeCallback } from "./vectorize"; -export { triggerVectorizeBook } from "./vectorize-trigger"; +export { + resetBookVectorization, + triggerVectorizeBook, + VectorizationCleanupError, +} from "./vectorize-trigger"; export type { VectorizeStatusCallback, VectorizeTriggerConfig, diff --git a/packages/core/src/rag/remote-embedding.ts b/packages/core/src/rag/remote-embedding.ts index fb34029b0..8c309cae3 100644 --- a/packages/core/src/rag/remote-embedding.ts +++ b/packages/core/src/rag/remote-embedding.ts @@ -15,6 +15,7 @@ export type RemoteEmbeddingFetch = (url: string, init: RequestInit) => Promise ({ + deleteChunks: vi.fn(), + deleteVectorIndexProvenance: vi.fn(), + insertChunks: vi.fn(), + setVectorIndexProvenance: vi.fn(), +})); + +const vectorDatabaseMocks = vi.hoisted(() => ({ + deleteByBookId: vi.fn(), + getStats: vi.fn(), + insert: vi.fn(), + isReady: vi.fn(), +})); + +const remoteEmbeddingMocks = vi.hoisted(() => ({ + requestRemoteEmbeddingBatch: vi.fn(), +})); + +const chunkerMocks = vi.hoisted(() => ({ + chunkContent: vi.fn(), +})); + +const eventBusMocks = vi.hoisted(() => ({ + emit: vi.fn(), +})); + +vi.mock("../db/database", () => databaseMocks); +vi.mock("./chunker", () => chunkerMocks); +vi.mock("./remote-embedding", () => remoteEmbeddingMocks); +vi.mock("./vector-db", () => ({ + getVectorDB: () => vectorDatabaseMocks, + hasVectorDB: () => true, +})); +vi.mock("../utils/event-bus", () => ({ eventBus: eventBusMocks })); + +import { + VectorizationCleanupError, + canStoreInSharedVectorDB, + resetBookVectorization, + triggerVectorizeBook, +} from "./vectorize-trigger"; + +beforeEach(() => { + vi.clearAllMocks(); + databaseMocks.deleteChunks.mockResolvedValue(undefined); + databaseMocks.deleteVectorIndexProvenance.mockResolvedValue(undefined); + databaseMocks.insertChunks.mockResolvedValue(undefined); + databaseMocks.setVectorIndexProvenance.mockResolvedValue(undefined); + chunkerMocks.chunkContent.mockReturnValue([ + { + id: "book-1-0-0", + bookId: "book-1", + chapterIndex: 0, + chapterTitle: "Chapter", + content: "content", + tokenCount: 1, + startCfi: "", + endCfi: "", + }, + ]); + vectorDatabaseMocks.deleteByBookId.mockResolvedValue(undefined); + vectorDatabaseMocks.getStats.mockResolvedValue({ totalVectors: 0, dimension: 2 }); + vectorDatabaseMocks.insert.mockResolvedValue(undefined); + vectorDatabaseMocks.isReady.mockResolvedValue(true); + remoteEmbeddingMocks.requestRemoteEmbeddingBatch.mockImplementation( + async (_model, texts: string[]) => ({ + ok: true, + embeddings: texts.map(() => [0.1, 0.2]), + }), + ); +}); describe("shared sqlite-vec dimension guard", () => { it("preserves a 384d book's acceleration index when a 1024d book is indexed", () => { @@ -12,3 +83,280 @@ describe("shared sqlite-vec dimension guard", () => { expect(canStoreInSharedVectorDB({ totalVectors: 120, dimension: 1024 }, 1024)).toBe(true); }); }); + +describe("failed vectorization cleanup", () => { + it("clears partial indexes and leaves the book unvectorized before rejecting", async () => { + const updates: Array<{ isVectorized: boolean; vectorizeProgress: number }> = []; + databaseMocks.setVectorIndexProvenance.mockRejectedValueOnce( + new Error("failed to save provenance"), + ); + + await expect( + triggerVectorizeBook( + "book-1", + [ + { + index: 0, + title: "Chapter", + content: "word ".repeat(400), + }, + ], + { + vectorModelEnabled: true, + vectorModelMode: "remote", + selectedBuiltinModelId: null, + remoteModel: { + url: "https://example.com/v1/embeddings", + apiKey: "test", + modelId: "test-model", + }, + }, + { + onBookUpdate: (_bookId, update) => updates.push(update), + }, + ), + ).rejects.toThrow("failed to save provenance"); + + expect(databaseMocks.deleteChunks).toHaveBeenCalledTimes(2); + expect(databaseMocks.deleteVectorIndexProvenance).toHaveBeenCalledTimes(2); + expect(vectorDatabaseMocks.deleteByBookId).toHaveBeenCalledTimes(2); + expect(updates.at(-1)).toEqual({ isVectorized: false, vectorizeProgress: 0 }); + expect(updates).not.toContainEqual({ isVectorized: true, vectorizeProgress: 1 }); + }); + + it("attempts every cleanup and resets the book flag when one store rejects", async () => { + const updates: Array<{ isVectorized: boolean; vectorizeProgress: number }> = []; + databaseMocks.deleteChunks.mockRejectedValueOnce(new Error("chunk cleanup failed")); + + await expect( + resetBookVectorization("book-1", { + onBookUpdate: (_bookId, update) => updates.push(update), + }), + ).rejects.toThrow("chunk cleanup failed"); + + expect(databaseMocks.deleteVectorIndexProvenance).toHaveBeenCalledWith("book-1"); + expect(vectorDatabaseMocks.deleteByBookId).toHaveBeenCalledWith("book-1"); + expect(updates).toEqual([{ isVectorized: false, vectorizeProgress: 0 }]); + }); + + it("waits for the unvectorized state write before cleanup resolves", async () => { + let releaseUpdate: (() => void) | undefined; + const updateReleased = new Promise((resolve) => { + releaseUpdate = resolve; + }); + + const cleanup = resetBookVectorization("book-1", { + onBookUpdate: async () => { + await updateReleased; + }, + }); + + const stateBeforeRelease = await Promise.race([ + cleanup.then(() => "resolved" as const), + new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 10)), + ]); + expect(stateBeforeRelease).toBe("pending"); + + releaseUpdate?.(); + await cleanup; + }); + + it("does not publish completion when the final vectorized-state write rejects", async () => { + const progressStatuses: string[] = []; + + await expect( + triggerVectorizeBook( + "book-1", + [{ index: 0, title: "Chapter", content: "content" }], + { + vectorModelEnabled: true, + vectorModelMode: "remote", + selectedBuiltinModelId: null, + remoteModel: { + url: "https://example.com/v1/embeddings", + apiKey: "test", + modelId: "test-model", + }, + }, + { + onBookUpdate: async (_bookId, update) => { + if (update.isVectorized) throw new Error("final state write failed"); + }, + }, + (progress) => progressStatuses.push(progress.status), + ), + ).rejects.toThrow("final state write failed"); + + expect(progressStatuses).not.toContain("completed"); + expect(progressStatuses.at(-1)).toBe("error"); + expect(eventBusMocks.emit).not.toHaveBeenCalledWith("vectorize:completed", expect.anything()); + expect(eventBusMocks.emit).toHaveBeenCalledWith("vectorize:error", { + bookId: "book-1", + error: "final state write failed", + }); + expect(databaseMocks.deleteChunks).toHaveBeenCalledTimes(2); + }); + + it("waits for the final vectorized-state write before publishing completion", async () => { + let releaseFinalWrite: (() => void) | undefined; + const finalWriteReleased = new Promise((resolve) => { + releaseFinalWrite = resolve; + }); + const progressStatuses: string[] = []; + + const vectorization = triggerVectorizeBook( + "book-1", + [{ index: 0, title: "Chapter", content: "content" }], + { + vectorModelEnabled: true, + vectorModelMode: "remote", + selectedBuiltinModelId: null, + remoteModel: { + url: "https://example.com/v1/embeddings", + apiKey: "test", + modelId: "test-model", + }, + }, + { + onBookUpdate: async (_bookId, update) => { + if (update.isVectorized) await finalWriteReleased; + }, + }, + (progress) => progressStatuses.push(progress.status), + ); + + await vi.waitFor(() => { + expect(databaseMocks.setVectorIndexProvenance).toHaveBeenCalledOnce(); + }); + expect(progressStatuses).not.toContain("completed"); + expect(eventBusMocks.emit).not.toHaveBeenCalledWith("vectorize:completed", expect.anything()); + + releaseFinalWrite?.(); + await vectorization; + + expect(progressStatuses.at(-1)).toBe("completed"); + expect(eventBusMocks.emit).toHaveBeenCalledWith("vectorize:completed", { + bookId: "book-1", + chunksCount: 1, + }); + }); + + it("cancels after a partial chunk write and removes every searchable residue", async () => { + const controller = new AbortController(); + const chapters = Array.from({ length: 51 }, (_, index) => ({ + index, + title: `Chapter ${index}`, + content: "content", + })); + databaseMocks.insertChunks.mockImplementationOnce(async () => { + controller.abort(); + }); + + await expect( + triggerVectorizeBook( + "book-1", + chapters, + { + vectorModelEnabled: true, + vectorModelMode: "remote", + selectedBuiltinModelId: null, + remoteModel: { + url: "https://example.com/v1/embeddings", + apiKey: "test", + modelId: "test-model", + }, + }, + { onBookUpdate: vi.fn() }, + undefined, + controller.signal, + ), + ).rejects.toMatchObject({ name: "AbortError" }); + + expect(databaseMocks.insertChunks).toHaveBeenCalledOnce(); + expect(databaseMocks.deleteChunks).toHaveBeenCalledTimes(2); + expect(databaseMocks.deleteVectorIndexProvenance).toHaveBeenCalledTimes(2); + expect(vectorDatabaseMocks.deleteByBookId).toHaveBeenCalledOnce(); + expect(databaseMocks.setVectorIndexProvenance).not.toHaveBeenCalled(); + expect(eventBusMocks.emit).not.toHaveBeenCalledWith("vectorize:completed", expect.anything()); + }); + + it("stops between embedding batches when cancelled", async () => { + const controller = new AbortController(); + remoteEmbeddingMocks.requestRemoteEmbeddingBatch.mockImplementationOnce(async () => { + controller.abort(); + return { ok: true, embeddings: Array.from({ length: 8 }, () => [0.1, 0.2]) }; + }); + + await expect( + triggerVectorizeBook( + "book-1", + Array.from({ length: 9 }, (_, index) => ({ + index, + title: `Chapter ${index}`, + content: "content", + })), + { + vectorModelEnabled: true, + vectorModelMode: "remote", + selectedBuiltinModelId: null, + remoteModel: { + url: "https://example.com/v1/embeddings", + apiKey: "test", + modelId: "test-model", + }, + }, + { onBookUpdate: vi.fn() }, + undefined, + controller.signal, + ), + ).rejects.toMatchObject({ name: "AbortError" }); + + expect(remoteEmbeddingMocks.requestRemoteEmbeddingBatch).toHaveBeenCalledOnce(); + expect(databaseMocks.insertChunks).not.toHaveBeenCalled(); + expect(databaseMocks.setVectorIndexProvenance).not.toHaveBeenCalled(); + expect(eventBusMocks.emit).not.toHaveBeenCalledWith("vectorize:completed", expect.anything()); + }); + + it("publishes error instead of cancelled when partial-index cleanup rejects", async () => { + const controller = new AbortController(); + const progressStatuses: string[] = []; + const onBookReset = vi.fn(); + databaseMocks.deleteChunks + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new Error("failed to delete partial chunks")); + databaseMocks.insertChunks.mockImplementationOnce(async () => { + controller.abort(); + }); + + const vectorization = triggerVectorizeBook( + "book-1", + [{ index: 0, title: "Chapter", content: "content" }], + { + vectorModelEnabled: true, + vectorModelMode: "remote", + selectedBuiltinModelId: null, + remoteModel: { + url: "https://example.com/v1/embeddings", + apiKey: "test", + modelId: "test-model", + }, + }, + { onBookUpdate: vi.fn(), onBookReset }, + (progress) => progressStatuses.push(progress.status), + controller.signal, + ); + + await expect(vectorization).rejects.toBeInstanceOf(VectorizationCleanupError); + expect(progressStatuses.at(-1)).toBe("error"); + expect(onBookReset).toHaveBeenCalledWith("book-1", { + isVectorized: false, + vectorizeProgress: 0, + }); + expect(eventBusMocks.emit).not.toHaveBeenCalledWith("vectorize:cancelled", expect.anything()); + expect(eventBusMocks.emit).not.toHaveBeenCalledWith("vectorize:completed", expect.anything()); + expect(eventBusMocks.emit).toHaveBeenCalledWith("vectorize:error", { + bookId: "book-1", + error: expect.stringContaining("failed to delete partial chunks"), + }); + }); +}); diff --git a/packages/core/src/rag/vectorize-trigger.ts b/packages/core/src/rag/vectorize-trigger.ts index c893abcd6..864911205 100644 --- a/packages/core/src/rag/vectorize-trigger.ts +++ b/packages/core/src/rag/vectorize-trigger.ts @@ -46,12 +46,51 @@ export interface VectorizeTriggerCallbacks { onBookUpdate: ( bookId: string, update: { isVectorized: boolean; vectorizeProgress: number }, - ) => void; + ) => void | Promise; + /** Reset visible/cache state after index deletion, even if durable persistence fails. */ + onBookReset?: ( + bookId: string, + update: { isVectorized: false; vectorizeProgress: 0 }, + ) => void | Promise; } /** Yield to the event loop so UI can repaint */ const yieldToUI = () => new Promise((r) => setTimeout(r, 0)); +export function createVectorizationAbortError(reason?: unknown): Error { + if (reason instanceof Error) return reason; + const error = new Error( + typeof reason === "string" && reason ? reason : "Vectorization cancelled", + ); + error.name = "AbortError"; + return error; +} + +export function throwIfVectorizationAborted(signal?: AbortSignal): void { + if (!signal?.aborted) return; + const error = createVectorizationAbortError(signal.reason); + if (error.name !== "AbortError") error.name = "AbortError"; + throw error; +} + +export function isVectorizationAbort(error: unknown): boolean { + return error instanceof Error && error.name === "AbortError"; +} + +export class VectorizationCleanupError extends Error { + readonly operationError: unknown; + readonly cleanupError: unknown; + + constructor(operationError: unknown, cleanupError: unknown) { + const cleanupMessage = + cleanupError instanceof Error ? cleanupError.message : String(cleanupError); + super(`Vectorization stopped, but partial index cleanup failed: ${cleanupMessage}`); + this.name = "VectorizationCleanupError"; + this.operationError = operationError; + this.cleanupError = cleanupError; + } +} + /** sqlite-vec currently owns one global vector dimension. */ export function canStoreInSharedVectorDB( stats: { totalVectors: number; dimension: number }, @@ -60,6 +99,35 @@ export function canStoreInSharedVectorDB( return stats.totalVectors === 0 || stats.dimension === embeddingDimension; } +/** Clear every persisted part of a book index and reset its library state. */ +export async function resetBookVectorization( + bookId: string, + callbacks: VectorizeTriggerCallbacks, +): Promise { + const results = await Promise.allSettled([ + deleteChunks(bookId), + deleteVectorIndexProvenance(bookId), + (async () => { + if (!hasVectorDB()) return; + const vectorDB = getVectorDB(); + if (vectorDB && (await vectorDB.isReady())) { + await vectorDB.deleteByBookId(bookId); + } + })(), + ]); + + invalidateChunkCache(bookId); + await (callbacks.onBookReset ?? callbacks.onBookUpdate)(bookId, { + isVectorized: false, + vectorizeProgress: 0, + }); + + const failure = results.find( + (result): result is PromiseRejectedResult => result.status === "rejected", + ); + if (failure) throw failure.reason; +} + /** * Trigger full vectorization for a book. * 1. Chunks chapter text into manageable pieces @@ -79,6 +147,7 @@ export async function triggerVectorizeBook( config: VectorizeTriggerConfig, callbacks: VectorizeTriggerCallbacks, onProgress?: VectorizeStatusCallback, + signal?: AbortSignal, ): Promise { if (!config.vectorModelEnabled) { throw new Error("Vector model is not enabled. Please enable it in Settings → Vector Model."); @@ -96,14 +165,16 @@ export async function triggerVectorizeBook( }; try { + throwIfVectorizationAborted(signal); // Update book state: vectorizing - callbacks.onBookUpdate(bookId, { + await callbacks.onBookUpdate(bookId, { isVectorized: false, vectorizeProgress: 0, }); eventBus.emit("vectorize:started", { bookId }); onProgress?.(progress); await yieldToUI(); + throwIfVectorizationAborted(signal); // Phase 1: Chunk chapters const allChunks: Array<{ @@ -121,6 +192,7 @@ export async function triggerVectorizeBook( const totalChapters = chapters.length; for (let i = 0; i < chapters.length; i++) { + throwIfVectorizationAborted(signal); const chapter = chapters[i]; const chunks = chunkContent( chapter.content, @@ -139,6 +211,7 @@ export async function triggerVectorizeBook( progress.totalChunks = totalChapters; onProgress?.(progress); await yieldToUI(); + throwIfVectorizationAborted(signal); } if (allChunks.length === 0) { @@ -158,22 +231,29 @@ export async function triggerVectorizeBook( config.selectedBuiltinModelId, progress, onProgress, + signal, ); } else { - await generateRemoteEmbeddings(allChunks, config, progress, onProgress); + await generateRemoteEmbeddings(allChunks, config, progress, onProgress, signal); } + throwIfVectorizationAborted(signal); // Phase 3: Store in database (batch insert for performance) progress.status = "indexing"; onProgress?.(progress); await yieldToUI(); + throwIfVectorizationAborted(signal); await deleteChunks(bookId); + throwIfVectorizationAborted(signal); await deleteVectorIndexProvenance(bookId); + throwIfVectorizationAborted(signal); // Insert in batches of 50 to avoid huge single transaction const insertBatchSize = 50; for (let i = 0; i < allChunks.length; i += insertBatchSize) { + throwIfVectorizationAborted(signal); await insertChunks(allChunks.slice(i, i + insertBatchSize)); + throwIfVectorizationAborted(signal); } // Store embeddings in vector database (sqlite-vec) @@ -182,6 +262,7 @@ export async function triggerVectorizeBook( const vectorDB = getVectorDB(); if (vectorDB && (await vectorDB.isReady())) { await vectorDB.deleteByBookId(bookId); + throwIfVectorizationAborted(signal); const vectorRecords: VectorRecord[] = allChunks.flatMap((c) => { const embedding = c.embedding; @@ -203,6 +284,7 @@ export async function triggerVectorizeBook( let canInsertIntoVectorDb = canStoreInSharedVectorDB(stats, detectedDimension); if (detectedDimension > 0 && vectorDB.reinit && stats.totalVectors === 0) { await vectorDB.reinit(detectedDimension); + throwIfVectorizationAborted(signal); canInsertIntoVectorDb = true; } else if (stats.dimension !== detectedDimension) { // sqlite-vec currently has one global vector dimension. Never drop @@ -214,6 +296,7 @@ export async function triggerVectorizeBook( } if (canInsertIntoVectorDb) { await vectorDB.insert(vectorRecords); + throwIfVectorizationAborted(signal); storedInVectorDb = true; } } @@ -227,39 +310,58 @@ export async function triggerVectorizeBook( console.warn("[Vectorize] Vector database not ready, skipping vector storage"); } } catch (err) { + throwIfVectorizationAborted(signal); console.error("[Vectorize] Failed to store vectors:", err); } } const provenance = getEmbeddingProvenance(config, allChunks[0]?.embedding?.length ?? 0); await setVectorIndexProvenance({ bookId, ...provenance, createdAt: Date.now() }); + throwIfVectorizationAborted(signal); // Invalidate search cache so next query picks up new embeddings invalidateChunkCache(bookId); - // Phase 4: Update state - progress.status = "completed"; - onProgress?.(progress); - - callbacks.onBookUpdate(bookId, { + // Phase 4: Persist the final state before publishing completion. + await callbacks.onBookUpdate(bookId, { isVectorized: true, vectorizeProgress: 1, }); + throwIfVectorizationAborted(signal); + + progress.status = "completed"; + onProgress?.(progress); eventBus.emit("vectorize:completed", { bookId, chunksCount: allChunks.length, }); } catch (err) { const message = err instanceof Error ? err.message : String(err); - progress.status = "error"; - progress.error = message; - onProgress?.(progress); + let cleanupError: unknown; + try { + await resetBookVectorization(bookId, callbacks); + } catch (errorDuringCleanup) { + cleanupError = errorDuringCleanup; + } - callbacks.onBookUpdate(bookId, { - isVectorized: false, - vectorizeProgress: 0, - }); - eventBus.emit("vectorize:error", { bookId, error: message }); + if (cleanupError) { + const failure = new VectorizationCleanupError(err, cleanupError); + progress.status = "error"; + progress.error = failure.message; + onProgress?.(progress); + eventBus.emit("vectorize:error", { bookId, error: failure.message }); + throw failure; + } + + const cancelled = isVectorizationAbort(err); + progress.status = cancelled ? "cancelled" : "error"; + progress.error = cancelled ? undefined : message; + onProgress?.(progress); + if (cancelled) { + eventBus.emit("vectorize:cancelled", { bookId }); + } else { + eventBus.emit("vectorize:error", { bookId, error: message }); + } throw err; } } @@ -269,12 +371,15 @@ function getEmbeddingProvenance( actualDimensions: number, ): EmbeddingProvenance { if (config.vectorModelMode === "builtin") { - const model = BUILTIN_EMBEDDING_MODELS.find((candidate) => candidate.id === config.selectedBuiltinModelId); + const model = BUILTIN_EMBEDDING_MODELS.find( + (candidate) => candidate.id === config.selectedBuiltinModelId, + ); if (!model) throw new Error("Cannot save provenance for an unknown built-in embedding model."); return { kind: "builtin", modelId: model.id, dimensions: actualDimensions || model.dimension }; } - if (!config.remoteModel) throw new Error("Cannot save provenance without a selected remote embedding model."); + if (!config.remoteModel) + throw new Error("Cannot save provenance without a selected remote embedding model."); return { kind: "remote", modelId: config.remoteModel.modelId, @@ -289,6 +394,7 @@ async function generateBuiltinEmbeddings( builtinModelId: string | null, progress: VectorizeProgress, onProgress?: VectorizeStatusCallback, + signal?: AbortSignal, ) { if (!builtinModelId) { throw new Error("No built-in model selected. Please select one in Settings → Vector Model."); @@ -298,19 +404,23 @@ async function generateBuiltinEmbeddings( if (!model) throw new Error(`Unknown built-in model: ${builtinModelId}`); // Ensure the model is loaded in the Worker + throwIfVectorizationAborted(signal); await loadEmbeddingPipeline(builtinModelId); + throwIfVectorizationAborted(signal); // Process in batches — Worker handles the heavy lifting off main thread const batchSize = 16; let globalProcessed = 0; for (let i = 0; i < chunks.length; i += batchSize) { + throwIfVectorizationAborted(signal); const batch = chunks.slice(i, i + batchSize); const texts = batch.map((c) => c.content); const batchOffset = i; // generateLocalEmbeddings now runs in Worker with per-item progress const embeddings = await generateLocalEmbeddings(builtinModelId, texts, (done, _total) => { + if (signal?.aborted) return; globalProcessed = batchOffset + done; progress.processedChunks = globalProcessed; eventBus.emit("vectorize:progress", { @@ -320,12 +430,14 @@ async function generateBuiltinEmbeddings( }); onProgress?.(progress); }); + throwIfVectorizationAborted(signal); for (let j = 0; j < batch.length; j++) { batch[j].embedding = embeddings[j]; } await yieldToUI(); + throwIfVectorizationAborted(signal); } } @@ -335,6 +447,7 @@ async function generateRemoteEmbeddings( config: VectorizeTriggerConfig, progress: VectorizeProgress, onProgress?: VectorizeStatusCallback, + signal?: AbortSignal, ) { const selectedModel = config.remoteModel; if (!selectedModel) { @@ -357,14 +470,17 @@ async function generateRemoteEmbeddings( > => { return requestRemoteEmbeddingBatch(selectedModel, inputTexts, { maxCharsPerInput: MAX_CHARS_PER_CHUNK, + signal, }); }; for (let i = 0; i < chunks.length; i += batchSize) { + throwIfVectorizationAborted(signal); const batch = chunks.slice(i, i + batchSize); const texts = batch.map((c) => c.content); const batchResult = await callEmbeddingApi(texts); + throwIfVectorizationAborted(signal); if (batchResult.ok) { for (let j = 0; j < batch.length; j++) { @@ -382,7 +498,9 @@ async function generateRemoteEmbeddings( `[Embedding] Batch failed (${batchResult.status}): ${batchResult.errorText.slice(0, 200)}. Retrying per-chunk.`, ); for (let j = 0; j < batch.length; j++) { + throwIfVectorizationAborted(signal); const single = await callEmbeddingApi([batch[j].content]); + throwIfVectorizationAborted(signal); if (single.ok) { batch[j].embedding = single.embeddings[0] ?? []; } else { @@ -405,5 +523,6 @@ async function generateRemoteEmbeddings( }); onProgress?.(progress); await yieldToUI(); + throwIfVectorizationAborted(signal); } } diff --git a/packages/core/src/types/rag.ts b/packages/core/src/types/rag.ts index bf2df05d1..9760aab60 100644 --- a/packages/core/src/types/rag.ts +++ b/packages/core/src/types/rag.ts @@ -69,6 +69,14 @@ export interface VectorizeProgress { bookId: string; totalChunks: number; processedChunks: number; - status: "idle" | "chunking" | "embedding" | "indexing" | "completed" | "error"; + status: + | "idle" + | "chunking" + | "embedding" + | "indexing" + | "completed" + | "cancelling" + | "cancelled" + | "error"; error?: string; } diff --git a/packages/core/src/utils/event-bus.ts b/packages/core/src/utils/event-bus.ts index d575a87a2..b919666b6 100644 --- a/packages/core/src/utils/event-bus.ts +++ b/packages/core/src/utils/event-bus.ts @@ -23,6 +23,7 @@ type EventMap = { "vectorize:started": { bookId: string }; "vectorize:progress": { bookId: string; progress: number; status: string }; "vectorize:completed": { bookId: string; chunksCount: number }; + "vectorize:cancelled": { bookId: string }; "vectorize:error": { bookId: string; error: string }; "tts:jump-to-current": { bookId: string; cfi: string; respond?: () => void }; "tts:open-lyrics-page": { bookId: string; respond?: () => void }; diff --git a/packages/foliate-js/mobi.js b/packages/foliate-js/mobi.js index 388a848f7..ac2529d7e 100644 --- a/packages/foliate-js/mobi.js +++ b/packages/foliate-js/mobi.js @@ -27,6 +27,8 @@ const PALMDOC_HEADER = { encryption: [12, 2, "uint"], }; +const ENCRYPTED_MOBI_ERROR = "Encrypted MOBI records are not supported"; + const MOBI_HEADER = { magic: [16, 4, "string"], length: [20, 4, "uint"], @@ -641,6 +643,8 @@ export class MOBI extends PDB { await super.open(file); // TODO: if (this.pdb.type === 'TEXt') this.headers = this.#getHeaders(await super.loadRecord(0)); + if (this.headers.palmdoc.encryption !== 0) + throw new Error(ENCRYPTED_MOBI_ERROR); this.#resourceStart = this.headers.mobi.resourceStart; let isKF8 = this.headers.mobi.version >= 8; if (!isKF8) { @@ -649,9 +653,12 @@ export class MOBI extends PDB { try { // it's a "combo" MOBI/KF8 file; try to open the KF8 part this.headers = this.#getHeaders(await super.loadRecord(boundary)); + if (this.headers.palmdoc.encryption !== 0) + throw new Error(ENCRYPTED_MOBI_ERROR); this.#start = boundary; isKF8 = true; } catch (e) { + if (e instanceof Error && e.message === ENCRYPTED_MOBI_ERROR) throw e; console.warn(e); console.warn("Failed to open KF8; falling back to MOBI"); }