feat: add local formula OCR and copy in reading mode

Author: reycn

bun.lock

@@ -95,7 +95,7 @@ const paths = await packager({
   arch,
   out: test ? '/tmp/pdfmathreader-slim-test-build' : resolve(root, 'release'),
   overwrite: true,
-  asar: { unpack: '**/*.node' },
+  asar: { unpack: '**/*.node', unpackDir: '**/onnxruntime-node/bin' },
   ignore: [
     /^\/release(?:\/|$)/,
     /^\/\.cache(?:\/|$)/,

electron/build/package.mjs

@@ -100,11 +100,24 @@ export async function stageApplication({
     : phase === 'dependencies'
       ? ['express', 'pdf-lib', '@firecrawl/pdf-inspector']
       : ['@firecrawl/pdf-inspector', 'pdf-lib'];
+  if (!names.includes('onnxruntime-node')) names.push('onnxruntime-node');
   if (test && !names.includes('pdf-lib')) names.push('pdf-lib');
   const fallback = !hasNativeInspector(platform, arch);
   if (fallback && !names.includes('pdfjs-dist')) names.push('pdfjs-dist');
   if (fallback && !names.includes('@thednp/dommatrix')) names.push('@thednp/dommatrix');
   const copied = await copyDependencies(root, stage, names, fallback);
+  // Ship only the target CPU runtime; other architectures multiply bundle size.
+  const onnxBin = join(stage, 'node_modules/onnxruntime-node/bin/napi-v6');
+  const { rm: removeRuntime } = await import('node:fs/promises');
+  for (const target of await readdir(onnxBin)) {
+    if (target !== platform)
+      await removeRuntime(join(onnxBin, target), { recursive: true, force: true });
+    else
+      for (const targetArch of await readdir(join(onnxBin, target)))
+        if (targetArch !== arch)
+          await removeRuntime(join(onnxBin, target, targetArch), { recursive: true, force: true });
+  }
+
   const browserLicenses = [
     '@fluentui/web-components',
     '@fluentui/tokens',
@@ -168,6 +181,7 @@ export async function stageApplication({
       mainFields: ['module', 'main'],
       external: [
         'electron',
+        'onnxruntime-node',
         // Keep PDF editing outside the startup bundles so dynamic imports also
         // avoid parsing the library when only the Start Page is requested.
         'pdf-lib',

electron/build/production-stage.mjs

@@ -29,6 +29,7 @@ const DEFAULT_PREFERENCES = Object.freeze({
   documentOpenMode: 'translation',
   interactionMode: 'reading',
   restoreDocuments: true,
+  formulaOcrEnabled: false,
   reduceResourceUsage: true,
   reduceBackgroundFrameRate: true,
   reuseTranslations: true,
@@ -227,6 +228,7 @@ const VALIDATORS = {
   reduceResourceUsage: (value) => typeof value === 'boolean',
   reduceBackgroundFrameRate: (value) => typeof value === 'boolean',
   reuseTranslations: (value) => typeof value === 'boolean',
+  formulaOcrEnabled: (value) => typeof value === 'boolean',
   emphasizeTopicSentences: (value) => typeof value === 'boolean',
   emphasizeInformation: (value) => typeof value === 'boolean',
   emphasizeResearchFindings: (value) => typeof value === 'boolean',

electron/main/services/preferences.mjs

@@ -24,6 +24,7 @@
     "@firecrawl/pdf-inspector": "latest",
     "@thednp/dommatrix": "3.1.2",
     "express": "^5.1.0",
+    "onnxruntime-node": "1.22.0",
     "pdf-lib": "^1.17.1",
     "pdfjs-dist": "^6.3.289"
   },

package.json

@@ -41,6 +41,8 @@ import {
 import { registerDocumentRoutes } from './http/document-routes.mjs';
 import { registerKernelRoutes } from './http/kernel-routes.mjs';
 import { registerTranslationRoutes } from './http/translation-routes.mjs';
+import { registerFormulaOcrRoutes } from './http/formula-ocr-routes.mjs';
+import { createFormulaOcrService } from './formula/formula-ocr-service.mjs';
 
 export async function startServer({
   port = 5173,
@@ -96,6 +98,9 @@ export async function startServer({
           args: [],
         }),
     });
+  const formulaOcr = createFormulaOcrService({
+    cacheDir: join(cacheDir, '..', 'models'),
+  });
   const documents = createDocumentStore();
   const documentCache = createDocumentCache(cacheDir);
   const providerLimiter = createLimiter(4, { label: 'provider' });
@@ -188,6 +193,7 @@ export async function startServer({
   registerAccessMiddleware(app, { token, origin: () => origin, development });
   registerPerformanceMiddleware(app, performanceTracker);
 
+  registerFormulaOcrRoutes(app, { service: formulaOcr });
   registerProviderPortRoute(app, isProviderPortOpen);
   registerCacheRoutes(app, { cacheManager });
   registerDeveloperRoutes(app, {
@@ -284,6 +290,7 @@ export async function startServer({
       layoutEntries.clear();
       documents.clear();
       server.closeAllConnections();
+      await formulaOcr.dispose();
       await engines.close();
       await localTranslator.close?.();
       await new Promise((r) => server.close(r));