Faster defaults: standard OCR by default, cheaper E-Books listing; fix main CI

backend/app/db.py

@@ -72,6 +72,7 @@ def transaction(database_path=None):
     cleaned_text TEXT,
     confidence_data TEXT,
     mean_confidence REAL,
+    word_count INTEGER,
     FOREIGN KEY (project_id) REFERENCES projects (id)
 );
 
@@ -158,11 +159,38 @@ def init_db(database_path=None):
             conn.execute('ALTER TABLE ocr_text ADD COLUMN confidence_data TEXT')
         if 'mean_confidence' not in columns:
             conn.execute('ALTER TABLE ocr_text ADD COLUMN mean_confidence REAL')
+        if 'word_count' not in columns:
+            conn.execute('ALTER TABLE ocr_text ADD COLUMN word_count INTEGER')
+        backfill_word_counts(conn)
         file_columns = {row['name'] for row in conn.execute('PRAGMA table_info(files)')}
         if 'archive_format' not in file_columns:
             conn.execute('ALTER TABLE files ADD COLUMN archive_format TEXT')
 
 
+EFFECTIVE_TEXT_SQL = "COALESCE(NULLIF(cleaned_text, ''), original_text, '')"
+
+
+def refresh_word_count(conn, project_id):
+    """Store the word count of a project's current text (cleaned if present, else OCR)."""
+    row = conn.execute(f'SELECT {EFFECTIVE_TEXT_SQL} AS text FROM ocr_text WHERE project_id = ?',
+                       (project_id,)).fetchone()
+    if row is not None:
+        conn.execute('UPDATE ocr_text SET word_count = ? WHERE project_id = ?', (len(row['text'].split()), project_id))
+
+
+def backfill_word_counts(conn, batch=200):
+    """Fill word_count for rows saved before the column existed (runs once per row)."""
+    while True:
+        rows = conn.execute(f'''
+            SELECT project_id, {EFFECTIVE_TEXT_SQL} AS text FROM ocr_text
+            WHERE word_count IS NULL AND (original_text IS NOT NULL OR cleaned_text IS NOT NULL)
+            LIMIT ?''', (batch,)).fetchall()
+        if not rows:
+            return
+        conn.executemany('UPDATE ocr_text SET word_count = ? WHERE project_id = ?',
+                         [(len(r['text'].split()), r['project_id']) for r in rows])
+
+
 def get_config_value(key, default=None):
     with transaction() as conn:
         row = conn.execute('SELECT value FROM app_config WHERE key = ?', (key,)).fetchone()

backend/app/routes/batch.py

@@ -3,7 +3,7 @@
 from flask import Blueprint, current_app, jsonify, request
 
 from .. import search_index
-from ..db import get_config_value, transaction
+from ..db import get_config_value, refresh_word_count, transaction
 from ..processors.batch_processor import BatchProcessor
 from ..security import ValidationError, display_filename, safe_upload_path
 from ..services.ocr import extract_document_text
@@ -83,6 +83,7 @@ def ocr_callback(project_id):
             with transaction() as conn:
                 conn.execute('UPDATE ocr_text SET original_text = ?, mean_confidence = ? WHERE project_id = ?',
                              (result['text'], result.get('mean_confidence'), project_id))
+                refresh_word_count(conn, project_id)
                 if result.get('ocr_pdf_path'):
                     conn.execute('UPDATE files SET ocr_path = ? WHERE project_id = ?',
                                  (result['ocr_pdf_path'], project_id))

backend/app/routes/documents.py

@@ -9,7 +9,7 @@
 from flask import Blueprint, Response, current_app, jsonify, request
 
 from .. import search_index
-from ..db import get_config_value, transaction
+from ..db import get_config_value, refresh_word_count, transaction
 from ..processors.metadata_extractor import extract_metadata
 from ..security import ValidationError
 from ..services import dublin_core
@@ -45,6 +45,7 @@ def save_cleaned_text(project_id):
 
     with transaction() as conn:
         conn.execute('UPDATE ocr_text SET cleaned_text = ? WHERE project_id = ?', (cleaned_text, project_id))
+        refresh_word_count(conn, project_id)
         set_status(conn, project_id, 'metadata')
         search_index.update(project_id, conn=conn)
     return jsonify({'success': True})

backend/app/routes/projects.py

@@ -12,6 +12,7 @@
 bp = Blueprint('projects', __name__, url_prefix='/api/projects')
 
 VALID_STATUSES = ('upload', 'ocr', 'cleanup', 'metadata', 'archived')
+SNIPPET_CHARS = 300
 
 
 def _servable_roots():
@@ -94,14 +95,15 @@ def list_ebooks():
                 f.cleaned_path,
                 f.final_path,
                 LENGTH(COALESCE(o.cleaned_text, o.original_text, '')) AS text_length,
-                o.original_text,
-                o.cleaned_text
+                COALESCE(o.word_count, 0) AS word_count,
+                -- Cut the preview in SQLite so full OCR texts never leave the database.
+                substr(COALESCE(NULLIF(o.cleaned_text, ''), o.original_text, ''), 1, ?) AS snippet
             FROM projects p
             LEFT JOIN metadata m ON p.id = m.project_id
             LEFT JOIN files f ON p.id = f.project_id
             LEFT JOIN ocr_text o ON p.id = o.project_id
             ORDER BY p.created_at DESC
-        ''').fetchall()
+        ''', (SNIPPET_CHARS + 1,)).fetchall()
 
     ebooks = []
     for r in rows:
@@ -121,16 +123,9 @@ def list_ebooks():
         has_ocr_pdf = bool(ocr_path and os.path.exists(ocr_path))
         has_searchable_pdf = has_final_pdf or has_ocr_pdf or bool(target_file)
 
-        cleaned = d.get('cleaned_text') or d.get('original_text') or ''
-        word_count = len(cleaned.split()) if cleaned else 0
-        snippet = (cleaned[:300] + '...') if len(cleaned) > 300 else cleaned
-
-        del d['original_text']
-        del d['cleaned_text']
-
+        snippet = d['snippet'] or ''
+        d['snippet'] = snippet[:SNIPPET_CHARS] + '...' if len(snippet) > SNIPPET_CHARS else snippet
         d['file_size'] = file_size
-        d['word_count'] = word_count
-        d['snippet'] = snippet
         d['has_searchable_pdf'] = has_searchable_pdf
         d['has_final_pdf'] = has_final_pdf
         d['has_pdf'] = bool(target_file)
@@ -167,8 +162,8 @@ def _ensure_searchable_pdf(project_id, project):
 
         try:
             if ext in current_app.config['IMAGE_EXTENSIONS']:
-                from PIL import Image
                 import pytesseract
+                from PIL import Image
                 with Image.open(original) as img:
                     pdf_bytes = pytesseract.image_to_pdf_or_hocr(img, extension='pdf')
                     with open(target_ocr_path, 'wb') as f:
@@ -197,8 +192,8 @@ def _ensure_searchable_pdf(project_id, project):
     if ocr_text and ocr_text.strip():
         try:
             from reportlab.lib.pagesizes import letter
-            from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer
-            from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
+            from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
+            from reportlab.platypus import Paragraph, SimpleDocTemplate, Spacer
             upload_folder = current_app.config['UPLOAD_FOLDER']
             stem = os.path.splitext(project['filename'])[0]
             target_ocr_path = os.path.join(upload_folder, f'ocr_{stem}.pdf')

backend/app/services/ocr_tasks.py

@@ -14,7 +14,7 @@
 from PIL import Image
 
 from .. import search_index
-from ..db import get_config_value, transaction
+from ..db import get_config_value, refresh_word_count, transaction
 from ..processors.advanced_ocr import AdvancedOCRProcessor
 from ..processors.glm_ocr import GlmOcrProcessor
 from ..processors.handwritten_to_pdf import HandwrittenToPDFConverter
@@ -51,6 +51,7 @@ def _save_ocr_result(project_id, text, confidence=None, ocr_pdf_path=None, mean_
                      (text, json.dumps(confidence) if confidence else None, mean_confidence, project_id))
         if ocr_pdf_path:
             conn.execute('UPDATE files SET ocr_path = ? WHERE project_id = ?', (ocr_pdf_path, project_id))
+        refresh_word_count(conn, project_id)
         set_status(conn, project_id, 'cleanup')
         search_index.update(project_id, conn=conn)
 

backend/app/services/pdfa.py

@@ -12,9 +12,9 @@
 born-digital PDFs keep their metadata but are not labelled PDF/A, because they
 can break rules a preflight cannot see.
 """
+import shutil
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
-import shutil
 
 try:
     import pikepdf

backend/tests/test_workflow.py

@@ -163,3 +163,28 @@ def test_metadata_extractor():
     result = extract_metadata(sample, 'ai_introduction.pdf')
     assert result['year'] == '2024'
     assert set(result) >= {'title', 'author', 'year', 'subject', 'keywords', 'confidence_scores'}
+
+
+def test_ebooks_listing_uses_stored_word_counts_and_short_snippets(client, app):
+    long_text = ' '.join(f'word{i}' for i in range(400))
+    pid = upload(client, 'notes.pdf', ('Title: Notes\n' + long_text).encode()).get_json()['project']['id']
+    client.post(f'/api/ocr/{pid}')
+    listed = {e['id']: e for e in client.get('/api/projects/ebooks').get_json()['ebooks']}[pid]
+    assert listed['word_count'] >= 400
+    assert listed['snippet'].endswith('...') and len(listed['snippet']) == 303
+    assert 'original_text' not in listed and 'cleaned_text' not in listed
+
+    # Editing the text updates the stored count.
+    client.post(f'/api/cleanup/{pid}', json={'cleaned_text': 'just three words'})
+    listed = {e['id']: e for e in client.get('/api/projects/ebooks').get_json()['ebooks']}[pid]
+    assert listed['word_count'] == 3 and listed['snippet'] == 'just three words'
+
+
+def test_word_counts_backfilled_for_existing_databases(make_app, app):
+    import sqlite3
+    with sqlite3.connect(app.config['DATABASE']) as conn:
+        conn.execute("INSERT INTO projects (id, filename, filepath, status) VALUES (900, 'old.pdf', '', 'cleanup')")
+        conn.execute("INSERT INTO ocr_text (project_id, original_text) VALUES (900, 'one two three four')")
+    restarted = make_app().test_client()
+    listed = {e['id']: e for e in restarted.get('/api/projects/ebooks').get_json()['ebooks']}[900]
+    assert listed['word_count'] == 4

src/pages/Cleanup.jsx

@@ -3,7 +3,7 @@
 import { useProject } from '../context/ProjectContext'
 import {
     Save, AlertCircle, ArrowRight, X, Download, Globe,
-    AlertTriangle, Sparkles, ZoomIn, ZoomOut, Maximize2,
+    AlertTriangle, Sparkles, ZoomIn, ZoomOut,
     ExternalLink, Eye, EyeOff, FileText, CheckCircle2, RotateCcw
 } from 'lucide-react'
 import WorkflowTracker from '../components/WorkflowTracker'
@@ -80,7 +80,7 @@
         if (projectId) {
             loadProject()
         }
     }, [projectId])
 
     const loadProject = async () => {
         try {

src/pages/Metadata.jsx

@@ -4,7 +4,7 @@
 import {
     Save, AlertCircle, ArrowRight, X, Sparkles, Check,
     FileText, Folder, Calendar, User, Tag, Eye, EyeOff,
-    ExternalLink, RefreshCw, Copy, CheckCircle2, ChevronRight
+    ExternalLink, Copy, CheckCircle2
 } from 'lucide-react'
 import WorkflowTracker from '../components/WorkflowTracker'
 import Modal from '../components/Modal'
@@ -45,7 +45,6 @@
     // AI Suggestions State
     const [suggestions, setSuggestions] = useState(null)
     const [extracting, setExtracting] = useState(false)
-    const [extractionError, setExtractionError] = useState(null)
 
     // Document Reference Pane State
     const [showRefPane, setShowRefPane] = useState(true)
@@ -57,7 +56,7 @@
             loadProject()
             loadSuggestions()
         }
     }, [projectId])
 
     const loadProject = async () => {
         try {
@@ -96,7 +95,6 @@
 
     const handleExtractAI = async () => {
         setExtracting(true)
-        setExtractionError(null)
 
         try {
             const res = await axios.post(`${API_URL}/metadata/extract/${projectId}`)
@@ -114,7 +112,6 @@
         } catch (err) {
             console.error('Failed to extract metadata:', err)
             const msg = err.response?.data?.error || 'Extraction failed'
-            setExtractionError(msg)
             addToast(msg, 'error')
         } finally {
             setExtracting(false)

src/pages/UploadOCR.jsx

@@ -37,7 +37,23 @@
     const [ocrResult, setOcrResult] = useState(null)
     const [pdfResult, setPdfResult] = useState(null)
     const [language, setLanguage] = useState('eng')
-    const [useAdvancedOCR, setUseAdvancedOCR] = useState(true)
+    // Standard OCR by default: Advanced layout analysis is about 3x slower per
+    // page and only helps documents with tables, forms or stamps.
+    const [useAdvancedOCR, setUseAdvancedOCR] = useState(() => {
+        try {
+            return localStorage.getItem('libradigit_advanced_ocr') === 'true'
+        } catch {
+            return false
+        }
+    })
+    const handleAdvancedToggle = (checked) => {
+        setUseAdvancedOCR(checked)
+        try {
+            localStorage.setItem('libradigit_advanced_ocr', String(checked))
+        } catch {
+            // Preference simply isn't remembered.
+        }
+    }
     const [activeEngine, setActiveEngine] = useState('tesseract') // 'tesseract' | 'glm-ocr'
 
     // Read active engine from settings once on mount
@@ -148,7 +164,7 @@
             }
         })()
         return () => controller.abort()
     }, [resumeProjectId])
 
     const finishJob = () => {
         setProcessing(false)
@@ -485,15 +501,16 @@
                                     </div>
                                     <div className="toggle-copy">
                                         <h4>Advanced Layout & Feature Analysis</h4>
-                                        <p>Detect structured tables, form fields, stamps, signatures, and auto-correct skew angle.</p>
+                                        <p>Detect structured tables, form fields, stamps, signatures, and auto-correct skew angle. About 3× slower per page — turn on for forms and tables.</p>
                                     </div>
                                 </div>
                                 <label className="dash-switch">
                                     <input
                                         type="checkbox"
                                         checked={useAdvancedOCR}
                                         disabled={activeEngine === 'glm-ocr'}
-                                        onChange={(e) => setUseAdvancedOCR(e.target.checked)}
+                                        aria-label="Advanced layout and feature analysis"
+                                        onChange={(e) => handleAdvancedToggle(e.target.checked)}
                                     />
                                     <span className="dash-slider"></span>
                                 </label>