From 0621362afcec3e1417f80442cf15bd4fc42a4385 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 6 Oct 2026 05:33:02 +0000 Subject: [PATCH] Faster defaults: standard OCR by default, cheaper E-Books listing; fix main CI Speed: - Upload page defaults to standard OCR. Advanced layout analysis measured ~3x slower per page (5.7s vs 1.9s on a 300 DPI page) and only helps tables/forms/stamps; the toggle now says so and remembers each user's choice (localStorage). - GET /api/projects/ebooks read every document's full OCR text to compute a word count and a 300-char snippet. Word counts are now stored (ocr_text.word_count, refreshed whenever OCR/cleanup/batch writes text, backfilled on startup for existing databases) and the snippet is cut in SQLite. 2,000 documents x 6,000 words: 579 ms -> 135 ms per request, same response. CI on main has been red since 3ae2fc5 (lint only): unused imports/state in Cleanup.jsx and Metadata.jsx (the extraction error was already shown as a toast), and import ordering in routes/projects.py and services/pdfa.py. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Jze3nDgZGqUXGceiu5RCAf --- backend/app/db.py | 28 ++++++++++++++++++++++++++++ backend/app/routes/batch.py | 3 ++- backend/app/routes/documents.py | 3 ++- backend/app/routes/projects.py | 25 ++++++++++--------------- backend/app/services/ocr_tasks.py | 3 ++- backend/app/services/pdfa.py | 2 +- backend/tests/test_workflow.py | 25 +++++++++++++++++++++++++ src/pages/Cleanup.jsx | 2 +- src/pages/Metadata.jsx | 5 +---- src/pages/UploadOCR.jsx | 23 ++++++++++++++++++++--- 10 files changed, 92 insertions(+), 27 deletions(-) diff --git a/backend/app/db.py b/backend/app/db.py index 536eee5..ed2529f 100644 --- a/backend/app/db.py +++ b/backend/app/db.py @@ -72,6 +72,7 @@ def transaction(database_path=None): cleaned_text TEXT, confidence_data TEXT, mean_confidence REAL, + word_count INTEGER, FOREIGN KEY (project_id) REFERENCES projects (id) ); @@ -158,11 +159,38 @@ def init_db(database_path=None): conn.execute('ALTER TABLE ocr_text ADD COLUMN confidence_data TEXT') if 'mean_confidence' not in columns: conn.execute('ALTER TABLE ocr_text ADD COLUMN mean_confidence REAL') + if 'word_count' not in columns: + conn.execute('ALTER TABLE ocr_text ADD COLUMN word_count INTEGER') + backfill_word_counts(conn) file_columns = {row['name'] for row in conn.execute('PRAGMA table_info(files)')} if 'archive_format' not in file_columns: conn.execute('ALTER TABLE files ADD COLUMN archive_format TEXT') +EFFECTIVE_TEXT_SQL = "COALESCE(NULLIF(cleaned_text, ''), original_text, '')" + + +def refresh_word_count(conn, project_id): + """Store the word count of a project's current text (cleaned if present, else OCR).""" + row = conn.execute(f'SELECT {EFFECTIVE_TEXT_SQL} AS text FROM ocr_text WHERE project_id = ?', + (project_id,)).fetchone() + if row is not None: + conn.execute('UPDATE ocr_text SET word_count = ? WHERE project_id = ?', (len(row['text'].split()), project_id)) + + +def backfill_word_counts(conn, batch=200): + """Fill word_count for rows saved before the column existed (runs once per row).""" + while True: + rows = conn.execute(f''' + SELECT project_id, {EFFECTIVE_TEXT_SQL} AS text FROM ocr_text + WHERE word_count IS NULL AND (original_text IS NOT NULL OR cleaned_text IS NOT NULL) + LIMIT ?''', (batch,)).fetchall() + if not rows: + return + conn.executemany('UPDATE ocr_text SET word_count = ? WHERE project_id = ?', + [(len(r['text'].split()), r['project_id']) for r in rows]) + + def get_config_value(key, default=None): with transaction() as conn: row = conn.execute('SELECT value FROM app_config WHERE key = ?', (key,)).fetchone() diff --git a/backend/app/routes/batch.py b/backend/app/routes/batch.py index bcb9629..e9759bc 100644 --- a/backend/app/routes/batch.py +++ b/backend/app/routes/batch.py @@ -3,7 +3,7 @@ from flask import Blueprint, current_app, jsonify, request from .. import search_index -from ..db import get_config_value, transaction +from ..db import get_config_value, refresh_word_count, transaction from ..processors.batch_processor import BatchProcessor from ..security import ValidationError, display_filename, safe_upload_path from ..services.ocr import extract_document_text @@ -83,6 +83,7 @@ def ocr_callback(project_id): with transaction() as conn: conn.execute('UPDATE ocr_text SET original_text = ?, mean_confidence = ? WHERE project_id = ?', (result['text'], result.get('mean_confidence'), project_id)) + refresh_word_count(conn, project_id) if result.get('ocr_pdf_path'): conn.execute('UPDATE files SET ocr_path = ? WHERE project_id = ?', (result['ocr_pdf_path'], project_id)) diff --git a/backend/app/routes/documents.py b/backend/app/routes/documents.py index a65f8dc..50a9d5a 100644 --- a/backend/app/routes/documents.py +++ b/backend/app/routes/documents.py @@ -9,7 +9,7 @@ from flask import Blueprint, Response, current_app, jsonify, request from .. import search_index -from ..db import get_config_value, transaction +from ..db import get_config_value, refresh_word_count, transaction from ..processors.metadata_extractor import extract_metadata from ..security import ValidationError from ..services import dublin_core @@ -45,6 +45,7 @@ def save_cleaned_text(project_id): with transaction() as conn: conn.execute('UPDATE ocr_text SET cleaned_text = ? WHERE project_id = ?', (cleaned_text, project_id)) + refresh_word_count(conn, project_id) set_status(conn, project_id, 'metadata') search_index.update(project_id, conn=conn) return jsonify({'success': True}) diff --git a/backend/app/routes/projects.py b/backend/app/routes/projects.py index 22c616a..b731de5 100644 --- a/backend/app/routes/projects.py +++ b/backend/app/routes/projects.py @@ -12,6 +12,7 @@ bp = Blueprint('projects', __name__, url_prefix='/api/projects') VALID_STATUSES = ('upload', 'ocr', 'cleanup', 'metadata', 'archived') +SNIPPET_CHARS = 300 def _servable_roots(): @@ -94,14 +95,15 @@ def list_ebooks(): f.cleaned_path, f.final_path, LENGTH(COALESCE(o.cleaned_text, o.original_text, '')) AS text_length, - o.original_text, - o.cleaned_text + COALESCE(o.word_count, 0) AS word_count, + -- Cut the preview in SQLite so full OCR texts never leave the database. + substr(COALESCE(NULLIF(o.cleaned_text, ''), o.original_text, ''), 1, ?) AS snippet FROM projects p LEFT JOIN metadata m ON p.id = m.project_id LEFT JOIN files f ON p.id = f.project_id LEFT JOIN ocr_text o ON p.id = o.project_id ORDER BY p.created_at DESC - ''').fetchall() + ''', (SNIPPET_CHARS + 1,)).fetchall() ebooks = [] for r in rows: @@ -121,16 +123,9 @@ def list_ebooks(): has_ocr_pdf = bool(ocr_path and os.path.exists(ocr_path)) has_searchable_pdf = has_final_pdf or has_ocr_pdf or bool(target_file) - cleaned = d.get('cleaned_text') or d.get('original_text') or '' - word_count = len(cleaned.split()) if cleaned else 0 - snippet = (cleaned[:300] + '...') if len(cleaned) > 300 else cleaned - - del d['original_text'] - del d['cleaned_text'] - + snippet = d['snippet'] or '' + d['snippet'] = snippet[:SNIPPET_CHARS] + '...' if len(snippet) > SNIPPET_CHARS else snippet d['file_size'] = file_size - d['word_count'] = word_count - d['snippet'] = snippet d['has_searchable_pdf'] = has_searchable_pdf d['has_final_pdf'] = has_final_pdf d['has_pdf'] = bool(target_file) @@ -167,8 +162,8 @@ def _ensure_searchable_pdf(project_id, project): try: if ext in current_app.config['IMAGE_EXTENSIONS']: - from PIL import Image import pytesseract + from PIL import Image with Image.open(original) as img: pdf_bytes = pytesseract.image_to_pdf_or_hocr(img, extension='pdf') with open(target_ocr_path, 'wb') as f: @@ -197,8 +192,8 @@ def _ensure_searchable_pdf(project_id, project): if ocr_text and ocr_text.strip(): try: from reportlab.lib.pagesizes import letter - from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer - from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle + from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet + from reportlab.platypus import Paragraph, SimpleDocTemplate, Spacer upload_folder = current_app.config['UPLOAD_FOLDER'] stem = os.path.splitext(project['filename'])[0] target_ocr_path = os.path.join(upload_folder, f'ocr_{stem}.pdf') diff --git a/backend/app/services/ocr_tasks.py b/backend/app/services/ocr_tasks.py index 641ab8c..eac1a16 100644 --- a/backend/app/services/ocr_tasks.py +++ b/backend/app/services/ocr_tasks.py @@ -14,7 +14,7 @@ from PIL import Image from .. import search_index -from ..db import get_config_value, transaction +from ..db import get_config_value, refresh_word_count, transaction from ..processors.advanced_ocr import AdvancedOCRProcessor from ..processors.glm_ocr import GlmOcrProcessor from ..processors.handwritten_to_pdf import HandwrittenToPDFConverter @@ -51,6 +51,7 @@ def _save_ocr_result(project_id, text, confidence=None, ocr_pdf_path=None, mean_ (text, json.dumps(confidence) if confidence else None, mean_confidence, project_id)) if ocr_pdf_path: conn.execute('UPDATE files SET ocr_path = ? WHERE project_id = ?', (ocr_pdf_path, project_id)) + refresh_word_count(conn, project_id) set_status(conn, project_id, 'cleanup') search_index.update(project_id, conn=conn) diff --git a/backend/app/services/pdfa.py b/backend/app/services/pdfa.py index 0b76e88..87ff138 100644 --- a/backend/app/services/pdfa.py +++ b/backend/app/services/pdfa.py @@ -12,9 +12,9 @@ born-digital PDFs keep their metadata but are not labelled PDF/A, because they can break rules a preflight cannot see. """ +import shutil from dataclasses import dataclass, field from datetime import datetime, timezone -import shutil try: import pikepdf diff --git a/backend/tests/test_workflow.py b/backend/tests/test_workflow.py index f864df4..5d6fbcd 100644 --- a/backend/tests/test_workflow.py +++ b/backend/tests/test_workflow.py @@ -163,3 +163,28 @@ def test_metadata_extractor(): result = extract_metadata(sample, 'ai_introduction.pdf') assert result['year'] == '2024' assert set(result) >= {'title', 'author', 'year', 'subject', 'keywords', 'confidence_scores'} + + +def test_ebooks_listing_uses_stored_word_counts_and_short_snippets(client, app): + long_text = ' '.join(f'word{i}' for i in range(400)) + pid = upload(client, 'notes.pdf', ('Title: Notes\n' + long_text).encode()).get_json()['project']['id'] + client.post(f'/api/ocr/{pid}') + listed = {e['id']: e for e in client.get('/api/projects/ebooks').get_json()['ebooks']}[pid] + assert listed['word_count'] >= 400 + assert listed['snippet'].endswith('...') and len(listed['snippet']) == 303 + assert 'original_text' not in listed and 'cleaned_text' not in listed + + # Editing the text updates the stored count. + client.post(f'/api/cleanup/{pid}', json={'cleaned_text': 'just three words'}) + listed = {e['id']: e for e in client.get('/api/projects/ebooks').get_json()['ebooks']}[pid] + assert listed['word_count'] == 3 and listed['snippet'] == 'just three words' + + +def test_word_counts_backfilled_for_existing_databases(make_app, app): + import sqlite3 + with sqlite3.connect(app.config['DATABASE']) as conn: + conn.execute("INSERT INTO projects (id, filename, filepath, status) VALUES (900, 'old.pdf', '', 'cleanup')") + conn.execute("INSERT INTO ocr_text (project_id, original_text) VALUES (900, 'one two three four')") + restarted = make_app().test_client() + listed = {e['id']: e for e in restarted.get('/api/projects/ebooks').get_json()['ebooks']}[900] + assert listed['word_count'] == 4 diff --git a/src/pages/Cleanup.jsx b/src/pages/Cleanup.jsx index 8726619..3cec029 100644 --- a/src/pages/Cleanup.jsx +++ b/src/pages/Cleanup.jsx @@ -3,7 +3,7 @@ import { useParams, useNavigate } from 'react-router-dom' import { useProject } from '../context/ProjectContext' import { Save, AlertCircle, ArrowRight, X, Download, Globe, - AlertTriangle, Sparkles, ZoomIn, ZoomOut, Maximize2, + AlertTriangle, Sparkles, ZoomIn, ZoomOut, ExternalLink, Eye, EyeOff, FileText, CheckCircle2, RotateCcw } from 'lucide-react' import WorkflowTracker from '../components/WorkflowTracker' diff --git a/src/pages/Metadata.jsx b/src/pages/Metadata.jsx index 475c216..1684a78 100644 --- a/src/pages/Metadata.jsx +++ b/src/pages/Metadata.jsx @@ -4,7 +4,7 @@ import { useProject } from '../context/ProjectContext' import { Save, AlertCircle, ArrowRight, X, Sparkles, Check, FileText, Folder, Calendar, User, Tag, Eye, EyeOff, - ExternalLink, RefreshCw, Copy, CheckCircle2, ChevronRight + ExternalLink, Copy, CheckCircle2 } from 'lucide-react' import WorkflowTracker from '../components/WorkflowTracker' import Modal from '../components/Modal' @@ -45,7 +45,6 @@ const Metadata = () => { // AI Suggestions State const [suggestions, setSuggestions] = useState(null) const [extracting, setExtracting] = useState(false) - const [extractionError, setExtractionError] = useState(null) // Document Reference Pane State const [showRefPane, setShowRefPane] = useState(true) @@ -96,7 +95,6 @@ const Metadata = () => { const handleExtractAI = async () => { setExtracting(true) - setExtractionError(null) try { const res = await axios.post(`${API_URL}/metadata/extract/${projectId}`) @@ -114,7 +112,6 @@ const Metadata = () => { } catch (err) { console.error('Failed to extract metadata:', err) const msg = err.response?.data?.error || 'Extraction failed' - setExtractionError(msg) addToast(msg, 'error') } finally { setExtracting(false) diff --git a/src/pages/UploadOCR.jsx b/src/pages/UploadOCR.jsx index 67f303c..f65e5a2 100644 --- a/src/pages/UploadOCR.jsx +++ b/src/pages/UploadOCR.jsx @@ -37,7 +37,23 @@ const UploadOCR = () => { const [ocrResult, setOcrResult] = useState(null) const [pdfResult, setPdfResult] = useState(null) const [language, setLanguage] = useState('eng') - const [useAdvancedOCR, setUseAdvancedOCR] = useState(true) + // Standard OCR by default: Advanced layout analysis is about 3x slower per + // page and only helps documents with tables, forms or stamps. + const [useAdvancedOCR, setUseAdvancedOCR] = useState(() => { + try { + return localStorage.getItem('libradigit_advanced_ocr') === 'true' + } catch { + return false + } + }) + const handleAdvancedToggle = (checked) => { + setUseAdvancedOCR(checked) + try { + localStorage.setItem('libradigit_advanced_ocr', String(checked)) + } catch { + // Preference simply isn't remembered. + } + } const [activeEngine, setActiveEngine] = useState('tesseract') // 'tesseract' | 'glm-ocr' // Read active engine from settings once on mount @@ -485,7 +501,7 @@ const UploadOCR = () => {

Advanced Layout & Feature Analysis

-

Detect structured tables, form fields, stamps, signatures, and auto-correct skew angle.

+

Detect structured tables, form fields, stamps, signatures, and auto-correct skew angle. About 3× slower per page — turn on for forms and tables.