Dashboard: real metrics, "needs your attention" panel, next-step actions

backend/app/db.py

@@ -71,6 +71,7 @@ def transaction(database_path=None):
     original_text TEXT,
     cleaned_text TEXT,
     confidence_data TEXT,
+    mean_confidence REAL,
     FOREIGN KEY (project_id) REFERENCES projects (id)
 );
 
@@ -155,6 +156,8 @@ def init_db(database_path=None):
         columns = {row['name'] for row in conn.execute('PRAGMA table_info(ocr_text)')}
         if 'confidence_data' not in columns:
             conn.execute('ALTER TABLE ocr_text ADD COLUMN confidence_data TEXT')
+        if 'mean_confidence' not in columns:
+            conn.execute('ALTER TABLE ocr_text ADD COLUMN mean_confidence REAL')
         file_columns = {row['name'] for row in conn.execute('PRAGMA table_info(files)')}
         if 'archive_format' not in file_columns:
             conn.execute('ALTER TABLE files ADD COLUMN archive_format TEXT')

backend/app/routes/batch.py

@@ -81,7 +81,8 @@ def ocr_callback(project_id):
             if result.get('unsupported'):
                 return False, result['text']
             with transaction() as conn:
-                conn.execute('UPDATE ocr_text SET original_text = ? WHERE project_id = ?', (result['text'], project_id))
+                conn.execute('UPDATE ocr_text SET original_text = ?, mean_confidence = ? WHERE project_id = ?',
+                             (result['text'], result.get('mean_confidence'), project_id))
                 if result.get('ocr_pdf_path'):
                     conn.execute('UPDATE files SET ocr_path = ? WHERE project_id = ?',
                                  (result['ocr_pdf_path'], project_id))

backend/app/routes/projects.py

@@ -28,9 +28,23 @@ def _first_existing(*paths):
 
 @bp.get('')
 def list_projects():
+    # Includes what the dashboard needs to suggest each document's next step.
     with transaction() as conn:
-        rows = conn.execute('SELECT * FROM projects ORDER BY created_at DESC').fetchall()
-    return jsonify({'projects': [dict(r) for r in rows]})
+        rows = conn.execute('''
+            SELECT p.*, m.title,
+                   f.final_path IS NOT NULL AS has_archive, f.archive_format,
+                   o.mean_confidence
+            FROM projects p
+            LEFT JOIN metadata m ON m.project_id = p.id
+            LEFT JOIN files f ON f.project_id = p.id
+            LEFT JOIN ocr_text o ON o.project_id = p.id
+            ORDER BY p.created_at DESC''').fetchall()
+    projects = []
+    for row in rows:
+        project = dict(row)
+        project['has_archive'] = bool(project['has_archive'])
+        projects.append(project)
+    return jsonify({'projects': projects})
 
 
 @bp.post('')

backend/app/services/ocr.py

@@ -41,12 +41,14 @@ def ocr_image(image, lang='eng'):
     """
     OCR one image with a single Tesseract pass.
 
-    Returns (text, low_confidence_words). Text is rebuilt from word boxes with
-    line and paragraph breaks, which matches image_to_string output, so the
-    confidence data comes for free instead of costing a second OCR run.
+    Returns (text, low_confidence_words, mean_confidence, word_count). Text is
+    rebuilt from word boxes with line and paragraph breaks, which matches
+    image_to_string output, so the confidence data comes for free instead of
+    costing a second OCR run.
     """
     data = pytesseract.image_to_data(image, lang=lang, output_type=pytesseract.Output.DICT)
     lines, order, low_conf = {}, [], []
+    conf_total, conf_count = 0.0, 0
     for i, word in enumerate(data['text']):
         if data['level'][i] != 5:
             continue
@@ -59,6 +61,9 @@ def ocr_image(image, lang='eng'):
             order.append(key)
         lines[key].append(word)
         conf = float(data['conf'][i])
+        if conf >= 0:
+            conf_total += conf
+            conf_count += 1
         if 0 <= conf < LOW_CONFIDENCE_THRESHOLD:
             low_conf.append({'word': word, 'conf': conf})
 
@@ -75,7 +80,8 @@ def ocr_image(image, lang='eng'):
         retry = pytesseract.image_to_string(image, lang=lang, config=HANDWRITTEN_CONFIG)
         if len(retry.strip()) > len(text.strip()):
             text = retry
-    return text, low_conf
+    mean = conf_total / conf_count if conf_count else None
+    return text, low_conf, mean, conf_count
 
 
 def _searchable_pdf_path(filepath):
@@ -90,6 +96,7 @@ def _extract_pdf_text_layer(doc):
 def _ocr_scanned_pdf(doc, filepath, lang, progress):
     """Render each page, OCR it and assemble a searchable (sandwich) PDF."""
     texts, confidence = [], []
+    conf_total, conf_count = 0.0, 0
     merged = pymupdf.open()
     has_pdf_pages = False
     total = len(doc)
@@ -104,7 +111,10 @@ def _ocr_scanned_pdf(doc, filepath, lang, progress):
             has_pdf_pages = True
         except Exception as e:
             current_app.logger.warning('Searchable PDF page %s failed: %s', page_idx + 1, e)
-        page_text, page_conf = ocr_image(img, lang)
+        page_text, page_conf, page_mean, page_words = ocr_image(img, lang)
+        if page_mean is not None:
+            conf_total += page_mean * page_words
+            conf_count += page_words
         for item in page_conf:
             item['page'] = page_idx + 1
         confidence.extend(page_conf)
@@ -116,7 +126,8 @@ def _ocr_scanned_pdf(doc, filepath, lang, progress):
         ocr_pdf_path = _searchable_pdf_path(filepath)
         merged.save(ocr_pdf_path, garbage=3, deflate=True)
     merged.close()
-    return '\n\n'.join(texts), confidence, ocr_pdf_path
+    mean = conf_total / conf_count if conf_count else None
+    return '\n\n'.join(texts), confidence, ocr_pdf_path, mean
 
 
 def _extract_pdf(filepath, lang, progress):
@@ -144,14 +155,14 @@ def _extract_pdf(filepath, lang, progress):
         if not tesseract_available():
             return {'text': text or TESSERACT_MISSING_MESSAGE, 'file_type': 'PDF (OCR unavailable)'}
         try:
-            ocr_text, confidence, ocr_pdf_path = _ocr_scanned_pdf(doc, filepath, lang, progress)
+            ocr_text, confidence, ocr_pdf_path, mean_conf = _ocr_scanned_pdf(doc, filepath, lang, progress)
         except Exception as e:
             current_app.logger.exception('Scanned PDF OCR failed')
             return {'text': text or f'No text found in PDF. OCR failed: {e}', 'file_type': 'PDF (OCR Failed)'}
 
     if ocr_text.strip():
         return {'text': ocr_text, 'file_type': 'PDF (Scanned/OCR)',
-                'confidence': confidence, 'ocr_pdf_path': ocr_pdf_path}
+                'confidence': confidence, 'ocr_pdf_path': ocr_pdf_path, 'mean_confidence': mean_conf}
     return {'text': text or 'No text found in PDF (OCR produced no results).', 'file_type': 'PDF (Empty)',
             'ocr_pdf_path': ocr_pdf_path}
 
@@ -172,7 +183,7 @@ def _extract_image(filepath, lang, engine, progress):
     try:
         with Image.open(filepath) as image:
             image.load()
-            text, confidence = ocr_image(image, lang)
+            text, confidence, mean_conf, _words = ocr_image(image, lang)
             ocr_pdf_path = None
             try:
                 pdf_bytes = pytesseract.image_to_pdf_or_hocr(image, extension='pdf', lang=lang)
@@ -184,7 +195,8 @@ def _extract_image(filepath, lang, engine, progress):
                 ocr_pdf_path = None
         if not text.strip():
             text = 'No text detected in image. Please ensure the image contains readable text.'
-        return {'text': text, 'file_type': 'image', 'confidence': confidence, 'ocr_pdf_path': ocr_pdf_path}
+        return {'text': text, 'file_type': 'image', 'confidence': confidence, 'ocr_pdf_path': ocr_pdf_path,
+                'mean_confidence': mean_conf}
     except Exception as e:
         message = f'Error extracting text from image: {e}'
         if 'tessdata' in str(e) or 'traineddata' in str(e):

backend/app/services/ocr_tasks.py

@@ -43,10 +43,12 @@ def _load_project_file(project_id):
     return project
 
 
-def _save_ocr_result(project_id, text, confidence=None, ocr_pdf_path=None):
+def _save_ocr_result(project_id, text, confidence=None, ocr_pdf_path=None, mean_confidence=None):
+    """mean_confidence is Tesseract's average word confidence (0-100), None when not measured."""
     with transaction() as conn:
-        conn.execute('UPDATE ocr_text SET original_text = ?, confidence_data = ? WHERE project_id = ?',
-                     (text, json.dumps(confidence) if confidence else None, project_id))
+        conn.execute('UPDATE ocr_text SET original_text = ?, confidence_data = ?, mean_confidence = ? '
+                     'WHERE project_id = ?',
+                     (text, json.dumps(confidence) if confidence else None, mean_confidence, project_id))
         if ocr_pdf_path:
             conn.execute('UPDATE files SET ocr_path = ? WHERE project_id = ?', (ocr_pdf_path, project_id))
         set_status(conn, project_id, 'cleanup')
@@ -60,7 +62,8 @@ def run_ocr(project_id, params, progress=no_progress):
 
     result = extract_document_text(project['filepath'], lang, engine, progress)
     progress(1, 1, 'Saving results')
-    _save_ocr_result(project_id, result['text'], result.get('confidence'), result.get('ocr_pdf_path'))
+    _save_ocr_result(project_id, result['text'], result.get('confidence'), result.get('ocr_pdf_path'),
+                     result.get('mean_confidence'))
     return {
         'success': True,
         'pages': 1,

backend/tests/test_workflow.py

@@ -26,6 +26,10 @@ def test_image_full_workflow(client, app, tmp_path):
     assert data['files']['ocr_path'] and os.path.exists(data['files']['ocr_path'])
 
     # Searchable immediately after OCR, with the snippet taken from the content.
+    listed = client.get('/api/projects').get_json()['projects'][0]
+    assert 80 <= listed['mean_confidence'] <= 100  # real Tesseract word confidence
+    assert listed['has_archive'] is False and listed['archive_format'] is None
+
     hits = client.get('/api/search?q=Library').get_json()
     assert hits['count'] == 1 and '<mark>' in hits['results'][0]['snippet']
 
@@ -53,6 +57,10 @@ def test_image_full_workflow(client, app, tmp_path):
     with fitz.open(final) as doc:
         assert doc.metadata['title'] == 'Annual Report'
 
+    listed = client.get('/api/projects').get_json()['projects'][0]
+    assert listed['has_archive'] is True and listed['archive_format'] == 'PDF/A-2b'
+    assert listed['title'] == 'Annual Report'
+
     served = client.get(f'/api/projects/{pid}/file?type=pdf')
     assert served.status_code == 200 and served.data.startswith(b'%PDF')
 

src/components/NeedsAttention.jsx

@@ -0,0 +1,124 @@
+import { useNavigate } from 'react-router-dom'
+import { ArrowRight, CheckCircle2, Loader2, Square } from 'lucide-react'
+import { displayTitle, nextStep } from '../utils/workflow'
+
+const UP_NEXT_LIMIT = 5
+
+const JOB_LABELS = {
+    ocr: 'OCR',
+    advanced_ocr: 'Advanced OCR',
+    handwritten_to_pdf: 'Handwriting to PDF',
+}
+
+/**
+ * "What needs me next": documents being processed right now, then the
+ * documents waiting on the user, oldest first.
+ */
+const NeedsAttention = ({ projects, jobs, onStopJob, onShowAll }) => {
+    const navigate = useNavigate()
+    const byId = new Map(projects.map(p => [p.id, p]))
+    const busy = new Set(jobs.map(job => job.project_id))
+
+    const waiting = projects
+        .filter(p => !busy.has(p.id) && nextStep(p).needsAction)
+        .sort((a, b) => new Date(a.updated_at) - new Date(b.updated_at))
+
+    if (jobs.length === 0 && waiting.length === 0) {
+        return (
+            <section className="attention-panel attention-clear" aria-label="Needs your attention">
+                <CheckCircle2 size={20} className="text-success" />
+                <span>All caught up — every document is archived.</span>
+            </section>
+        )
+    }
+
+    return (
+        <section className="attention-panel" aria-label="Needs your attention">
+            {jobs.length > 0 && (
+                <div className="attention-group">
+                    <h2 className="attention-heading">Processing now</h2>
+                    <ul className="attention-list">
+                        {jobs.map(job => {
+                            const project = byId.get(job.project_id)
+                            const pct = Math.round(job.progress || 0)
+                            return (
+                                <li key={job.id} className="attention-row">
+                                    <Loader2 size={18} className="spin-anim text-primary" aria-hidden="true" />
+                                    <div className="attention-main">
+                                        <button
+                                            type="button"
+                                            className="attention-title link-like"
+                                            onClick={() => navigate(`/upload?project=${job.project_id}`)}
+                                        >
+                                            {project ? displayTitle(project) : `Document ${job.project_id}`}
+                                        </button>
+                                        <span className="attention-hint">
+                                            {JOB_LABELS[job.kind] || job.kind} · {job.status === 'queued' ? 'Queued' : job.message}
+                                        </span>
+                                        <div
+                                            className="attention-progress"
+                                            role="progressbar"
+                                            aria-label={`Progress for ${project ? displayTitle(project) : 'document'}`}
+                                            aria-valuemin={0}
+                                            aria-valuemax={100}
+                                            aria-valuenow={pct}
+                                        >
+                                            <div className="attention-progress-fill" style={{ width: `${pct}%` }} />
+                                        </div>
+                                    </div>
+                                    <span className="attention-pct">{job.status === 'queued' ? '—' : `${pct}%`}</span>
+                                    <button
+                                        type="button"
+                                        className="btn-dash-secondary btn-sm"
+                                        onClick={() => onStopJob(job)}
+                                        disabled={job.message === 'Cancelling…'}
+                                    >
+                                        <Square size={12} />
+                                        <span>Stop</span>
+                                    </button>
+                                </li>
+                            )
+                        })}
+                    </ul>
+                </div>
+            )}
+
+            {waiting.length > 0 && (
+                <div className="attention-group">
+                    <h2 className="attention-heading">
+                        Up next <span className="tab-count">{waiting.length}</span>
+                    </h2>
+                    <ul className="attention-list">
+                        {waiting.slice(0, UP_NEXT_LIMIT).map(project => {
+                            const step = nextStep(project)
+                            return (
+                                <li key={project.id} className="attention-row">
+                                    <span className={`attention-dot step-${step.key}`} aria-hidden="true" />
+                                    <div className="attention-main">
+                                        <span className="attention-title">{displayTitle(project)}</span>
+                                        <span className="attention-hint">{step.hint}</span>
+                                    </div>
+                                    <button
+                                        type="button"
+                                        className="btn-dash-primary btn-sm"
+                                        onClick={() => navigate(step.path)}
+                                    >
+                                        <span>{step.label}</span>
+                                        <ArrowRight size={14} />
+                                    </button>
+                                </li>
+                            )
+                        })}
+                    </ul>
+                    {waiting.length > UP_NEXT_LIMIT && (
+                        <button type="button" className="attention-more link-like" onClick={onShowAll}>
+                            Show all {waiting.length} waiting documents
+                        </button>
+                    )}
+                </div>
+            )}
+        </section>
+    )
+}
+
+export default NeedsAttention

src/hooks/useActiveJobs.js

@@ -0,0 +1,48 @@
+import { useEffect, useRef, useState } from 'react'
+import axios from 'axios'
+import { API_URL } from '../config'
+
+const FAST_POLL_MS = 1500 // while something is running
+const IDLE_POLL_MS = 8000 // picks up jobs started from another page or window
+
+/**
+ * Background jobs that are queued or running, kept fresh by polling.
+ * onFinished() is called when a job leaves the active list, so callers can
+ * reload the documents it changed.
+ */
+export function useActiveJobs(onFinished) {
+    const [jobs, setJobs] = useState([])
+    const onFinishedRef = useRef(onFinished)
+    onFinishedRef.current = onFinished
+
+    useEffect(() => {
+        let cancelled = false
+        let timer
+        let previous = new Set()
+
+        const poll = async () => {
+            let active = previous.size > 0
+            try {
+                const { data } = await axios.get(`${API_URL}/jobs`, { params: { active: true } })
+                if (cancelled) return
+                const ids = new Set(data.jobs.map(job => job.id))
+                const finished = [...previous].some(id => !ids.has(id))
+                previous = ids
+                active = ids.size > 0
+                setJobs(data.jobs)
+                if (finished) onFinishedRef.current?.()
+            } catch {
+                // Backend unavailable: keep the last known list and retry.
+            }
+            if (!cancelled) timer = setTimeout(poll, active ? FAST_POLL_MS : IDLE_POLL_MS)
+        }
+
+        poll()
+        return () => {
+            cancelled = true
+            clearTimeout(timer)
+        }
+    }, [])
+
+    return jobs
+}