From 85fc7f39eb90f22ea4e61ddb8f66fa1cc53bfde2 Mon Sep 17 00:00:00 2001 From: krishnendu07-code Date: Fri, 12 Jun 2026 00:53:42 +0530 Subject: [PATCH 1/2] Handle camera permission errors gracefully --- src/pages/ScannerPage.tsx | 467 ++++++++++++++++++++++++-------------- 1 file changed, 293 insertions(+), 174 deletions(-) diff --git a/src/pages/ScannerPage.tsx b/src/pages/ScannerPage.tsx index c9e09e1..959ef0d 100644 --- a/src/pages/ScannerPage.tsx +++ b/src/pages/ScannerPage.tsx @@ -1,24 +1,32 @@ -import { useState, useEffect, useCallback, useRef } from 'react'; -import { useNavigate } from 'react-router-dom'; -import { Camera, Zap, RotateCcw, FlashlightOff, Flashlight, SwitchCamera, Upload } from 'lucide-react'; -import StatusTerminal from '../components/StatusTerminal'; -import { api, isAuthenticated } from '../lib/api'; -import { FishFreshnessInference } from '../fusionInference.js'; -import type { ScanResult } from '../lib/types'; +import { useState, useEffect, useCallback, useRef } from "react"; +import { useNavigate } from "react-router-dom"; +import { + Camera, + Zap, + RotateCcw, + FlashlightOff, + Flashlight, + SwitchCamera, + Upload, +} from "lucide-react"; +import StatusTerminal from "../components/StatusTerminal"; +import { api, isAuthenticated } from "../lib/api"; +import { FishFreshnessInference } from "../fusionInference.js"; +import type { ScanResult } from "../lib/types"; // ───────────────────────────────────────────────────────────────────────────── // Types // ───────────────────────────────────────────────────────────────────────────── -type ScanPhase = 'idle' | 'processing' | 'done' | 'error'; -type InferenceMode = 'cloud' | 'edge' | null; +type ScanPhase = "idle" | "processing" | "done" | "error"; +type InferenceMode = "cloud" | "edge" | null; /** Normalised result — maps both HF backend and ONNX outputs to one shape. */ interface DisplayResult { - label: 'Fresh' | 'Moderate' | 'Spoiled'; - freshness: number; // 0–100 integer - grade: string; // A+, A, B, C, D (cloud) or derived (edge) - confidence: string; // formatted percentage string + label: "Fresh" | "Moderate" | "Spoiled"; + freshness: number; // 0–100 integer + grade: string; // A+, A, B, C, D (cloud) or derived (edge) + confidence: string; // formatted percentage string } // ───────────────────────────────────────────────────────────────────────────── @@ -26,17 +34,17 @@ interface DisplayResult { // ───────────────────────────────────────────────────────────────────────────── function deriveGrade(freshness: number): string { - if (freshness >= 92) return 'A+'; - if (freshness >= 80) return 'A'; - if (freshness >= 65) return 'B'; - if (freshness >= 50) return 'C'; - return 'D'; + if (freshness >= 92) return "A+"; + if (freshness >= 80) return "A"; + if (freshness >= 65) return "B"; + if (freshness >= 50) return "C"; + return "D"; } function labelColor(label: string) { - if (label === 'Fresh') return 'text-neon'; - if (label === 'Moderate') return 'text-secondary'; - return 'text-error'; + if (label === "Fresh") return "text-neon"; + if (label === "Moderate") return "text-secondary"; + return "text-error"; } // ───────────────────────────────────────────────────────────────────────────── @@ -50,9 +58,12 @@ let engineLoading = false; async function getEngine(): Promise { if (engineReady && engineInstance) return engineInstance; if (engineLoading) { - await new Promise(resolve => { + await new Promise((resolve) => { const poll = setInterval(() => { - if (engineReady) { clearInterval(poll); resolve(); } + if (engineReady) { + clearInterval(poll); + resolve(); + } }, 100); }); return engineInstance!; @@ -70,17 +81,17 @@ async function getEngine(): Promise { // ───────────────────────────────────────────────────────────────────────────── async function captureVideoBlob(video: HTMLVideoElement): Promise { - const canvas = document.createElement('canvas'); - canvas.width = video.videoWidth || 640; + const canvas = document.createElement("canvas"); + canvas.width = video.videoWidth || 640; canvas.height = video.videoHeight || 480; - canvas.getContext('2d')?.drawImage(video, 0, 0); - return new Promise(resolve => canvas.toBlob(resolve, 'image/jpeg', 0.92)); + canvas.getContext("2d")?.drawImage(video, 0, 0); + return new Promise((resolve) => canvas.toBlob(resolve, "image/jpeg", 0.92)); } async function blobToImageElement(blob: Blob): Promise { return new Promise((resolve, reject) => { const img = new Image(); - img.onload = () => resolve(img); + img.onload = () => resolve(img); img.onerror = reject; img.src = URL.createObjectURL(blob); }); @@ -94,43 +105,80 @@ export default function ScannerPage() { const navigate = useNavigate(); // ── State ────────────────────────────────────────────────────────────────── - const [scanPhase, setScanPhase] = useState('idle'); + const [scanPhase, setScanPhase] = useState("idle"); const [inferenceMode, setInferenceMode] = useState(null); - const [result, setResult] = useState(null); - const [error, setError] = useState(''); - const [flashOn, setFlashOn] = useState(false); - const [facingMode, setFacingMode] = useState<'environment' | 'user'>('environment'); - const [progress, setProgress] = useState(0); - const [copied, setCopied] = useState(false); - const [previewUrl, setPreviewUrl] = useState(null); - - const videoRef = useRef(null); + const [result, setResult] = useState(null); + const [error, setError] = useState(""); + const [flashOn, setFlashOn] = useState(false); + const [facingMode, setFacingMode] = useState<"environment" | "user">( + "environment", + ); + const [progress, setProgress] = useState(0); + const [copied, setCopied] = useState(false); + const [previewUrl, setPreviewUrl] = useState(null); + const [cameraError, setCameraError] = useState(null); + + const videoRef = useRef(null); const fileInputRef = useRef(null); - const progressRef = useRef | null>(null); - const streamRef = useRef(null); + const progressRef = useRef | null>(null); + const streamRef = useRef(null); // ── Pre-warm ONNX engine on mount (runs in background) ──────────────────── - useEffect(() => { getEngine().catch(console.error); }, []); + useEffect(() => { + getEngine().catch(console.error); + }, []); // ── Camera stream ────────────────────────────────────────────────────────── useEffect(() => { - if (scanPhase !== 'idle') return; + if (scanPhase !== "idle") return; let cancelled = false; const currentVideo = videoRef.current; - navigator.mediaDevices.getUserMedia({ video: { facingMode } }) - .then(stream => { - if (cancelled) { stream.getTracks().forEach(t => t.stop()); return; } + navigator.mediaDevices + .getUserMedia({ video: { facingMode } }) + .then((stream) => { + if (cancelled) { + stream.getTracks().forEach((t) => t.stop()); + return; + } + + setCameraError(null); streamRef.current = stream; - if (currentVideo) currentVideo.srcObject = stream; + + if (currentVideo) { + currentVideo.srcObject = stream; + } }) - .catch(err => { if (!cancelled) console.error('Camera error:', err); }); + .catch((err) => { + if (cancelled) return; + + console.error("Camera error:", err); + + if (err instanceof DOMException) { + if (err.name === "NotAllowedError") { + setCameraError("Camera permission was denied"); + } else if (err.name === "NotFoundError") { + setCameraError( + "No camera was found on this device. Please connect a camera and try again.", + ); + } else { + setCameraError( + "Unable to access the camera. Please check your browser permissions and try again.", + ); + } + } else { + setCameraError("Something went wrong while accessing the camera."); + } + }); return () => { cancelled = true; - streamRef.current?.getTracks().forEach(t => t.stop()); + streamRef.current?.getTracks().forEach((t) => t.stop()); streamRef.current = null; - if (currentVideo) currentVideo.srcObject = null; + + if (currentVideo) { + currentVideo.srcObject = null; + } }; }, [facingMode, scanPhase]); @@ -138,7 +186,7 @@ export default function ScannerPage() { const startProgress = useCallback(() => { setProgress(0); progressRef.current = setInterval(() => { - setProgress(prev => prev >= 90 ? prev : prev + Math.random() * 5 + 1); + setProgress((prev) => (prev >= 90 ? prev : prev + Math.random() * 5 + 1)); }, 120); }, []); @@ -149,153 +197,192 @@ export default function ScannerPage() { // ── Stop camera ──────────────────────────────────────────────────────────── const stopCamera = useCallback(() => { - streamRef.current?.getTracks().forEach(t => t.stop()); + streamRef.current?.getTracks().forEach((t) => t.stop()); streamRef.current = null; if (videoRef.current) videoRef.current.srcObject = null; }, []); // ── Core: run hybrid inference on a single blob ──────────────────────────── - const runScan = useCallback(async (blob: Blob) => { - setScanPhase('processing'); - startProgress(); - setError(''); - setInferenceMode(null); + const runScan = useCallback( + async (blob: Blob) => { + setScanPhase("processing"); + startProgress(); + setError(""); + setInferenceMode(null); + + // Store preview + const url = URL.createObjectURL(blob); + setPreviewUrl(url); + stopCamera(); + + try { + // ── Path A: online — try HF backend first ───────────────────────────── + const online = await api.scanOnline(blob); + + if (online) { + // Cloud inference succeeded — use typed ScanResult fields directly + const s: ScanResult = online.scan; + const freshness = s.freshness_index; + stopProgress(100); + setInferenceMode("cloud"); + setResult({ + label: s.is_fresh + ? freshness >= 80 + ? "Fresh" + : "Moderate" + : "Spoiled", + freshness, + grade: s.grade, + confidence: `${Math.round((s.confidence ?? 0.9) * 100)}%`, + }); - // Store preview - const url = URL.createObjectURL(blob); - setPreviewUrl(url); - stopCamera(); + if (s.scan_id) { + sessionStorage.setItem("lastScanId", s.scan_id); + } + setScanPhase("done"); + setTimeout(() => navigate("/analysis"), 1800); + return; + } - try { - // ── Path A: online — try HF backend first ───────────────────────────── - const online = await api.scanOnline(blob); + // ── Path B: offline — fall back to ONNX ─────────────────────────────── + setInferenceMode("edge"); + const imgEl = await blobToImageElement(blob); + const engine = await getEngine(); + const fusion = await engine.predictSingle(imgEl); - if (online) { - // Cloud inference succeeded — use typed ScanResult fields directly - const s: ScanResult = online.scan; - const freshness = s.freshness_index; stopProgress(100); - setInferenceMode('cloud'); + const freshness = Math.round(fusion.fusedScore * 100); setResult({ - label: s.is_fresh ? (freshness >= 80 ? 'Fresh' : 'Moderate') : 'Spoiled', + label: fusion.label, freshness, - grade: s.grade, - confidence: `${Math.round((s.confidence ?? 0.9) * 100)}%`, + grade: deriveGrade(freshness), + confidence: fusion.confidence, }); - - if (s.scan_id) { - sessionStorage.setItem('lastScanId', s.scan_id); - } - setScanPhase('done'); - setTimeout(() => navigate('/analysis'), 1800); - return; + setScanPhase("done"); + + // Best-effort backend save (non-blocking, offline-safe) + const canvas = document.createElement("canvas"); + canvas.width = 224; + canvas.height = 224; + canvas.getContext("2d")?.drawImage(imgEl, 0, 0, 224, 224); + canvas.toBlob( + async (saveBlob) => { + if (!saveBlob) return; + try { + const saved = await api.submitScan(saveBlob, { + freshness_label: fusion.label, + fused_score: fusion.fusedScore, + source: "edge_onnx", + }); + if ((saved?.scan as unknown as { scan_id?: string })?.scan_id) { + sessionStorage.setItem( + "lastScanId", + (saved.scan as unknown as { scan_id: string }).scan_id, + ); + } + } catch { + /* offline or backend down — result still shown locally */ + } + }, + "image/jpeg", + 0.85, + ); + + setTimeout(() => navigate("/analysis"), 1800); + } catch (err) { + stopProgress(0); + const msg = err instanceof Error ? err.message : "Inference failed."; + const isNotFish = + msg.includes("NOT_A_FISH") || + msg.includes("not appear to contain a fish"); + setError( + isNotFish + ? "NOT_A_FISH: No fish detected. Please photograph a fish." + : msg, + ); + setScanPhase("error"); } - - // ── Path B: offline — fall back to ONNX ─────────────────────────────── - setInferenceMode('edge'); - const imgEl = await blobToImageElement(blob); - const engine = await getEngine(); - const fusion = await engine.predictSingle(imgEl); - - stopProgress(100); - const freshness = Math.round(fusion.fusedScore * 100); - setResult({ - label: fusion.label, - freshness, - grade: deriveGrade(freshness), - confidence: fusion.confidence, - }); - setScanPhase('done'); - - // Best-effort backend save (non-blocking, offline-safe) - const canvas = document.createElement('canvas'); - canvas.width = 224; canvas.height = 224; - canvas.getContext('2d')?.drawImage(imgEl, 0, 0, 224, 224); - canvas.toBlob(async saveBlob => { - if (!saveBlob) return; - try { - const saved = await api.submitScan(saveBlob, { - freshness_label: fusion.label, - fused_score: fusion.fusedScore, - source: 'edge_onnx', - }); - if ((saved?.scan as unknown as { scan_id?: string })?.scan_id) { - sessionStorage.setItem('lastScanId', (saved.scan as unknown as { scan_id: string }).scan_id); - } - } catch { /* offline or backend down — result still shown locally */ } - }, 'image/jpeg', 0.85); - - setTimeout(() => navigate('/analysis'), 1800); - - } catch (err) { - stopProgress(0); - const msg = err instanceof Error ? err.message : 'Inference failed.'; - const isNotFish = msg.includes('NOT_A_FISH') || msg.includes('not appear to contain a fish'); - setError(isNotFish ? 'NOT_A_FISH: No fish detected. Please photograph a fish.' : msg); - setScanPhase('error'); - } - }, [startProgress, stopProgress, stopCamera, navigate]); + }, + [startProgress, stopProgress, stopCamera, navigate], + ); // ── Camera capture ───────────────────────────────────────────────────────── const captureFrame = useCallback(async () => { - if (!isAuthenticated()) { navigate('/auth'); return; } + if (!isAuthenticated()) { + navigate("/auth"); + return; + } const video = videoRef.current; if (!video) return; const blob = await captureVideoBlob(video); - if (!blob) { setError('Failed to capture frame.'); return; } + if (!blob) { + setError("Failed to capture frame."); + return; + } await runScan(blob); }, [runScan, navigate]); // ── File upload ──────────────────────────────────────────────────────────── const handleUploadClick = useCallback(() => { - if (!isAuthenticated()) { navigate('/auth'); return; } + if (!isAuthenticated()) { + navigate("/auth"); + return; + } fileInputRef.current?.click(); }, [navigate]); - const handleFileChange = useCallback(async (e: React.ChangeEvent) => { - const file = e.target.files?.[0]; - if (!file) return; - if (fileInputRef.current) fileInputRef.current.value = ''; - await runScan(file); - }, [runScan]); + const handleFileChange = useCallback( + async (e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + if (!file) return; + if (fileInputRef.current) fileInputRef.current.value = ""; + await runScan(file); + }, + [runScan], + ); // ── Reset ────────────────────────────────────────────────────────────────── const resetScan = useCallback(() => { - setScanPhase('idle'); + setScanPhase("idle"); setResult(null); - setError(''); + setError(""); setInferenceMode(null); setProgress(0); - if (previewUrl) { URL.revokeObjectURL(previewUrl); setPreviewUrl(null); } + if (previewUrl) { + URL.revokeObjectURL(previewUrl); + setPreviewUrl(null); + } }, [previewUrl]); const toggleCamera = useCallback(() => { - setFacingMode(prev => prev === 'environment' ? 'user' : 'environment'); + setFacingMode((prev) => (prev === "environment" ? "user" : "environment")); }, []); // ── Derived ──────────────────────────────────────────────────────────────── - const isScanning = scanPhase === 'processing'; - const scanComplete = scanPhase === 'done'; - const freshness = result?.freshness ?? null; + const isScanning = scanPhase === "processing"; + const scanComplete = scanPhase === "done"; + const freshness = result?.freshness ?? null; const terminalMessages = (() => { - if (isScanning && inferenceMode === 'edge') return ['MODE: EDGE_ONNX', 'RUNNING_LOCAL_INFERENCE...']; - if (isScanning && inferenceMode === 'cloud') return ['MODE: CLOUD_API', 'CONNECTING_TO_HF...']; - if (isScanning) return ['DETECTING_MODEL...', 'PLEASE_WAIT']; - if (scanComplete && inferenceMode === 'edge') return ['MODEL: EDGE_ONNX', 'DEVICE: ON_DEVICE', 'LATENCY: <50ms']; - if (scanComplete && inferenceMode === 'cloud') return ['MODEL: CLOUD_API', 'DEVICE: HF_INFERENCE']; - if (scanPhase === 'error') return ['SCAN_SEQ: FAILED', 'CHECK_SPECIMEN']; - return ['SYSTEM: READY', 'POINT_CAMERA_AT_FISH']; + if (isScanning && inferenceMode === "edge") + return ["MODE: EDGE_ONNX", "RUNNING_LOCAL_INFERENCE..."]; + if (isScanning && inferenceMode === "cloud") + return ["MODE: CLOUD_API", "CONNECTING_TO_HF..."]; + if (isScanning) return ["DETECTING_MODEL...", "PLEASE_WAIT"]; + if (scanComplete && inferenceMode === "edge") + return ["MODEL: EDGE_ONNX", "DEVICE: ON_DEVICE", "LATENCY: <50ms"]; + if (scanComplete && inferenceMode === "cloud") + return ["MODEL: CLOUD_API", "DEVICE: HF_INFERENCE"]; + if (scanPhase === "error") return ["SCAN_SEQ: FAILED", "CHECK_SPECIMEN"]; + return ["SYSTEM: READY", "POINT_CAMERA_AT_FISH"]; })(); // ── Render ───────────────────────────────────────────────────────────────── return (
- {/* ── Viewport ──────────────────────────────────────────────────── */}
- {/* Preview or live camera */} {previewUrl && !isScanning ? ( Captured + ) : cameraError ? ( +
+
+

+ Camera Access Needed +

+ +

{cameraError}

+ +

+ Click the camera icon in your browser's address bar, allow + camera access, and refresh the page. +

+
+
) : (