diff --git a/.github/workflows/search-eval.yml b/.github/workflows/search-eval.yml
new file mode 100644
index 0000000..af06fb3
--- /dev/null
+++ b/.github/workflows/search-eval.yml
@@ -0,0 +1,62 @@
+name: Search quality
+
+# Typed search with the real vision-language weights: does a query find the photos
+# it should? Compares SigLIP with SigLIP 2, one phrasing with several, and English
+# with Persian queries, on public labelled images. See scripts/eval_search.py.
+on:
+ workflow_dispatch:
+ push:
+ branches: ["claude/**", "release/**"]
+ paths:
+ - "scripts/eval_search.py"
+ - "scripts/calibrate.py"
+ - "src/vie/models.py"
+ - "src/vie/default.yaml"
+ - ".github/workflows/search-eval.yml"
+
+jobs:
+ eval:
+ name: ${{ matrix.model }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 120
+ strategy:
+ fail-fast: false
+ matrix:
+ model:
+ - google/siglip-so400m-patch14-384
+ - google/siglip2-so400m-patch14-384
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+ - name: Install the CPU stack
+ run: |
+ python -m pip install --upgrade pip
+ pip install "torch==2.5.1" "torchvision==0.20.1" \
+ --index-url https://download.pytorch.org/whl/cpu
+ pip install -e ".[models,cpu]"
+ pip freeze | grep -iE "^(torch|transformers|tokenizers|sentencepiece)="
+ - name: Cache model weights
+ uses: actions/cache@v4
+ with:
+ path: ~/.cache/huggingface
+ key: hf-${{ matrix.model }}-v1
+ - name: Fetch the labelled set
+ run: python scripts/fetch_calibration_data.py benchmarks/search-data --per-label 4
+ - name: Evaluate
+ # Whole frame only: the multi-crop layout is 6x the crops, too slow for a CPU runner.
+ run: python scripts/eval_search.py benchmarks/search-data --model ${{ matrix.model }} --layout 1
+ - name: Food and species thresholds on this model
+ # The food, species and food-gate thresholds were calibrated on SigLIP; this
+ # reports precision and recall at the configured values for each model.
+ run: |
+ python scripts/calibrate.py benchmarks/search-data --model ${{ matrix.model }} \
+ --out benchmarks/calibration-${{ strategy.job-index }}.json
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: search-eval-${{ strategy.job-index }}
+ path: |
+ benchmarks/search-eval-*.json
+ benchmarks/calibration-*.json
diff --git a/.github/workflows/showcase.yml b/.github/workflows/showcase.yml
new file mode 100644
index 0000000..f1a8e11
--- /dev/null
+++ b/.github/workflows/showcase.yml
@@ -0,0 +1,60 @@
+name: Showcase
+
+# Builds the search showcase (food, species, free text, Persian, photos-and-videos
+# concept search) with the real weights on public data, and commits the cards to
+# assets/showcase/search on the same branch. See scripts/make_search_showcase.py.
+on:
+ workflow_dispatch:
+ push:
+ branches: ["claude/**"]
+ paths:
+ - "scripts/make_search_showcase.py"
+ - ".github/workflows/showcase.yml"
+
+permissions:
+ contents: write
+
+jobs:
+ showcase:
+ runs-on: ubuntu-latest
+ timeout-minutes: 120
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ github.ref_name }}
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+ - name: Install the CPU stack
+ run: |
+ sudo apt-get update && sudo apt-get install -y libgl1 libglib2.0-0 libfribidi0 libraqm0
+ python -m pip install --upgrade pip
+ pip install "torch==2.5.1" "torchvision==0.20.1" \
+ --index-url https://download.pytorch.org/whl/cpu
+ pip install -e ".[models,cpu,video]" arabic-reshaper python-bidi
+ - name: Cache model weights
+ uses: actions/cache@v4
+ with:
+ path: |
+ ~/.cache/huggingface
+ ~/.cache/torch
+ ~/.insightface
+ key: showcase-models-v1
+ - name: Fetch public data
+ run: |
+ python scripts/fetch_calibration_data.py data/public --per-label 2
+ mkdir -p data/videos
+ for name in classroom face-demographics-walking-and-pause; do
+ curl -sSL -o "data/videos/$name.mp4" \
+ "https://raw.githubusercontent.com/intel-iot-devkit/sample-videos/master/$name.mp4"
+ done
+ - name: Build the showcase
+ run: python scripts/make_search_showcase.py --data data/public --videos data/videos
+ - name: Commit the cards
+ run: |
+ git config user.name "github-actions[bot]"
+ git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+ git add assets/showcase/search
+ git commit -m "docs: search showcase built with the real models" || exit 0
+ git pull --rebase origin "${{ github.ref_name }}"
+ git push origin "HEAD:${{ github.ref_name }}"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d4447d0..580366c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,89 @@
Each version is a git tag. Check one out to get that state of the project.
+## v0.6.0: search the whole archive by what it shows
+
+**One concept search over photos and videos.** `Engine.search_everything("a theatre
+performance")` ranks photos and video moments together on SigLIP's calibrated
+probability that the description fits, keeps the strongest moments in view, and
+draws video tiles with their timestamp. The notebook has a section for it.
+
+**Food and unknown species no longer touch pixels at query time.** Each detected
+animal and dish is encoded at index time, padded exactly as search pads it, in the
+same batched pass as the scene regions (schema v4). A food query, or a species
+outside ImageNet, used to decode every candidate image and run the image tower on
+its crops, so it took minutes on thousands of photos; it is now a matrix product
+over stored vectors through the paths' own ranking rule. An older index is still
+served by cropping.
+
+**Persian queries and SigLIP 2.** `google/siglip2-so400m-patch14-384` is supported
+(its tokenizer declares no length, so the 64 tokens both models trained on are now
+explicit) and understands Persian. A query is averaged over several phrasings
+(`text.query_templates`). `scripts/eval_search.py` and the *Search quality* workflow
+measure both choices with the real weights.
+
+**Measured defaults.** SigLIP 2 is the default vision-language model: equal on English
+queries (mAP 1.0 both), 0.945 against 0.738 on Persian, and with thresholds calibrated
+for it (`model_overrides`, food 0.92, species 0.84) level with SigLIP on food and species
+(precision 0.90–0.94 against 0.89 over two runs; species 0.95 both).
+Each model's thresholds follow it automatically, and a threshold set explicitly still
+wins. A Persian query is encoded as typed, since wrapping it in phrasings never helped.
+`text.min_probability: 0.001` lets an absent concept return nothing. Changing the model
+means `vie index --rebuild`; the index refuses to mix models.
+
+**Video faces at a fixed cost per person.** Each person keeps up to four exemplar
+views (the most typical, then the least alike) and their on-screen intervals,
+instead of a vector for every sighting; face search returns all of a person's
+intervals.
+
+## v0.5.0: one engine, more accurate faces, output worth showing
+
+**Faces, measured against student-face-gate.** `scripts/face_benchmark.py` puts ArcFace
+(this repo) and SFace (the gate) on identical aligned crops. ArcFace wins on every measure
+(d′ 8.08 against 5.86; 96.9% against 83.6% of hard head poses at each model's own
+threshold, with no wrong person accepted by either), so it stays. Two ideas came over from
+the gate, each measured here:
+
+* **Mirror averaging**, on by default (`face.flip_tta`): every face is embedded as seen and
+ mirrored. Hard poses 96.9% -> 97.6%, d′ 8.08 -> 8.43. One extra 112x112 pass per face.
+* **Several query photos**: `vie search face --image a.jpg --image b.jpg`. Each indexed face
+ keeps its best score over the query photos.
+
+Turning the recipe on re-indexes only the images that have faces, not the whole archive.
+
+**`vie.api.Engine`.** Every capability behind one object that loads each model once:
+incremental image and video indexing, the four searches, video search, result cards and
+annotated video. It refuses to prune when most of the archive vanishes at once, which is a
+disconnected drive far more often than a real deletion.
+
+**Output.** `vie.report.showcase` renders a search as a presentation-grade image, cropped
+around the matched subject. `vie.video.render` writes an annotated MP4: tracked faces,
+names from reference photos (with a margin over the runner-up, as in the gate) or grouped
+people, shot labels and a presence timeline, plus a card of who appeared for how long.
+
+**Found in a strict review of the whole codebase, and fixed:**
+
+* **Face indexing was doing 2.2x the work it needed.** InsightFace runs every model in
+ its pack on every face by default, including two landmark models and age/gender that
+ nothing here reads. Loading only detection and recognition: 1144 -> 527 ms on a
+ four-face 1080p frame, with identical faces and vectors.
+* **Unchanged files could be re-hashed forever.** A file whose content matched but
+ whose mtime had moved (a copy, a Drive sync, an extracted zip) was skipped without the
+ new mtime being stored, so every later run read it in full again. A gallery that
+ moved kept pointing at its old paths. Both are now recorded on the first such run.
+* **Pruning is guarded in the pipeline,** so `vie index` is protected too, not only the
+ notebook: if fewer than half of the indexed files are on disk, nothing is deleted.
+* **The annotated video ignored rotation flags** (portrait phone videos came out
+ sideways) and gave up on a damaged file; it now uses the analysis decoder for both.
+* **Persian names and file names** are shaped right to left on cards and videos.
+* Comments that still described CLIP, MobileNetV3 and a 768-d space now describe what
+ runs: SigLIP so400m, ConvNeXt-Tiny and the margin rule.
+
+**`notebooks/0_run_everything.ipynb`.** The whole system with one *Run all* in Colab. The
+index lives on Google Drive, so the first run reads the whole dataset and later runs read
+only new or changed files: 37 photos took 18.4 s, and with one photo added the next run
+took 1.1 s; two videos took 67.8 s the first time and 0.0 s the next. Every stage reports its own failure and the rest carry on.
+
## v0.4.0: video understanding
New `vie.video` package and `vie video` commands: shot detection, keyframes, zero-shot
diff --git a/README.md b/README.md
index 4ff1bb6..7e8e966 100644
--- a/README.md
+++ b/README.md
@@ -26,22 +26,48 @@ Faces, animals, food, plain-English descriptions, and moments inside a video, al
| Ask with | Example | You get |
|:--|:--|:--|
-| A face | `vie search face --image person.jpg` | Every photo of that person |
+| A face | `vie search face --image a.jpg --image b.jpg` | Every photo of that person; more photos find more |
| An animal | `vie search animal --query "red panda"` | Photos with the animal boxed |
| A dish | `vie search food --query "ghormeh sabzi"` | Photos of it, even with no detector class for it |
| A description | `vie search text --query "a man with a blue backpack"` | The closest matches |
+| A concept, everywhere | `Engine.search_everything("a theatre performance")` | Photos and video moments together, ranked on one scale |
| A video | `vie video analyze clip.mp4 --html report.html` | Shots, topics, people, speech and a summary |
+| An annotated video | `Engine.render_video(...)` | An MP4 with every face tracked, named or grouped, on a timeline |
| A moment | `vie video search --query "a goal being scored"` | Timestamps across all your videos |
## See it work
+
+
+
+
+Named from three reference photos each, taken from other recordings. Corridor and classroom: [MP4](assets/showcase/corridor_people.mp4) · [MP4](assets/showcase/classroom_people.mp4)
+
+
+
+| Face search with two query photos | Every face in a classroom, grouped |
+|:--|:--|
+|  |  |
+| **Who was in the video, and for how long** | **Corridor camera** |
+|  |  |
+
+Real output of the current code, regenerated by scripts/make_showcase.py. Footage: intel-iot-devkit/sample-videos, CC BY 4.0.
+
+Screenshots from the original notebooks
+
| Face search | Species search |
|:--|:--|
|  |  |
| **Food search** | **Free-text search** |
|  |  |
-Screenshots from the original notebooks, so the scores shown predate the current scoring.
+\n",
+ "\n",
+ "# 🚀 سیستم کامل — همهچیز با یک «Run all»\n",
+ "\n",
+ "[](https://colab.research.google.com/github/ParsaVictor/visual-intelligence-engine/blob/main/notebooks/0_run_everything.ipynb)\n",
+ "\n",
+ "این نوتبوک کل سیستم را از اول تا آخر اجرا میکند:\n",
+ "\n",
+ "| مرحله | چه میکند |\n",
+ "|:--|:--|\n",
+ "| ۱. آمادهسازی | Drive را وصل میکند، GPU را پیدا میکند، بسته را نصب میکند |\n",
+ "| ۲. ایندکس عکسها | **بار اول همهی دیتاست را میخواند؛ از بار دوم فقط عکسهای جدید یا تغییرکرده را** |\n",
+ "| ۳. جستجو | چهره، حیوان، غذا و متن آزاد، با خروجی تصویری آمادهی ارائه. غذا و گونهی ناشناخته از بردارهای ذخیرهشده جواب داده میشوند و هیچ عکسی باز نمیشود |\n",
+ "| ۴. ویدیو | تحلیل ویدیوها، گزارش HTML و **ویدیوی خروجی با کادر چهره، نام افراد و خط زمانی** |\n",
+ "| ۵. جستجوی مفهوم | یک توضیح («تئاتر»، «کلاس درس») و همهی عکسها و لحظههای ویدیو که آن را نشان میدهند، در یک فهرست |\n",
+ "| ۶. خلاصه | همهی خروجیها در Drive، با زمان هر مرحله |\n",
+ "\n",
+ "## نحوهی اجرا (از گوشی هم میشود)\n",
+ "\n",
+ "1. منوی **Runtime → Change runtime type → T4 GPU**\n",
+ "2. عکسها را در Drive در پوشهی `MyDrive/vie/dataset` بگذارید (پوشه یا یک فایل `zip`). زیرپوشه هم اشکالی ندارد.\n",
+ "3. **Runtime → Run all**. اگر Drive اجازه خواست، تأیید کنید.\n",
+ "\n",
+ "اگر پوشهی دیتاست خالی باشد، نوتبوک یک مجموعهی کوچک عمومی دانلود میکند تا همهی مراحل باز هم اجرا شوند.\n",
+ "\n",
+ "## چرا بار دوم خیلی سریع است\n",
+ "\n",
+ "ایندکس روی **Google Drive** ذخیره میشود، نه روی دیسک موقت Colab که با هر بار بسته شدن پاک میشود.\n",
+ "هر عکس با اندازه و زمان تغییرش شناخته میشود؛ عکسی که عوض نشده **اصلاً باز نمیشود**،\n",
+ "و اگر هیچ عکس جدیدی نباشد **هیچ مدلی هم بار نمیشود**. اگر وسط کار قطع شود، کار انجامشده هر چند دقیقه در Drive ذخیره شده و ادامهی کار از همانجا شروع میشود.\n",
+ "\n",
+ "## دقت\n",
+ "\n",
+ "| بخش | مدل | چرا |\n",
+ "|:--|:--|:--|\n",
+ "| چهره | ArcFace (InsightFace `buffalo_l`) **+ میانگین با تصویر آینهای** | در مقایسهی مستقیم با SFace (ریپازیتوری گیت دانشجویی)، روی چهرههای با زاویهی سخت ٪۹۷٫۶ در برابر ٪۸۳٫۶ و بدون هیچ پذیرش اشتباه. جزئیات در `results/face_benchmark.json` |\n",
+ "| جستجوی چهره با چند عکس | بیشترین شباهت بین عکسهای پرسوجو | در گیت دانشجویی از ٪۸۶ به ٪۹۶ رسید |\n",
+ "| متن، حیوان، غذا | SigLIP 2 روی GPU | اندازهگیری با مدل واقعی. جستجوی انگلیسی برابر با SigLIP (mAP ۱٫۰)، جستجوی فارسی ۰٫۹۴۵ در برابر ۰٫۷۳۸، و در غذا و حیوان همسطح. روی CPU مدل سبکتر انتخاب میشود |\n",
+ "| «چیزی پیدا نشد» | کف احتمال ۰٫۰۰۱ | مفهومی که در آرشیو نیست، به جای چند نتیجهی بیربط، هیچ نتیجهای برنمیگرداند |\n",
+ "| جستجوی متنی | میانگین چند جملهبندی از هر پرسوجو | توصیف پایدارتر یک مفهوم نسبت به یک جملهی تنها |\n",
+ "\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9107f47f",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## ⚙️ تنظیمات\n",
+ "\n",
+ "**فقط همین سلول را تغییر دهید.** بقیهی سلولها خودکارند.\n",
+ "\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "efc3b663",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ═══════════════════ تنظیمات ═══════════════════\n",
+ "DRIVE_ROOT = \"/content/drive/MyDrive/vie\" # همهچیز اینجا میماند و بین اجراها حفظ میشود\n",
+ "DATASET = f\"{DRIVE_ROOT}/dataset\" # پوشهی عکسها، یا مسیر یک فایل .zip\n",
+ "VIDEOS = f\"{DRIVE_ROOT}/videos\" # ویدیوها (اختیاری)\n",
+ "FACE_QUERIES = f\"{DRIVE_ROOT}/queries\" # queries/<نام فرد>/*.jpg ← ۱ تا ۳ عکس از هر نفر\n",
+ "KNOWN_PEOPLE = f\"{DRIVE_ROOT}/people\" # people/<نام>/*.jpg ← برای نامگذاری افراد در ویدیو\n",
+ "\n",
+ "ANIMAL_QUERIES = [\"cat\", \"dog\", \"bird\", \"horse\", \"zebra\"]\n",
+ "FOOD_QUERIES = [\"pizza\", \"kebab\", \"salad\", \"cake\"]\n",
+ "TEXT_QUERIES = [\"a man with a blue backpack\", \"people smiling at a party\",\n",
+ " \"a sunset over the sea\", \"a person reading a book\"]\n",
+ "VIDEO_QUERIES = [\"people talking\", \"a person walking in a corridor\"]\n",
+ "# جستجوی مفهوم: یک توضیح، همهی عکسها و لحظههای ویدیو که آن را نشان میدهند.\n",
+ "# با مدل پیشفرض روی GPU (SigLIP 2) فارسی هم میشود نوشت.\n",
+ "CONCEPT_QUERIES = [\"a theatre performance on stage\", \"people in a classroom\",\n",
+ " \"a celebration with friends\", \"تئاتر\", \"کلاس درس\"]\n",
+ "\n",
+ "QUALITY = \"auto\" # auto | best | english | fast (auto: SigLIP 2 روی GPU که فارسی هم میفهمد، مدل سبک روی CPU)\n",
+ "RENDER_VIDEOS = True # ساخت ویدیوی خروجی با کادر و نام افراد\n",
+ "MAX_RENDER_SECONDS = 180 # سقف طول هر ویدیوی خروجی، برای صرفهجویی در زمان\n",
+ "DEMO_IF_EMPTY = True # اگر دیتاست خالی بود، دادهی نمونهی عمومی دانلود شود\n",
+ "REPO_REF = \"claude/image-processing-optimization-7abxze\" # شاخه، برچسب یا commit؛ اگر نبود main\n",
+ "# نسخهی v0.5.0 (پیش از جستجوی مفهوم و SigLIP 2): REPO_REF = \"c3ef80c8e11f722908f1463eb7e950cd029cd7dd\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "19bb0656",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## ۱. آمادهسازی\n",
+ "\n",
+ "بار اول ۳ تا ۵ دقیقه طول میکشد. هر خطایی که پیش بیاید با پیام فارسی نشان داده میشود و اجرا متوقف نمیشود.\n",
+ "\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "31321d9d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os, sys, json, time, shutil, sqlite3, subprocess, traceback, contextlib\n",
+ "from pathlib import Path\n",
+ "\n",
+ "IN_COLAB = \"google.colab\" in sys.modules\n",
+ "if IN_COLAB:\n",
+ " from google.colab import drive\n",
+ " drive.mount(\"/content/drive\")\n",
+ "elif DRIVE_ROOT.startswith(\"/content/\"):\n",
+ " DRIVE_ROOT = os.path.abspath(\"vie_drive\") # خارج از Colab: یک پوشهی محلی\n",
+ " DATASET, VIDEOS = f\"{DRIVE_ROOT}/dataset\", f\"{DRIVE_ROOT}/videos\"\n",
+ " FACE_QUERIES, KNOWN_PEOPLE = f\"{DRIVE_ROOT}/queries\", f\"{DRIVE_ROOT}/people\"\n",
+ "\n",
+ "GPU = subprocess.run(\"nvidia-smi -L\", shell=True, capture_output=True, text=True)\n",
+ "HAS_GPU = GPU.returncode == 0 and \"GPU\" in GPU.stdout\n",
+ "print(\"GPU:\", GPU.stdout.strip() if HAS_GPU else \"ندارد — روی CPU کندتر اجرا میشود \"\n",
+ " \"(Runtime → Change runtime type → T4 GPU)\")\n",
+ "\n",
+ "SRC = Path(\"/content/visual-intelligence-engine\" if IN_COLAB else \"vie-src\").resolve()\n",
+ "URL = \"https://github.com/ParsaVictor/visual-intelligence-engine.git\"\n",
+ "def fetch_ref(ref):\n",
+ " \"\"\"یک شاخه، برچسب یا شناسهی commit را در SRC میگذارد.\"\"\"\n",
+ " shutil.rmtree(SRC, ignore_errors=True)\n",
+ " if subprocess.run([\"git\", \"clone\", \"-q\", \"--depth\", \"1\", \"-b\", ref, URL, str(SRC)]).returncode == 0:\n",
+ " return True\n",
+ " # شناسهی commit با clone -b نمیآید؛ مستقیم fetch میشود.\n",
+ " shutil.rmtree(SRC, ignore_errors=True)\n",
+ " SRC.mkdir(parents=True)\n",
+ " git = lambda *a: subprocess.run([\"git\", \"-C\", str(SRC), *a], capture_output=True).returncode == 0\n",
+ " return git(\"init\", \"-q\") and git(\"remote\", \"add\", \"origin\", URL) and \\\n",
+ " git(\"fetch\", \"-q\", \"--depth\", \"1\", \"origin\", ref) and git(\"checkout\", \"-q\", \"FETCH_HEAD\")\n",
+ "\n",
+ "if not fetch_ref(REPO_REF):\n",
+ " print(f\"⚠️ نسخهی {REPO_REF} پیدا نشد؛ از main استفاده میشود\")\n",
+ " assert fetch_ref(\"main\"), \"دریافت کد از گیتهاب ممکن نشد\"\n",
+ "print(\"کد:\", subprocess.run([\"git\", \"-C\", str(SRC), \"log\", \"-1\", \"--format=%h %s\"],\n",
+ " capture_output=True, text=True).stdout.strip())\n",
+ "\n",
+ "def pip(*args):\n",
+ " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", *args],\n",
+ " capture_output=True, text=True)\n",
+ " if r.returncode:\n",
+ " print(r.stderr[-2000:])\n",
+ " return r.returncode == 0\n",
+ "\n",
+ "t0 = time.time()\n",
+ "pip(\"-e\", f\"{SRC}[models,video]\")\n",
+ "# دقیقاً یکی از دو نسخهی onnxruntime، وگرنه با هم تداخل دارند\n",
+ "subprocess.run([sys.executable, \"-m\", \"pip\", \"uninstall\", \"-y\", \"-q\", \"onnxruntime\", \"onnxruntime-gpu\"],\n",
+ " capture_output=True)\n",
+ "pip(\"onnxruntime-gpu\" if HAS_GPU else \"onnxruntime\")\n",
+ "pip(\"arabic-reshaper\", \"python-bidi\") # نوشتن درست نامهای فارسی روی تصویر، اگر Pillow خودش نتواند\n",
+ "\n",
+ "# ── بررسی نصب در یک فرایند تازه، تا خطای نصب همینجا دیده شود نه وسط کار ──\n",
+ "CHECK = \"\"\"\n",
+ "import cv2, numpy, onnxruntime, insightface, torch, transformers, av, PIL\n",
+ "assert cv2.imdecode(numpy.frombuffer(cv2.imencode('.png', numpy.zeros((4, 4, 3), 'uint8'))[1], 'uint8'), 1) is not None\n",
+ "print('opencv', cv2.__version__, '| torch', torch.__version__, '| cuda', torch.cuda.is_available(),\n",
+ " '| onnx', onnxruntime.get_available_providers()[:2])\n",
+ "\"\"\"\n",
+ "probe = subprocess.run([sys.executable, \"-c\", CHECK], capture_output=True, text=True)\n",
+ "if probe.returncode and \"cv2\" in probe.stderr:\n",
+ " # Colab ships opencv-python and insightface brings the headless build; when the\n",
+ " # two disagree, cv2 breaks. Reinstalling one build cleanly fixes it.\n",
+ " print(\"OpenCV ناسازگار بود؛ دوباره نصب میشود…\")\n",
+ " pip(\"--force-reinstall\", \"--no-deps\", \"opencv-python-headless\")\n",
+ " probe = subprocess.run([sys.executable, \"-c\", CHECK], capture_output=True, text=True)\n",
+ "print(probe.stdout.strip() or probe.stderr[-1500:])\n",
+ "if HAS_GPU and \"CUDAExecutionProvider\" not in probe.stdout:\n",
+ " print(\"⚠️ مدل چهره روی GPU اجرا نمیشود (onnxruntime-gpu)؛ روی CPU کار میکند ولی کندتر.\")\n",
+ "\n",
+ "sys.path.insert(0, str(SRC / \"src\")) # نصب editable در kernel در حال اجرا بدون این دیده نمیشود\n",
+ "import vie\n",
+ "print(f\"✅ Visual Intelligence Engine {vie.__version__} آماده شد در {time.time() - t0:.0f} ثانیه\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d332940b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ── ابزارهای کمکی نوتبوک ───────────────────────────────────────────\n",
+ "from IPython.display import HTML, Image as IPImage, Video, display\n",
+ "\n",
+ "STATUS = [] # خلاصهی نهایی: (مرحله، وضعیت، ثانیه، توضیح)\n",
+ "\n",
+ "@contextlib.contextmanager\n",
+ "def stage(title):\n",
+ " \"\"\"هر مرحله جدا اجرا میشود؛ خطای یک مرحله بقیه را متوقف نمیکند.\"\"\"\n",
+ " started = time.time()\n",
+ " display(HTML(f'▶ {title}
'))\n",
+ " try:\n",
+ " yield\n",
+ " STATUS.append((title, \"✅\", time.time() - started, \"\"))\n",
+ " except Exception as exc:\n",
+ " STATUS.append((title, \"⚠️\", time.time() - started, f\"{type(exc).__name__}: {exc}\"))\n",
+ " display(HTML(f'⚠️ این مرحله انجام نشد: '\n",
+ " f'{type(exc).__name__}: {exc}
بقیهی مراحل ادامه پیدا میکنند.
'))\n",
+ " traceback.print_exc(limit=3)\n",
+ "\n",
+ "def say(text):\n",
+ " display(HTML(f'{text}
'))\n",
+ "\n",
+ "def show_image(path, width=1100):\n",
+ " if Path(path).exists():\n",
+ " display(IPImage(filename=str(path), width=width))\n",
+ "\n",
+ "def show_video(path, width=900):\n",
+ " path = Path(path)\n",
+ " if not path.exists():\n",
+ " return\n",
+ " if path.stat().st_size < 40e6:\n",
+ " display(Video(str(path), embed=True, width=width, html_attributes=\"controls muted\"))\n",
+ " else:\n",
+ " say(f\"ویدیو بزرگتر از آن است که اینجا پخش شود؛ در Drive ببینید: {path}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8f688a99",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### دیتاست، ایندکس و مدلها\n",
+ "\n",
+ "- ایندکس اصلی در `DRIVE_ROOT/index` میماند. هنگام کار یک کپی روی دیسک سریع Colab ساخته میشود، چون SQLite روی Drive کند و شکننده است. هر چند دقیقه و در پایان هر مرحله، کپی به Drive برگردانده میشود.\n",
+ "- برای هر مدل یک ایندکس جدا ساخته میشود، پس عوض کردن `QUALITY` هیچوقت ایندکس را خراب نمیکند.\n",
+ "- مدلها یک بار دانلود میشوند و در `DRIVE_ROOT/models` میمانند.\n",
+ "\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c4476d8e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import zipfile, urllib.request\n",
+ "\n",
+ "for d in (DRIVE_ROOT, VIDEOS, FACE_QUERIES, KNOWN_PEOPLE, f\"{DRIVE_ROOT}/index\", f\"{DRIVE_ROOT}/outputs\"):\n",
+ " os.makedirs(d, exist_ok=True)\n",
+ "OUT = Path(f\"{DRIVE_ROOT}/outputs\")\n",
+ "LOCAL = Path(\"/content/vie_local\" if IN_COLAB else \"vie_local\").resolve()\n",
+ "LOCAL.mkdir(parents=True, exist_ok=True)\n",
+ "\n",
+ "# ── دیتاست: پوشه یا zip ──────────────────────────────────────────────\n",
+ "GALLERY = Path(DATASET)\n",
+ "if DATASET.lower().endswith(\".zip\"):\n",
+ " GALLERY = LOCAL / \"dataset\"\n",
+ " marker = GALLERY / \".extracted\"\n",
+ " zstat = os.stat(DATASET)\n",
+ " stamp = f\"{zstat.st_size}-{zstat.st_mtime_ns}\"\n",
+ " if not marker.exists() or marker.read_text() != stamp:\n",
+ " print(\"از حالت فشرده خارج میشود…\")\n",
+ " with zipfile.ZipFile(DATASET) as z:\n",
+ " z.extractall(GALLERY)\n",
+ " # extractall stamps every file \"now\", which would make the whole dataset\n",
+ " # look changed each session. Restoring the archive's own times lets the\n",
+ " # index skip unchanged photos without even reading them.\n",
+ " for info in z.infolist():\n",
+ " target = GALLERY / info.filename\n",
+ " if not info.is_dir() and target.exists():\n",
+ " when = time.mktime(info.date_time + (0, 0, -1))\n",
+ " os.utime(target, (when, when))\n",
+ " marker.write_text(stamp)\n",
+ "GALLERY.mkdir(parents=True, exist_ok=True)\n",
+ "\n",
+ "IMAGE_EXT = {\".jpg\", \".jpeg\", \".png\", \".webp\", \".bmp\", \".tif\", \".tiff\"}\n",
+ "def count_images(root):\n",
+ " return sum(1 for p in Path(root).rglob(\"*\") if p.suffix.lower() in IMAGE_EXT)\n",
+ "\n",
+ "# ── دادهی نمونه اگر دیتاست خالی است ─────────────────────────────────\n",
+ "RAW = \"https://raw.githubusercontent.com\"\n",
+ "DEMO_VIDEOS = [\"head-pose-face-detection-female-and-male.mp4\", \"face-demographics-walking-and-pause.mp4\",\n",
+ " \"classroom.mp4\"]\n",
+ "def fetch(url, dest):\n",
+ " dest = Path(dest)\n",
+ " if not dest.exists():\n",
+ " dest.parent.mkdir(parents=True, exist_ok=True)\n",
+ " urllib.request.urlretrieve(url, dest)\n",
+ " return dest\n",
+ "\n",
+ "if DEMO_IF_EMPTY and count_images(GALLERY) == 0:\n",
+ " say(\"دیتاست خالی است؛ یک مجموعهی کوچک عمومی (ویدیوهای CC BY 4.0 اینتل و عکسهای نمونه) دانلود میشود.\")\n",
+ " import av\n",
+ " for name in DEMO_VIDEOS + [\"head-pose-face-detection-female.mp4\"]:\n",
+ " fetch(f\"{RAW}/intel-iot-devkit/sample-videos/master/{name}\", LOCAL / \"demo_videos\" / name)\n",
+ " for name in (\"dog.jpg\", \"horses.jpg\", \"eagle.jpg\", \"giraffe.jpg\", \"person.jpg\"):\n",
+ " try:\n",
+ " fetch(f\"{RAW}/pjreddie/darknet/master/data/{name}\", GALLERY / \"animals\" / name)\n",
+ " except Exception as exc:\n",
+ " print(\"skip\", name, exc)\n",
+ " def stills(video, every, prefix):\n",
+ " with av.open(str(video)) as c:\n",
+ " nxt = 0.0\n",
+ " for f in c.decode(video=0):\n",
+ " if f.time is not None and f.time >= nxt:\n",
+ " f.to_image().save(GALLERY / \"people\" / f\"{prefix}_{f.time:06.1f}.jpg\", quality=92)\n",
+ " nxt += every\n",
+ " (GALLERY / \"people\").mkdir(parents=True, exist_ok=True)\n",
+ " for name, every in ((\"head-pose-face-detection-female-and-male.mp4\", 10),\n",
+ " (\"face-demographics-walking-and-pause.mp4\", 8), (\"classroom.mp4\", 8)):\n",
+ " stills(LOCAL / \"demo_videos\" / name, every, Path(name).stem[:12])\n",
+ " q = Path(FACE_QUERIES) / \"demo_person\"\n",
+ " if not q.exists():\n",
+ " q.mkdir(parents=True)\n",
+ " with av.open(str(LOCAL / \"demo_videos\" / \"head-pose-face-detection-female.mp4\")) as c:\n",
+ " for f in c.decode(video=0):\n",
+ " if f.time and f.time >= 1.0:\n",
+ " f.to_image().save(q / \"front.jpg\"); break\n",
+ " for name in DEMO_VIDEOS[:2]:\n",
+ " if not (Path(VIDEOS) / name).exists():\n",
+ " shutil.copy(LOCAL / \"demo_videos\" / name, Path(VIDEOS) / name)\n",
+ "\n",
+ "print(f\"📁 دیتاست: {GALLERY} — {count_images(GALLERY):,} عکس\")\n",
+ "\n",
+ "# ── انتخاب مدل ────────────────────────────────────────────────────────\n",
+ "BEST_MODEL = \"google/siglip2-so400m-patch14-384\"\n",
+ "AUTO_ON_GPU = \"best\" # بر اساس scripts/eval_search.py\n",
+ "# best: SigLIP 2، دقیقترین و چندزبانه · english: SigLIP قبلی · fast: مدل سبک برای CPU\n",
+ "# آستانههای غذا و حیوان برای هر مدل جدا اندازهگیری شدهاند و خودکار اعمال میشوند.\n",
+ "MODELS = {\"best\": BEST_MODEL, \"multilingual\": BEST_MODEL,\n",
+ " \"english\": \"google/siglip-so400m-patch14-384\", \"fast\": \"google/siglip-base-patch16-224\"}\n",
+ "VLM = MODELS.get(QUALITY) or (MODELS[AUTO_ON_GPU] if HAS_GPU else MODELS[\"fast\"])\n",
+ "slug = VLM.split(\"/\")[-1]\n",
+ "DB_DRIVE = Path(f\"{DRIVE_ROOT}/index/index-{slug}.db\")\n",
+ "DB_LOCAL = LOCAL / DB_DRIVE.name\n",
+ "\n",
+ "def sqlite_copy(src, dst):\n",
+ " \"\"\"کپی سازگار یک پایگاه SQLite، حتی وسط کار، و جایگزینی اتمی.\n",
+ "\n",
+ " نسخهی پشتیبان اول روی دیسک محلی ساخته میشود، چون SQLite روی Drive کند و\n",
+ " شکننده است. بعد یک فایل کامل کپی و با یک rename جایگزین میشود، پس فایل مقصد\n",
+ " هیچوقت نیمهکاره نمیماند.\n",
+ " \"\"\"\n",
+ " snap = LOCAL / (Path(src).name + \".snapshot\")\n",
+ " snap.unlink(missing_ok=True)\n",
+ " a, b = sqlite3.connect(src), sqlite3.connect(snap)\n",
+ " try:\n",
+ " a.backup(b)\n",
+ " finally:\n",
+ " a.close(); b.close()\n",
+ " tmp = Path(str(dst) + \".part\")\n",
+ " shutil.copyfile(snap, tmp)\n",
+ " os.replace(tmp, dst)\n",
+ "\n",
+ "if DB_DRIVE.exists() and (not DB_LOCAL.exists() or DB_DRIVE.stat().st_mtime > DB_LOCAL.stat().st_mtime):\n",
+ " sqlite_copy(DB_DRIVE, DB_LOCAL)\n",
+ " print(\"♻️ ایندکس قبلی از Drive بازیابی شد\")\n",
+ "\n",
+ "_last_save = [0.0]\n",
+ "def save_index(force=True):\n",
+ " if DB_LOCAL.exists() and (force or time.time() - _last_save[0] > 180):\n",
+ " sqlite_copy(DB_LOCAL, DB_DRIVE)\n",
+ " _last_save[0] = time.time()\n",
+ "\n",
+ "from vie.api import Engine\n",
+ "from vie.logging_setup import configure\n",
+ "configure(\"WARNING\")\n",
+ "engine = Engine.from_overrides({\n",
+ " \"paths\": {\"gallery\": str(GALLERY), \"videos\": str(VIDEOS), \"database\": str(DB_LOCAL),\n",
+ " \"model_cache\": f\"{DRIVE_ROOT}/models\"},\n",
+ " \"runtime\": {\"device\": \"auto\", \"low_memory\": not HAS_GPU},\n",
+ " \"models\": {\"vision_language\": VLM},\n",
+ " \"indexing\": {\"batch_size\": 16 if HAS_GPU else 4, \"workers\": 8, \"commit_every\": 50},\n",
+ " \"video\": {\"captions\": False, \"transcript\": False},\n",
+ "})\n",
+ "CFG = engine.config\n",
+ "before = engine.counts()\n",
+ "say(f\"مدل تصویر-متن: {VLM} · چهره: ArcFace + آینه · \"\n",
+ " f\"در ایندکس: {before['images']:,} عکس، {before['faces']:,} چهره، \"\n",
+ " f\"{before['videos']} ویدیو\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "277c5e90",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## ۲. ایندکس عکسها\n",
+ "\n",
+ "- **بار اول** همهی عکسها خوانده میشوند: چهره، حیوان و غذا تشخیص داده میشوند و بردار تصویر ساخته میشود.\n",
+ "- **از بار دوم** فقط عکسهای جدید یا تغییرکرده خوانده میشوند.\n",
+ "- عکسی که از دیتاست پاک شده باشد از ایندکس هم حذف میشود. اما اگر ناگهان بیشتر دیتاست غیب شده باشد (مثلاً Drive هنوز وصل نشده)، سیستم چیزی را پاک نمیکند.\n",
+ "\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d1e0b500",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with stage(\"ایندکس عکسها\"):\n",
+ " from tqdm.auto import tqdm\n",
+ " bar = tqdm(desc=\"عکسهای جدید\", unit=\"img\")\n",
+ " def on_progress(stats):\n",
+ " bar.n = stats.indexed; bar.refresh()\n",
+ " save_index(force=False) # هر ۳ دقیقه یک نسخهی امن در Drive\n",
+ " first = before[\"images\"] == 0\n",
+ " INDEX_REPORT = engine.index_images(progress=on_progress)\n",
+ " bar.close()\n",
+ " save_index()\n",
+ " r = INDEX_REPORT\n",
+ " if not r.pruning:\n",
+ " say(\"⚠️ بیشتر عکسهای ایندکسشده در پوشه پیدا نشدند، پس هیچ چیز از ایندکس حذف نشد. \"\n",
+ " \"مسیر DATASET یا اتصال Drive را بررسی کنید.\")\n",
+ " after = engine.counts()\n",
+ " kind = \"اولین ایندکس کامل\" if first else \"بهروزرسانی افزایشی\"\n",
+ " say(f\"{kind}: {r.scanned:,} عکس پیدا شد · {r.indexed:,} عکس تازه پردازش شد · \"\n",
+ " f\"{r.skipped:,} عکس بدون تغییر بود و اصلاً باز نشد · {r.pruned} حذف · \"\n",
+ " f\"{r.failed} خطا · {r.seconds:.1f} ثانیه\"\n",
+ " + (f\" · {r.images_per_second:.1f} عکس در ثانیه\" if r.indexed else \"\"))\n",
+ " say(f\"اکنون در ایندکس: {after['images']:,} عکس · {after['faces']:,} چهره · \"\n",
+ " f\"{after['animals']:,} عکس با حیوان · {after['food']:,} عکس با غذا\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "39c1bdac",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with stage(\"اثبات افزایشی بودن: اجرای دوباره\"):\n",
+ " again = engine.index_images()\n",
+ " say(f\"اجرای دوباره روی همان دیتاست: {again.indexed} عکس پردازش شد، \"\n",
+ " f\"{again.skipped:,} عکس رد شد، در {again.seconds:.2f} ثانیه. \"\n",
+ " \"عکس جدید که اضافه کنید، فقط همان پردازش میشود.\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b01efb70",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## ۳. جستجو، با خروجی آمادهی ارائه\n",
+ "\n",
+ "برای هر جستجو یک تصویر (PNG) و یک گزارش HTML در `DRIVE_ROOT/outputs` ساخته میشود.\n",
+ "\n",
+ "- **چهره:** برای هر نفر یک پوشه در `queries` بسازید و ۱ تا ۳ عکس از او بگذارید. سیستم با همهی عکسها جستجو میکند و برای هر چهره در آرشیو بهترین شباهت را نگه میدارد. اگر پوشه خالی باشد، پرتکرارترین چهرهی آرشیو انتخاب میشود.\n",
+ "\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5955f6a1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "CARDS = OUT / \"cards\"; REPORTS = OUT / \"reports\"\n",
+ "CARDS.mkdir(exist_ok=True); REPORTS.mkdir(exist_ok=True)\n",
+ "SHOWN = []\n",
+ "\n",
+ "def present(result, name):\n",
+ " card = engine.save_card(result, CARDS / f\"{name}.png\")\n",
+ " engine.save_html(result, REPORTS / f\"{name}.html\")\n",
+ " SHOWN.append(card)\n",
+ " best = f\"{result.matches[0].display_score * 100:.1f}٪\" if result.matches else \"—\"\n",
+ " say(f\"«{result.query}» → {len(result.matches)} نتیجه · بهترین {best} · \"\n",
+ " f\"{result.milliseconds:.0f} میلیثانیه\")\n",
+ " show_image(card)\n",
+ "\n",
+ "def safe_name(text):\n",
+ " return \"\".join(c if c.isalnum() else \"-\" for c in text)[:40].strip(\"-\")\n",
+ "\n",
+ "with stage(\"جستجوی چهره\"):\n",
+ " people = [p for p in sorted(Path(FACE_QUERIES).iterdir()) if p.is_dir()]\n",
+ " jobs = [(p.name, sorted(x for x in p.iterdir() if x.suffix.lower() in IMAGE_EXT)[:5]) for p in people]\n",
+ " jobs = [(n, imgs) for n, imgs in jobs if imgs]\n",
+ " if not jobs:\n",
+ " # چهرهی پرتکرار بریده میشود، چون در عکس گروهی چهرهی بزرگتر شاید کس دیگری باشد\n",
+ " auto = engine.most_photographed_face(crop_to=OUT / \"queries\" / \"most-seen-person.jpg\")\n",
+ " jobs = [(\"most-seen-person\", [auto])] if auto else []\n",
+ " if not jobs:\n",
+ " say(\"هیچ چهرهای در ایندکس نیست.\")\n",
+ " for name, images in jobs:\n",
+ " present(engine.search_face(images, top_k=12), f\"face-{safe_name(name)}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "db13fe63",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with stage(\"جستجوی حیوان\"):\n",
+ " for q in ANIMAL_QUERIES:\n",
+ " result = engine.search_animal(q, top_k=8)\n",
+ " if result.matches:\n",
+ " present(result, f\"animal-{safe_name(q)}\")\n",
+ " else:\n",
+ " say(f\"«{q}»: در آرشیو پیدا نشد\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "764007eb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with stage(\"جستجوی غذا\"):\n",
+ " for q in FOOD_QUERIES:\n",
+ " result = engine.search_food(q, top_k=8)\n",
+ " if result.matches:\n",
+ " present(result, f\"food-{safe_name(q)}\")\n",
+ " else:\n",
+ " say(f\"«{q}»: در آرشیو پیدا نشد\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c6d04182",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with stage(\"جستجوی متنی آزاد\"):\n",
+ " for q in TEXT_QUERIES:\n",
+ " present(engine.search_text(q, top_k=8), f\"text-{safe_name(q)}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9c9969e9",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## ۴. ویدیو: تحلیل، گزارش و ویدیوی خروجی\n",
+ "\n",
+ "فقط ویدیوهای **جدید یا تغییرکرده** پردازش میشوند. برای هر کدام این خروجیها ساخته میشود:\n",
+ "\n",
+ "- **گزارش HTML:** نماها، موضوع، افراد، حیوان و غذا روی خط زمانی، و خلاصه به انگلیسی و فارسی.\n",
+ "- **ویدیوی خروجی MP4:** کادر هر چهره، نام فرد (اگر در `people/<نام>/` عکس مرجع گذاشته باشید)، برچسب نما و خط زمانی حضور هر نفر.\n",
+ "- **کارت افراد:** چه کسی، چقدر و از چه زمانی در ویدیو بوده است.\n",
+ "\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8a13d9de",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "VID_OUT = OUT / \"videos\"; VID_OUT.mkdir(exist_ok=True)\n",
+ "RENDERED = []\n",
+ "\n",
+ "with stage(\"ویدیوها\"):\n",
+ " from vie.report.video import write_video_report\n",
+ " from vie.video.render import render_people_card\n",
+ " known = engine.known_people(KNOWN_PEOPLE)\n",
+ " if known:\n",
+ " say(\"افراد مرجع برای نامگذاری: \" + \"، \".join(f\"{k} ({len(v)} عکس)\" for k, v in known.items()))\n",
+ "\n",
+ " def after(key, path, analysis):\n",
+ " name = safe_name(Path(key).stem)\n",
+ " write_video_report(analysis, VID_OUT / f\"{name}.html\")\n",
+ " say(f\"🎬 {key}: {analysis.summary.get('fa') or analysis.summary.get('en', '')}\")\n",
+ " if RENDER_VIDEOS:\n",
+ " result = engine.render_video(path, VID_OUT / f\"{name}.mp4\", known=known or None,\n",
+ " analysis=analysis, title=Path(key).stem[:40],\n",
+ " max_seconds=MAX_RENDER_SECONDS,\n",
+ " detect_every=1 if HAS_GPU else 3)\n",
+ " card = VID_OUT / f\"{name}-people.png\"\n",
+ " render_people_card(result, title=Path(key).stem[:40]).save(card)\n",
+ " RENDERED.append((result.path, card))\n",
+ " say(f\"ویدیوی خروجی: {result.frames:,} فریم با سرعت {result.fps_processed:.1f} فریم در ثانیه · \"\n",
+ " + \"، \".join(f\"{p.label}: {p.seconds:.0f} ثانیه\" for p in result.people[:6]))\n",
+ " save_index()\n",
+ "\n",
+ " vstats = engine.index_videos(on_analysed=after)\n",
+ " save_index()\n",
+ " say(f\"ویدیوها: {vstats.scanned} پیدا شد · {vstats.indexed} ویدیوی تازه تحلیل شد · \"\n",
+ " f\"{vstats.skipped} بدون تغییر رد شد · {vstats.failed} خطا\")\n",
+ " if not RENDERED:\n",
+ " done = sorted(VID_OUT.glob(\"*.mp4\"))\n",
+ " RENDERED = [(p, p.with_name(p.stem + \"-people.png\")) for p in done]\n",
+ " if done:\n",
+ " say(\"ویدیوی تازهای نبود؛ خروجیهای قبلی نمایش داده میشوند.\")\n",
+ " for video, card in RENDERED[:3]:\n",
+ " show_video(video)\n",
+ " show_image(card)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4437a018",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with stage(\"جستجو در ویدیوها\"):\n",
+ " from vie.video.understand import format_clock\n",
+ " if engine.counts()[\"videos\"] == 0:\n",
+ " say(\"ویدیویی در ایندکس نیست.\")\n",
+ " for q in VIDEO_QUERIES:\n",
+ " moments = engine.search_videos(q, top_k=5)\n",
+ " rows = \"\".join(f\"| {m.video_key} | {format_clock(m.start)}–{format_clock(m.end)} | \"\n",
+ " f\"{m.score:.3f} | {m.kind} |
\" for m in moments)\n",
+ " say(f\"«{q}»\" + (f'' if rows else \": چیزی پیدا نشد\"))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "180c9625",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "### جستجوی مفهوم در کل آرشیو\n",
+ "\n",
+ "یک توضیح، مثلاً «a theatre performance» یا با مدل چندزبانه «تئاتر». جواب شامل **همهی عکسها و لحظههای ویدیو** است که آن مفهوم را نشان میدهند، در یک فهرست مرتب. روی کاشی هر لحظهی ویدیو، زمانش نوشته شده است. این جستجو در ایندکس انجام میشود و هیچ عکس یا ویدیویی باز نمیشود، به جز همان چند فریمی که روی کارت نشان داده میشوند.\n",
+ "\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fd804a89",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with stage(\"جستجوی مفهوم در عکسها و ویدیوها\"):\n",
+ " from vie.vlm import has_right_to_left, is_multilingual\n",
+ " for q in CONCEPT_QUERIES:\n",
+ " if has_right_to_left(q) and not is_multilingual(VLM):\n",
+ " say(f\"«{q}» رد شد: مدل فعلی فقط انگلیسی میفهمد. برای فارسی QUALITY = \\\"best\\\" روی GPU\")\n",
+ " continue\n",
+ " result = engine.search_everything(q, top_k=12, frames_dir=OUT / \"frames\")\n",
+ " present(result, f\"concept-{safe_name(q)}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3e2cbf78",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "## ۵. خلاصه\n",
+ "\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9431b9c1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "save_index()\n",
+ "rows = \"\".join(f\"| {s} | {t} | {sec:.1f}s | {note[:160]} |
\"\n",
+ " for t, s, sec, note in STATUS)\n",
+ "final = engine.counts()\n",
+ "say(f\"\"\"
\n",
+ "در ایندکس: {final['images']:,} عکس · {final['faces']:,} چهره · {final['videos']} ویدیو
\n",
+ "خروجیها در Drive: {OUT} — {len(list(CARDS.glob('*.png')))} کارت جستجو،\n",
+ "{len(list(VID_OUT.glob('*.mp4')))} ویدیوی خروجی، گزارشهای HTML
\n",
+ "دفعهی بعد: عکس یا ویدیوی جدید را به همان پوشهها اضافه کنید و دوباره Run all بزنید؛ فقط موارد جدید پردازش میشوند.\"\"\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "accelerator": "GPU",
+ "colab": {
+ "gpuType": "T4",
+ "name": "0_run_everything.ipynb",
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/notebooks/README.md b/notebooks/README.md
index dca144a..3be3aa0 100644
--- a/notebooks/README.md
+++ b/notebooks/README.md
@@ -5,8 +5,14 @@ also lives in `src/vie/` on purpose: these are meant to be read top to bottom
and run in Colab without installing the package, so each one carries its own
config, model loading and helpers.
+**Start with `0_run_everything.ipynb`.** It installs the maintained package and runs the
+whole system on your Google Drive with one *Run all*: incremental indexing (only new files
+after the first run), all searches, video analysis, annotated output videos and
+presentation cards. The others below are standalone, read-top-to-bottom explanations.
+
| Notebook | Covers |
|:--|:--|
+| `0_run_everything.ipynb` | **The complete system in Colab**, on the current package |
| `1_face_recognition.ipynb` | InsightFace `buffalo_l`, 512-d ArcFace, indexing and search |
| `2_animal_recognition.ipynb` | MegaDetector + classifier + CLIP fallback, with a cell that **proves** the original species-mapping defect against the real ImageNet class list |
| `3_food_recognition.ipynb` | RT-DETR-X + CLIP contrastive prompts |
diff --git a/results/face_benchmark.json b/results/face_benchmark.json
new file mode 100644
index 0000000..7ea22e6
--- /dev/null
+++ b/results/face_benchmark.json
@@ -0,0 +1,60 @@
+{
+ "faces": 776,
+ "genuine_pairs": 17956,
+ "hard_pairs": 3716,
+ "impostor_pairs": 18472,
+ "models": {
+ "ArcFace buffalo_l (this repo)": {
+ "eer": 0.0,
+ "separation_d_prime": 8.08,
+ "genuine_mean": 0.777,
+ "impostor_max": 0.308,
+ "tar_at_far0": 1.0,
+ "hard_tar_at_far0": 1.0,
+ "threshold": 0.46,
+ "tar_at_threshold": 0.9856,
+ "hard_tar_at_threshold": 0.9691,
+ "far_at_threshold": 0.0,
+ "ms_per_face_cpu": 77.5
+ },
+ "ArcFace buffalo_l + mirror": {
+ "eer": 0.0,
+ "separation_d_prime": 8.43,
+ "genuine_mean": 0.787,
+ "impostor_max": 0.301,
+ "tar_at_far0": 1.0,
+ "hard_tar_at_far0": 1.0,
+ "threshold": 0.46,
+ "tar_at_threshold": 0.9888,
+ "hard_tar_at_threshold": 0.9755,
+ "far_at_threshold": 0.0,
+ "ms_per_face_cpu": 217.0
+ },
+ "SFace (student-face-gate)": {
+ "eer": 0.002,
+ "separation_d_prime": 5.86,
+ "genuine_mean": 0.776,
+ "impostor_max": 0.397,
+ "tar_at_far0": 0.9775,
+ "hard_tar_at_far0": 0.9779,
+ "threshold": 0.55,
+ "tar_at_threshold": 0.8975,
+ "hard_tar_at_threshold": 0.8361,
+ "far_at_threshold": 0.0,
+ "ms_per_face_cpu": 12.0
+ },
+ "SFace + mirror (student-face-gate)": {
+ "eer": 0.0021,
+ "separation_d_prime": 6.07,
+ "genuine_mean": 0.787,
+ "impostor_max": 0.402,
+ "tar_at_far0": 0.9816,
+ "hard_tar_at_far0": 0.9812,
+ "threshold": 0.55,
+ "tar_at_threshold": 0.9115,
+ "hard_tar_at_threshold": 0.8687,
+ "far_at_threshold": 0.0,
+ "ms_per_face_cpu": 13.3
+ }
+ }
+}
\ No newline at end of file
diff --git a/results/search_eval.json b/results/search_eval.json
new file mode 100644
index 0000000..fe5cda1
--- /dev/null
+++ b/results/search_eval.json
@@ -0,0 +1,122 @@
+{
+ "source": "Search quality workflow, run 42239834130, scripts/eval_search.py --layout 1",
+ "set": "fetch_calibration_data.py --per-label 4: 12 Food-101 dishes, cats, dogs, mini-ImageNet artefacts as distractors",
+ "queries": 14,
+ "models": {
+ "google/siglip-so400m-patch14-384": {
+ "label alone": {
+ "mAP": 1.0,
+ "top1": 1.0,
+ "r_precision": 1.0
+ },
+ "ensemble": {
+ "mAP": 1.0,
+ "top1": 1.0,
+ "r_precision": 1.0
+ },
+ "persian alone": {
+ "mAP": 0.7384,
+ "top1": 0.7143,
+ "r_precision": 0.6786
+ },
+ "persian ensemble": {
+ "mAP": 0.422,
+ "top1": 0.4286,
+ "r_precision": 0.3464
+ },
+ "nothing_found": {
+ "present_min": 0.0229,
+ "present_median": 0.8633,
+ "absent_max": 0.3612,
+ "absent_median": 1.7e-05,
+ "a theatre stage with actors": 0.00084
+ },
+ "seconds_to_embed_cpu": 325.2,
+ "at_configured_thresholds": {
+ "food (0.98)": {
+ "precision": 0.8889,
+ "recall": 1.0
+ },
+ "animal (0.91)": {
+ "precision": 0.9524,
+ "recall": 1.0
+ },
+ "food gate (-1.0)": {
+ "precision": 1.0,
+ "recall": 1.0
+ }
+ }
+ },
+ "google/siglip2-so400m-patch14-384": {
+ "label alone": {
+ "mAP": 1.0,
+ "top1": 1.0,
+ "r_precision": 1.0
+ },
+ "ensemble": {
+ "mAP": 1.0,
+ "top1": 1.0,
+ "r_precision": 1.0
+ },
+ "persian alone": {
+ "mAP": 0.9445,
+ "top1": 0.9286,
+ "r_precision": 0.9107
+ },
+ "persian ensemble": {
+ "mAP": 0.9373,
+ "top1": 1.0,
+ "r_precision": 0.875
+ },
+ "nothing_found": {
+ "present_min": 0.0092,
+ "present_median": 0.7466,
+ "absent_max": 0.0877,
+ "absent_median": 1.2e-05,
+ "a theatre stage with actors": 0.00072
+ },
+ "seconds_to_embed_cpu": 355.8,
+ "at_configured_thresholds": {
+ "food (0.98, SigLIP's)": {
+ "precision": 0.9348,
+ "recall": 0.8958
+ },
+ "animal (0.91, SigLIP's)": {
+ "precision": 1.0,
+ "recall": 0.9
+ },
+ "food (0.92, best F1 margin 2.49)": {
+ "precision": 0.94,
+ "recall": 0.9792
+ },
+ "animal (0.84, best F1 margin 1.62)": {
+ "precision": 0.9524,
+ "recall": 1.0
+ },
+ "food gate (-1.0)": {
+ "precision": 0.96,
+ "recall": 1.0
+ }
+ }
+ }
+ },
+ "calibration_run": "Search quality workflow, run 42241763687, scripts/calibrate.py",
+ "confirmation_run": {
+ "run": 42245151851,
+ "note": "same thresholds, sample of 44 dish images instead of 48",
+ "google/siglip2-so400m-patch14-384": {
+ "food (0.92)": {
+ "precision": 0.898,
+ "recall": 1.0
+ },
+ "animal (0.84)": {
+ "precision": 0.95,
+ "recall": 0.95
+ },
+ "food gate (-1.0)": {
+ "precision": 0.9565,
+ "recall": 1.0
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/scripts/eval_search.py b/scripts/eval_search.py
new file mode 100644
index 0000000..5b2ed05
--- /dev/null
+++ b/scripts/eval_search.py
@@ -0,0 +1,210 @@
+"""Measure typed search: does a query find the photos it should, and only those?
+
+ python scripts/fetch_calibration_data.py DATA --per-label 6
+ python scripts/eval_search.py DATA --model google/siglip2-so400m-patch14-384
+
+Every label in the labelled set (Food-101 dishes, cats, dogs) becomes a query, and
+the images carrying that label are the right answers. Images are embedded exactly as
+the index embeds them (whole frame plus multi-crop regions, best region wins), so the
+numbers describe the product, not a simplified copy of it.
+
+Reported per query variant:
+
+* ``mAP``: mean average precision over the whole ranking, the standard retrieval
+ measure. 1.0 means every right image ranks above every wrong one.
+* ``top1``: how often the first result is right.
+* ``r_precision``: precision within the first R results, R = number of right images.
+
+Variants: the label alone, the label averaged over ``text.query_templates`` (prompt
+ensembling), and the same in Persian, which only a multilingual model can answer.
+
+For a "nothing found" answer it also reports SigLIP's calibrated probability of the
+best right image for present concepts, against the best image at all for concepts
+that are absent from the set ("a theatre stage", "a football match"). A floor between
+the two lets search say "nothing here" instead of returning its five least-bad images.
+
+The set is small: read differences of a few points as noise, differences of ten as real.
+"""
+
+from __future__ import annotations
+
+import argparse
+import copy
+import json
+import os
+import time
+from pathlib import Path
+
+import numpy as np
+import yaml
+from PIL import Image
+
+from vie.config import DEFAULT_CONFIG_PATH, Config
+from vie.models import VisionLanguageEncoder, load_all
+from vie.quality import crop_regions
+from vie.vlm import default_score_mode
+
+#: Food-101 classes and the two animals, in Persian, for the multilingual variant.
+PERSIAN = {
+ "apple pie": "پای سیب", "baby back ribs": "دنده کبابی", "baklava": "باقلوا",
+ "beef carpaccio": "کارپاچیو گوشت", "beef tartare": "تارتار گوشت",
+ "beet salad": "سالاد چغندر", "beignets": "دونات فرانسوی", "bibimbap": "بیبیمباپ",
+ "bread pudding": "پودینگ نان", "breakfast burrito": "بوریتو صبحانه",
+ "bruschetta": "بروسکتا", "caesar salad": "سالاد سزار", "cannoli": "کانولی",
+ "caprese salad": "سالاد کاپرزه", "carrot cake": "کیک هویج", "ceviche": "سویچه",
+ "cheesecake": "چیزکیک", "cheese plate": "بشقاب پنیر", "chicken curry": "کاری مرغ",
+ "chicken quesadilla": "کسادیای مرغ", "chicken wings": "بال مرغ",
+ "chocolate cake": "کیک شکلاتی", "chocolate mousse": "موس شکلات", "churros": "چوروس",
+ "clam chowder": "سوپ صدف", "club sandwich": "کلاب ساندویچ", "crab cakes": "کیک خرچنگ",
+ "creme brulee": "کرم بروله", "croque madame": "کروک مادام", "cup cakes": "کاپ کیک",
+ "deviled eggs": "تخممرغ شکمپر", "donuts": "دونات", "dumplings": "دامپلینگ",
+ "edamame": "ادامامه", "eggs benedict": "تخممرغ بندیکت", "escargots": "حلزون",
+ "falafel": "فلافل", "filet mignon": "فیله مینیون", "fish and chips": "ماهی و سیبزمینی",
+ "foie gras": "فوآگرا", "french fries": "سیبزمینی سرخکرده",
+ "french onion soup": "سوپ پیاز", "french toast": "نان تست فرانسوی",
+ "fried calamari": "کالاماری سوخاری", "fried rice": "برنج سرخکرده",
+ "frozen yogurt": "ماست یخزده", "garlic bread": "نان سیر", "gnocchi": "نیوکی",
+ "greek salad": "سالاد یونانی", "grilled cheese sandwich": "ساندویچ پنیر کبابی",
+ "grilled salmon": "ماهی سالمون کبابی", "guacamole": "گواکامولی", "gyoza": "گیوزا",
+ "hamburger": "همبرگر", "hot and sour soup": "سوپ ترش و تند", "hot dog": "هاتداگ",
+ "huevos rancheros": "تخممرغ مکزیکی", "hummus": "حمص", "ice cream": "بستنی",
+ "lasagna": "لازانیا", "lobster bisque": "سوپ خرچنگ", "lobster roll sandwich": "ساندویچ خرچنگ",
+ "macaroni and cheese": "ماکارونی و پنیر", "macarons": "ماکارون", "miso soup": "سوپ میسو",
+ "mussels": "صدف", "nachos": "ناچو", "omelette": "املت", "onion rings": "حلقه پیاز",
+ "oysters": "صدف خوراکی", "pad thai": "پد تای", "paella": "پائیا", "pancakes": "پنکیک",
+ "panna cotta": "پاناکوتا", "peking duck": "اردک پکنی", "pho": "سوپ فو", "pizza": "پیتزا",
+ "pork chop": "استیک خوک", "poutine": "پوتین", "prime rib": "دنده گوساله",
+ "pulled pork sandwich": "ساندویچ گوشت ریشریش", "ramen": "رامن", "ravioli": "راویولی",
+ "red velvet cake": "کیک رد ولوت", "risotto": "ریزوتو", "samosa": "سمبوسه",
+ "sashimi": "ساشیمی", "scallops": "گوش ماهی", "seaweed salad": "سالاد جلبک",
+ "shrimp and grits": "میگو و بلغور", "spaghetti bolognese": "اسپاگتی بلونز",
+ "spaghetti carbonara": "اسپاگتی کاربونارا", "spring rolls": "اسپرینگ رول",
+ "steak": "استیک", "strawberry shortcake": "کیک توتفرنگی", "sushi": "سوشی",
+ "tacos": "تاکو", "takoyaki": "تاکویاکی", "tiramisu": "تیرامیسو",
+ "tuna tartare": "تارتار ماهی تن", "waffles": "وافل",
+ "cat": "گربه", "dog": "سگ",
+}
+PERSIAN_TEMPLATES = ["{query}", "عکسی از {query}", "تصویری از {query}"]
+
+#: Concepts with no image in the set, to see what search returns for "nothing".
+ABSENT = ["a theatre stage with actors", "a football match in a stadium",
+ "a snowy mountain", "a wedding ceremony", "a busy city street at night",
+ "a classroom with students", "a car race", "people dancing at a concert"]
+
+
+def average_precision(scores: np.ndarray, relevant: np.ndarray) -> float:
+ order = np.argsort(-scores, kind="stable")
+ hits = relevant[order]
+ if not hits.any():
+ return 0.0
+ precision_at_hit = np.cumsum(hits) / (np.arange(len(hits)) + 1)
+ return float(precision_at_hit[hits].mean())
+
+
+def evaluate(vectors: list[np.ndarray], labels: list[str], queries: dict[str, np.ndarray]
+ ) -> dict:
+ """Rank every image for every query, scoring an image by its best region."""
+ aps, top1, rprec = [], [], []
+ for label, query in queries.items():
+ scores = np.array([float((regions @ query).max()) for regions in vectors])
+ relevant = np.array([lbl == label for lbl in labels])
+ r = int(relevant.sum())
+ if not r:
+ continue
+ order = np.argsort(-scores, kind="stable")
+ aps.append(average_precision(scores, relevant))
+ top1.append(bool(relevant[order[0]]))
+ rprec.append(float(relevant[order[:r]].mean()))
+ return {"queries": len(aps), "mAP": round(float(np.mean(aps)), 4),
+ "top1": round(float(np.mean(top1)), 4),
+ "r_precision": round(float(np.mean(rprec)), 4)}
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("data", type=Path)
+ parser.add_argument("--model", default="google/siglip-so400m-patch14-384")
+ parser.add_argument("--out", type=Path, default=None)
+ parser.add_argument("--layout", default=None,
+ help="crop layout; default the config's. '1' (whole frame) is 6x "
+ "faster, which a CPU runner needs for the 877M-parameter models")
+ args = parser.parse_args()
+
+ raw = copy.deepcopy(yaml.safe_load(DEFAULT_CONFIG_PATH.read_text(encoding="utf-8")))
+ raw["models"]["vision_language"] = args.model
+ raw["models"]["score_mode"] = default_score_mode(args.model)
+ raw["runtime"]["low_memory"] = bool(os.environ.get("VIE_LOW_MEMORY"))
+ raw["indexing"]["batch_size"] = 6
+ config = Config.from_dict(raw)
+ encoder = VisionLanguageEncoder(load_all(config))
+ layout = args.layout or (config.crop_layout if config.multi_crop else "1")
+
+ paths = sorted(p for p in args.data.rglob("*.jpg") if p.is_file())
+ labels = [p.relative_to(args.data).parts[1].replace("_", " ") for p in paths]
+ kinds = [p.relative_to(args.data).parts[0] for p in paths]
+ started = time.perf_counter()
+ vectors: list[np.ndarray] = []
+ for start in range(0, len(paths), 4):
+ # BGR, as the index sees images.
+ images = [np.asarray(Image.open(p).convert("RGB"))[:, :, ::-1].copy()
+ for p in paths[start:start + 4]]
+ regions = [crop_regions(im.shape[1], im.shape[0], layout) for im in images]
+ for found in encoder.embed_batch(images, regions):
+ vectors.append(np.vstack([v for _, v in found]))
+ print(f" embedded {len(vectors)}/{len(paths)} in {time.perf_counter() - started:.0f} s",
+ flush=True)
+ embed_s = time.perf_counter() - started
+ whole = [v[:1] for v in vectors] # region 0 is always the whole frame
+ print(f"{args.model}: {len(paths)} images, {embed_s:.0f} s to embed", flush=True)
+
+ targets = sorted({lbl for lbl, kind in zip(labels, kinds, strict=True) if kind != "other"})
+ templates = list(config.query_templates)
+ variants = {
+ "label alone": {t: encoder.encode_query(t, ["{query}"]) for t in targets},
+ "ensemble": {t: encoder.encode_query(t, templates) for t in targets},
+ }
+ translated = [t for t in targets if t in PERSIAN]
+ if translated:
+ variants["persian alone"] = {t: encoder.encode_query(PERSIAN[t], ["{query}"])
+ for t in translated}
+ variants["persian ensemble"] = {t: encoder.encode_query(PERSIAN[t], PERSIAN_TEMPLATES)
+ for t in translated}
+
+ report: dict = {"model": args.model, "layout": layout, "images": len(paths),
+ "queries": len(targets),
+ "templates": templates, "seconds_to_embed": round(embed_s, 1),
+ "multi_crop": {}, "whole_frame_only": {}}
+ for name, queries in variants.items():
+ report["multi_crop"][name] = evaluate(vectors, labels, queries)
+ report["whole_frame_only"][name] = evaluate(whole, labels, queries)
+
+ # What a "nothing found" floor would have to separate.
+ calibrated = getattr(encoder, "calibrated", lambda cosine: None)
+ present, absent = [], []
+ for label, query in variants["ensemble"].items():
+ best = max(float((v @ query).max()) for v, lbl in zip(vectors, labels, strict=True)
+ if lbl == label)
+ present.append(calibrated(best))
+ for concept in ABSENT:
+ query = encoder.encode_query(concept, templates)
+ absent.append(calibrated(max(float((v @ query).max()) for v in vectors)))
+ if all(p is not None for p in present + absent):
+ report["nothing_found"] = {
+ "present_best_probability": {"min": min(present), "median": float(np.median(present))},
+ "absent_best_probability": {"max": max(absent), "median": float(np.median(absent))},
+ "absent_queries": dict(zip(ABSENT, absent, strict=True)),
+ }
+
+ print(json.dumps(report, indent=2, ensure_ascii=False), flush=True)
+ print(f"\n{'variant':18s} {'multi-crop mAP':>15s} {'top1':>6s} {'whole mAP':>10s}")
+ for name in variants:
+ mc, wf = report["multi_crop"][name], report["whole_frame_only"][name]
+ print(f"{name:18s} {mc['mAP']:>15.3f} {mc['top1']:>6.2f} {wf['mAP']:>10.3f}")
+ out = args.out or Path(f"benchmarks/search-eval-{args.model.split('/')[-1]}.json")
+ out.parent.mkdir(parents=True, exist_ok=True)
+ out.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/face_benchmark.py b/scripts/face_benchmark.py
new file mode 100644
index 0000000..d1b6690
--- /dev/null
+++ b/scripts/face_benchmark.py
@@ -0,0 +1,207 @@
+"""Head-to-head face recognition benchmark: ArcFace (this repo) vs SFace (student-face-gate).
+
+Both recognisers see exactly the same aligned 112x112 crops, cut by the same
+detector with the same five landmarks, so the comparison isolates the recogniser.
+
+Ground truth comes from public CC-BY 4.0 test videos (intel-iot-devkit/sample-videos)
+without any hand labelling:
+
+* **genuine pairs**: two frames of a single-person video. The person turns their
+ head through large angles, so pairs with a big yaw difference are hard.
+* **impostor pairs**: two different faces in the *same frame* are necessarily two
+ different people, plus every pair across the two single-person videos (a woman
+ and a man).
+
+This is a stress test of pose, not a replacement for LFW or IJB-C. It is small,
+and the numbers are a comparison between models, not a promise of field accuracy.
+
+ python scripts/face_benchmark.py --videos DIR --sface face_recognition_sface_2021dec.onnx
+"""
+
+from __future__ import annotations
+
+import argparse
+import itertools
+import json
+import time
+from pathlib import Path
+
+import numpy as np
+
+SINGLE_PERSON = {
+ "head-pose-face-detection-female.mp4": "female",
+ "head-pose-face-detection-male.mp4": "male",
+}
+CROWD = [
+ "classroom.mp4",
+ "face-demographics-walking.mp4",
+ "face-demographics-walking-and-pause.mp4",
+ "people-detection.mp4",
+ "head-pose-face-detection-female-and-male.mp4",
+]
+
+
+def extract(videos: Path, every: float, app) -> list[dict]:
+ import av
+ from insightface.utils import face_align
+
+ faces: list[dict] = []
+ for name in list(SINGLE_PERSON) + CROWD:
+ path = videos / name
+ if not path.exists():
+ continue
+ container = av.open(str(path))
+ next_t, frame_no = 0.0, 0
+ for frame in container.decode(video=0):
+ t = float(frame.time or 0.0)
+ if t + 1e-6 < next_t:
+ continue
+ next_t = t + every
+ image = frame.to_ndarray(format="bgr24")
+ found = [f for f in app.get(image)
+ if f.det_score >= 0.5
+ and min(f.bbox[2] - f.bbox[0], f.bbox[3] - f.bbox[1]) >= 40]
+ if name in SINGLE_PERSON and found:
+ found = [max(found, key=lambda f: (f.bbox[2] - f.bbox[0])
+ * (f.bbox[3] - f.bbox[1]))]
+ for f in found:
+ yaw = float(f.pose[1]) if getattr(f, "pose", None) is not None else 0.0
+ faces.append({
+ "video": name, "frame": frame_no, "time": t, "yaw": yaw,
+ "identity": SINGLE_PERSON.get(name),
+ "crop": face_align.norm_crop(image, landmark=f.kps, image_size=112),
+ })
+ frame_no += 1
+ container.close()
+ return faces
+
+
+def unit(x: np.ndarray) -> np.ndarray:
+ return x / np.linalg.norm(x, axis=-1, keepdims=True)
+
+
+def embed_arcface(rec, crops, flip: bool) -> np.ndarray:
+ out = []
+ for start in range(0, len(crops), 64):
+ batch = crops[start:start + 64]
+ feats = rec.get_feat(batch)
+ if flip:
+ feats = feats + rec.get_feat([np.ascontiguousarray(c[:, ::-1]) for c in batch])
+ out.append(feats)
+ return unit(np.vstack(out).astype(np.float32))
+
+
+def embed_sface(model_path: str, crops, flip: bool) -> np.ndarray:
+ import cv2
+
+ sface = cv2.FaceRecognizerSF.create(model_path, "")
+ out = []
+ for crop in crops:
+ v = sface.feature(crop).ravel()
+ if flip:
+ v = v + sface.feature(np.ascontiguousarray(crop[:, ::-1])).ravel()
+ out.append(v)
+ return unit(np.vstack(out).astype(np.float32))
+
+
+def pairs(faces: list[dict]):
+ genuine, hard, impostor = [], [], []
+ by_id: dict[str, list[int]] = {}
+ for i, f in enumerate(faces):
+ if f["identity"]:
+ by_id.setdefault(f["identity"], []).append(i)
+ for members in by_id.values():
+ for a, b in itertools.combinations(members, 2):
+ genuine.append((a, b))
+ if abs(faces[a]["yaw"] - faces[b]["yaw"]) >= 30:
+ hard.append((a, b))
+ ids = list(by_id.values())
+ for x, y in itertools.combinations(ids, 2):
+ impostor.extend(itertools.product(x, y))
+ frames: dict[tuple[str, int], list[int]] = {}
+ for i, f in enumerate(faces):
+ if not f["identity"]:
+ frames.setdefault((f["video"], f["frame"]), []).append(i)
+ for members in frames.values():
+ impostor.extend(itertools.combinations(members, 2))
+ return genuine, hard, impostor
+
+
+def scores(emb: np.ndarray, idx) -> np.ndarray:
+ a, b = np.array(idx).T
+ return np.einsum("ij,ij->i", emb[a], emb[b])
+
+
+def metrics(gen: np.ndarray, hard: np.ndarray, imp: np.ndarray, threshold: float) -> dict:
+ # Every observed score is a candidate threshold. Sorted arrays and searchsorted
+ # give FAR and FRR at all of them in O(n log n) instead of a loop over each.
+ thresholds = np.unique(np.concatenate([gen, imp]))
+ imp_sorted, gen_sorted = np.sort(imp), np.sort(gen)
+ far = 1.0 - np.searchsorted(imp_sorted, thresholds, side="left") / len(imp)
+ frr = np.searchsorted(gen_sorted, thresholds, side="left") / len(gen)
+ k = int(np.argmin(np.abs(far - frr)))
+ zero_far = float(imp.max()) + 1e-6
+ return {
+ "eer": round(float((far[k] + frr[k]) / 2), 4),
+ "separation_d_prime": round(float((gen.mean() - imp.mean())
+ / np.sqrt((gen.var() + imp.var()) / 2)), 2),
+ "genuine_mean": round(float(gen.mean()), 3),
+ "impostor_max": round(float(imp.max()), 3),
+ "tar_at_far0": round(float((gen >= zero_far).mean()), 4),
+ "hard_tar_at_far0": round(float((hard >= zero_far).mean()), 4),
+ "threshold": threshold,
+ "tar_at_threshold": round(float((gen >= threshold).mean()), 4),
+ "hard_tar_at_threshold": round(float((hard >= threshold).mean()), 4),
+ "far_at_threshold": round(float((imp >= threshold).mean()), 6),
+ }
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--videos", type=Path, required=True)
+ parser.add_argument("--sface", required=True)
+ parser.add_argument("--every", type=float, default=0.5)
+ parser.add_argument("--out", type=Path, default=Path("results/face_benchmark.json"))
+ args = parser.parse_args()
+
+ from insightface.app import FaceAnalysis
+
+ app = FaceAnalysis(name="buffalo_l", providers=["CPUExecutionProvider"],
+ allowed_modules=["detection", "landmark_3d_68"])
+ app.prepare(ctx_id=-1, det_size=(640, 640))
+ faces = extract(args.videos, args.every, app)
+ crops = [f["crop"] for f in faces]
+ genuine, hard, impostor = pairs(faces)
+ print(f"{len(faces)} faces, {len(genuine)} genuine pairs ({len(hard)} with yaw gap >= 30), "
+ f"{len(impostor)} impostor pairs")
+
+ full = FaceAnalysis(name="buffalo_l", providers=["CPUExecutionProvider"],
+ allowed_modules=["detection", "recognition"])
+ full.prepare(ctx_id=-1, det_size=(640, 640))
+ rec = full.models["recognition"]
+
+ variants = {
+ # Each model at the operating threshold its own project ships with.
+ "ArcFace buffalo_l (this repo)": (lambda: embed_arcface(rec, crops, False), 0.46),
+ "ArcFace buffalo_l + mirror": (lambda: embed_arcface(rec, crops, True), 0.46),
+ "SFace (student-face-gate)": (lambda: embed_sface(args.sface, crops, False), 0.55),
+ "SFace + mirror (student-face-gate)": (lambda: embed_sface(args.sface, crops, True), 0.55),
+ }
+ report = {"faces": len(faces), "genuine_pairs": len(genuine), "hard_pairs": len(hard),
+ "impostor_pairs": len(impostor), "models": {}}
+ for name, (embed, threshold) in variants.items():
+ started = time.perf_counter()
+ emb = embed()
+ per_face = (time.perf_counter() - started) / len(crops) * 1000
+ result = metrics(scores(emb, genuine), scores(emb, hard), scores(emb, impostor), threshold)
+ result["ms_per_face_cpu"] = round(per_face, 1)
+ report["models"][name] = result
+ print(f"{name:38s} {json.dumps(result)}")
+
+ args.out.parent.mkdir(parents=True, exist_ok=True)
+ args.out.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+ print(f"written to {args.out}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/fetch_calibration_data.py b/scripts/fetch_calibration_data.py
index 309b18e..2be78e6 100644
--- a/scripts/fetch_calibration_data.py
+++ b/scripts/fetch_calibration_data.py
@@ -60,13 +60,15 @@ def harvest(
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("out", type=Path)
+ parser.add_argument("--per-label", type=int, default=3,
+ help="images per dish; animals get this times 8/3 (search eval: 6)")
args = parser.parse_args()
# Food-101 is ordered by class, 250 validation images each.
harvest(args.out, "food", "ethz/food101", "default", "validation", "label",
- [i * 250 for i in range(0, 101, 9)], 100, 3)
+ [i * 250 for i in range(0, 101, 9)], 100, args.per_label)
harvest(args.out, "animal", "microsoft/cats_vs_dogs", "default", "train", "labels",
- [0, 12000], 100, 8)
+ [0, 12000], 100, max(8, args.per_label * 8 // 3))
# WordNet ids: n03 and n04 are artefacts, n01 and n02 mostly animals.
harvest(args.out, "other", "timm/mini-imagenet", "default", "validation", "label",
list(range(0, 11000, 500)), 20, 2, lambda label: label.startswith(("n03", "n04")))
diff --git a/scripts/make_search_showcase.py b/scripts/make_search_showcase.py
new file mode 100644
index 0000000..21b7d29
--- /dev/null
+++ b/scripts/make_search_showcase.py
@@ -0,0 +1,113 @@
+"""Regenerate assets/showcase/search/: food, species, free text, Persian, concept.
+
+Runs the whole system with the real weights (SigLIP 2, ConvNeXt, MegaDetector,
+RT-DETR, InsightFace) on public data, through :class:`vie.api.Engine`, exactly as
+the notebook does, and saves one presentation card per search.
+
+ python scripts/fetch_calibration_data.py DATA --per-label 2
+ python scripts/make_search_showcase.py --data DATA --videos VIDEOS --out assets/showcase/search
+
+DATA holds Food-101, cats-vs-dogs and mini-ImageNet images (see fetch_calibration_data.py);
+VIDEOS holds intel-iot-devkit/sample-videos files (CC BY 4.0). The *Showcase* workflow
+runs this on a CPU runner and commits the cards.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import shutil
+import time
+from pathlib import Path
+
+from eval_search import PERSIAN
+
+from vie.api import Engine
+from vie.video.decode import frames_at
+
+
+def build_gallery(data: Path, videos: Path, gallery: Path) -> list[str]:
+ """A small mixed archive: dishes, pets, unrelated objects and classroom stills."""
+ from PIL import Image
+
+ if gallery.exists():
+ shutil.rmtree(gallery)
+ dishes = sorted(p for p in (data / "food").iterdir() if p.is_dir())[:6]
+ picks = [(d, 2) for d in dishes]
+ picks += [(data / "animal" / "cat", 3), (data / "animal" / "dog", 3)]
+ picks += [(d, 1) for d in sorted((data / "other").iterdir())[:4]]
+ for folder, count in picks:
+ for i, source in enumerate(sorted(folder.glob("*.jpg"))[:count]):
+ target = gallery / folder.parent.name / f"{folder.name}_{i}.jpg"
+ target.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy(source, target)
+ for name, times in (("classroom.mp4", [4, 14, 24]),
+ ("face-demographics-walking-and-pause.mp4", [30, 60])):
+ for frame in frames_at(videos / name, times, max_side=1280):
+ target = gallery / "stills" / f"{Path(name).stem[:12]}_{frame.time:05.1f}.jpg"
+ target.parent.mkdir(parents=True, exist_ok=True)
+ Image.fromarray(frame.image[:, :, ::-1]).save(target, quality=92)
+ return [d.name.replace("_", " ") for d in dishes]
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--data", type=Path, required=True)
+ parser.add_argument("--videos", type=Path, required=True)
+ parser.add_argument("--out", type=Path, default=Path("assets/showcase/search"))
+ parser.add_argument("--work", type=Path, default=Path("data/search-showcase"))
+ args = parser.parse_args()
+
+ gallery, clips = args.work / "gallery", args.work / "videos"
+ dishes = build_gallery(args.data, args.videos, gallery)
+ clips.mkdir(parents=True, exist_ok=True)
+ shutil.copy(args.videos / "classroom.mp4", clips / "classroom.mp4")
+
+ engine = Engine.from_overrides({
+ "paths": {"gallery": str(gallery), "videos": str(clips),
+ "database": str(args.work / "index.db")},
+ "runtime": {"device": "cpu", "low_memory": True},
+ "indexing": {"batch_size": 4, "workers": 4},
+ "video": {"max_duration": 30, "captions": False, "transcript": False},
+ })
+ report: dict = {"model": engine.config.models["vision_language"], "searches": []}
+ started = time.perf_counter()
+ report["index"] = engine.index_images().summary()
+ report["videos"] = engine.index_videos().summary()
+ report["index_seconds"] = round(time.perf_counter() - started, 1)
+ print(report["index"], "|", report["videos"], flush=True)
+
+ frames = args.work / "frames"
+ persian_dish = next((PERSIAN[d] for d in dishes if d in PERSIAN), None)
+ searches = [
+ ("food", dishes[0], lambda q: engine.search_food(q, top_k=8)),
+ ("food", dishes[1], lambda q: engine.search_food(q, top_k=8)),
+ ("animal", "cat", lambda q: engine.search_animal(q, top_k=8)),
+ ("animal", "a puppy", lambda q: engine.search_animal(q, top_k=8)),
+ ("text", "a bowl of soup", lambda q: engine.search_text(q, top_k=8)),
+ ("concept", "people sitting in a classroom",
+ lambda q: engine.search_everything(q, top_k=8, min_moments=2, frames_dir=frames)),
+ ("concept", "گربه", lambda q: engine.search_everything(q, top_k=8, frames_dir=frames)),
+ ("concept", "a theatre stage with actors",
+ lambda q: engine.search_everything(q, top_k=8, frames_dir=frames)),
+ ]
+ if persian_dish:
+ searches.append(("concept", persian_dish,
+ lambda q: engine.search_everything(q, top_k=8, frames_dir=frames)))
+
+ args.out.mkdir(parents=True, exist_ok=True)
+ for number, (kind, query, run) in enumerate(searches, 1):
+ result = run(query)
+ name = f"{number:02d}-{kind}"
+ engine.save_card(result, args.out / f"{name}.jpg")
+ entry = {"card": f"{name}.jpg", "kind": kind, "query": query,
+ "matches": len(result.matches), "milliseconds": round(result.milliseconds, 1),
+ "top": [(m.file_name, round(m.display_score, 4)) for m in result.matches[:5]]}
+ report["searches"].append(entry)
+ print(json.dumps(entry, ensure_ascii=False), flush=True)
+ (args.out / "report.json").write_text(json.dumps(report, indent=2, ensure_ascii=False),
+ encoding="utf-8")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/make_showcase.py b/scripts/make_showcase.py
new file mode 100644
index 0000000..bb3c5a2
--- /dev/null
+++ b/scripts/make_showcase.py
@@ -0,0 +1,141 @@
+"""Regenerate the face showcase in assets/showcase/ from public CC-BY 4.0 videos.
+
+Needs only InsightFace and PyAV (no torch, no GPU). Uses the same face recipe the
+index uses (ArcFace buffalo_l with mirror averaging), the same search ranking
+(:func:`vie.search.face.rank`) and the same renderers the notebook uses.
+
+ python scripts/make_showcase.py --videos DIR --out assets/showcase
+
+DIR holds files from https://github.com/intel-iot-devkit/sample-videos (CC BY 4.0).
+"""
+
+from __future__ import annotations
+
+import argparse
+import time
+from pathlib import Path
+
+from vie.adapters import face_vectors
+from vie.config import Config
+from vie.report.showcase import CardStats, save_search_card
+from vie.search import face as face_search
+from vie.video.render import render_annotated_video, render_people_card
+
+
+def face_app():
+ from insightface.app import FaceAnalysis
+
+ app = FaceAnalysis(name="buffalo_l", providers=["CPUExecutionProvider"],
+ allowed_modules=["detection", "recognition"])
+ app.prepare(ctx_id=-1, det_size=(640, 640))
+ return app
+
+
+def detector(app, min_px: int = 24):
+ def detect(image):
+ found = [f for f in app.get(image)
+ if f.det_score >= 0.5
+ and min(f.bbox[2] - f.bbox[0], f.bbox[3] - f.bbox[1]) >= min_px]
+ return list(zip([tuple(float(v) for v in f.bbox[:4]) for f in found],
+ face_vectors(app, image, found, True), strict=True))
+ return detect
+
+
+def stills(video: Path, out: Path, times: list[float], prefix: str) -> list[Path]:
+ """Save frames at the given seconds as JPEGs: a stand-in photo archive."""
+ import av
+
+ out.mkdir(parents=True, exist_ok=True)
+ wanted = sorted(times)
+ saved: list[Path] = []
+ with av.open(str(video)) as container:
+ for frame in container.decode(video=0):
+ if not wanted:
+ break
+ if frame.time is not None and frame.time >= wanted[0]:
+ path = out / f"{prefix}_{wanted.pop(0):05.1f}s.jpg"
+ frame.to_image().save(path, quality=92)
+ saved.append(path)
+ return saved
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--videos", type=Path, required=True)
+ parser.add_argument("--out", type=Path, default=Path("assets/showcase"))
+ parser.add_argument("--work", type=Path, default=Path("data/showcase"))
+ args = parser.parse_args()
+ config = Config.load()
+ app = face_app()
+ detect = detector(app)
+ v = args.videos
+
+ # ── a photo archive, cut from the videos ──────────────────────────
+ gallery = args.work / "gallery"
+ photos = (
+ stills(v / "head-pose-face-detection-female.mp4", gallery,
+ [8, 22, 37, 51, 66, 80, 95, 110, 124], "lobby_cam1")
+ + stills(v / "head-pose-face-detection-male.mp4", gallery,
+ [10, 30, 50, 70, 90, 110], "lobby_cam2")
+ + stills(v / "head-pose-face-detection-female-and-male.mp4", gallery,
+ [15, 45, 75, 105, 135], "desk_cam")
+ + stills(v / "face-demographics-walking-and-pause.mp4", gallery,
+ [6, 12, 25, 40, 55, 70, 85], "corridor")
+ + stills(v / "classroom.mp4", gallery, [3, 12, 21, 30], "classroom")
+ )
+ queries = stills(v / "head-pose-face-detection-female.mp4", args.work / "queries",
+ [1.0, 129.0], "query")
+
+ import cv2
+
+ indexed = []
+ for path in photos:
+ image = cv2.imread(str(path))
+ for box, vector in detect(image):
+ indexed.append((path.name, tuple(int(b) for b in box), vector, str(path)))
+ query_vectors = [detect(cv2.imread(str(q))) for q in queries]
+ query_vectors = [max(found, key=lambda d: (d[0][2] - d[0][0]) * (d[0][3] - d[0][1]))[1]
+ for found in query_vectors if found]
+
+ started = time.perf_counter()
+ best: dict[str, object] = {}
+ for vector in query_vectors:
+ for match in face_search.rank(vector, indexed, config):
+ if match.file_name not in best or match.score > best[match.file_name].score:
+ best[match.file_name] = match
+ matches = sorted(best.values(), key=lambda m: -m.score)
+ elapsed = (time.perf_counter() - started) * 1000
+ print(f"{len(indexed)} faces in {len(photos)} photos; {len(matches)} matches")
+ args.out.mkdir(parents=True, exist_ok=True)
+ save_search_card(matches, args.out / "face-search.jpg", kind="face",
+ query="The same person, across cameras and poses",
+ query_images=queries,
+ stats=CardStats(search_ms=elapsed, indexed_images=len(photos)),
+ max_results=12)
+
+ # ── videos ────────────────────────────────────────────────────────
+ known = {}
+ for name, video, at in (("Student 1042", "head-pose-face-detection-female.mp4", [2, 40, 128]),
+ ("Student 2177", "head-pose-face-detection-male.mp4", [3, 45, 120])):
+ refs = stills(v / video, args.work / "enrolled" / name, at, "ref")
+ known[name] = [max(found, key=lambda d: (d[0][2] - d[0][0]) * (d[0][3] - d[0][1]))[1]
+ for found in (detect(cv2.imread(str(r))) for r in refs) if found]
+
+ jobs = [
+ ("desk_named.mp4", "head-pose-face-detection-female-and-male.mp4",
+ "Front desk: enrolled students", known, 45),
+ ("corridor_people.mp4", "face-demographics-walking-and-pause.mp4",
+ "Corridor: who passed, and when", None, 90),
+ ("classroom_people.mp4", "classroom.mp4", "Classroom: every face, grouped", None, 20),
+ ]
+ for out_name, src, title, names, seconds in jobs:
+ result = render_annotated_video(v / src, args.work / out_name, detect=detect, known=names,
+ title=title, max_seconds=seconds, detect_every=2,
+ max_side=1280)
+ print(out_name, result.frames, "frames", f"{result.fps_processed:.1f} fps",
+ [(p.label, round(p.seconds, 1)) for p in result.people])
+ render_people_card(result, title=title).save(args.out / out_name.replace(".mp4", ".png"))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/vie/__init__.py b/src/vie/__init__.py
index 65bc0bb..7891529 100644
--- a/src/vie/__init__.py
+++ b/src/vie/__init__.py
@@ -1,6 +1,6 @@
"""Visual Intelligence Engine — multimodal search over an image archive."""
-__version__ = "0.4.0"
+__version__ = "0.6.0"
from vie.config import Config, ConfigError
from vie.scoring import Match, best_per_image, contrastive_score
diff --git a/src/vie/adapters.py b/src/vie/adapters.py
index baa1ef1..cb315b5 100644
--- a/src/vie/adapters.py
+++ b/src/vie/adapters.py
@@ -23,6 +23,49 @@
log = get_logger("adapters")
+def face_vectors(app: Any, image: Any, faces: Sequence[Any], flip: bool) -> list[Any]:
+ """Unit identity vectors for detected faces, optionally mirror-averaged.
+
+ InsightFace has already embedded each face as seen. With ``flip`` the aligned
+ crop is mirrored and embedded once more, in one batch for the whole image, and
+ the two raw vectors are summed before normalising. A face seen at an angle
+ then carries evidence from both of its sides, which is where single-view
+ embeddings lose the most (see scripts/face_benchmark.py).
+ """
+ import numpy as np
+
+ if not faces:
+ return []
+ recogniser = getattr(app, "models", {}).get("recognition")
+ if (not flip or recogniser is None
+ or any(getattr(face, "kps", None) is None for face in faces)):
+ return [face.normed_embedding for face in faces]
+
+ from insightface.utils import face_align
+
+ size = recogniser.input_size[0]
+ mirrored = [
+ np.ascontiguousarray(
+ face_align.norm_crop(image, landmark=face.kps, image_size=size)[:, ::-1]
+ )
+ for face in faces
+ ]
+ extra = np.asarray(recogniser.get_feat(mirrored), dtype=np.float32).reshape(len(faces), -1)
+ out = []
+ for face, other in zip(faces, extra, strict=True):
+ seen = getattr(face, "embedding", None)
+ if seen is None: # a FaceAnalysis built without its recognition module
+ seen = recogniser.get_feat(
+ [face_align.norm_crop(image, landmark=face.kps, image_size=size)]
+ )[0]
+ # Raw vectors are summed before normalising, as InsightFace's own flip test
+ # does: a raw norm tracks how much identity a view carries, so the clearer
+ # side of a turned face counts for more.
+ vector = np.asarray(seen, dtype=np.float32).ravel() + other
+ out.append(vector / max(float(np.linalg.norm(vector)), 1e-12))
+ return out
+
+
class InsightFaceDetector:
"""InsightFace ``buffalo_l``: RetinaFace detection + ArcFace embeddings.
@@ -38,18 +81,20 @@ def __init__(self, bundle: ModelBundle, config: Config) -> None:
self.rejected = 0
def detect(self, image: Any) -> list[tuple[Box, Any]]:
- out: list[tuple[Box, Any]] = []
- for face in self.bundle.face_app.get(image):
+ app = self.bundle.face_app
+ kept: list[tuple[Box, Any]] = []
+ for face in app.get(image):
box = tuple(int(v) for v in face.bbox[:4])
score = float(getattr(face, "det_score", 1.0))
if not is_usable_face(score, box, self.config.face.min_det_score,
self.config.face.min_face_px):
self.rejected += 1
continue
- # normed_embedding is unit length, which is what makes a dot
- # product equal cosine similarity downstream.
- out.append((box, face.normed_embedding))
- return out
+ kept.append((box, face))
+ # Unit length either way, which is what makes a dot product equal cosine
+ # similarity downstream.
+ vectors = face_vectors(app, image, [f for _, f in kept], self.config.face.flip_tta)
+ return [(box, vector) for (box, _), vector in zip(kept, vectors, strict=True)]
class MegaDetectorAdapter:
diff --git a/src/vie/api.py b/src/vie/api.py
new file mode 100644
index 0000000..34c5827
--- /dev/null
+++ b/src/vie/api.py
@@ -0,0 +1,516 @@
+"""One object for the whole system: index, search, understand video, present.
+
+The CLI starts a process per command, so every command loads its models again.
+A notebook, a demo or a service wants the opposite: load each model once and use
+it for everything. :class:`Engine` is that, built only from the tested pieces the
+CLI uses, so both paths give identical answers.
+
+ from vie.api import Engine
+ engine = Engine.from_overrides({"paths": {"gallery": "/data/photos"}})
+ report = engine.index_images() # first run: everything; later: only new files
+ result = engine.search_face(["me.jpg"])
+ engine.save_card(result, "face.png")
+
+Speed comes from not doing work twice:
+
+* an image whose size and modification time are unchanged is skipped without
+ being opened, and an index that is up to date loads no model at all;
+* every model loads on first use and stays loaded for the life of the engine;
+* searches narrow candidates in SQL before any model runs.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass, field, replace
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any
+
+import yaml
+
+from vie.config import DEFAULT_CONFIG_PATH, Config, merge
+from vie.logging_setup import get_logger
+from vie.scoring import Match
+
+log = get_logger("api")
+
+
+def load_config(overrides: dict | None = None, base: str | Path | None = None) -> Config:
+ """The packaged defaults (or ``base``), with ``overrides`` merged on top."""
+ path = Path(base) if base else DEFAULT_CONFIG_PATH
+ with path.open("r", encoding="utf-8") as handle:
+ raw = yaml.safe_load(handle) or {}
+ raw = merge(raw, overrides or {})
+ # What the caller set explicitly wins over the selected model's thresholds:
+ # re-applying it as that model's override puts it last.
+ model = raw["models"]["vision_language"]
+ explicit = {k: v for k, v in (overrides or {}).items()
+ if k not in ("models", "model_overrides")}
+ if explicit:
+ raw.setdefault("model_overrides", {})
+ raw["model_overrides"][model] = merge(raw["model_overrides"].get(model) or {}, explicit)
+ config = Config.from_dict(raw)
+ config.validate()
+ return config
+
+
+@dataclass
+class IndexReport:
+ scanned: int
+ indexed: int
+ skipped: int
+ pruned: int
+ failed: int
+ seconds: float
+ #: False when pruning was off or refused; see :func:`vie.pipeline.build_index`.
+ pruning: bool = True
+
+ @property
+ def images_per_second(self) -> float:
+ return self.indexed / self.seconds if self.seconds else 0.0
+
+ def summary(self) -> str:
+ return (f"{self.scanned} found · {self.indexed} newly indexed · {self.skipped} unchanged "
+ f"(skipped) · {self.pruned} removed · {self.failed} failed · {self.seconds:.1f}s")
+
+
+@dataclass
+class SearchResult:
+ kind: str
+ query: str
+ matches: list[Match]
+ milliseconds: float
+ query_images: list[Path] = field(default_factory=list)
+ #: Per result, a short tag drawn on its tile, such as "VIDEO 01:24"; same order.
+ badges: list[str | None] = field(default_factory=list)
+
+
+class Engine:
+ """Every capability of the system behind one long-lived object."""
+
+ def __init__(self, config: Config) -> None:
+ self.config = config
+ self._bundle: Any = None
+ self._encoder: Any = None
+ self._video_engines: dict[Any, Any] = {}
+
+ @classmethod
+ def from_overrides(cls, overrides: dict | None = None,
+ base: str | Path | None = None) -> Engine:
+ return cls(load_config(overrides, base))
+
+ # ── models, each loaded once ──────────────────────────────────────
+
+ @property
+ def bundle(self) -> Any:
+ if self._bundle is None:
+ from vie.models import load_all
+
+ self._bundle = load_all(self.config)
+ return self._bundle
+
+ @property
+ def encoder(self) -> Any:
+ if self._encoder is None:
+ from vie.models import VisionLanguageEncoder
+
+ self._encoder = VisionLanguageEncoder(self.bundle)
+ return self._encoder
+
+ def _runtime(self, _config: Config | None = None) -> tuple[Any, Any, Any]:
+ return self.bundle, self.encoder, self.encoder
+
+ # ── the archive ───────────────────────────────────────────────────
+
+ def counts(self) -> dict[str, int]:
+ """How much is indexed. Opens no model."""
+ from vie.database import Index
+
+ if not Path(self.config.database_path).exists():
+ return {"images": 0, "faces": 0, "animals": 0, "food": 0, "videos": 0}
+ index = Index(self.config.database_path)
+ index.initialise()
+ with index.connect() as conn:
+ def one(sql: str) -> int:
+ try:
+ return int(conn.execute(sql).fetchone()[0])
+ except Exception: # noqa: BLE001 - a table that does not exist yet
+ return 0
+ return {
+ "images": one("SELECT COUNT(*) FROM gallery_meta"),
+ "faces": one("SELECT COUNT(*) FROM face_embeddings"),
+ "animals": one("SELECT COUNT(*) FROM gallery_meta WHERE has_animal = 1"),
+ "food": one("SELECT COUNT(*) FROM gallery_meta WHERE has_food = 1"),
+ "videos": one("SELECT COUNT(*) FROM videos"),
+ }
+
+ def index_images(
+ self,
+ *,
+ rebuild: bool = False,
+ rehash: bool = False,
+ prune: bool = True,
+ progress: Callable[[Any], None] | None = None,
+ ) -> IndexReport:
+ """Bring the index up to date with the gallery folder.
+
+ The first run reads every image. After that only new or changed files are
+ read; everything else is skipped from its size and modification time, and if
+ nothing changed no model is even loaded.
+
+ Pruning removes rows for files no longer on disk, but :func:`build_index`
+ refuses it when most of the archive seems to have vanished at once: that is a
+ disconnected drive or a mistyped path far more often than a real deletion.
+ """
+ from vie.adapters import build_detectors
+ from vie.models import SpeciesClassifier
+ from vie.pipeline import build_index
+
+ config = self.config
+ bundle = self.bundle
+ detectors = build_detectors(bundle, config)
+ encoder = self.encoder
+ classifier = SpeciesClassifier(bundle)
+ started = time.perf_counter()
+ stats = build_index(
+ config,
+ loader=detectors.loader,
+ size_of=detectors.size_of,
+ face_detector=detectors.face,
+ animal_detector=detectors.animal,
+ food_detector=detectors.food,
+ embed_batch=encoder.embed_batch,
+ food_gate=lambda vectors: encoder.food_gate_scores(vectors, config),
+ classify_crops=classifier.classify,
+ crop=detectors.crop,
+ class_names=lambda: bundle.class_names,
+ prune=prune,
+ rebuild=rebuild,
+ rehash=rehash,
+ on_progress=progress,
+ )
+ return IndexReport(stats.scanned, stats.indexed, stats.skipped, stats.pruned,
+ stats.failed, time.perf_counter() - started,
+ pruning=prune and not stats.prune_refused)
+
+ # ── search ────────────────────────────────────────────────────────
+
+ def _search(self, kind: str, *, query: str = "", images: Sequence[Path] = (),
+ top_k: int | None = None) -> SearchResult:
+ from vie.query import run_search
+
+ args = SimpleNamespace(kind=kind, query=query, top_k=top_k,
+ image=[Path(p) for p in images] or None)
+ started = time.perf_counter()
+ matches = run_search(args, self.config, self._runtime)
+ shown = query or " + ".join(Path(p).name for p in images)
+ return SearchResult(kind, shown, matches, (time.perf_counter() - started) * 1000,
+ [Path(p) for p in images])
+
+ def search_face(self, images: str | Path | Sequence[str | Path],
+ top_k: int | None = None) -> SearchResult:
+ """Every photo of this person. Several photos of them find more."""
+ if isinstance(images, str | Path):
+ images = [images]
+ return self._search("face", images=[Path(p) for p in images], top_k=top_k)
+
+ def search_animal(self, query: str, top_k: int | None = None) -> SearchResult:
+ return self._search("animal", query=query, top_k=top_k)
+
+ def search_food(self, query: str, top_k: int | None = None) -> SearchResult:
+ return self._search("food", query=query, top_k=top_k)
+
+ def search_text(self, query: str, top_k: int | None = None) -> SearchResult:
+ return self._search("text", query=query, top_k=top_k or self.config.top_k)
+
+ def search_everything(
+ self,
+ query: str,
+ *,
+ top_k: int = 12,
+ min_moments: int = 4,
+ frames_dir: str | Path | None = None,
+ ) -> SearchResult:
+ """One description, every photo and every video moment that shows it.
+
+ Photos are scored by their best region and video moments by their keyframes,
+ both through SigLIP's calibrated probability that the description fits, so
+ they rank on one scale. Video keyframes carry a whole-frame vector only, which
+ reads a little lower than a photo's best region, so the strongest
+ ``min_moments`` moments are kept even when photos outscore them.
+
+ Each moment becomes a result whose picture is the frame at its start, grabbed
+ from the video into ``frames_dir`` (a temporary folder by default).
+ """
+ import tempfile
+
+ from vie.query import query_vector
+
+ started = time.perf_counter()
+ photos = self.search_text(query, top_k=top_k).matches if self.counts()["images"] else []
+ moments: list[Any] = []
+ if self.counts()["videos"]:
+ from vie.video.store import VideoStore
+
+ store = VideoStore(self.config)
+ store.initialise()
+ with store.connect() as conn:
+ store.index.check_embedding_model(conn, self.config.models["vision_language"])
+ vector = query_vector(self.encoder, query, self.config)
+ moments = store.search_visual(conn, vector, top_k=top_k)
+ calibrate = getattr(self.encoder, "calibrated", None)
+
+ def probability(cosine: float) -> float:
+ value = calibrate(cosine) if calibrate is not None else None
+ return cosine if value is None else value
+
+ frames = Path(frames_dir or tempfile.mkdtemp(prefix="vie-frames-"))
+ clips: list[tuple[Match, str]] = []
+ floor = self.config.min_probability if calibrate is not None else 0.0
+ for moment in moments:
+ if floor and probability(moment.score) < floor:
+ continue # the same "nothing found" floor as photos
+ picture = grab_frame(moment.file_path, moment.start, frames)
+ if picture is None:
+ continue
+ clock = f"{int(moment.start // 60)}:{int(moment.start % 60):02d}"
+ clips.append((Match(file_name=f"{Path(moment.video_key).stem} @ {clock}",
+ file_path=str(picture), score=probability(moment.score)),
+ f"VIDEO {clock}"))
+
+ ranked = [(m.display_score, m, None) for m in photos]
+ ranked += [(m.score, m, badge) for m, badge in clips]
+ ranked.sort(key=lambda item: -item[0])
+ chosen = ranked[:top_k]
+ missing = [item for item in ranked[top_k:] if item[2] is not None]
+ have = sum(1 for item in chosen if item[2] is not None)
+ for item in missing[: max(0, min_moments - have)]:
+ # Make room by dropping the weakest photo, keeping the order by score.
+ photo_positions = [i for i, entry in enumerate(chosen) if entry[2] is None]
+ if not photo_positions:
+ break
+ chosen.pop(photo_positions[-1])
+ chosen.append(item)
+ chosen.sort(key=lambda item: -item[0])
+ matches = [replace(m, confidence=score) for score, m, _ in chosen]
+ return SearchResult("concept", query, matches, (time.perf_counter() - started) * 1000,
+ badges=[badge for _, _, badge in chosen])
+
+ def most_photographed_face(self, crop_to: str | Path | None = None,
+ sample: int = 3000) -> Path | None:
+ """A photo of whoever appears most often: a sensible demo query.
+
+ Counts, for a sample of indexed faces, how many others clear the match
+ threshold, and picks the best-connected face, preferring a large one.
+
+ A query photo is searched by its *largest* face, which in a group photo
+ need not be the face that was picked. Give ``crop_to`` and the picked face
+ is saved there, padded, and that file is returned instead of the photo.
+ """
+ import numpy as np
+
+ from vie.database import Index
+
+ if not Path(self.config.database_path).exists():
+ return None
+ index = Index(self.config.database_path)
+ with index.connect() as conn:
+ rows = list(index.iter_faces(conn))
+ scales = index.scales(conn)
+ if not rows:
+ return None
+ rng = np.random.default_rng(self.config.seed)
+ if len(rows) > sample:
+ rows = [rows[i] for i in rng.choice(len(rows), sample, replace=False)]
+ vectors = np.vstack([np.asarray(r[2], dtype=np.float32) for r in rows])
+ degree = (vectors @ vectors.T >= self.config.face.match_threshold).sum(axis=1)
+ area = np.array([(b[2] - b[0]) * (b[3] - b[1]) for _, b, _, _ in rows], dtype=float)
+ # lexsort's primary key is the last one: connections first, then size.
+ name, box, _, file_path = rows[int(np.lexsort((-area, -degree))[0])]
+ if crop_to is None:
+ return Path(file_path)
+
+ from vie.report.showcase import open_rgb
+
+ photo = open_rgb(file_path, max_side=100_000)
+ # Boxes are stored at the scale the index saw; map back to the file.
+ scale = scales.get(name, 1.0) or 1.0
+ x1, y1, x2, y2 = (v / scale for v in box)
+ pad = 0.6 * max(x2 - x1, y2 - y1)
+ region = (max(0, int(x1 - pad)), max(0, int(y1 - pad)),
+ min(photo.width, int(x2 + pad)), min(photo.height, int(y2 + pad)))
+ out = Path(crop_to)
+ out.parent.mkdir(parents=True, exist_ok=True)
+ photo.crop(region).save(out, quality=95)
+ return out
+
+ # ── presentation ──────────────────────────────────────────────────
+
+ def save_card(self, result: SearchResult, path: str | Path, **kwargs: Any) -> Path:
+ """A presentation-grade PNG or JPEG of one search."""
+ from vie.report.showcase import CardStats, save_search_card
+
+ return save_search_card(
+ result.matches, path, kind=result.kind, query=result.query,
+ query_images=result.query_images, badges=result.badges or None,
+ stats=CardStats(search_ms=result.milliseconds,
+ indexed_images=self.counts()["images"]),
+ **kwargs,
+ )
+
+ def save_html(self, result: SearchResult, path: str | Path) -> Path:
+ from vie.report import write_search_report
+
+ return write_search_report(result.matches, path, kind=result.kind, query=result.query)
+
+ # ── faces as a function, for video ────────────────────────────────
+
+ def face_detect(self) -> Callable[[Any], list[tuple[Any, Any]]]:
+ """``frame -> [(box, unit vector)]`` with exactly the index's face recipe."""
+ from vie.adapters import InsightFaceDetector
+
+ detector = InsightFaceDetector(self.bundle, self.config)
+ return detector.detect
+
+ def known_people(self, folder: str | Path) -> dict[str, list[Any]]:
+ """Reference vectors from ``folder//*.jpg``, one entry per person."""
+ from vie.pipeline import IMAGE_SUFFIXES
+ from vie.query import query_face_embedding
+
+ root = Path(folder)
+ people: dict[str, list[Any]] = {}
+ if not root.is_dir():
+ return people
+ for person in sorted(p for p in root.iterdir() if p.is_dir()):
+ vectors = []
+ for photo in sorted(person.iterdir()):
+ if photo.suffix.lower() not in IMAGE_SUFFIXES:
+ continue
+ try:
+ vectors.append(query_face_embedding(photo, self.config, self.bundle))
+ except ValueError as exc:
+ log.warning("%s", exc)
+ if vectors:
+ people[person.name] = vectors
+ return people
+
+ # ── video ─────────────────────────────────────────────────────────
+
+ def video_engine(self, stages: Any = None) -> Any:
+ from vie.video.library import Stages, build_engine
+
+ stages = stages or Stages.from_config(self.config)
+ if stages not in self._video_engines:
+ self._video_engines[stages] = build_engine(self.config, self.bundle, self.encoder,
+ stages)
+ return self._video_engines[stages]
+
+ def index_videos(self, *, stages: Any = None, prune: bool = True,
+ on_analysed: Callable[[str, Path, Any], None] | None = None) -> Any:
+ """Analyse only new or changed videos; ``on_analysed`` sees each one."""
+ from vie.video.library import index_videos
+
+ return index_videos(self.config, self.video_engine(stages), prune=prune,
+ on_analysed=on_analysed)
+
+ def analyse_video(self, path: str | Path, *, max_duration: float | None = None,
+ stages: Any = None) -> Any:
+ if max_duration is None:
+ return self.video_engine(stages).analyse(path)
+ from vie.video.library import Stages, build_engine
+
+ config = replace(self.config, video=replace(self.config.video, max_duration=max_duration))
+ engine = build_engine(config, self.bundle, self.encoder,
+ stages or Stages.from_config(config))
+ return engine.analyse(path)
+
+ def render_video(self, source: str | Path, output: str | Path, *,
+ known: dict[str, list[Any]] | None = None, analysis: Any = None,
+ title: str | None = None, **kwargs: Any) -> Any:
+ """An annotated MP4: tracked faces, names or people, labels and a timeline."""
+ from vie.video.render import render_annotated_video
+
+ return render_annotated_video(
+ source, output, detect=self.face_detect(), known=known, title=title,
+ shot_labels=(shot_labeller(analysis, self.config.video.label_threshold)
+ if analysis is not None else None),
+ match_threshold=self.config.face.match_threshold,
+ cluster_threshold=self.config.video.face_cluster_threshold,
+ **kwargs,
+ )
+
+ def search_videos(self, query: str | None = None, image: str | Path | None = None,
+ top_k: int = 10) -> list[Any]:
+ """Moments across every indexed video, by description and speech, or by face.
+
+ Give exactly one of ``query`` or ``image``. For a query, transcript hits come
+ first, as in ``vie video search``: a spoken word is an exact match, while a
+ visual score is a cosine on another scale, so the two are not merged by score.
+ """
+ from vie.query import query_face_embedding, query_vector
+ from vie.video.store import VideoStore
+
+ if (query is None) == (image is None):
+ raise ValueError("give exactly one of query or image")
+ store = VideoStore(self.config)
+ store.initialise()
+ with store.connect() as conn:
+ store.index.check_embedding_model(conn, self.config.models["vision_language"])
+ if image is not None:
+ vector = query_face_embedding(image, self.config, self.bundle)
+ return store.search_face(conn, vector, top_k)
+ spoken = store.search_speech(conn, query, top_k)
+ seen = store.search_visual(conn, query_vector(self.encoder, query, self.config),
+ top_k=top_k)
+ return (spoken + seen)[:top_k]
+
+
+def grab_frame(video: str | Path, at: float, folder: Path, max_side: int = 1280) -> Path | None:
+ """Save the frame at ``at`` seconds as a JPEG in ``folder``; ``None`` if unreadable.
+
+ Through the analysis decoder, so a rotated phone video comes out upright.
+ """
+ from PIL import Image
+
+ from vie.video.decode import frames_at
+
+ target = folder / f"{Path(video).stem}-{at:08.2f}.jpg"
+ if target.exists():
+ return target
+ try:
+ frame = next(iter(frames_at(video, [at], max_side=max_side)), None)
+ except Exception: # noqa: BLE001 - a missing or damaged video loses one tile, not the card
+ log.warning("could not read %s at %.1f s", video, at)
+ return None
+ if frame is None:
+ return None
+ folder.mkdir(parents=True, exist_ok=True)
+ Image.fromarray(frame.image[:, :, ::-1]).save(target, quality=90)
+ return target
+
+
+def shot_labeller(analysis: Any, threshold: float = 0.4) -> Callable[[float], list[str]]:
+ """``time -> the current shot's strongest labels``, for the video's header.
+
+ ``threshold`` is the same presence share as ``video.label_threshold``: a label
+ the analysis would not call present in a shot is not shown on it either.
+ """
+ import bisect
+
+ shots = sorted(getattr(analysis, "shots", []) or [], key=lambda shot: shot.start)
+ starts = [shot.start for shot in shots]
+ labels = {item.shot: item for item in getattr(analysis, "shot_labels", []) or []}
+
+ def at(t: float) -> list[str]:
+ position = bisect.bisect_right(starts, t) - 1
+ if position < 0 or t > shots[position].end + 1e-6:
+ return []
+ found = labels.get(shots[position].index)
+ if found is None:
+ return []
+ ranked = sorted(found.labels.values(), key=lambda pair: -pair[1])
+ return [label for label, score in ranked if score >= threshold][:2]
+
+ return at
diff --git a/src/vie/cli.py b/src/vie/cli.py
index 1cda507..55376a7 100644
--- a/src/vie/cli.py
+++ b/src/vie/cli.py
@@ -1,7 +1,7 @@
"""Command-line interface.
vie index
- vie search face --image query.jpg
+ vie search face --image query.jpg [--image another.jpg ...]
vie search animal --query "red panda"
vie search food --query "French fries"
vie search text --query "a man with a blue backpack" --html results.html
@@ -71,7 +71,8 @@ def _build_parser() -> argparse.ArgumentParser:
kinds = search.add_subparsers(dest="kind", required=True)
face = kinds.add_parser("face", help="find a person by photo", parents=[common])
- face.add_argument("--image", required=True, type=Path)
+ face.add_argument("--image", required=True, type=Path, action="append",
+ help="a photo of the person; repeat with more photos for better recall")
face.add_argument("--top-k", type=int, default=None)
face.add_argument("--html", type=Path, default=None, help="write a visual report here")
@@ -213,7 +214,8 @@ def _run_search(args: argparse.Namespace, config: Config, as_json: bool) -> int:
if args.html is not None:
from vie.report import write_search_report
- query = str(args.image) if args.kind == "face" else args.query
+ query = (" + ".join(Path(p).name for p in args.image) if args.kind == "face"
+ else args.query)
write_search_report(matches, args.html, kind=args.kind, query=query)
log.info("report written to %s", args.html)
_render(matches, as_json)
@@ -293,6 +295,7 @@ def _run_video(args: argparse.Namespace, config: Config, as_json: bool) -> int:
def _run_video_search(args: argparse.Namespace, config: Config, as_json: bool) -> int:
from dataclasses import asdict
+ from vie.query import query_vector
from vie.video.store import VideoStore
from vie.video.understand import format_clock
@@ -321,7 +324,7 @@ def _run_video_search(args: argparse.Namespace, config: Config, as_json: bool) -
else:
moments = store.search_speech(conn, args.query, args.top_k)
moments += store.search_visual(
- conn, encoder.encode_text(args.query), top_k=args.top_k
+ conn, query_vector(encoder, args.query, config), top_k=args.top_k
)
if as_json:
diff --git a/src/vie/config.py b/src/vie/config.py
index d83ca8c..4de0c42 100644
--- a/src/vie/config.py
+++ b/src/vie/config.py
@@ -38,6 +38,32 @@ class ConfigError(ValueError):
"""Raised when a configuration is internally inconsistent."""
+def merge(base: dict, extra: dict) -> dict:
+ """Deep-merge ``extra`` over ``base`` without mutating either."""
+ import copy
+
+ out = copy.deepcopy(base)
+ for key, value in (extra or {}).items():
+ if isinstance(value, dict) and isinstance(out.get(key), dict):
+ out[key] = merge(out[key], value)
+ else:
+ out[key] = copy.deepcopy(value)
+ return out
+
+
+def with_model_overrides(raw: dict[str, Any]) -> dict[str, Any]:
+ """Apply ``model_overrides[]`` on top of the rest of the config.
+
+ Thresholds are on each model's own scale: SigLIP 2's logit margins run narrower
+ than SigLIP's, so the same number means a stricter cut. Each model's measured
+ thresholds live under its name and follow it automatically, so switching models
+ never silently keeps another model's thresholds.
+ """
+ model = (raw.get("models") or {}).get("vision_language", "")
+ override = (raw.get("model_overrides") or {}).get(model)
+ return merge(raw, override) if override else raw
+
+
@dataclass(frozen=True)
class FaceConfig:
match_threshold: float
@@ -47,6 +73,14 @@ class FaceConfig:
det_size_small: int
small_image_px: int
max_index_dimension: int
+ # Embed each face and its mirror image and average the two. Measured in
+ # scripts/face_benchmark.py; costs one extra recognition pass per face.
+ flip_tta: bool = True
+
+ @property
+ def recipe(self) -> str:
+ """What produced the stored face vectors, so a change can be noticed."""
+ return "arcface" + ("+flip" if self.flip_tta else "")
@dataclass(frozen=True)
@@ -126,6 +160,7 @@ class VideoConfig:
whisper_model: str = "small"
whisper_device: str = "auto"
face_cluster_threshold: float = 0.45
+ face_exemplars: int = 4
min_face_sightings: int = 2
presence_gap: float = 12.0
label_threshold: float = 0.40
@@ -162,6 +197,10 @@ class Config:
database_path: Path
model_cache: str
models: dict[str, str]
+ #: Phrasings a free-text query is averaged over; see ``encode_query``.
+ query_templates: tuple[str, ...] = ("{query}",)
+ #: Below this calibrated probability a free-text result is not shown at all.
+ min_probability: float = 0.0
# ---- construction -------------------------------------------------
@@ -178,6 +217,7 @@ def load(cls, path: str | Path | None = None) -> Config:
@classmethod
def from_dict(cls, raw: dict[str, Any]) -> Config:
+ raw = with_model_overrides(raw)
return cls(
raw=raw,
face=FaceConfig(**raw["face"]),
@@ -199,6 +239,8 @@ def from_dict(cls, raw: dict[str, Any]) -> Config:
database_path=Path(raw["paths"]["database"]),
model_cache=str(raw["paths"].get("model_cache", "") or ""),
models=dict(raw["models"]),
+ query_templates=tuple(raw["text"].get("query_templates") or ("{query}",)),
+ min_probability=float(raw["text"].get("min_probability", 0.0)),
)
# ---- validation ---------------------------------------------------
@@ -206,6 +248,8 @@ def from_dict(cls, raw: dict[str, Any]) -> Config:
def validate(self) -> None:
"""Enforce the index/search invariants. Raises :class:`ConfigError`."""
problems: list[str] = []
+ if not self.query_templates or any("{query}" not in t for t in self.query_templates):
+ problems.append("text.query_templates must each contain {query}")
# A candidate the gate rejects is unreachable at search time, so the
# gate must never be the stricter of the two.
diff --git a/src/vie/database.py b/src/vie/database.py
index 735be18..7599671 100644
--- a/src/vie/database.py
+++ b/src/vie/database.py
@@ -34,7 +34,7 @@
from vie.embeddings import from_blob, to_blob
-SCHEMA_VERSION = 3
+SCHEMA_VERSION = 4
class SchemaError(RuntimeError):
@@ -68,7 +68,9 @@ class EmbeddingModelMismatch(SchemaError):
-- Boxes are stored in the coordinates of the capped image the models saw.
-- Dividing by this factor maps them back onto the original file.
index_scale REAL NOT NULL DEFAULT 1.0,
- mtime_ns INTEGER NOT NULL DEFAULT 0
+ mtime_ns INTEGER NOT NULL DEFAULT 0,
+ -- 1 once the image's object vectors are stored (see object_embeddings).
+ objects_embedded INTEGER NOT NULL DEFAULT 0
);
-- (file_name, bbox) is the natural key: one row per detected face. Without it
@@ -123,6 +125,22 @@ class EmbeddingModelMismatch(SchemaError):
FOREIGN KEY (file_name) REFERENCES gallery_meta (file_name) ON DELETE CASCADE
);
+-- A vision-language vector of each detected animal and dish, cropped and padded
+-- exactly as the search paths crop them. With these stored, a food query or an
+-- out-of-vocabulary species query is a matrix product: no image is decoded and no
+-- image model runs at query time.
+CREATE TABLE IF NOT EXISTS object_embeddings (
+ file_name TEXT NOT NULL,
+ kind TEXT NOT NULL, -- 'animal' or 'food'
+ bbox TEXT NOT NULL, -- the detector's box
+ crop TEXT NOT NULL, -- the padded box that was encoded
+ confidence REAL NOT NULL,
+ class_id INTEGER,
+ embedding BLOB NOT NULL,
+ PRIMARY KEY (file_name, kind, bbox),
+ FOREIGN KEY (file_name) REFERENCES gallery_meta (file_name) ON DELETE CASCADE
+);
+
CREATE INDEX IF NOT EXISTS idx_meta_face ON gallery_meta (has_face);
CREATE INDEX IF NOT EXISTS idx_meta_animal ON gallery_meta (has_animal);
CREATE INDEX IF NOT EXISTS idx_meta_food ON gallery_meta (has_food);
@@ -132,6 +150,7 @@ class EmbeddingModelMismatch(SchemaError):
#: Every table derived from an image, and so cleared when that image changes.
DERIVED_TABLES = (
"face_embeddings", "clip_embeddings", "animal_boxes", "animal_predictions", "food_boxes",
+ "object_embeddings",
)
_ADDED_COLUMNS = {
@@ -140,6 +159,9 @@ class EmbeddingModelMismatch(SchemaError):
"food_vlm_score": "REAL NOT NULL DEFAULT 0.0",
"index_scale": "REAL NOT NULL DEFAULT 1.0",
"mtime_ns": "INTEGER NOT NULL DEFAULT 0",
+ # 1 once an image's object vectors were written; 0 for images indexed before,
+ # which the search paths still serve by cropping at query time.
+ "objects_embedded": "INTEGER NOT NULL DEFAULT 0",
}
@@ -327,12 +349,14 @@ def upsert_image(
food_vlm_score: float = 0.0,
index_scale: float = 1.0,
mtime_ns: int = 0,
+ objects_embedded: bool = False,
) -> None:
conn.execute(
"""INSERT INTO gallery_meta
(file_name, file_path, content_hash, file_size, has_face, has_animal, has_food,
- food_by_detector, food_by_vlm, food_vlm_score, index_scale, mtime_ns)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ food_by_detector, food_by_vlm, food_vlm_score, index_scale, mtime_ns,
+ objects_embedded)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (file_name) DO UPDATE SET
file_path = excluded.file_path, content_hash = excluded.content_hash,
file_size = excluded.file_size, has_face = excluded.has_face,
@@ -340,10 +364,11 @@ def upsert_image(
food_by_detector = excluded.food_by_detector,
food_by_vlm = excluded.food_by_vlm,
food_vlm_score = excluded.food_vlm_score,
- index_scale = excluded.index_scale, mtime_ns = excluded.mtime_ns""",
+ index_scale = excluded.index_scale, mtime_ns = excluded.mtime_ns,
+ objects_embedded = excluded.objects_embedded""",
(file_name, file_path, hash_, size, int(has_face), int(has_animal), int(has_food),
int(food_by_detector), int(food_by_vlm), float(food_vlm_score),
- float(index_scale), int(mtime_ns)),
+ float(index_scale), int(mtime_ns), int(objects_embedded)),
)
def add_face(
@@ -424,6 +449,64 @@ def known_files(self, conn: sqlite3.Connection) -> dict[str, tuple[str, int, int
)
}
+ def add_object(
+ self, conn: sqlite3.Connection, file_name: str, kind: str, box: tuple,
+ crop: tuple, confidence: float, class_id: int | None, embedding: np.ndarray,
+ ) -> None:
+ conn.execute(
+ "INSERT OR REPLACE INTO object_embeddings "
+ "(file_name, kind, bbox, crop, confidence, class_id, embedding) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?)",
+ (file_name, kind, format_box(box), format_box(crop), float(confidence),
+ class_id, to_blob(embedding)),
+ )
+
+ def object_vectors(
+ self, conn: sqlite3.Connection, kind: str
+ ) -> dict[str, list[tuple[tuple, tuple, float, int | None, np.ndarray]]]:
+ """``file_name -> [(box, crop, confidence, class_id, vector)]`` for one kind."""
+ found: dict[str, list[tuple[tuple, tuple, float, int | None, np.ndarray]]] = {}
+ for name, bbox, crop, confidence, class_id, blob in conn.execute(
+ "SELECT file_name, bbox, crop, confidence, class_id, embedding "
+ "FROM object_embeddings WHERE kind = ?", (kind,)
+ ):
+ found.setdefault(name, []).append(
+ (parse_box(bbox), parse_box(crop), confidence, class_id, from_blob(blob))
+ )
+ return found
+
+ def embedded_images(self, conn: sqlite3.Connection) -> set[str]:
+ """Images whose object vectors are stored, so search needs no pixels."""
+ return {r[0] for r in conn.execute(
+ "SELECT file_name FROM gallery_meta WHERE objects_embedded = 1")}
+
+ def whole_vectors(self, conn: sqlite3.Connection, names: set[str]) -> dict[str, np.ndarray]:
+ """The stored whole-frame vector of each named image."""
+ if not names:
+ return {}
+ return {
+ name: from_blob(blob)
+ for name, blob in conn.execute(
+ "SELECT file_name, embedding FROM clip_embeddings WHERE region = 'whole'")
+ if name in names
+ }
+
+ def refresh_files(
+ self, conn: sqlite3.Connection, rows: list[tuple[str, str, int, int]]
+ ) -> None:
+ """Record where unchanged files now are, and their current size and mtime.
+
+ ``rows`` is ``(file_name, file_path, size, mtime_ns)`` for files whose content
+ hash matched. Without this, a file whose mtime moved (a copy, a sync, an
+ extracted archive) is re-hashed on every run, and a gallery that moved keeps
+ paths to where it used to be.
+ """
+ conn.executemany(
+ "UPDATE gallery_meta SET file_path = ?, file_size = ?, mtime_ns = ? "
+ "WHERE file_name = ?",
+ [(path, size, mtime, name) for name, path, size, mtime in rows],
+ )
+
def candidates(self, conn: sqlite3.Connection, flag: str) -> list[GalleryRow]:
if flag not in {"has_face", "has_animal", "has_food"}:
raise ValueError(f"unknown flag {flag!r}")
diff --git a/src/vie/default.yaml b/src/vie/default.yaml
index 8ac9973..9acd321 100644
--- a/src/vie/default.yaml
+++ b/src/vie/default.yaml
@@ -51,8 +51,13 @@ models:
# openai/clip-vit-large-patch14 428M params ~75.5%
# google/siglip-base-patch16-224 203M params ~76.2% (runs on a laptop CPU)
# google/siglip-so400m-patch14-384 877M params ~83.2% (wants a GPU)
+ # google/siglip2-so400m-patch14-384 1.1B params (wants a GPU; multilingual)
+ # Measured here (results/search_eval.json, Search quality workflow): SigLIP 2 ties
+ # SigLIP on English queries (mAP 1.0 both) and leads on Persian (0.945 vs 0.738);
+ # with its own thresholds below it is level on food and species (precision 0.90-0.94
+ # vs 0.89 over two runs, species 0.95 both), so Persian decides.
# Changing this requires `vie index --rebuild`; the index refuses to mix models.
- vision_language: google/siglip-so400m-patch14-384
+ vision_language: google/siglip2-so400m-patch14-384
# "sigmoid" for SigLIP, "softmax" for CLIP -- they are trained with
# different objectives and their logits are not interchangeable.
score_mode: sigmoid
@@ -62,6 +67,9 @@ models:
face:
# ArcFace embeddings are L2-normalised, so a dot product is cosine similarity.
+ # Measured in scripts/face_benchmark.py: the strongest different-person pair
+ # scored 0.30, while 98.9% of same-person pairs (97.6% at hard head poses)
+ # scored at least 0.46. Lower it for recall, raise it to be stricter.
match_threshold: 0.46
# Quality gate at index time. Deliberately permissive: the goal is only to
@@ -75,6 +83,11 @@ face:
small_image_px: 400
# Image cap, in pixels, applied to the longest side before any model runs.
max_index_dimension: 1600
+ # Embed every face twice, as seen and mirrored, and average. Borrowed from
+ # student-face-gate, where it was measured on LFW; re-measured here for ArcFace
+ # in scripts/face_benchmark.py. Costs one extra 112x112 recognition pass per
+ # face, which is small next to detection. Changing it: `vie index --rebuild`.
+ flip_tta: true
animal:
# MegaDetector v5a raw YOLOv5 head: 0=animal, 1=person, 2=vehicle.
@@ -153,6 +166,22 @@ food:
text:
top_k: 5
+ # A free-text query is encoded in each of these phrasings and the vectors are
+ # averaged (prompt ensembling), which describes a concept more steadily than one
+ # wording. Measured in scripts/eval_search.py; ["{query}"] turns it off.
+ query_templates:
+ - "{query}"
+ - "a photo of {query}"
+ - "a picture showing {query}"
+ - "a scene with {query}"
+
+ # "Nothing found" instead of the least-bad few. SigLIP's calibrated probability
+ # that the description fits, measured in scripts/eval_search.py on both SigLIP and
+ # SigLIP 2: the right image of a present concept scored at least 0.009, while the
+ # best image for 7 of 8 absent concepts scored under 0.001 (median 0.00001).
+ # 0 shows everything. Applies to SigLIP models only, which calibrate.
+ min_probability: 0.001
+
# Multi-crop embeddings. One vector for the whole frame plus a 2x2 grid and
# a centre crop; a query scores against the best-matching region.
#
@@ -196,6 +225,10 @@ video:
whisper_device: auto # auto | cpu | cuda
face_cluster_threshold: 0.45 # the same ArcFace scale as face.match_threshold
+ # Face vectors kept per person per video, chosen to differ (frontal, turned, lit
+ # differently). Searching for someone compares against these; when they appear is
+ # stored as plain intervals. 4 keeps 1000 videos of ten people each under 100 MB.
+ face_exemplars: 4
min_face_sightings: 2 # one glimpse is more likely a false detection
presence_gap: 12.0 # seconds apart that still count as continuous
@@ -252,3 +285,15 @@ video:
- someone driving
- animals moving
- text and graphics on screen
+
+# Thresholds on each model's own scale, applied when that model is selected above.
+# SigLIP 2's logit margins run narrower than SigLIP's (median food margin 6.3 vs 8.5),
+# so SigLIP's 0.98 / 0.91 kept its precision but lost a tenth of the right answers.
+# Calibrated with scripts/calibrate.py on the labelled set, in the Search quality
+# workflow; the sections above hold SigLIP so400m's values.
+model_overrides:
+ google/siglip2-so400m-patch14-384:
+ food:
+ match_threshold: 0.92 # precision 0.90-0.94, recall 0.98-1.00 (0.98: 0.935 / 0.896)
+ animal:
+ match_threshold: 0.84 # precision 0.95, recall 0.95-1.00 (0.91: 1.000 / 0.900)
diff --git a/src/vie/models.py b/src/vie/models.py
index 975a0b5..286dee0 100644
--- a/src/vie/models.py
+++ b/src/vie/models.py
@@ -36,6 +36,9 @@
log = get_logger("models")
+#: SigLIP and SigLIP 2 both trained their text tower on exactly 64 tokens.
+SIGLIP_TEXT_TOKENS = 64
+
#: Pinned so a change to yolov5 master cannot alter detection behaviour.
YOLOV5_PIN = "v7.0"
@@ -198,7 +201,12 @@ def build() -> Any:
kwargs: dict[str, Any] = {}
if self.cache_dir:
kwargs["root"] = str(self.cache_dir / "insightface")
- app = FaceAnalysis(name=self.config.models["face_pack"], providers=providers, **kwargs)
+ # The pack also ships two landmark models and an age/gender model, which
+ # FaceAnalysis runs on every face by default. Nothing here reads their
+ # output; skipping them halved the time per face (1144 -> 527 ms on a
+ # four-face 1080p frame, CPU) with identical detections and embeddings.
+ app = FaceAnalysis(name=self.config.models["face_pack"], providers=providers,
+ allowed_modules=["detection", "recognition"], **kwargs)
size = self.config.face.det_size_large
app.prepare(ctx_id=0 if runtime.is_cuda else -1, det_size=(size, size))
log.info("loaded InsightFace %s", self.config.models["face_pack"])
@@ -343,10 +351,16 @@ def is_siglip(self) -> bool:
def _tokenize(self, texts: Sequence[str]) -> Any:
# SigLIP was trained on text padded to its full 64 tokens, and its text
# tower has no attention mask. padding=True produces a different, wrong
- # embedding with no error at all.
- padding = "max_length" if self.is_siglip else True
+ # embedding with no error at all. The length is given explicitly because
+ # SigLIP 2's Gemma tokenizer declares no maximum of its own, so
+ # "max_length" alone would pad to an effectively unbounded length.
+ if self.is_siglip:
+ return self.bundle.clip_processor(
+ text=list(texts), return_tensors="pt", padding="max_length",
+ max_length=SIGLIP_TEXT_TOKENS, truncation=True,
+ )
return self.bundle.clip_processor(
- text=list(texts), return_tensors="pt", padding=padding, truncation=True
+ text=list(texts), return_tensors="pt", padding=True, truncation=True
)
def encode_images(self, images: Sequence[Any], batch_size: int | None = None):
@@ -388,6 +402,29 @@ def encode_texts(self, texts: Sequence[str]):
def encode_text(self, text: str):
return self.encode_texts([text])[0]
+ def encode_query(self, text: str, templates: Sequence[str] | None = None):
+ """One vector for a search query, averaged over several phrasings of it.
+
+ "theatre" alone, "a photo of theatre" and "a scene showing theatre" land in
+ slightly different places; their mean is a steadier description of the
+ concept than any one of them (prompt ensembling). The templates come from
+ ``text.query_templates``; with the single template ``{query}`` this is
+ exactly :meth:`encode_text`.
+ """
+ import numpy as np
+
+ from vie.vlm import has_right_to_left
+
+ if templates is None and has_right_to_left(text):
+ # Measured (scripts/eval_search.py): English phrasings around a Persian
+ # query, or Persian ones, never beat the query as typed, and cost SigLIP
+ # up to 0.32 mAP. A right-to-left query is encoded as it is.
+ templates = ["{query}"]
+ templates = list(templates or self.bundle.config.query_templates or ["{query}"])
+ prompts = [t.format(query=text) for t in templates]
+ mean = self.encode_texts(prompts).mean(axis=0)
+ return (mean / max(float(np.linalg.norm(mean)), 1e-12)).astype(np.float32)
+
def encode_image(self, image: Any):
"""Encode one BGR array or RGB PIL image."""
return self.encode_images([bgr_to_pil(image) if hasattr(image, "shape") else image])[0]
diff --git a/src/vie/pipeline.py b/src/vie/pipeline.py
index 53eccc7..cc0134c 100644
--- a/src/vie/pipeline.py
+++ b/src/vie/pipeline.py
@@ -26,8 +26,8 @@
from typing import Any
from vie.config import Config
-from vie.database import Index, content_hash
-from vie.geometry import pad_box
+from vie.database import Index, content_hash, format_box
+from vie.geometry import Box, pad_box
from vie.indexing import BoxDetector, FaceDetector, ImageAnalysis, analyse_batch
from vie.logging_setup import get_logger
from vie.quality import crop_regions
@@ -71,11 +71,14 @@ class IndexStats:
skipped: int = 0
pruned: int = 0
failed: int = 0
+ #: Pruning was requested but refused, because too much of the index vanished.
+ prune_refused: bool = False
def summary(self) -> str:
return (
f"scanned={self.scanned} indexed={self.indexed} skipped={self.skipped} "
f"pruned={self.pruned} failed={self.failed}"
+ + (" prune=refused" if self.prune_refused else "")
)
@@ -144,14 +147,60 @@ class _Components:
crop: Callable[[Any, Any], Any] | None
-def _embed(items: list[_Item], config: Config, parts: _Components) -> list[list[tuple[str, Any]]]:
+#: Separates an object region's kind from its box in a region name: "food|10,20,90,80".
+OBJECT_SEP = "|"
+
+
+def object_regions(
+ size: tuple[int, int], analysis: ImageAnalysis, config: Config
+) -> list[tuple[str, Box, Box]]:
+ """``(name, detector box, padded crop)`` for every animal and dish to embed.
+
+ Padded exactly as :func:`vie.search.animal.select_crops` and
+ :func:`vie.search.food.select_crops` pad at query time, so a stored vector is the
+ vector those paths would have computed. Size floors were applied at detection;
+ the confidence floor is applied at query time, against the stored confidence.
+ Food boxes outside ``food.search_classes`` (a fork, a cup) are never searched,
+ so they are not embedded either.
+ """
+ width, height = size
+ out: list[tuple[str, Box, Box]] = []
+ for box, _ in analysis.animal_boxes:
+ crop = pad_box(box, width, height, ratio=config.animal.padding_ratio)
+ out.append((f"animal{OBJECT_SEP}{format_box(box)}", box, crop))
+ searchable = set(config.food.search_classes)
+ for box, _, class_id in analysis.food_detections:
+ if class_id is not None and class_id not in searchable:
+ continue
+ crop = pad_box(box, width, height, pixels=config.food.padding_px)
+ out.append((f"food{OBJECT_SEP}{format_box(box)}", box, crop))
+ return out
+
+
+def _embed(
+ items: list[_Item], analyses: list[ImageAnalysis], config: Config, parts: _Components
+) -> list[list[tuple[str, Any]]]:
+ """Every region vector of every image, objects included, in one batched pass."""
if parts.embed_batch is not None:
layout = config.crop_layout if config.multi_crop else "1"
- regions = [crop_regions(*item.size, layout) for item in items]
+ regions = [
+ crop_regions(*item.size, layout)
+ + [(name, crop) for name, _, crop in object_regions(item.size, analysis, config)]
+ for item, analysis in zip(items, analyses, strict=True)
+ ]
return parts.embed_batch([item.image for item in items], regions)
return [[("whole", parts.embed_image(item.image))] for item in items]
+def _split_objects(
+ vectors: list[tuple[str, Any]],
+) -> tuple[list[tuple[str, Any]], dict[str, Any]]:
+ """Separate scene regions (text search) from object regions (food and animals)."""
+ scene = [(name, v) for name, v in vectors if OBJECT_SEP not in name]
+ objects = {name: v for name, v in vectors if OBJECT_SEP in name}
+ return scene, objects
+
+
def _food_gate_scores(
vectors: list[list[tuple[str, Any]]], config: Config, parts: _Components
) -> list[float]:
@@ -201,6 +250,7 @@ def _write(
predictions: list[tuple[Any, int, float]],
config: Config,
gate_enabled: bool,
+ objects: dict[str, Any] | None = None,
) -> None:
food_by_vlm = gate_enabled and gate >= config.food.clip_gate_threshold
stat = item.path.stat()
@@ -221,6 +271,7 @@ def _write(
food_vlm_score=gate,
index_scale=item.scale,
mtime_ns=stat.st_mtime_ns,
+ objects_embedded=objects is not None,
)
for box, embedding in analysis.faces:
index.add_face(conn, item.key, box, embedding)
@@ -232,6 +283,16 @@ def _write(
index.add_food_box(conn, item.key, box, confidence, class_id)
for name, vector in regions:
index.add_clip(conn, item.key, vector, region=name)
+ if objects is not None:
+ confidence = {("animal", box): (c, None) for box, c in analysis.animal_boxes}
+ confidence.update({("food", box): (c, k) for box, c, k in analysis.food_detections})
+ for name, box, crop in object_regions(item.size, analysis, config):
+ vector = objects.get(name)
+ if vector is None: # an empty crop the encoder skipped
+ continue
+ kind = name.split(OBJECT_SEP, 1)[0]
+ score, class_id = confidence[(kind, box)]
+ index.add_object(conn, item.key, kind, box, crop, score, class_id, vector)
def _index_batch(
@@ -245,7 +306,10 @@ def _index_batch(
animal_detector=parts.animal_detector,
food_detector=parts.food_detector,
)
- vectors = _embed(items, config, parts)
+ embedded = _embed(items, analyses, config, parts)
+ split = [_split_objects(regions) for regions in embedded]
+ vectors = [scene for scene, _ in split]
+ objects = [found if parts.embed_batch is not None else None for _, found in split]
gates = _food_gate_scores(vectors, config, parts)
predictions = _classify_animals(items, analyses, config, parts)
except Exception:
@@ -261,11 +325,12 @@ def _index_batch(
return
gate_enabled = parts.food_gate is not None and config.food.clip_gate_enabled
- for item, analysis, regions, gate, found in zip(
- items, analyses, vectors, gates, predictions, strict=True
+ for item, analysis, regions, gate, found, stored in zip(
+ items, analyses, vectors, gates, predictions, objects, strict=True
):
try:
- _write(index, conn, item, analysis, regions, gate, found, config, gate_enabled)
+ _write(index, conn, item, analysis, regions, gate, found, config, gate_enabled,
+ stored)
stats.indexed += 1
except Exception:
log.exception("failed to store %s", item.key)
@@ -274,6 +339,19 @@ def _index_batch(
on_progress(stats)
+def _stale_face_images(index: Index, conn: Any, config: Config) -> set[str]:
+ """Images whose face vectors came from a different recipe than the config's.
+
+ Mixing recipes in one index would quietly shift every score, so those images
+ are re-indexed. Only images that have faces pay for it, not the whole archive.
+ An index written before the recipe was recorded used plain ArcFace.
+ """
+ found = index.get_meta(conn, "face_recipe") or "arcface"
+ if found == config.face.recipe:
+ return set()
+ return {row[0] for row in conn.execute("SELECT file_name FROM gallery_meta WHERE has_face = 1")}
+
+
def build_index(
config: Config,
*,
@@ -293,6 +371,7 @@ def build_index(
batch_size: int | None = None,
commit_every: int | None = None,
prune: bool = True,
+ prune_guard: float = 0.5,
rebuild: bool = False,
rehash: bool = False,
on_progress: Callable[[IndexStats], None] | None = None,
@@ -302,6 +381,9 @@ def build_index(
All model access is injected, which keeps this function testable end to end
without a GPU: the orchestration logic is what carries the risk here, not the
model calls.
+
+ ``prune`` drops rows for files no longer on disk, unless fewer than
+ ``prune_guard`` of the indexed files are still there (``0`` disables the guard).
"""
if embed_image is None and embed_batch is None:
raise ValueError("build_index needs embed_batch or embed_image")
@@ -333,30 +415,60 @@ def build_index(
index.bind_embedding_model(conn, config.models["vision_language"])
if prune:
- stats.pruned = index.prune_missing(conn, set(keys.values()))
- if stats.pruned:
- log.info("pruned %d rows for files no longer on disk", stats.pruned)
+ # An unmounted drive or a mistyped path looks exactly like "every photo
+ # was deleted". Rows cost minutes of GPU each to rebuild, so pruning most
+ # of the index at once is refused rather than trusted.
+ indexed = conn.execute("SELECT COUNT(*) FROM gallery_meta").fetchone()[0]
+ if indexed and len(keys) < indexed * prune_guard:
+ stats.prune_refused = True
+ log.warning(
+ "only %d of %d indexed images are on disk; not pruning. Check the "
+ "gallery path; to really drop them, run with --rebuild.",
+ len(keys), indexed,
+ )
+ else:
+ stats.pruned = index.prune_missing(conn, set(keys.values()))
+ if stats.pruned:
+ log.info("pruned %d rows for files no longer on disk", stats.pruned)
known = index.known_files(conn)
+ stored_paths = dict(conn.execute("SELECT file_name, file_path FROM gallery_meta"))
+ stale_faces = _stale_face_images(index, conn, config)
+ if stale_faces:
+ log.info("face recipe changed to %s: re-indexing the %d images with faces",
+ config.face.recipe, len(stale_faces))
trust_mtime = settings.trust_mtime and not rehash
pending: list[tuple[Path, str]] = []
+ refreshed: list[tuple[str, str, int, int]] = []
for path in paths:
key = keys[path]
stat = path.stat()
seen = known.get(key)
+ if key in stale_faces:
+ pending.append((path, content_hash(path)))
+ continue
if (
trust_mtime and seen is not None
and seen[1] == stat.st_size and seen[2] == stat.st_mtime_ns
):
stats.skipped += 1
+ if stored_paths.get(key) != str(path):
+ refreshed.append((key, str(path), stat.st_size, stat.st_mtime_ns))
continue
digest = content_hash(path)
if seen is not None and seen[0] == digest:
+ # Same content under a new mtime or path: remember both, so the
+ # next run skips it from its stat alone instead of hashing again.
stats.skipped += 1
+ refreshed.append((key, str(path), stat.st_size, stat.st_mtime_ns))
continue
pending.append((path, digest))
+ if refreshed:
+ index.refresh_files(conn, refreshed)
+ conn.commit()
if not pending:
+ index.set_meta(conn, "face_recipe", config.face.recipe)
log.info("index is up to date (%s)", stats.summary())
return stats
@@ -383,6 +495,9 @@ def decoded_items() -> Iterator[_Item]:
conn.commit() # checkpoint: an interruption keeps this much
last_commit = stats.indexed
log.info("checkpoint at %d images", stats.indexed)
+ # Recorded only once every stale image was redone, so an interrupted run
+ # picks the rest up next time.
+ index.set_meta(conn, "face_recipe", config.face.recipe)
log.info("indexing complete (%s)", stats.summary())
return stats
diff --git a/src/vie/query.py b/src/vie/query.py
index 8df6272..e35e5ea 100644
--- a/src/vie/query.py
+++ b/src/vie/query.py
@@ -90,37 +90,73 @@ def query_face_embedding(image_path: Any, config: Config, bundle: Any) -> Any:
app = bundle.face_app
ctx = 0 if bundle.runtime.is_cuda else -1
- def detect() -> list[tuple[Any, Any]]:
- return [(tuple(int(v) for v in f.bbox[:4]), f.normed_embedding) for f in app.get(image)]
+ from vie.adapters import face_vectors
- app.prepare(ctx_id=ctx, det_size=face_search.detection_size(width, height, config))
- faces = detect()
- if not faces:
- # Retry once at full detection size: small or tightly cropped portraits
- # are the common failure and this recovers most of them.
- app.prepare(ctx_id=ctx, det_size=(config.face.det_size_large, config.face.det_size_large))
+ def detect() -> list[tuple[Any, Any]]:
+ found = list(app.get(image))
+ if not found:
+ return []
+ # Only the subject needs the mirrored pass, not every bystander.
+ subject = max(found, key=lambda f: (f.bbox[2] - f.bbox[0]) * (f.bbox[3] - f.bbox[1]))
+ vector = face_vectors(app, image, [subject], config.face.flip_tta)[0]
+ return [(tuple(int(v) for v in subject.bbox[:4]), vector)]
+
+ try:
+ app.prepare(ctx_id=ctx, det_size=face_search.detection_size(width, height, config))
faces = detect()
+ if not faces:
+ # Retry once at full detection size: small or tightly cropped portraits
+ # are the common failure and this recovers most of them.
+ size = config.face.det_size_large
+ app.prepare(ctx_id=ctx, det_size=(size, size))
+ faces = detect()
+ finally:
+ # The bundle is shared with indexing and video, which expect the large size.
+ size = config.face.det_size_large
+ app.prepare(ctx_id=ctx, det_size=(size, size))
if not faces:
- raise ValueError("no face detected in the query image")
+ raise ValueError(f"no face detected in the query image {Path(image_path).name}")
return face_search.select_query_face(faces)[1]
+def query_images(value: Any) -> list[Path]:
+ """``--image`` may be given once or several times."""
+ if value is None:
+ return []
+ items = value if isinstance(value, list | tuple) else [value]
+ return [Path(item) for item in items]
+
+
def _search_face(args, config: Config, index: Index, load_runtime) -> list[Match]:
- if args.image is None:
+ images = query_images(args.image)
+ if not images:
raise ValueError("a face search needs a query image")
- if not Path(args.image).is_file():
- raise FileNotFoundError(f"query image not found: {args.image}")
+ for image in images:
+ if not image.is_file():
+ raise FileNotFoundError(f"query image not found: {image}")
bundle, _, _ = load_runtime(config)
- embedding = query_face_embedding(args.image, config, bundle)
+ embeddings = [query_face_embedding(image, config, bundle) for image in images]
with index.connect() as conn:
catalog = VectorCatalog.open(conn, "face_embeddings", config.search, _ann_cache(config))
- return face_search.rank_catalog(embedding, catalog, _file_paths(conn), config)
+ return face_search.rank_catalog_multi(embeddings, catalog, _file_paths(conn), config)
# ── free text ──────────────────────────────────────────────────────────
+def query_vector(encoder: Any, text: str, config: Config) -> Any:
+ """The vector a typed query is searched with, by text and video search alike."""
+ from vie.vlm import has_right_to_left, is_multilingual
+
+ model = config.models["vision_language"]
+ if has_right_to_left(text) and not is_multilingual(model):
+ log.warning("%s understands English only, so this query will match poorly; "
+ "google/siglip2-so400m-patch14-384 understands Persian", model)
+ encode = getattr(encoder, "encode_query", None)
+ return encode(text) if encode is not None else encoder.encode_text(text)
+
+
def _search_text(args, config: Config, index: Index, load_runtime) -> list[Match]:
query = args.query.strip()
if not query:
@@ -130,9 +166,16 @@ def _search_text(args, config: Config, index: Index, load_runtime) -> list[Match
catalog = VectorCatalog.open(conn, "clip_embeddings", config.search, _ann_cache(config))
paths = _file_paths(conn)
_, _, encoder = load_runtime(config)
- vector = encoder.encode_text(query)
+ vector = query_vector(encoder, query, config)
matches = text_search.rank_catalog(vector, catalog, paths, config, args.top_k)
- return text_search.with_confidence(matches, getattr(encoder, "calibrated", None))
+ calibrate = getattr(encoder, "calibrated", None)
+ matches = text_search.with_confidence(matches, calibrate)
+ if config.min_probability > 0 and calibrate is not None and calibrate(0.0) is not None:
+ found = len(matches)
+ matches = [m for m in matches if m.display_score >= config.min_probability]
+ if found and not matches:
+ log.info("nothing cleared text.min_probability=%s", config.min_probability)
+ return matches
# ── species ────────────────────────────────────────────────────────────
@@ -164,11 +207,18 @@ def _search_animal(args, config: Config, index: Index, load_runtime) -> list[Mat
log.info("no stored prediction cleared the floor; trying the open vocabulary")
rows = index.candidates(conn, "has_animal")
stored = index.animal_boxes(conn)
+ embedded = index.embedded_images(conn)
+ vectors = index.object_vectors(conn, "animal")
log.info("%d candidate images after the has_animal prefilter", len(rows))
- return _score_crops(
- query, rows, stored, animal_search.select_crops, animal_search.rank_with_clip,
- animal_search.Candidate, config, load_runtime,
+ fresh = [row for row in rows if row.file_name in embedded]
+ older = [row for row in rows if row.file_name not in embedded]
+ return best_per_image(
+ _score_stored(query, fresh, vectors, {}, "animal", animal_search.rank_with_clip,
+ animal_search.Candidate, config, load_runtime)
+ + _score_crops(query, older, stored, animal_search.select_crops,
+ animal_search.rank_with_clip, animal_search.Candidate, config,
+ load_runtime)
)
@@ -182,14 +232,90 @@ def _search_food(args, config: Config, index: Index, load_runtime) -> list[Match
with index.connect() as conn:
rows = index.candidates(conn, "has_food")
stored = index.food_boxes(conn, config.food.search_classes)
+ embedded = index.embedded_images(conn)
+ vectors = index.object_vectors(conn, "food")
+ # For a gate-admitted image with no dish box above the search floor, the
+ # whole frame is scored instead, as the crop path does.
+ wholes = index.whole_vectors(conn, {r.file_name for r in rows if r.file_name in embedded})
log.info("%d candidate images after the has_food prefilter", len(rows))
- return _score_crops(
- query, rows, stored, food_search.select_crops, food_search.rank,
- food_search.Candidate, config, load_runtime, whole_frame_fallback=True,
+ fresh = [row for row in rows if row.file_name in embedded]
+ older = [row for row in rows if row.file_name not in embedded]
+ return best_per_image(
+ _score_stored(query, fresh, vectors, wholes, "food", food_search.rank,
+ food_search.Candidate, config, load_runtime)
+ + _score_crops(query, older, stored, food_search.select_crops, food_search.rank,
+ food_search.Candidate, config, load_runtime, whole_frame_fallback=True)
)
+class _StoredVectors:
+ """A ``CropScorer`` over vectors computed at index time instead of over pixels.
+
+ The search paths' ranking functions take crops and a scorer; handing them stored
+ vectors and this scorer reuses their exact decision rule, while the work per
+ candidate drops from decoding an image and running the image tower to one dot
+ product per prompt.
+ """
+
+ def __init__(self, encoder: Any) -> None:
+ self.encoder = encoder
+
+ def score(self, vectors: list[Any], prompts: list[str]) -> list[list[float]]:
+ import numpy as np
+
+ if not vectors:
+ return []
+ texts = self.encoder.encode_texts(prompts)
+ return self.encoder.logits_from_similarity(np.vstack(vectors) @ texts.T).tolist()
+
+
+def _score_stored(
+ query: str,
+ rows: list[Any],
+ objects: dict[str, list[tuple[Any, Any, float, int | None, Any]]],
+ wholes: dict[str, Any],
+ kind: str,
+ rank: Any,
+ candidate_type: Any,
+ config: Config,
+ load_runtime: Any,
+) -> list[Match]:
+ """Score images indexed with object vectors: no image is opened.
+
+ The floors :func:`_score_crops` applies through ``select_crops`` are applied
+ here to the stored boxes: confidence at the search threshold, and for food only
+ the searchable classes. Size floors and padding were applied at index time.
+ """
+ if not rows:
+ return []
+ section = config.animal if kind == "animal" else config.food
+ searchable = set(config.food.search_classes) if kind == "food" else None
+ candidates: list[Any] = []
+ vectors: list[Any] = []
+ whole_frame: set[str] = set()
+ for row in rows:
+ kept = 0
+ for _, crop, confidence, class_id, vector in objects.get(row.file_name, []):
+ if confidence < section.search_confidence:
+ continue
+ if searchable is not None and class_id is not None and class_id not in searchable:
+ continue
+ candidates.append(candidate_type(row.file_name, row.file_path, crop, confidence))
+ vectors.append(vector)
+ kept += 1
+ if not kept and row.file_name in wholes:
+ candidates.append(candidate_type(row.file_name, row.file_path, (0, 0, 0, 0), 1.0))
+ vectors.append(wholes[row.file_name])
+ whole_frame.add(row.file_name)
+ if not candidates:
+ return []
+ _, scorer, _ = load_runtime(config)
+ matches = rank(query, candidates, vectors, _StoredVectors(scorer), config)
+ # A whole-frame match has no box worth drawing.
+ return [replace(m, box=None) if m.file_name in whole_frame else m for m in matches]
+
+
def _score_crops(
query: str,
rows: list[Any],
@@ -212,6 +338,7 @@ def _score_crops(
if not rows:
return []
+ log.info("%d images indexed before object vectors; cropping them at query time", len(rows))
_, scorer, _ = load_runtime(config)
step = max(1, config.search.crop_batch)
diff --git a/src/vie/report/showcase.py b/src/vie/report/showcase.py
new file mode 100644
index 0000000..6c1781f
--- /dev/null
+++ b/src/vie/report/showcase.py
@@ -0,0 +1,391 @@
+"""Presentation-grade result images: one PNG per search, ready to put in a deck.
+
+The HTML reports are for working with results. These are for showing them: a
+fixed-size image that looks the same in a slide, a chat message or a README.
+Same visual language as :mod:`vie.report.theme`: near-black ink, one warm signal
+colour, hairlines, and the photographs as the brightest thing on the page.
+
+Each tile is cropped around the detection rather than the centre of the photo,
+so the face, animal or dish that matched is what the viewer sees first.
+
+Persian and Arabic text (file names, people's names) is shaped right to left by
+Pillow's raqm layout when the build has it, which the PyPI wheels do; otherwise
+by ``arabic-reshaper`` and ``python-bidi`` if installed. Without either, Arabic
+script would come out as disconnected letters, so :func:`display_text` is applied
+to every string drawn.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+from vie import __version__
+from vie.scoring import Match
+
+INK = (11, 13, 16)
+INK_2 = (18, 21, 26)
+INK_3 = (26, 30, 37)
+HAIR = (38, 43, 51)
+PAPER = (238, 240, 242)
+MUTED = (152, 160, 171)
+FAINT = (98, 106, 117)
+SIGNAL = (255, 180, 84)
+SIGNAL_INK = (28, 20, 5)
+LANES = [(255, 180, 84), (108, 196, 255), (143, 227, 136), (229, 140, 255), (255, 138, 122),
+ (255, 222, 110), (120, 230, 214), (255, 160, 200)]
+
+KIND_TITLES = {
+ "face": "Face search",
+ "animal": "Species search",
+ "food": "Dish search",
+ "text": "Free-text search",
+ "concept": "Concept search · photos and videos",
+ "video": "Video understanding",
+}
+
+
+#: What a score means, per search path, for the footnote and the score caption.
+SCORE_NOTES = {
+ "face": ("similarity", "ArcFace cosine similarity · the same person scores above "
+ "the match threshold"),
+ "animal": ("confidence", "classifier probability, or SigLIP margin over rival prompts"),
+ "food": ("confidence", "SigLIP margin over rival prompts"),
+ "text": ("confidence", "SigLIP's calibrated probability that the description fits"),
+ "concept": ("confidence", "SigLIP's calibrated probability that the description fits · "
+ "video tiles show the moment's first frame"),
+}
+
+
+@lru_cache(maxsize=64)
+def font(size: int, bold: bool = False) -> Any:
+ """A real TrueType face wherever one exists; Pillow's default otherwise.
+
+ DejaVu, which matplotlib ships, comes first: it covers Latin, Persian and
+ Arabic in one face. Cached, because looking a face up walks font folders.
+ """
+ from PIL import ImageFont
+
+ names = (["Inter-SemiBold.ttf", "DejaVuSans-Bold.ttf", "LiberationSans-Bold.ttf",
+ "Arial Bold.ttf", "arialbd.ttf", "segoeuib.ttf"] if bold else
+ ["Inter-Regular.ttf", "DejaVuSans.ttf", "LiberationSans-Regular.ttf",
+ "Arial.ttf", "arial.ttf", "segoeui.ttf"])
+ candidates: list[Path] = []
+ try:
+ import matplotlib
+
+ candidates += [Path(matplotlib.get_data_path()) / "fonts" / "ttf" / n for n in names]
+ except Exception: # noqa: BLE001 - matplotlib is optional here
+ pass
+ for root in ("/usr/share/fonts", "/usr/local/share/fonts", "C:/Windows/Fonts",
+ "/Library/Fonts", "/System/Library/Fonts"):
+ base = Path(root)
+ if base.is_dir():
+ for name in names:
+ candidates += list(base.rglob(name))[:1]
+ for path in candidates:
+ try:
+ return ImageFont.truetype(str(path), size)
+ except OSError:
+ continue
+ try:
+ return ImageFont.load_default(size)
+ except TypeError: # Pillow < 10.1
+ return ImageFont.load_default()
+
+
+def _has_raqm() -> bool:
+ try:
+ from PIL import features
+
+ return bool(features.check("raqm"))
+ except Exception: # noqa: BLE001
+ return False
+
+
+def display_text(text: str) -> str:
+ """Text ready for Pillow, with right-to-left scripts shaped where needed."""
+ if not any("\u0590" <= ch <= "\u08ff" or "\ufb1d" <= ch <= "\ufeff" for ch in text):
+ return text
+ if _has_raqm():
+ return text # Pillow shapes and orders it itself
+ try:
+ import arabic_reshaper
+ from bidi.algorithm import get_display
+
+ return get_display(arabic_reshaper.reshape(text))
+ except ImportError:
+ return text
+
+
+def _text_width(draw: Any, text: str, face: Any) -> int:
+ left, _, right, _ = draw.textbbox((0, 0), text, font=face)
+ return right - left
+
+
+def _ellipsize(draw: Any, text: str, face: Any, width: int, middle: bool = False) -> str:
+ """Shorten to fit ``width``. ``middle`` keeps the end, where file names differ."""
+ if _text_width(draw, text, face) <= width:
+ return text
+ if middle:
+ head, tail = text[: len(text) // 2], text[len(text) // 2:]
+ while head and tail and _text_width(draw, head + "…" + tail, face) > width:
+ head, tail = head[:-1], tail[1:]
+ return head + "…" + tail
+ while text and _text_width(draw, text + "…", face) > width:
+ text = text[:-1]
+ return text + "…"
+
+
+def open_rgb(path: str | Path, max_side: int = 1600) -> Any:
+ """Open any image as RGB, honouring EXIF rotation, capped in size."""
+ from PIL import Image, ImageOps
+
+ with Image.open(path) as handle:
+ image = ImageOps.exif_transpose(handle).convert("RGB")
+ image.thumbnail((max_side, max_side))
+ return image
+
+
+def focus_crop(image: Any, size: tuple[int, int], box: Sequence[int] | None,
+ box_scale: float = 1.0) -> tuple[Any, tuple[int, int, int, int] | None]:
+ """Cover-fit ``image`` into ``size``, centred on ``box`` when there is one.
+
+ Returns the tile and the box in tile coordinates. The box is kept whole when it
+ fits; a subject bigger than the tile is zoomed out to fit instead.
+ """
+ from PIL import Image
+
+ tw, th = size
+ w, h = image.size
+ scale = max(tw / w, th / h)
+ if box is not None:
+ bx1, by1, bx2, by2 = (v * box_scale for v in box)
+ bw, bh = max(1.0, bx2 - bx1), max(1.0, by2 - by1)
+ # Leave room around the subject: it should fill about half the tile.
+ scale = max(scale, min(tw / (bw * 2.2), th / (bh * 2.2)))
+ scale = min(scale, max(tw / w, th / h) * 3.0)
+ cx, cy = (bx1 + bx2) / 2, (by1 + by2) / 2
+ else:
+ cx, cy = w / 2, h / 2
+ resized = image.resize((max(1, round(w * scale)), max(1, round(h * scale))),
+ Image.Resampling.LANCZOS)
+ rw, rh = resized.size
+ left = int(min(max(cx * scale - tw / 2, 0), max(rw - tw, 0)))
+ top = int(min(max(cy * scale - th / 2, 0), max(rh - th, 0)))
+ tile = Image.new("RGB", size, INK_3)
+ tile.paste(resized.crop((left, top, left + tw, top + th)),
+ ((tw - min(tw, rw)) // 2, (th - min(th, rh)) // 2))
+ if box is None:
+ return tile, None
+ x1, y1, x2, y2 = (round(v * box_scale * scale) for v in box)
+ return tile, (x1 - left, y1 - top, x2 - left, y2 - top)
+
+
+def rounded(image: Any, radius: int) -> Any:
+ from PIL import Image, ImageDraw
+
+ mask = Image.new("L", image.size, 0)
+ ImageDraw.Draw(mask).rounded_rectangle((0, 0, *image.size), radius=radius, fill=255)
+ out = Image.new("RGBA", image.size)
+ out.paste(image, (0, 0), mask)
+ return out
+
+
+def draw_brackets(draw: Any, box: Sequence[int], colour: tuple[int, int, int],
+ width: int = 3, frac: float = 0.24) -> None:
+ """Corner brackets plus a hairline: reads as a detection without hiding it."""
+ x1, y1, x2, y2 = box
+ draw.rectangle(box, outline=colour + (110,), width=1)
+ arm = max(8, int(min(x2 - x1, y2 - y1) * frac))
+ for (x, y, dx, dy) in ((x1, y1, 1, 1), (x2, y1, -1, 1), (x1, y2, 1, -1), (x2, y2, -1, -1)):
+ draw.line([(x, y), (x + dx * arm, y)], fill=colour, width=width)
+ draw.line([(x, y), (x, y + dy * arm)], fill=colour, width=width)
+
+
+def pill(draw: Any, xy: tuple[int, int], text: str, face: Any,
+ fill: tuple[int, int, int], ink: tuple[int, int, int], pad: tuple[int, int] = (10, 5)
+ ) -> tuple[int, int, int, int]:
+ x, y = xy
+ text = display_text(text)
+ width = _text_width(draw, text, face)
+ _, top, _, bottom = draw.textbbox((0, 0), text, font=face)
+ rect = (x, y, x + width + pad[0] * 2, y + (bottom - top) + pad[1] * 2)
+ draw.rounded_rectangle(rect, radius=(rect[3] - rect[1]) // 2, fill=fill)
+ draw.text((x + pad[0], y + pad[1] - top), text, font=face, fill=ink)
+ return rect
+
+
+@dataclass
+class CardStats:
+ """Optional numbers for the header, all shown only when given."""
+
+ search_ms: float | None = None
+ indexed_images: int | None = None
+ candidates: int | None = None
+
+
+def render_search_card(
+ matches: Sequence[Match],
+ *,
+ kind: str,
+ query: str,
+ query_images: Sequence[str | Path] = (),
+ stats: CardStats | None = None,
+ columns: int = 4,
+ max_results: int = 8,
+ width: int = 1600,
+ badges: Sequence[str | None] | None = None,
+) -> Any:
+ """Render a search as one image. Returns a Pillow image.
+
+ ``badges`` tags results by position, e.g. ``"VIDEO 1:24"`` on a video moment.
+ """
+ from PIL import Image, ImageDraw
+
+ shown = list(matches[:max_results])
+ margin, gap = 56, 22
+ side = 300 if query_images else 0
+ grid_w = width - margin * 2 - (side + gap if side else 0)
+ # The column count stays fixed whatever the number of results, so a card with one
+ # match has the same tile size as a card with eight, not one tile the full width.
+ columns = max(1, columns)
+ tile_w = (grid_w - gap * (columns - 1)) // columns
+ tile_h = int(tile_w * 0.75)
+ caption_h = 92
+ rows = max(1, -(-len(shown) // columns))
+ header_h = 240
+ body_h = rows * (tile_h + caption_h) + (rows - 1) * gap if shown else 220
+ if side:
+ body_h = max(body_h, 60 + len(query_images[:3]) * (side + 16))
+ height = header_h + body_h + 120
+
+ canvas = Image.new("RGB", (width, height), INK)
+ draw = ImageDraw.Draw(canvas, "RGBA")
+ f_eyebrow, f_title = font(18, True), font(46, True)
+ f_name, f_small, f_score = font(19, True), font(16), font(26, True)
+
+ # ── header ────────────────────────────────────────────────────────
+ draw.text((margin, 50), KIND_TITLES.get(kind, "Search").upper(), font=f_eyebrow, fill=SIGNAL)
+ title = _ellipsize(draw, display_text(query), f_title, width - margin * 2)
+ draw.text((margin, 80), title, font=f_title, fill=PAPER)
+ chips: list[str] = [f"{len(matches)} match" + ("" if len(matches) == 1 else "es")]
+ if matches:
+ chips.append(f"best {max(m.display_score for m in matches) * 100:.1f}%")
+ if stats and stats.search_ms is not None:
+ chips.append(f"search {stats.search_ms:.0f} ms" if stats.search_ms >= 1
+ else f"search {stats.search_ms:.2f} ms")
+ if stats and stats.indexed_images is not None:
+ chips.append(f"{stats.indexed_images:,} images indexed")
+ x = margin
+ for n, chip in enumerate(chips):
+ w = _text_width(draw, chip, f_small) + 28
+ draw.rounded_rectangle((x, 150, x + w, 184), radius=17, fill=INK_2,
+ outline=SIGNAL if n == 0 else HAIR, width=1)
+ draw.text((x + 14, 157), chip, font=f_small, fill=PAPER)
+ x += w + 10
+ draw.line([(margin, header_h - 26), (width - margin, header_h - 26)], fill=HAIR, width=1)
+
+ top = header_h
+ left = margin
+ # ── query column ─────────────────────────────────────────────────
+ if side:
+ count = min(3, len(query_images))
+ heading = "QUERY" if count == 1 else f"QUERY · {count} PHOTOS"
+ draw.text((margin, top), heading, font=f_eyebrow, fill=MUTED)
+ y = top + 34
+ for path in query_images[:3]:
+ try:
+ tile, _ = focus_crop(open_rgb(path), (side, side), None)
+ except Exception: # noqa: BLE001 - one unreadable query must not stop the card
+ continue
+ masked = rounded(tile, 14)
+ canvas.paste(masked, (margin, y), masked)
+ draw.rounded_rectangle((margin, y, margin + side, y + side), radius=14,
+ outline=SIGNAL, width=2)
+ y += side + 16
+ left = margin + side + gap
+ draw.line([(left - gap // 2, top), (left - gap // 2, top + body_h)], fill=HAIR, width=1)
+
+ # ── results ───────────────────────────────────────────────────────
+ if not shown:
+ draw.text((left, top + 60), "Nothing in the index cleared the threshold.",
+ font=font(24), fill=MUTED)
+ from vie.report.theme import original_size
+
+ caption, note = SCORE_NOTES.get(kind, ("score", ""))
+ for i, match in enumerate(shown):
+ r, c = divmod(i, columns)
+ x0 = left + c * (tile_w + gap)
+ y0 = top + r * (tile_h + caption_h + gap)
+ try:
+ photo = open_rgb(match.file_path, 2400)
+ # Boxes are in original-file pixels; the photo here may be smaller.
+ original = original_size(match.file_path)
+ box_scale = photo.size[0] / original[0] if original and original[0] else 1.0
+ tile, box = focus_crop(photo, (tile_w, tile_h), match.box, box_scale)
+ except Exception: # noqa: BLE001
+ tile, box = Image.new("RGB", (tile_w, tile_h), INK_3), None
+ layer = Image.new("RGBA", tile.size, (0, 0, 0, 0))
+ tdraw = ImageDraw.Draw(layer)
+ if box is not None:
+ draw_brackets(tdraw, box, SIGNAL if i == 0 else PAPER, width=4 if i == 0 else 3)
+ # Soft shade at the top so the rank badge reads on any photo.
+ for k in range(46):
+ tdraw.line([(0, k), (tile_w, k)], fill=(0, 0, 0, int(120 * (1 - k / 46))))
+ tile = Image.alpha_composite(tile.convert("RGBA"), layer)
+ masked = rounded(tile.convert("RGB"), 14)
+ canvas.paste(masked, (x0, y0), masked)
+ draw.rounded_rectangle((x0, y0, x0 + tile_w, y0 + tile_h), radius=14, outline=HAIR, width=1)
+ pill(draw, (x0 + 12, y0 + 12), f"{i + 1:02d}", f_small,
+ SIGNAL if i == 0 else (0, 0, 0, 150), SIGNAL_INK if i == 0 else PAPER)
+ badge = badges[i] if badges is not None and i < len(badges) else None
+ if badge:
+ text = "▶ " + badge
+ bw = _text_width(draw, text, f_small) + 20
+ pill(draw, (x0 + tile_w - bw - 12, y0 + 12), text, f_small, LANES[1], SIGNAL_INK)
+
+ cy = y0 + tile_h + 14
+ value = max(0.0, min(1.0, match.display_score))
+ score = f"{value * 100:.1f}%"
+ sw = _text_width(draw, score, f_score)
+ draw.text((x0 + tile_w - sw, cy - 2), score, font=f_score,
+ fill=SIGNAL if i == 0 else PAPER)
+ name = _ellipsize(draw, display_text(Path(match.file_name).stem), f_name,
+ tile_w - sw - 16, middle=True)
+ draw.text((x0, cy), name, font=f_name, fill=PAPER)
+ cw = _text_width(draw, caption, f_small)
+ draw.text((x0 + tile_w - cw, cy + 30), caption, font=f_small, fill=FAINT)
+ # The species or dish that matched; for free text the label is only the
+ # name of the crop region, which means nothing to a reader.
+ if match.label and kind in ("animal", "food"):
+ draw.text((x0, cy + 28),
+ _ellipsize(draw, display_text(match.label), f_small, tile_w - cw - 16),
+ font=f_small, fill=MUTED)
+ by = cy + 60
+ draw.rounded_rectangle((x0, by, x0 + tile_w, by + 6), radius=3, fill=INK_3)
+ draw.rounded_rectangle((x0, by, x0 + max(6, int(tile_w * value)), by + 6), radius=3,
+ fill=SIGNAL if i == 0 else MUTED)
+
+ # ── footer ────────────────────────────────────────────────────────
+ fy = height - 70
+ draw.line([(margin, fy), (width - margin, fy)], fill=HAIR, width=1)
+ draw.text((margin, fy + 20), f"Visual Intelligence Engine {__version__}", font=f_small,
+ fill=FAINT)
+ draw.text((width - margin - _text_width(draw, note, f_small), fy + 20), note, font=f_small,
+ fill=FAINT)
+ return canvas
+
+
+def save_search_card(matches: Sequence[Match], path: str | Path, **kwargs: Any) -> Path:
+ path = Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ image = render_search_card(matches, **kwargs)
+ if path.suffix.lower() in {".jpg", ".jpeg"}:
+ image.save(path, quality=90, optimize=True, progressive=True)
+ else:
+ image.save(path, optimize=True)
+ return path
diff --git a/src/vie/scoring.py b/src/vie/scoring.py
index 5c841f7..7211835 100644
--- a/src/vie/scoring.py
+++ b/src/vie/scoring.py
@@ -3,12 +3,12 @@
The single best idea in the original project was in the food module: instead of
thresholding a raw cosine similarity, it scored the user's query against two
rival prompts and took a softmax. That gives the model an explicit "none of the
-above" option, so the number that comes out is a calibrated posterior rather
-than an uncalibrated distance.
+above" option.
-The animal module did not do this — it used an ImageNet classifier with no
-confidence floor at all, and shipped 21.5% top-1 results as confirmed matches.
-This module makes the good pattern the shared default.
+:func:`contrastive_score` is that softmax, which is right for CLIP. SigLIP is
+scored differently, by its margin over the strongest rival; both live in
+:func:`vie.vlm.hypothesis_score`, which is what the search paths call. This module
+keeps the softmax and the :class:`Match` record every path returns.
"""
from __future__ import annotations
diff --git a/src/vie/search/animal.py b/src/vie/search/animal.py
index 6c5f84c..58e57e8 100644
--- a/src/vie/search/animal.py
+++ b/src/vie/search/animal.py
@@ -1,28 +1,28 @@
-"""Animal search — a hybrid of a supervised classifier and CLIP.
+"""Animal search: a hybrid of a supervised classifier and a vision-language model.
Why hybrid
----------
Two models answer "which animal is this", and they are good at different things:
-=================== ========== =========== ========== ==================
-model params GFLOPs/crop ImageNet vocabulary
-=================== ========== =========== ========== ==================
-MobileNetV3-Large 5.5 M 0.217 74.0 % 398 animal classes
-CLIP ViT-L/14 428 M ~81 ~75.5 % open
-=================== ========== =========== ========== ==================
+===================== ======== =========== ==================================
+model params ImageNet vocabulary
+===================== ======== =========== ==================================
+ConvNeXt-Tiny 28.6 M 82.5 % top-1 398 ImageNet animal classes
+SigLIP so400m 877 M ~83 % 0-shot open: any name a person can type
+===================== ======== =========== ==================================
-CLIP costs roughly **373x the compute per crop** for about 1.5 points of
-top-1 — a bad trade when the species is one the classifier already knows.
-But it is the *only* option when it is not: ``quokka`` is not an ImageNet
-class, so no supervised ImageNet model can ever return it.
+Published figures, not measured here. The classifier is about as accurate on the
+species it knows and far cheaper, and it runs once per crop at index time. SigLIP
+is the *only* option for a species outside ImageNet: ``quokka`` is not a class, so
+no ImageNet classifier can ever return it.
-So the classifier is the primary path and CLIP is the fallback:
+So the classifier is the primary path and SigLIP is the fallback:
.. code-block:: text
query -> resolve against ImageNet vocabulary
- in vocabulary -> precomputed predictions (pure SQL, no inference)
- out of it -> CLIP over candidate crops (expensive, rare)
+ in vocabulary -> stored predictions (pure SQL, no model)
+ out of it -> SigLIP over candidate crops (rival prompts, rare)
The classifier also moves to **index time**. Each animal crop is classified
once when the image is ingested and its top-k predictions are stored, so an
@@ -56,14 +56,14 @@
class CropScorer(Protocol):
- """Scores image crops against text prompts (CLIP)."""
+ """Scores image crops against text prompts (the vision-language model)."""
def score(self, crops: Sequence[object], prompts: Sequence[str]) -> list[list[float]]:
...
class CropClassifier(Protocol):
- """Classifies image crops into a fixed vocabulary (MobileNetV3)."""
+ """Classifies image crops into a fixed vocabulary (ImageNet-1k; ConvNeXt-Tiny by default)."""
def classify(self, crops: Sequence[object], top_k: int) -> list[list[tuple[int, float]]]:
"""Return ``(class_id, probability)`` pairs per crop, best first."""
diff --git a/src/vie/search/face.py b/src/vie/search/face.py
index 5a88d2c..4b646d9 100644
--- a/src/vie/search/face.py
+++ b/src/vie/search/face.py
@@ -83,6 +83,40 @@ def rank_catalog(
return best_per_image(matches)
+def rank_catalog_multi(
+ query_embeddings: list[np.ndarray],
+ catalog: VectorCatalog,
+ file_paths: dict[str, str],
+ config: Config,
+) -> list[Match]:
+ """Search with several photos of the same person; each face keeps its best score.
+
+ One photo is one pose under one light. student-face-gate measured what a second
+ and third photo buy at a fixed threshold, with no wrong person admitted:
+ 86.1% -> 92.8% -> 96.3% of genuine visits accepted. Taking the maximum over
+ the query photos is the same rule its matcher applies to enrolled templates.
+ """
+ if not query_embeddings:
+ raise ValueError("no query embedding")
+ if len(query_embeddings) == 1:
+ return rank_catalog(query_embeddings[0], catalog, file_paths, config)
+ exact = catalog.backend.name == "exact"
+ k = len(catalog) if exact else config.search.ann_candidates
+ best: dict[tuple[str, str], float] = {}
+ for embedding in query_embeddings:
+ for name, bbox, score in catalog.search(embedding, k):
+ key = (name, bbox)
+ if score > best.get(key, -1.0):
+ best[key] = score
+ matches = [
+ Match(file_name=name, file_path=file_paths.get(name, name), score=score,
+ box=parse_box(bbox))
+ for (name, bbox), score in best.items()
+ if score >= config.face.match_threshold
+ ]
+ return best_per_image(matches)
+
+
def select_query_face(faces: list[tuple[Box, np.ndarray]]) -> tuple[Box, np.ndarray] | None:
"""Pick the subject of a query photo: the largest detected face.
diff --git a/src/vie/search/food.py b/src/vie/search/food.py
index e433b78..ab12ee3 100644
--- a/src/vie/search/food.py
+++ b/src/vie/search/food.py
@@ -1,13 +1,15 @@
-"""Food search — detect tableware, then recognise the dish with CLIP.
-
-This module's decision rule is the one good idea the original project already
-had, kept intact: score the user's query against explicit rival prompts and take
-a softmax, rather than thresholding a raw similarity. That gives the model a
-"none of the above" option, which is why its demo scores read 96-99% while the
-free-text path's raw cosines read 23%.
-
-The animal path has since been rebuilt on the same pattern, so three of the four
-search paths now share one recognition rule.
+"""Food search: find plates and bowls, then recognise the dish with SigLIP.
+
+The decision rule is the best idea the original project had, kept: score the query
+against explicit rival prompts ("a completely different food", "an empty plate")
+instead of thresholding a raw similarity, which gives the model a way to say "none
+of these". :func:`vie.vlm.hypothesis_score` turns the logits into one number: a
+softmax posterior for CLIP, and for SigLIP the sigmoid of the margin over the
+strongest rival, which ``scripts/calibrate.py`` measured as the more stable cut.
+
+Candidates come from RT-DETR boxes on COCO's tableware and food classes, plus whole
+images the zero-cost SigLIP food gate admitted at index time, since a plate of
+ghormeh sabzi fires none of COCO's ten Western food classes.
Fixed here: the search pass used a different COCO class list and a different
confidence than indexing, which re-introduced class 56 (``chair``) — furniture
diff --git a/src/vie/search/text.py b/src/vie/search/text.py
index b58bfe4..5683744 100644
--- a/src/vie/search/text.py
+++ b/src/vie/search/text.py
@@ -1,17 +1,17 @@
-"""Open-vocabulary text search over CLIP image embeddings.
+"""Open-vocabulary text search over vision-language image embeddings.
-Both sides of the comparison are unit-length CLIP vectors in the same 768-d
-projected space, so the dot product is cosine similarity.
+Both sides are unit-length vectors from the configured model (SigLIP so400m by
+default, 1152-d; CLIP also works), so the dot product is cosine similarity.
-The change from the original is presentational but not cosmetic. It displayed
-raw cosine as a percentage, so its best result read ``23.30%`` — a perfectly
-healthy CLIP score that looks like failure to anyone reading it. CLIP cosines
-occupy a narrow band (roughly 0.15-0.35 for genuine matches) because the
-contrastive objective never required them to span [0, 1].
+Ranking uses the raw cosine. Display does not: the original showed the cosine as a
+percentage, so its best hit read ``23.30%``, a healthy score that looks like
+failure, because these models' cosines live in a narrow band well below 1.
+:func:`with_confidence` attaches what a reader should see instead: SigLIP's own
+calibrated probability when the model has a learned bias, and a relative
+confidence across the results otherwise.
-:func:`rank` therefore returns the raw similarity *and* a relative confidence
-computed across the candidate set, so the UI can show something interpretable
-without pretending the cosine itself is a probability.
+Each image is stored as several region vectors (multi-crop) and scores as its
+best region, so a small subject is not averaged away by the rest of the frame.
"""
from __future__ import annotations
diff --git a/src/vie/video/library.py b/src/vie/video/library.py
index ef5ba58..5272138 100644
--- a/src/vie/video/library.py
+++ b/src/vie/video/library.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from typing import Any
@@ -67,11 +68,19 @@ def summary(self) -> str:
)
-def index_videos(config: Config, engine: VideoEngine, *, prune: bool = True) -> VideoIndexStats:
+def index_videos(
+ config: Config,
+ engine: VideoEngine,
+ *,
+ prune: bool = True,
+ on_analysed: Callable[[str, Path, Any], None] | None = None,
+) -> VideoIndexStats:
"""Analyse every new or changed video, committing after each one.
A video can take minutes, so losing one to an interruption is the most this
can cost. A file that fails is counted and logged, and the run carries on.
+ ``on_analysed(key, path, analysis)`` runs after each video is stored, so a
+ caller can write its report while the analysis is still in memory.
"""
store = VideoStore(config)
store.initialise()
@@ -99,10 +108,18 @@ def index_videos(config: Config, engine: VideoEngine, *, prune: bool = True) ->
stats.skipped += 1
continue
try:
- store.save(conn, key, path, digest, engine.analyse(path))
+ analysis = engine.analyse(path)
+ store.save(conn, key, path, digest, analysis)
conn.commit()
stats.indexed += 1
except Exception:
log.exception("failed to analyse %s", key)
stats.failed += 1
+ continue
+ if on_analysed is not None:
+ try:
+ on_analysed(key, path, analysis)
+ except Exception:
+ # The analysis is stored; a failed report must not undo that.
+ log.exception("post-processing failed for %s", key)
return stats
diff --git a/src/vie/video/people.py b/src/vie/video/people.py
index 77eb269..bb3975a 100644
--- a/src/vie/video/people.py
+++ b/src/vie/video/people.py
@@ -50,6 +50,33 @@ def size(s: Sighting) -> int:
return max(self.sightings, key=size)
+def choose_exemplars(sightings: Sequence[Sighting], k: int) -> list[Sighting]:
+ """Up to ``k`` views that together cover how a person looked in a video.
+
+ The first is the most typical view (closest to the person's mean vector); each
+ next one is the view least like any already chosen (farthest-point sampling),
+ so a turned head or a change of light is kept rather than four near-copies of
+ the same frontal face. Searching against these instead of every sighting keeps
+ a person findable at a small, fixed cost per person.
+ """
+ if k <= 0 or not sightings:
+ return []
+ if len(sightings) <= k:
+ return list(sightings)
+ vectors = np.vstack([s.embedding for s in sightings]).astype(np.float32)
+ vectors /= np.linalg.norm(vectors, axis=1, keepdims=True)
+ mean = vectors.mean(axis=0)
+ chosen = [int(np.argmax(vectors @ mean))]
+ nearest = vectors @ vectors[chosen[0]]
+ while len(chosen) < k:
+ candidate = int(np.argmin(nearest))
+ if candidate in chosen:
+ break
+ chosen.append(candidate)
+ nearest = np.maximum(nearest, vectors @ vectors[candidate])
+ return [sightings[i] for i in sorted(chosen, key=lambda i: sightings[i].time)]
+
+
def merge_times(times: Sequence[float], gap: float, pad: float = 0.0) -> list[Interval]:
"""Join timestamps closer than ``gap`` into intervals."""
if not times:
diff --git a/src/vie/video/render.py b/src/vie/video/render.py
new file mode 100644
index 0000000..2e994ee
--- /dev/null
+++ b/src/vie/video/render.py
@@ -0,0 +1,500 @@
+"""An annotated copy of a video: who is on screen, when, and what the shot is about.
+
+The analysis pipeline looks at a few keyframes per shot, which is what makes it
+fast. This module is the opposite trade: it visits every frame (detecting faces on
+every ``detect_every``-th and tracking in between) to produce a watchable MP4 for a
+demo or a review.
+
+Identity, in order of preference:
+
+1. **Named people.** Give ``known`` a few reference vectors per name, and a face
+ is labelled when its track's mean vector clears ``match_threshold`` against
+ that person and beats the runner-up by ``margin``. The margin rule comes from
+ student-face-gate, where it removed wrong-person accepts at low thresholds.
+2. **Anonymous people.** Otherwise tracks are grouped into "Person 1, 2, ..." by
+ average similarity, on the same ArcFace scale as the rest of the system.
+
+A track's identity is decided from its *mean* embedding over every frame seen so
+far, not from one frame, so a single blurred or turned frame cannot relabel it.
+
+Frames come from :func:`vie.video.decode.iter_decoded`, the decoder the analysis
+uses, so a portrait phone video is upright and a truncated file still renders up to
+the damage. The output has no audio track: it is for watching who appears, and
+copying audio would tie the encoder to the source's codec.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass, field
+from fractions import Fraction
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from vie.logging_setup import get_logger
+
+log = get_logger("video.render")
+
+
+@dataclass
+class Track:
+ id: int
+ box: tuple[float, float, float, float]
+ velocity: tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0)
+ vector_sum: np.ndarray | None = None
+ views: int = 0
+ missed: int = 0
+ person: int | None = None
+ name: str | None = None
+ score: float = 0.0
+
+ @property
+ def vector(self) -> np.ndarray | None:
+ if self.vector_sum is None:
+ return None
+ return self.vector_sum / max(float(np.linalg.norm(self.vector_sum)), 1e-12)
+
+
+@dataclass
+class PersonRecord:
+ id: int
+ label: str
+ colour: tuple[int, int, int]
+ vector_sum: np.ndarray
+ seconds: float = 0.0
+ first: float = 0.0
+ last: float = 0.0
+ present: list[float] = field(default_factory=list)
+ thumbnail: Any = None
+ best_area: int = 0
+ best_quality: float = 0.0
+
+ @property
+ def vector(self) -> np.ndarray:
+ return self.vector_sum / max(float(np.linalg.norm(self.vector_sum)), 1e-12)
+
+
+@dataclass
+class RenderResult:
+ path: Path
+ frames: int
+ seconds: float
+ people: list[PersonRecord]
+ fps_processed: float
+
+
+def iou(a: Sequence[float], b: Sequence[float]) -> float:
+ ix1, iy1, ix2, iy2 = max(a[0], b[0]), max(a[1], b[1]), min(a[2], b[2]), min(a[3], b[3])
+ inter = max(0.0, ix2 - ix1) * max(0.0, iy2 - iy1)
+ union = (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - inter
+ return inter / union if union > 0 else 0.0
+
+
+class FaceTracker:
+ """IoU tracking with appearance as a tie-breaker, and identity by mean vector."""
+
+ def __init__(
+ self,
+ *,
+ known: dict[str, Sequence[np.ndarray]] | None = None,
+ match_threshold: float = 0.46,
+ margin: float = 0.05,
+ cluster_threshold: float = 0.45,
+ max_missed: int = 8,
+ ) -> None:
+ self.known = {
+ name: np.vstack([v / np.linalg.norm(v) for v in vectors]).astype(np.float32)
+ for name, vectors in (known or {}).items() if len(vectors)
+ }
+ self.match_threshold = match_threshold
+ self.margin = margin
+ self.cluster_threshold = cluster_threshold
+ self.max_missed = max_missed
+ self.tracks: list[Track] = []
+ self.people: list[PersonRecord] = []
+ self._next_track = 1
+
+ def predict(self) -> None:
+ """Move every track along its velocity, for frames with no detection."""
+ for t in self.tracks:
+ t.box = tuple(b + v for b, v in zip(t.box, t.velocity, strict=True))
+
+ def update(self, detections: Sequence[tuple[Sequence[float], np.ndarray]]) -> None:
+ unmatched = list(range(len(detections)))
+ pairs = []
+ for ti, track in enumerate(self.tracks):
+ for di, (box, vector) in enumerate(detections):
+ overlap = iou(track.box, box)
+ appearance = float(track.vector @ vector) if track.vector is not None else 0.0
+ if overlap > 0.15 or appearance > 0.55:
+ pairs.append((overlap + 0.5 * appearance, ti, di))
+ used_t: set[int] = set()
+ for _, ti, di in sorted(pairs, reverse=True):
+ if ti in used_t or di not in unmatched:
+ continue
+ used_t.add(ti)
+ unmatched.remove(di)
+ self._absorb(self.tracks[ti], *detections[di])
+ for ti, track in enumerate(self.tracks):
+ if ti not in used_t:
+ track.missed += 1
+ self.tracks = [t for t in self.tracks if t.missed <= self.max_missed]
+ for di in unmatched:
+ track = Track(self._next_track, tuple(float(v) for v in detections[di][0]))
+ self._next_track += 1
+ self._absorb(track, *detections[di], fresh=True)
+ self.tracks.append(track)
+ for track in self.tracks:
+ if track.missed == 0:
+ self._identify(track)
+
+ def _absorb(self, track: Track, box: Sequence[float], vector: np.ndarray,
+ fresh: bool = False) -> None:
+ box = tuple(float(v) for v in box)
+ if not fresh:
+ # Smooth the box a little: detections jitter by a few pixels per frame.
+ track.velocity = tuple(0.5 * (n - o) for n, o in zip(box, track.box, strict=True))
+ box = tuple(0.6 * n + 0.4 * o for n, o in zip(box, track.box, strict=True))
+ track.box = box
+ track.missed = 0
+ track.vector_sum = vector.copy() if track.vector_sum is None else track.vector_sum + vector
+ track.views += 1
+
+ def _identify(self, track: Track) -> None:
+ vector = track.vector
+ if vector is None:
+ return
+ if self.known:
+ scores = sorted(((float((m @ vector).max()), name) for name, m in self.known.items()),
+ reverse=True)
+ best, name = scores[0]
+ runner = scores[1][0] if len(scores) > 1 else -1.0
+ if best >= self.match_threshold and best - runner >= self.margin:
+ track.name, track.score = name, best
+ track.person = self._person_for_name(name, vector)
+ else:
+ track.name, track.score, track.person = None, best, None
+ return
+ if track.person is not None:
+ person = self.people[track.person - 1]
+ track.score = float(person.vector @ vector)
+ if track.views % 4 == 0:
+ person.vector_sum = person.vector_sum + vector
+ return
+ if track.views < 3:
+ return # a glimpse or two is more often a false detection than a person
+ best, choice = self.cluster_threshold, None
+ for person in self.people:
+ score = float(person.vector @ vector)
+ busy = any(t.person == person.id and t is not track and t.missed == 0
+ for t in self.tracks)
+ if score >= best and not busy:
+ best, choice = score, person
+ if choice is None:
+ choice = self._new_person(f"Person {len(self.people) + 1}", vector)
+ best = 1.0
+ track.person, track.score = choice.id, best
+ track.name = choice.label
+
+ def _person_for_name(self, name: str, vector: np.ndarray) -> int:
+ for person in self.people:
+ if person.label == name:
+ return person.id
+ return self._new_person(name, vector).id
+
+ def _new_person(self, label: str, vector: np.ndarray) -> PersonRecord:
+ from vie.report.showcase import LANES
+
+ person = PersonRecord(len(self.people) + 1, label,
+ LANES[len(self.people) % len(LANES)], vector.copy())
+ self.people.append(person)
+ return person
+
+
+def _encoder(path: Path, width: int, height: int, fps: float):
+ """An H.264 writer where the build has one, MPEG-4 Part 2 otherwise."""
+ import av
+
+ # faststart moves the index to the front, so a browser can play while loading.
+ container = av.open(str(path), mode="w", options={"movflags": "+faststart"})
+ for codec in ("libx264", "h264", "libopenh264", "mpeg4"):
+ try:
+ # A fraction keeps 29.97 fps at 29.97; rounding to 30 drifts 3.6 s an hour.
+ stream = container.add_stream(codec, rate=Fraction(fps).limit_denominator(1001))
+ break
+ except Exception: # noqa: BLE001 - try the next encoder this build has
+ continue
+ else:
+ raise RuntimeError("no usable video encoder in this PyAV build")
+ stream.width, stream.height = width, height
+ stream.pix_fmt = "yuv420p"
+ if stream.codec_context.name == "libx264":
+ stream.options = {"crf": "20", "preset": "veryfast"}
+ return container, stream
+
+
+def _sharpness(patch: np.ndarray) -> float:
+ """Variance of the Laplacian, squashed to [0, 1): blur scores near zero."""
+ import cv2
+
+ if patch.size == 0:
+ return 0.0
+ gray = cv2.cvtColor(patch, cv2.COLOR_BGR2GRAY)
+ value = float(cv2.Laplacian(gray, cv2.CV_64F).var())
+ return value / (value + 60.0)
+
+
+def _format_clock(seconds: float) -> str:
+ seconds = max(0.0, seconds)
+ return f"{int(seconds // 60):02d}:{seconds % 60:05.2f}"
+
+
+def render_annotated_video(
+ source: str | Path,
+ output: str | Path,
+ *,
+ detect: Callable[[np.ndarray], list[tuple[Sequence[float], np.ndarray]]],
+ known: dict[str, Sequence[np.ndarray]] | None = None,
+ title: str | None = None,
+ shot_labels: Callable[[float], list[str]] | None = None,
+ detect_every: int = 2,
+ max_seconds: float | None = None,
+ max_side: int = 1280,
+ match_threshold: float = 0.46,
+ cluster_threshold: float = 0.45,
+ progress: Callable[[float], None] | None = None,
+ min_seconds: float = 1.0,
+) -> RenderResult:
+ """Write an annotated MP4 and return who appeared in it.
+
+ ``detect`` takes a BGR frame and returns ``(box, unit_vector)`` per face, which
+ is what :class:`vie.adapters.InsightFaceDetector` already produces, so faces here
+ use exactly the recipe the index uses.
+ """
+ import itertools
+ import time
+
+ import av
+ from PIL import Image, ImageDraw
+
+ from vie.report.showcase import (
+ FAINT,
+ HAIR,
+ INK,
+ MUTED,
+ PAPER,
+ SIGNAL,
+ SIGNAL_INK,
+ display_text,
+ draw_brackets,
+ font,
+ pill,
+ )
+ from vie.video import decode
+
+ source, output = Path(source), Path(output)
+ output.parent.mkdir(parents=True, exist_ok=True)
+ tracker = FaceTracker(known=known, match_threshold=match_threshold,
+ cluster_threshold=cluster_threshold,
+ max_missed=max(4, detect_every * 4))
+
+ info = decode.probe(source)
+ fps = info.fps or 25.0
+ duration = info.duration
+ if max_seconds:
+ duration = min(duration, max_seconds) if duration else max_seconds
+ decoded = decode.iter_decoded(source, max_side=max_side)
+ first = next(decoded, None)
+ if first is None:
+ raise decode.VideoDecodeError(f"{source} has no decodable frame")
+ # From a decoded frame, not the header: rotation swaps width and height.
+ out_h, out_w = (v // 2 * 2 for v in first.image.shape[:2])
+ # Shown in the strip and counted in the header only once seen this long, so a
+ # false detection that lasts a few frames never takes a lane.
+ min_frames = max(1, int(fps * min_seconds / 2))
+ unit = max(14, out_h // 34)
+ lane_pitch = int(unit * 1.15)
+ strip_h = (lane_pitch * 5 + unit) // 2 * 2 # room for five people, always even
+ writer, out_stream = _encoder(output, out_w, out_h + strip_h, fps)
+
+ f_title, f_label = font(int(unit * 1.25), True), font(unit, True)
+ f_small = font(int(unit * .85))
+ started = time.perf_counter()
+ frames = 0
+ t = 0.0
+ try:
+ for frame in itertools.chain([first], decoded):
+ t = frame.time
+ if max_seconds and t > max_seconds:
+ break
+ image = np.ascontiguousarray(frame.image[:out_h, :out_w])
+ detected_now = frames % max(1, detect_every) == 0
+ if detected_now:
+ tracker.update(detect(image))
+ else:
+ tracker.predict()
+
+ canvas = Image.new("RGB", (out_w, out_h + strip_h), INK)
+ canvas.paste(Image.fromarray(image[:, :, ::-1]), (0, 0))
+ draw = ImageDraw.Draw(canvas, "RGBA")
+
+ on_screen = 0
+ for track in tracker.tracks:
+ if track.missed > detect_every:
+ continue
+ x1, y1, x2, y2 = (int(v) for v in track.box)
+ person = tracker.people[track.person - 1] if track.person else None
+ colour = person.colour if person else (200, 205, 212)
+ draw_brackets(draw, (x1, y1, x2, y2), colour, width=max(2, unit // 6))
+ label = (f"{track.name} {track.score * 100:.0f}%" if track.name
+ else "unknown" if tracker.known else "…")
+ pill(draw, (x1, max(0, y1 - unit * 2)), label, f_label,
+ colour if person else (40, 44, 52),
+ SIGNAL_INK if person else PAPER, (unit // 2, unit // 4))
+ if person is not None:
+ on_screen += 1
+ if not person.present or t - person.present[-1] > 1e-6:
+ person.present.append(t)
+ area = (x2 - x1) * (y2 - y1)
+ # Thumbnails only from a real detection, never a predicted box, and
+ # the sharpest large view: the largest alone is usually motion blur.
+ if (detected_now and track.missed == 0 and x1 >= 0 and y1 >= 0
+ and area * 2 > person.best_area):
+ face = image[y1:y2, x1:x2]
+ quality = area * _sharpness(face)
+ if face.size and quality > person.best_quality:
+ pad = int((x2 - x1) * 0.3)
+ crop = image[max(0, y1 - pad):y2 + pad, max(0, x1 - pad):x2 + pad]
+ person.thumbnail = Image.fromarray(crop[:, :, ::-1].copy())
+ person.best_quality = quality
+ person.best_area = max(person.best_area, area)
+
+ # ── heads-up display ─────────────────────────────────────
+ hud_h = int(unit * 3.4)
+ for k in range(hud_h):
+ draw.line([(0, k), (out_w, k)], fill=(0, 0, 0, int(170 * (1 - k / hud_h))))
+ draw.text((unit, unit * 0.6), display_text((title or source.stem)[:60]), font=f_title,
+ fill=PAPER)
+ clock = _format_clock(t)
+ cw = draw.textbbox((0, 0), clock, font=f_title)[2]
+ draw.text((out_w - cw - unit, unit * 0.6), clock, font=f_title, fill=SIGNAL)
+ # On the same dark backing as the chips, so it reads over a bright window.
+ mark = "Visual Intelligence Engine"
+ mw = draw.textbbox((0, 0), mark, font=f_small)[2] + unit
+ pill(draw, (out_w - mw - unit, int(unit * 2.3)), mark, f_small, (0, 0, 0, 140),
+ MUTED, (unit // 2, unit // 5))
+ lanes = [p for p in tracker.people if len(p.present) >= min_frames]
+ chips = [f"{on_screen} on screen",
+ f"{len(lanes)} {'person' if len(lanes) == 1 else 'people'} so far"]
+ if shot_labels is not None:
+ chips = shot_labels(t)[:2] + chips
+ x = unit
+ for chip in chips:
+ r = pill(draw, (x, int(unit * 2.3)), chip, f_small, (0, 0, 0, 140), PAPER,
+ (unit // 2, unit // 5))
+ x = r[2] + unit // 2
+
+ # ── presence timeline ────────────────────────────────────
+ top = out_h
+ draw.line([(0, top), (out_w, top)], fill=HAIR, width=1)
+ lanes = lanes[:5] # in order of first appearance, so lanes never reshuffle
+ lane_h = max(4, lane_pitch - 6)
+ span = duration or max(t, 1.0)
+ label_w = int(unit * 7)
+ for li, person in enumerate(lanes):
+ ly = top + unit // 2 + li * lane_pitch
+ draw.text((unit, ly - 3), display_text(person.label[:14]), font=f_small, fill=MUTED)
+ for seen in person.present:
+ px = label_w + int((out_w - label_w - unit) * min(1.0, seen / span))
+ draw.line([(px, ly), (px, ly + lane_h)], fill=person.colour, width=2)
+ if not lanes:
+ draw.text((unit, top + strip_h // 3), "no faces yet", font=f_small, fill=FAINT)
+ head = label_w + int((out_w - label_w - unit) * min(1.0, t / span))
+ draw.line([(head, top + 4), (head, top + strip_h - 4)], fill=SIGNAL, width=2)
+
+ out = av.VideoFrame.from_image(canvas)
+ for packet in out_stream.encode(out):
+ writer.mux(packet)
+ frames += 1
+ if progress is not None and frames % 30 == 0 and span:
+ progress(min(1.0, t / span))
+ for packet in out_stream.encode():
+ writer.mux(packet)
+ finally:
+ writer.close()
+ decoded.close()
+
+ step = 1.0 / fps
+ for person in tracker.people:
+ if person.present:
+ person.first, person.last = person.present[0], person.present[-1]
+ person.seconds = len(person.present) * step
+ # Under ``min_seconds`` on screen is more often a false detection than a person.
+ # Kept in order of first appearance, which is also the order of their numbers.
+ people = sorted((p for p in tracker.people if p.seconds >= min_seconds),
+ key=lambda p: p.first)
+ elapsed = time.perf_counter() - started
+ log.info("annotated %d frames in %.1fs -> %s", frames, elapsed, output)
+ return RenderResult(output, frames, t, people, frames / elapsed if elapsed else 0.0)
+
+
+def render_people_card(result: RenderResult, *, title: str, width: int = 1600) -> Any:
+ """One image summarising who appeared and for how long."""
+ from PIL import Image, ImageDraw
+
+ from vie.report.showcase import (
+ FAINT,
+ HAIR,
+ INK,
+ INK_3,
+ MUTED,
+ PAPER,
+ SIGNAL,
+ display_text,
+ focus_crop,
+ font,
+ rounded,
+ )
+
+ people = result.people[:12]
+ margin, gap = 56, 36
+ # Balanced rows (7 people: 4 + 3, not 5 + 2), tiles as large as the width allows.
+ rows = max(1, -(-len(people) // 6))
+ cols = max(1, -(-len(people) // rows))
+ tile = min(300, (width - margin * 2 - gap * (cols - 1)) // cols)
+ height = 230 + rows * (tile + 140) + 80
+ canvas = Image.new("RGB", (width, height), INK)
+ draw = ImageDraw.Draw(canvas, "RGBA")
+ draw.text((margin, 50), "VIDEO UNDERSTANDING · PEOPLE", font=font(18, True), fill=SIGNAL)
+ draw.text((margin, 80), display_text(title[:60]), font=font(46, True), fill=PAPER)
+ draw.text((margin, 142), f"{len(result.people)} people · {result.frames:,} frames · "
+ f"{_format_clock(result.seconds)} analysed · {result.fps_processed:.1f} frames/s",
+ font=font(18), fill=MUTED)
+ draw.line([(margin, 190), (width - margin, 190)], fill=HAIR, width=1)
+ span = max(result.seconds, 1e-6)
+ for i, person in enumerate(people):
+ r, c = divmod(i, cols)
+ x, y = margin + c * (tile + gap), 220 + r * (tile + 140)
+ if person.thumbnail is not None:
+ face, _ = focus_crop(person.thumbnail, (tile, tile), None)
+ else:
+ face = Image.new("RGB", (tile, tile), INK_3)
+ masked = rounded(face, 16)
+ canvas.paste(masked, (x, y), masked)
+ draw.rounded_rectangle((x, y, x + tile, y + tile), radius=16, outline=person.colour,
+ width=3)
+ draw.text((x, y + tile + 12), display_text(person.label), font=font(22, True), fill=PAPER)
+ draw.text((x, y + tile + 42), f"{person.seconds:.1f}s on screen", font=font(15),
+ fill=MUTED)
+ draw.text((x, y + tile + 62), f"first seen {_format_clock(person.first)}", font=font(15),
+ fill=FAINT)
+ by = y + tile + 94
+ draw.rounded_rectangle((x, by, x + tile, by + 10), radius=5, fill=INK_3)
+ for seen in person.present[::2]:
+ px = x + int(tile * min(1.0, seen / span))
+ draw.line([(px, by), (px, by + 10)], fill=person.colour, width=1)
+ draw.text((margin, height - 50), "Faces are grouped by appearance; names come only from "
+ "reference photos you supply.", font=font(15), fill=FAINT)
+ return canvas
diff --git a/src/vie/video/store.py b/src/vie/video/store.py
index 352123d..c4f971a 100644
--- a/src/vie/video/store.py
+++ b/src/vie/video/store.py
@@ -19,7 +19,7 @@
from vie.database import Index, format_box
from vie.embeddings import from_blob, to_blob
from vie.video.engine import VideoAnalysis
-from vie.video.people import Interval, merge_times
+from vie.video.people import Interval, choose_exemplars, merge_times
VIDEO_SCHEMA = """
CREATE TABLE IF NOT EXISTS videos (
@@ -51,6 +51,17 @@
FOREIGN KEY (video_key) REFERENCES videos (video_key) ON DELETE CASCADE
);
+-- When each person is on screen, as intervals. Their face vectors are only the few
+-- exemplars in video_faces, so this is what a face search reports back.
+CREATE TABLE IF NOT EXISTS video_person_spans (
+ video_key TEXT NOT NULL,
+ person INTEGER NOT NULL,
+ start REAL NOT NULL,
+ end REAL NOT NULL,
+ PRIMARY KEY (video_key, person, start),
+ FOREIGN KEY (video_key) REFERENCES videos (video_key) ON DELETE CASCADE
+);
+
CREATE TABLE IF NOT EXISTS video_transcript (
video_key TEXT NOT NULL,
start REAL NOT NULL,
@@ -180,14 +191,21 @@ def save(
"VALUES (?, ?, ?, ?)",
(key, frame.time, frame.shot, to_blob(frame.vector)),
)
+ exemplars = self.config.video.face_exemplars
for person in analysis.people:
- for sighting in person.sightings:
+ for sighting in choose_exemplars(person.sightings, exemplars):
conn.execute(
"INSERT OR REPLACE INTO video_faces "
"(video_key, time, bbox, person, embedding) VALUES (?, ?, ?, ?, ?)",
(key, sighting.time, format_box(sighting.box), person.id,
to_blob(sighting.embedding)),
)
+ for span in person.intervals:
+ conn.execute(
+ "INSERT OR REPLACE INTO video_person_spans "
+ "(video_key, person, start, end) VALUES (?, ?, ?, ?)",
+ (key, person.id, span.start, span.end),
+ )
for segment in analysis.transcript:
conn.execute(
"INSERT OR REPLACE INTO video_transcript (video_key, start, end, text) "
@@ -273,30 +291,51 @@ def search_speech(
def search_face(
self, conn: sqlite3.Connection, query: np.ndarray, top_k: int = 10
) -> list[Moment]:
- """Where a given face appears, as intervals, across every video."""
+ """Where a given face appears, as intervals, across every video.
+
+ A person matches when any of their stored exemplar views clears the face
+ threshold, and the answer is every interval that person is on screen. An
+ index from before exemplars kept every sighting and has no spans; there the
+ matching sightings themselves are merged into intervals, as before.
+ """
rows = conn.execute(
- "SELECT f.video_key, v.file_path, f.time, f.embedding FROM video_faces f "
+ "SELECT f.video_key, v.file_path, f.time, f.person, f.embedding FROM video_faces f "
"JOIN videos v USING (video_key)"
).fetchall()
if not rows:
return []
- matrix = np.vstack([from_blob(r[3]) for r in rows])
+ matrix = np.vstack([from_blob(r[4]) for r in rows])
scores = matrix @ np.asarray(query, dtype=np.float32).ravel()
threshold = self.config.face.match_threshold
+ spans: dict[tuple[str, int], list[tuple[float, float]]] = {}
+ for key, person, start, end in conn.execute(
+ "SELECT video_key, person, start, end FROM video_person_spans"
+ ):
+ spans.setdefault((key, person), []).append((start, end))
+
+ best: dict[tuple[str, int], float] = {}
hits: dict[str, list[tuple[float, float]]] = {}
paths: dict[str, str] = {}
- for (key, path, time, _), score in zip(rows, scores.tolist(), strict=True):
- if score >= threshold:
+ for (key, path, time, person, _), score in zip(rows, scores.tolist(), strict=True):
+ if score < threshold:
+ continue
+ paths[key] = path
+ if (key, person) in spans:
+ best[(key, person)] = max(score, best.get((key, person), -1.0))
+ else:
hits.setdefault(key, []).append((time, score))
- paths[key] = path
- moments: list[Moment] = []
+ moments: list[Moment] = [
+ Moment(key, paths[key], start, end, score, "face")
+ for (key, person), score in best.items()
+ for start, end in spans[(key, person)]
+ ]
for key, entries in hits.items():
for span in _spans(entries, self.config.video.presence_gap):
inside = [s for t, s in entries if span.start <= t <= span.end]
moments.append(Moment(key, paths[key], span.start, span.end, max(inside), "face"))
- moments.sort(key=lambda m: m.score, reverse=True)
+ moments.sort(key=lambda m: (-m.score, m.video_key, m.start))
return moments[:top_k]
diff --git a/src/vie/vlm.py b/src/vie/vlm.py
index 887197f..9f94155 100644
--- a/src/vie/vlm.py
+++ b/src/vie/vlm.py
@@ -48,6 +48,16 @@ def is_siglip(model_name: str) -> bool:
return "siglip" in model_name.lower()
+def is_multilingual(model_name: str) -> bool:
+ """SigLIP 2 was trained on multilingual WebLI; SigLIP and CLIP on English."""
+ return "siglip2" in model_name.lower()
+
+
+def has_right_to_left(text: str) -> bool:
+ """Persian, Arabic or Hebrew script anywhere in ``text``."""
+ return any("\u0590" <= ch <= "\u08ff" or "\ufb1d" <= ch <= "\ufeff" for ch in text)
+
+
def default_score_mode(model_name: str) -> str:
return SIGMOID if is_siglip(model_name) else SOFTMAX
diff --git a/tests/test_integration.py b/tests/test_integration.py
index f6691e2..3ea3aac 100644
--- a/tests/test_integration.py
+++ b/tests/test_integration.py
@@ -260,7 +260,9 @@ def test_inference_is_batched(raw_config, tmp_path) -> None:
stats = stack.index(batch_size=4)
assert stats.indexed == 6
assert stack.encoder.forward_passes == 2
- assert stack.encoder.batch_sizes == [24, 12] # four and two images, six crops each
+ # Four and two images, six scene crops each, plus the fake animal every red image
+ # holds: its object vector rides in the same forward pass, not a second one.
+ assert stack.encoder.batch_sizes == [28, 14]
assert stack.food.batches == [4, 2]
diff --git a/tests/test_v05.py b/tests/test_v05.py
new file mode 100644
index 0000000..da2a96a
--- /dev/null
+++ b/tests/test_v05.py
@@ -0,0 +1,499 @@
+"""v0.5: face accuracy, the Engine API, result cards and the annotated video."""
+
+from __future__ import annotations
+
+import copy
+from pathlib import Path
+from types import SimpleNamespace
+
+import numpy as np
+import pytest
+from fakes import FakeStack, write_png
+
+from vie.config import Config
+from vie.database import Index
+from vie.scoring import Match
+
+
+def unit(v) -> np.ndarray:
+ v = np.asarray(v, dtype=np.float32)
+ return v / np.linalg.norm(v)
+
+
+@pytest.fixture()
+def cfg(raw_config: dict, tmp_path: Path) -> Config:
+ root = tmp_path / "gallery"
+ write_png(root / "red.png", (230, 20, 20))
+ write_png(root / "blue.png", (20, 20, 230))
+ raw = copy.deepcopy(raw_config)
+ raw["paths"] = {"gallery": str(root), "database": str(tmp_path / "index.db")}
+ return Config.from_dict(raw)
+
+
+# ── mirror averaging ──────────────────────────────────────────────────
+
+
+def test_mirror_vector_is_the_normalised_sum_of_both_views() -> None:
+ pytest.importorskip("insightface")
+ from vie.adapters import face_vectors
+
+ seen, mirrored = np.array([3.0, 0, 0, 0]), np.array([0, 4.0, 0, 0], dtype=np.float32)
+
+ class Rec:
+ input_size = (112, 112)
+
+ def get_feat(self, crops):
+ assert crops[0].shape == (112, 112, 3)
+ return np.vstack([mirrored for _ in crops])
+
+ face = SimpleNamespace(embedding=seen, normed_embedding=unit(seen),
+ kps=np.array([[38, 52], [74, 52], [56, 72], [42, 92], [70, 92]],
+ dtype=np.float32))
+ app = SimpleNamespace(models={"recognition": Rec()})
+ image = np.zeros((160, 160, 3), dtype=np.uint8)
+ (vector,) = face_vectors(app, image, [face], flip=True)
+ np.testing.assert_allclose(vector, unit([3.0, 4.0, 0, 0]), rtol=1e-6)
+ (plain,) = face_vectors(app, image, [face], flip=False)
+ np.testing.assert_allclose(plain, unit(seen))
+
+
+def test_mirror_falls_back_to_the_plain_vector_without_a_recogniser() -> None:
+ from vie.adapters import face_vectors
+
+ face = SimpleNamespace(normed_embedding=unit([1, 2, 3]))
+ assert face_vectors(SimpleNamespace(), None, [face], flip=True)[0] is face.normed_embedding
+
+
+# ── several query photos ──────────────────────────────────────────────
+
+
+class Catalog:
+ def __init__(self, rows):
+ self.rows = rows
+ self.backend = SimpleNamespace(name="exact")
+
+ def __len__(self):
+ return len(self.rows)
+
+ def search(self, query, k):
+ scored = [(name, bbox, float(vec @ query)) for name, bbox, vec in self.rows]
+ return sorted(scored, key=lambda r: -r[2])[:k]
+
+
+def test_a_second_query_photo_finds_the_pose_the_first_missed(cfg) -> None:
+ from vie.search.face import rank_catalog_multi
+
+ frontal, profile = unit([1, 0, 0]), unit([0, 1, 0])
+ catalog = Catalog([("front.jpg", "0,0,10,10", frontal),
+ ("side.jpg", "0,0,10,10", unit([0.1, 1, 0])),
+ ("stranger.jpg", "0,0,10,10", unit([0, 0, 1]))])
+ paths = {}
+ one = rank_catalog_multi([frontal], catalog, paths, cfg)
+ both = rank_catalog_multi([frontal, profile], catalog, paths, cfg)
+ assert [m.file_name for m in one] == ["front.jpg"]
+ assert sorted(m.file_name for m in both) == ["front.jpg", "side.jpg"]
+
+
+def test_face_search_accepts_several_images_end_to_end(cfg, tmp_path) -> None:
+ from vie.query import run_search
+
+ stack = FakeStack(cfg)
+ stack.index()
+ query = cfg.gallery_path / "blue.png"
+ args = SimpleNamespace(kind="face", query="", image=[query, query], top_k=None)
+ assert [m.file_name for m in run_search(args, cfg, stack.load_runtime)] == ["blue.png"]
+
+
+# ── changing the face recipe re-indexes only images with faces ───────
+
+
+def test_a_new_face_recipe_reindexes_only_the_images_with_faces(cfg) -> None:
+ stack = FakeStack(cfg)
+ assert stack.index().indexed == 2
+ assert stack.index().indexed == 0 # incremental: nothing changed, nothing read
+ index = Index(cfg.database_path)
+ with index.connect() as conn:
+ assert index.get_meta(conn, "face_recipe") == "arcface+flip"
+ index.set_meta(conn, "face_recipe", "arcface") # an index from before v0.5
+ stats = stack.index()
+ assert stats.indexed == 1 # blue.png has the face; red.png is left alone
+ assert stack.index().indexed == 0
+
+
+# ── the Engine ────────────────────────────────────────────────────────
+
+
+def test_load_config_merges_overrides_over_the_packaged_defaults(tmp_path) -> None:
+ from vie.api import load_config
+
+ config = load_config({"paths": {"gallery": str(tmp_path)}, "face": {"flip_tta": False}})
+ assert config.gallery_path == tmp_path
+ assert config.face.flip_tta is False
+ assert config.face.match_threshold == 0.46 # untouched defaults survive
+
+
+def test_pruning_most_of_the_index_at_once_is_refused(cfg) -> None:
+ """An unmounted drive looks like every photo was deleted; rows must survive it."""
+ stack = FakeStack(cfg)
+ stack.index()
+ for name in ("red.png", "blue.png"):
+ (cfg.gallery_path / name).unlink()
+ stats = stack.index()
+ assert stats.prune_refused and stats.pruned == 0
+ with Index(cfg.database_path).connect() as conn:
+ assert conn.execute("SELECT COUNT(*) FROM gallery_meta").fetchone()[0] == 2
+ forced = stack.index(prune_guard=0.0)
+ assert forced.pruned == 2 and not forced.prune_refused
+
+
+def test_engine_reports_a_refused_prune(cfg, monkeypatch) -> None:
+ import vie.adapters
+ import vie.models
+ import vie.pipeline
+ from vie.api import Engine
+
+ monkeypatch.setattr(vie.pipeline, "build_index", lambda config, **kw: SimpleNamespace(
+ scanned=0, indexed=0, skipped=0, pruned=0, failed=0, prune_refused=True))
+ monkeypatch.setattr(vie.adapters, "build_detectors", lambda b, c: SimpleNamespace(
+ loader=None, size_of=None, face=None, animal=None, food=None, crop=None))
+ monkeypatch.setattr(vie.models, "SpeciesClassifier", lambda b: SimpleNamespace(classify=None))
+ engine = Engine(cfg)
+ engine._bundle, engine._encoder = SimpleNamespace(), SimpleNamespace(
+ embed_batch=None, food_gate_scores=None)
+ assert engine.index_images().pruning is False
+
+
+def test_auto_face_query_is_a_crop_of_the_chosen_face(cfg, tmp_path) -> None:
+ from PIL import Image
+
+ from vie.api import Engine
+
+ FakeStack(cfg).index()
+ engine = Engine(cfg)
+ assert engine.most_photographed_face() == cfg.gallery_path / "blue.png"
+ crop = engine.most_photographed_face(crop_to=tmp_path / "query.jpg")
+ with Image.open(crop) as image:
+ # The fake face is 50x60 inside a 120x90 photo; padded, then clipped to it.
+ assert image.size[0] <= 120 and image.size[1] <= 90 and image.size[0] > 50
+
+
+def test_video_search_needs_exactly_one_of_query_or_image(cfg) -> None:
+ from vie.api import Engine
+
+ with pytest.raises(ValueError, match="exactly one"):
+ Engine(cfg).search_videos()
+
+
+# ── presentation ──────────────────────────────────────────────────────
+
+
+def test_result_card_renders_with_boxes_and_query(tmp_path) -> None:
+ pytest.importorskip("PIL")
+ from vie.report.showcase import CardStats, save_search_card
+
+ photos = [write_png(tmp_path / f"p{i}.png", (40 * i, 90, 160), (640, 480)) for i in range(5)]
+ matches = [Match(p.name, str(p), 0.9 - i * 0.1, (100, 80, 300, 320), label="zebra")
+ for i, p in enumerate(photos)]
+ out = save_search_card(matches, tmp_path / "card.png", kind="animal", query="zebra",
+ query_images=[photos[0]], stats=CardStats(search_ms=3.2,
+ indexed_images=5))
+ from PIL import Image
+
+ with Image.open(out) as image:
+ assert image.size[0] == 1600 and image.size[1] > 600
+
+
+def test_focus_crop_keeps_the_subject_in_the_tile() -> None:
+ from PIL import Image
+
+ from vie.report.showcase import focus_crop
+
+ image = Image.new("RGB", (2000, 1000))
+ _, box = focus_crop(image, (400, 300), (1700, 400, 1900, 600))
+ x1, y1, x2, y2 = box
+ assert 0 <= x1 < x2 <= 400 and 0 <= y1 < y2 <= 300
+
+
+# ── the annotated video ───────────────────────────────────────────────
+
+
+def test_tracker_keeps_identities_apart_and_stable() -> None:
+ from vie.video.render import FaceTracker
+
+ a, b = unit([1, 0, 0, 0]), unit([0, 1, 0, 0])
+ tracker = FaceTracker()
+ for step in range(6):
+ tracker.update([((10 + step, 10, 60 + step, 60), a), ((200, 10, 250, 60), b)])
+ labels = sorted(t.name for t in tracker.tracks)
+ assert labels == ["Person 1", "Person 2"]
+ assert len(tracker.people) == 2
+
+
+def test_tracker_names_only_with_a_margin_over_the_runner_up() -> None:
+ from vie.video.render import FaceTracker
+
+ known = {"Ava": [unit([1, 0, 0])], "Ben": [unit([0.98, 0.2, 0])]}
+ tracker = FaceTracker(known=known, margin=0.05)
+ tracker.update([((0, 0, 50, 50), unit([1, 0.1, 0]))])
+ assert tracker.tracks[0].name is None # the two references are too alike to call it
+ tracker = FaceTracker(known={"Ava": [unit([1, 0, 0])], "Ben": [unit([0, 1, 0])]})
+ tracker.update([((0, 0, 50, 50), unit([1, 0.1, 0]))])
+ assert tracker.tracks[0].name == "Ava"
+
+
+def test_render_writes_a_playable_video(tmp_path) -> None:
+ av = pytest.importorskip("av")
+ from vie.video.render import render_annotated_video, render_people_card
+
+ source = tmp_path / "in.mp4"
+ container = av.open(str(source), mode="w")
+ stream = container.add_stream("mpeg4", rate=10)
+ stream.width, stream.height, stream.pix_fmt = 160, 120, "yuv420p"
+ for i in range(30):
+ frame = np.full((120, 160, 3), 40 + i * 3, dtype=np.uint8)
+ for packet in stream.encode(av.VideoFrame.from_ndarray(frame, format="rgb24")):
+ container.mux(packet)
+ for packet in stream.encode():
+ container.mux(packet)
+ container.close()
+
+ face = unit([1, 0, 0, 0])
+ result = render_annotated_video(source, tmp_path / "out.mp4",
+ detect=lambda img: [((40, 30, 90, 90), face)],
+ title="test", detect_every=2)
+ assert result.frames == 30
+ assert [p.label for p in result.people] == ["Person 1"]
+ with av.open(str(result.path)) as check:
+ assert sum(1 for _ in check.decode(video=0)) == 30
+ assert render_people_card(result, title="test").size[0] == 1600
+
+
+def test_shot_labeller_reads_labels_of_the_current_shot() -> None:
+ from vie.api import shot_labeller
+
+ analysis = SimpleNamespace(
+ shots=[SimpleNamespace(index=1, start=5.0, end=9.0),
+ SimpleNamespace(index=0, start=0.0, end=5.0)],
+ shot_labels=[SimpleNamespace(shot=0, labels={"topic": ("an interview", 0.8),
+ "setting": ("in a studio", 0.6)}),
+ SimpleNamespace(shot=1, labels={"topic": ("a lecture", 0.3)})],
+ )
+ at = shot_labeller(analysis, threshold=0.4)
+ assert at(2.0) == ["an interview", "in a studio"]
+ assert at(6.0) == [] # below the presence threshold
+ assert at(20.0) == [] # past the last shot
+ assert shot_labeller(analysis, threshold=0.2)(6.0) == ["a lecture"]
+
+
+def test_right_to_left_text_is_prepared_and_latin_is_untouched() -> None:
+ from vie.report.showcase import display_text, font
+
+ assert display_text("Person 1 93%") == "Person 1 93%"
+ assert isinstance(display_text("دانشجو ۱۰۴۲"), str)
+ assert font(20) is font(20) # cached: looking up a face walks font folders
+
+
+# ── incremental runs stay incremental ─────────────────────────────────
+
+
+def test_a_touched_but_unchanged_file_is_hashed_once_then_never_again(cfg, monkeypatch) -> None:
+ """A sync or a copy moves the mtime. Hashing then must happen once, not forever."""
+ import os
+
+ import vie.pipeline as pipeline
+
+ stack = FakeStack(cfg)
+ stack.index()
+ target = cfg.gallery_path / "red.png"
+ os.utime(target, ns=(target.stat().st_atime_ns, target.stat().st_mtime_ns + 10**9))
+ hashed = []
+ real = pipeline.content_hash
+ monkeypatch.setattr(pipeline, "content_hash", lambda p: hashed.append(p) or real(p))
+ assert stack.index().indexed == 0
+ assert hashed == [target]
+ assert stack.index().indexed == 0
+ assert hashed == [target] # the new mtime was remembered
+
+
+def test_a_moved_gallery_keeps_its_index_and_points_at_the_new_place(
+ cfg, raw_config, tmp_path
+) -> None:
+ import shutil
+
+ FakeStack(cfg).index()
+ moved = tmp_path / "moved"
+ shutil.copytree(cfg.gallery_path, moved) # copytree keeps contents, not always mtimes
+ shutil.rmtree(cfg.gallery_path)
+ raw = copy.deepcopy(raw_config)
+ raw["paths"] = {"gallery": str(moved), "database": str(cfg.database_path)}
+ stats = FakeStack(Config.from_dict(raw)).index()
+ assert stats.indexed == 0 and stats.pruned == 0
+ with Index(cfg.database_path).connect() as conn:
+ paths = {Path(p).parent for (p,) in conn.execute("SELECT file_path FROM gallery_meta")}
+ assert paths == {moved}
+
+
+# ── food and unknown species are answered from stored vectors ─────────
+
+
+@pytest.fixture()
+def food_cfg(raw_config: dict, tmp_path: Path) -> Config:
+ root = tmp_path / "gallery"
+ write_png(root / "red.png", (230, 20, 20))
+ write_png(root / "green.png", (20, 230, 20))
+ write_png(root / "blue.png", (20, 20, 230))
+ raw = copy.deepcopy(raw_config)
+ raw["paths"] = {"gallery": str(root), "database": str(tmp_path / "index.db")}
+ return Config.from_dict(raw)
+
+
+def _no_pixels(monkeypatch) -> None:
+ import vie.adapters
+
+ def refuse(*args, **kwargs):
+ raise AssertionError("a query decoded an image")
+
+ monkeypatch.setattr(vie.adapters, "load_image", refuse)
+
+
+def test_object_vectors_are_stored_at_index_time(food_cfg) -> None:
+ FakeStack(food_cfg).index()
+ with Index(food_cfg.database_path).connect() as conn:
+ rows = conn.execute("SELECT file_name, kind FROM object_embeddings").fetchall()
+ flags = dict(conn.execute("SELECT file_name, objects_embedded FROM gallery_meta"))
+ assert rows == [("red.png", "animal")] # the fake detector's one animal
+ assert set(flags.values()) == {1}
+
+
+@pytest.mark.parametrize(("kind", "query", "expected"), [
+ ("food", "kebab", ["green.png"]), # gate-admitted, scored on its whole frame
+ ("animal", "red quokka", None), # out of vocabulary: the vector path
+])
+def test_food_and_unknown_species_open_no_image(food_cfg, monkeypatch, kind, query,
+ expected) -> None:
+ from vie.query import run_search
+
+ stack = FakeStack(food_cfg)
+ stack.index()
+ reference = run_search(SimpleNamespace(kind=kind, query=query, image=None, top_k=None),
+ food_cfg, stack.load_runtime)
+ _no_pixels(monkeypatch)
+ found = run_search(SimpleNamespace(kind=kind, query=query, image=None, top_k=None),
+ food_cfg, stack.load_runtime)
+ assert [m.file_name for m in found] == [m.file_name for m in reference]
+ if expected is not None:
+ assert [m.file_name for m in found] == expected
+
+
+def test_an_older_index_is_still_served_by_cropping(food_cfg) -> None:
+ from vie.query import run_search
+
+ stack = FakeStack(food_cfg)
+ stack.index()
+ request = SimpleNamespace(kind="food", query="kebab", image=None, top_k=None)
+ new = run_search(request, food_cfg, stack.load_runtime)
+ with Index(food_cfg.database_path).connect() as conn:
+ conn.execute("UPDATE gallery_meta SET objects_embedded = 0")
+ conn.execute("DELETE FROM object_embeddings")
+ old = run_search(request, food_cfg, stack.load_runtime)
+ assert [m.file_name for m in old] == [m.file_name for m in new] == ["green.png"]
+ assert old[0].score == pytest.approx(new[0].score, abs=1e-5)
+
+
+def test_an_unknown_species_is_found_from_its_stored_vector(raw_config, tmp_path,
+ monkeypatch) -> None:
+ from vie.query import run_search
+
+ write_png(tmp_path / "g" / "red.png", (230, 20, 20))
+ write_png(tmp_path / "g" / "blue.png", (20, 20, 230))
+ raw = copy.deepcopy(raw_config)
+ raw["paths"] = {"gallery": str(tmp_path / "g"), "database": str(tmp_path / "i.db")}
+ raw["animal"]["match_threshold"] = 0.5
+ config = Config.from_dict(raw)
+ stack = FakeStack(config)
+ stack.index()
+ _no_pixels(monkeypatch)
+ found = run_search(SimpleNamespace(kind="animal", query="red quokka", image=None,
+ top_k=None), config, stack.load_runtime)
+ assert [m.file_name for m in found] == ["red.png"] and found[0].box is not None
+
+
+# ── one description, photos and video moments together ────────────────
+
+
+def test_concept_search_returns_photos_and_video_moments(raw_config, tmp_path) -> None:
+ pytest.importorskip("av")
+ from test_video import write_video
+
+ from vie.api import Engine
+ from vie.report.showcase import save_search_card
+ from vie.video.engine import VideoEngine
+ from vie.video.library import index_videos
+
+ write_png(tmp_path / "gallery" / "red.png", (230, 20, 20))
+ write_png(tmp_path / "gallery" / "blue.png", (20, 20, 230))
+ write_video(tmp_path / "videos" / "clip.mp4", [((230, 20, 20), 4), ((20, 230, 20), 4)])
+ raw = copy.deepcopy(raw_config)
+ raw["paths"] = {"gallery": str(tmp_path / "gallery"), "videos": str(tmp_path / "videos"),
+ "database": str(tmp_path / "index.db")}
+ raw["video"]["sample_seconds"] = 0.5
+ raw["video"]["keyframe_seconds"] = 2.0
+ config = Config.from_dict(raw)
+ stack = FakeStack(config)
+ stack.index()
+ index_videos(config, VideoEngine(config, stack.encoder, on_stage=lambda s: None))
+
+ engine = Engine(config)
+ engine._bundle, engine._encoder = stack.bundle, stack.encoder
+ result = engine.search_everything("something red", top_k=4, min_moments=1,
+ frames_dir=tmp_path / "frames")
+ names = [m.file_name for m in result.matches]
+ assert "red.png" in names
+ videos = [i for i, badge in enumerate(result.badges) if badge]
+ assert videos and names[videos[0]].startswith("clip @ 0:0")
+ assert Path(result.matches[videos[0]].file_path).is_file() # the grabbed frame
+ card = save_search_card(result.matches, tmp_path / "card.png", kind="concept",
+ query=result.query, badges=result.badges)
+ assert card.is_file()
+
+
+# ── measured search defaults ──────────────────────────────────────────
+
+
+def test_a_persian_query_is_encoded_as_typed() -> None:
+ from vie.models import VisionLanguageEncoder
+
+ seen: list[list[str]] = []
+ encoder = VisionLanguageEncoder.__new__(VisionLanguageEncoder)
+ encoder.bundle = SimpleNamespace(config=SimpleNamespace(
+ query_templates=("{query}", "a photo of {query}")))
+ encoder.encode_texts = lambda prompts: seen.append(list(prompts)) or np.ones((len(prompts), 4))
+ encoder.encode_query("تئاتر")
+ encoder.encode_query("theatre")
+ assert seen == [["تئاتر"], ["theatre", "a photo of theatre"]]
+
+
+def test_nothing_found_floor_hides_improbable_text_results(cfg, raw_config) -> None:
+ from vie.query import run_search
+
+ stack = FakeStack(cfg)
+ stack.index()
+ request = SimpleNamespace(kind="text", query="something red", image=None, top_k=5)
+ shown = run_search(request, cfg, stack.load_runtime)
+ raw = copy.deepcopy(raw_config)
+ raw["paths"] = {"gallery": str(cfg.gallery_path), "database": str(cfg.database_path)}
+ raw["text"]["min_probability"] = 0.999
+ strict = Config.from_dict(raw)
+ assert shown and run_search(request, strict, stack.load_runtime) == []
+
+
+def test_each_model_brings_its_own_thresholds_and_explicit_settings_win() -> None:
+ from vie.api import load_config
+
+ v2 = load_config({"models": {"vision_language": "google/siglip2-so400m-patch14-384"}})
+ v1 = load_config({"models": {"vision_language": "google/siglip-so400m-patch14-384"}})
+ assert (v2.food.match_threshold, v2.animal.match_threshold) == (0.92, 0.84)
+ assert (v1.food.match_threshold, v1.animal.match_threshold) == (0.98, 0.91)
+ mine = load_config({"models": {"vision_language": "google/siglip2-so400m-patch14-384"},
+ "food": {"match_threshold": 0.95}})
+ assert mine.food.match_threshold == 0.95 and mine.animal.match_threshold == 0.84
diff --git a/tests/test_video.py b/tests/test_video.py
index 8a408dd..19bcb34 100644
--- a/tests/test_video.py
+++ b/tests/test_video.py
@@ -755,3 +755,31 @@ def test_cli_analyze_missing_file_is_a_clean_error(cfg, tmp_path, monkeypatch) -
def test_cli_search_needs_exactly_one_query(cfg, tmp_path) -> None:
code = cli.main(["video", "search", "--config", str(_config_file(cfg, tmp_path / "c.yaml"))])
assert code == 2
+
+
+def test_exemplars_are_the_typical_view_then_the_most_different() -> None:
+ from vie.video.people import Sighting, choose_exemplars
+
+ frontal = [_vector(1.0, 0.02 * i, 0.0) for i in range(8)]
+ turned = _vector(0.6, 0.0, 0.8)
+ sightings = [Sighting(float(i), (0, 0, 10, 10), v) for i, v in enumerate(frontal)]
+ sightings.append(Sighting(99.0, (0, 0, 10, 10), turned))
+ picked = choose_exemplars(sightings, 2)
+ assert len(picked) == 2 and picked[-1].time == 99.0 # the turned head is kept
+ assert choose_exemplars(sightings[:1], 4) == sightings[:1]
+ assert choose_exemplars(sightings, 0) == []
+
+
+def test_face_storage_is_capped_per_person_and_search_still_finds_them(
+ clip: Path, cfg: Config
+) -> None:
+ index_videos(cfg, engine(cfg, face_detector=FakeFaces()))
+ store = VideoStore(cfg)
+ with store.connect() as conn:
+ per_person = conn.execute(
+ "SELECT person, COUNT(*) FROM video_faces GROUP BY person").fetchall()
+ spans = conn.execute("SELECT COUNT(*) FROM video_person_spans").fetchone()[0]
+ moments = store.search_face(conn, np.eye(512, dtype=np.float32)[0])
+ assert per_person and all(count <= cfg.video.face_exemplars for _, count in per_person)
+ assert spans >= 1
+ assert moments and moments[0].kind == "face" and moments[0].start >= 11.0