MCP server, model discovery, CI/packaging lanes, and parallel vision preprocessing

.github/workflows/docker.yml

@@ -11,18 +11,40 @@ env:
 
 jobs:
   build:
-    name: Build & push (linux/amd64)
+    name: Build & push (${{ matrix.variant }}, linux/amd64)
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - variant: cuda
+            dockerfile: Dockerfile
+            suffix: ""
+          - variant: rocm
+            dockerfile: Dockerfile.rocm
+            suffix: "-rocm"
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
 
+      # The ROCm devel base image plus the build layers overflow the runner's
+      # default free space; clear the preinstalled toolchains we don't use.
+      - name: Free disk space
+        if: matrix.variant == 'rocm'
+        run: |
+          sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc
+          sudo apt-get remove -y '^dotnet-.*' '^llvm-.*' google-chrome-stable || true
+          sudo apt-get autoremove -y
+          df -h /
+
       - name: Docker metadata (tags)
         id: meta
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.IMAGE }}
+          flavor: |
+            suffix=${{ matrix.suffix }},onlatest=true
           tags: |
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
@@ -42,9 +64,10 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           context: .
+          file: ${{ matrix.dockerfile }}
           platforms: linux/amd64
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=${{ matrix.variant }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.variant }}

.github/workflows/release.yml

@@ -74,12 +74,20 @@ jobs:
             runner: ubuntu-22.04
             suffix: ""
             gpu_deps: ""
+            build_deb: true
           # Vulkan — needs Ubuntu 24.04 for glslc/spirv-headers. The resulting binary
           # needs glibc 2.39+ and a Vulkan driver at runtime (Mesa RADV/ANV, etc.).
           - name: x86_64-linux-vulkan
             runner: ubuntu-24.04
             suffix: "-vulkan"
             gpu_deps: "glslc glslang-tools libvulkan-dev spirv-headers"
+          # ROCm — HIP SDK from AMD's own repo (the ROCm install step below);
+          # build.rs auto-enables the HIP backend when hipcc is on PATH.
+          - name: x86_64-linux-rocm
+            runner: ubuntu-22.04
+            suffix: "-rocm"
+            gpu_deps: ""
+            gpu_backend: rocm
 
     steps:
       - uses: actions/checkout@v4
@@ -109,6 +117,27 @@ jobs:
           sudo apt-get update -q
           sudo apt-get install -y build-essential cmake clang libclang-dev ninja-build ${{ matrix.gpu_deps }}
 
+      - name: Install ROCm HIP SDK
+        if: matrix.gpu_backend == 'rocm'
+        run: |
+          sudo mkdir -p /etc/apt/keyrings
+          wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key \
+            | sudo gpg --dearmor -o /etc/apt/keyrings/rocm.gpg
+          echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] \
+            https://repo.radeon.com/rocm/apt/6.2 jammy main" \
+            | sudo tee /etc/apt/sources.list.d/rocm.list
+          # Give ROCm repo higher priority than Ubuntu's packages.
+          # Ubuntu 22.04 ships rocminfo 5.0.0-1 (upstream versioning) which
+          # apt prefers over ROCm's 1.0.0.6XXXX (Radeon versioning).
+          # Pin-Priority 1001 forces ROCm packages even when they look like a downgrade.
+          printf 'Package: *\nPin: origin repo.radeon.com\nPin-Priority: 1001\n' \
+            | sudo tee /etc/apt/preferences.d/rocm.pref
+          sudo apt-get update -q
+          sudo apt-get install -y rocm-hip-sdk
+          echo "/opt/rocm/bin" >> "$GITHUB_PATH"
+          echo "/opt/rocm/lib/llvm/bin" >> "$GITHUB_PATH"
+          echo "HIPCC=/opt/rocm/bin/hipcc" >> "$GITHUB_ENV"
+
       - name: Build
         run: cargo build --release --bin fox --bin fox-bench
 
@@ -133,12 +162,42 @@ jobs:
           echo "ASSET=$TARBALL" >> "$GITHUB_ENV"
           echo "ASSET_SHA256=${TARBALL}.sha256" >> "$GITHUB_ENV"
 
+      - name: Install cargo-deb
+        if: matrix.build_deb
+        run: cargo install cargo-deb --locked
+
+      - name: Build .deb package
+        if: matrix.build_deb
+        shell: bash
+        run: |
+          cargo deb --no-build
+          DEB=$(ls target/debian/*.deb)
+          sha256sum "$DEB" > "${DEB}.sha256"
+          echo "DEB_ASSET=$DEB" >> "$GITHUB_ENV"
+          echo "DEB_ASSET_SHA256=${DEB}.sha256" >> "$GITHUB_ENV"
+
+      - name: The .deb must actually install its binary
+        # Same reasoning as the tarball smoke test below: unpack what is about to
+        # be published and run it, instead of trusting the asset glob.
+        if: matrix.build_deb
+        shell: bash
+        run: |
+          SMOKE="$(mktemp -d)"
+          dpkg-deb -x "$DEB_ASSET" "$SMOKE"
+          LD_LIBRARY_PATH="$SMOKE/usr/lib" "$SMOKE/usr/bin/fox" --version
+          echo "packaged .deb installs a binary that starts and reports its version"
+
       - name: The tarball must actually run
         # Unpack what is about to be published, somewhere else, and start it. v0.20.3
         # shipped a binary that died on a missing `libmtmd.so.0` and nothing noticed,
         # because packaging was only ever checked by reading the glob. A release that
         # cannot print its own version is not a release.
         shell: bash
+        # The ROCm leg's libggml-hip.so pulls in the HIP runtime from /opt/rocm,
+        # which the runner has but the default loader path does not; empty and
+        # harmless on the other legs.
+        env:
+          LD_LIBRARY_PATH: ${{ matrix.gpu_backend == 'rocm' && '/opt/rocm/lib' || '' }}
         run: |
           SMOKE="$(mktemp -d)"
           tar xzf "$ASSET" -C "$SMOKE"
@@ -155,9 +214,13 @@ jobs:
           #   curl -fsSL .../releases/latest/download/install.sh | sh
           # and that URL 404'd, because the only assets uploaded were the tarball and
           # its checksum. The advertised way to install fox did not work.
+          # DEB_ASSET is only set on the build_deb leg; softprops skips the
+          # resulting empty lines on the others.
           files: |
             ${{ env.ASSET }}
             ${{ env.ASSET_SHA256 }}
+            ${{ env.DEB_ASSET }}
+            ${{ env.DEB_ASSET_SHA256 }}
             install.sh
             install.ps1
 

.github/workflows/secret-scanning.yml

@@ -0,0 +1,24 @@
+name: Secret Scanning
+
+on:
+  push:
+    branches: [main, develop]
+  pull_request:
+    branches: [main, develop]
+
+env:
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+
+jobs:
+  secret-scan:
+    name: Secret Scan
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: TruffleHog secret scan
+        uses: trufflesecurity/trufflehog@v3.88.26
+        with:
+          extra_args: --only-verified

.github/workflows/update-llamacpp.yml

@@ -0,0 +1,98 @@
+name: Update llama.cpp
+
+on:
+  schedule:
+    - cron: '0 6 * * 1' # Monday 6 AM UTC
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+  issues: write
+
+env:
+  SUBMODULE_PATH: vendor/llama.cpp
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+        with:
+          token: ${{ secrets.PAT_TOKEN }}
+          submodules: recursive
+          fetch-depth: 0
+
+      - name: Check for upstream updates
+        id: check
+        working-directory: ${{ env.SUBMODULE_PATH }}
+        run: |
+          git fetch origin master
+          CURRENT=$(git rev-parse HEAD)
+          LATEST=$(git rev-parse origin/master)
+          BEHIND=$(git rev-list --count HEAD..origin/master)
+          echo "current=$CURRENT" >> "$GITHUB_OUTPUT"
+          echo "latest=$LATEST" >> "$GITHUB_OUTPUT"
+          echo "behind=$BEHIND" >> "$GITHUB_OUTPUT"
+          if [ "$CURRENT" = "$LATEST" ]; then
+            echo "Already up to date with llama.cpp master"
+          else
+            echo "$BEHIND new commits available"
+          fi
+
+      - name: Update submodule
+        if: steps.check.outputs.behind != '0'
+        working-directory: ${{ env.SUBMODULE_PATH }}
+        run: git checkout origin/master
+
+      - name: Verify build
+        if: steps.check.outputs.behind != '0'
+        run: |
+          FOX_SKIP_LLAMA=1 rustup run stable cargo check 2>&1
+        env:
+          CARGO_TERM_COLOR: always
+
+      - name: Create pull request
+        if: steps.check.outputs.behind != '0'
+        run: |
+          BEHIND=${{ steps.check.outputs.behind }}
+          SHORT_OLD=$(echo "${{ steps.check.outputs.current }}" | cut -c1-10)
+          SHORT_NEW=$(echo "${{ steps.check.outputs.latest }}" | cut -c1-10)
+          BRANCH="auto/llama-cpp-${SHORT_NEW}"
+
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          # Close any existing auto-update PRs
+          gh pr list --label "llama.cpp-update" --state open --json number --jq '.[].number' \
+            | xargs -I{} gh pr close {} --comment "Superseded by newer update" 2>/dev/null || true
+
+          git push origin --delete "$BRANCH" 2>/dev/null || true
+          git checkout -b "$BRANCH"
+          git add "${{ env.SUBMODULE_PATH }}"
+          git commit -m "$(cat <<EOF
+          chore: update llama.cpp submodule (+${BEHIND} commits)
+
+          ${SHORT_OLD} → ${SHORT_NEW}
+          EOF
+          )"
+          git push origin "$BRANCH"
+
+          gh pr create \
+            --title "chore: update llama.cpp (+${BEHIND} upstream commits)" \
+            --label "llama.cpp-update" \
+            --body "$(cat <<EOF
+          ## Summary
+          - Updates \`vendor/llama.cpp\` submodule to latest master
+          - **${BEHIND}** new upstream commits (\`${SHORT_OLD}\` → \`${SHORT_NEW}\`)
+          - [Compare changes](https://github.com/ggml-org/llama.cpp/compare/${SHORT_OLD}...${SHORT_NEW})
+
+          ## Test plan
+          - [ ] CI passes (\`cargo check\` verified in workflow)
+          - [ ] CUDA Docker build succeeds
+          - [ ] Smoke test: serve a model and run a completion
+          EOF
+          )"
+        env:
+          GH_TOKEN: ${{ secrets.PAT_TOKEN }}

Cargo.toml

@@ -51,6 +51,7 @@ dashmap = "6"
 toml = "0.8"
 bytes = "1"
 dirs = "5"
+walkdir = "2"
 # `json` is NOT a default feature, and without it the `tojson` filter is unknown.
 # Every native tool-use chat template (Qwen, Hermes, Mistral) renders its tool
 # listing with `{{ tool | tojson }}`, so leaving it out makes those templates fail
@@ -82,3 +83,24 @@ tokio-test = "0.4"
 tempfile = "3"
 tower = { version = "0.4", features = ["util"] }
 http-body-util = "0.1"
+
+[package.metadata.deb]
+maintainer = "Ferrumox Contributors"
+copyright = "2026 Ferrumox contributors"
+depends = "$auto"
+suggests = "nvidia-cuda-toolkit"
+section = "utils"
+priority = "optional"
+extended-description = """\
+High-performance LLM inference engine built in Rust. \
+Provides OpenAI and Ollama compatible APIs, continuous batching, \
+KV cache management, and GPU acceleration via llama.cpp."""
+# `lib*.so*`, not a hand-written list of prefixes — the release tarball once
+# shipped without `libmtmd.so.0` because its glob named `libggml*`/`libllama*`
+# only, and the installed binary could not start (see release.yml).
+assets = [
+    ["target/release/fox", "usr/bin/", "755"],
+    ["target/release/fox-bench", "usr/bin/", "755"],
+    ["target/release/lib*.so*", "usr/lib/", "644"],
+    ["fox.service", "lib/systemd/system/fox@.service", "644"],
+]

docs/cli/serve.md

@@ -67,6 +67,7 @@ fox serve --json-logs --port 8080 --max-models 2 --keep-alive-secs 600
 | `--spec-draft-len <N>` | `FOX_SPEC_DRAFT_LEN` | `4` | Maximum draft tokens proposed per speculative step. |
 | `--draft-model <NAME>` | `FOX_DRAFT_MODEL` | — | Name/path of a smaller model to use as the speculative-decoding draft proposer instead of n-gram lookup — generalizes speculation to any text, not just repetitive output. Requires `--speculative true` (ignored with a startup warning otherwise). The draft and target must share the same tokenizer — checked at load time, fails loudly on mismatch. Loaded once alongside the target and kept resident for the process lifetime; not subject to LRU eviction or VRAM budgeting — size both models to fit. `--spec-ngram` is ignored in this mode. |
 | `--mmproj <NAME>` | `FOX_MMPROJ` | — | Name/path of a paired mmproj (vision projector) GGUF, enabling image input (OpenAI `image_url` / Ollama `images`) via llama.cpp's `mtmd` library. Resolved the same way as `--model-path`/`--draft-model` (alias, disk match, or a direct file path). One global pairing — matched against whatever model is currently loaded, like `--draft-model`. Only `data:` base64 image URIs are accepted (no remote fetch). See `docs/design/vision-support.md`. |
+| `--vision-contexts <N>` | `FOX_VISION_CONTEXTS` | `1` | Number of vision (mtmd) contexts loaded with the mmproj. `1` = concurrent requests take turns CLIP-encoding their images; `N` = up to N encode in parallel, at roughly one mmproj's VRAM footprint per extra context. Repeated images are additionally served from a per-model CLIP cache, so a conversation that re-sends its history's images every turn only pays each encode once. Ignored without `--mmproj`. |
 | `--lora-modules <NAME>=<PATH>[:<SCALE>][,...]` | `FOX_LORA_MODULES` | — | Comma-separated LoRA adapters loaded onto the primary model (`--model-path`) at startup, e.g. `finetune=/path/adapter.gguf:0.8,other=/other.gguf`. `SCALE` defaults to `1.0`. Select an adapter per-request by passing its `NAME` as the `model` field in `/v1/chat/completions` or `/api/chat` — the request is served by the primary model with that adapter applied. Requests without a recognized adapter name use the primary model unmodified. Adapters are a property of the whole context, not per-sequence: concurrent requests on different adapters are grouped and processed as separate sub-batches (see `docs/design/lora-support.md`), and prefix caching is skipped for any request carrying an adapter selection. |
 | `--gpu-memory-fraction <F>` | `FOX_GPU_MEMORY_FRACTION` | `0.85` | Fraction of GPU VRAM reserved for the KV cache. Must be between 0.0 and 1.0. The remaining memory is left for model weights and other allocations. |
 | `--type-kv <TYPE>` | `FOX_TYPE_KV` | `f16` | KV cache element type for both K and V: `f16`, `q8_0`, or `q4_0`. |

src/api/auth.rs

@@ -83,6 +83,7 @@ mod tests {
                     n_gpu_layers: -1,
                     moe_offload_cpu: false,
                     mmproj: None,
+                    vision_contexts: 1,
                     mtp_model: None,
                     lora_modules: Vec::new(),
                     primary_model: None,