diff --git a/mseries/local-ai/ENGINE-SCOREBOARD.md b/mseries/local-ai/ENGINE-SCOREBOARD.md index d9944345..9427b856 100644 --- a/mseries/local-ai/ENGINE-SCOREBOARD.md +++ b/mseries/local-ai/ENGINE-SCOREBOARD.md @@ -51,6 +51,54 @@ Each block is one model and load; each panel has its own scale. **First token** | | Qwen3.5-4B MLX 4-bit | MLX (omarchy-mlx, in-process) | `defaults (MLX_OMARCHY_BATCH_WORK fallback 5000)` | 32 | 18.55 | 0.73 | 2.38 | 3 | reference | not gated | `7.2.11-elementary-aurora72-wip4` | Honeykrisp Mesa 7faf04c065 | 2026-10-28 | [2026-10-28-engine-compare-qwen4b-short.json](receipts/2026-10-28-engine-compare-qwen4b-short.json) | | | Qwen3.5-4B MLX 4-bit | mlx-omarchy-serve, mlx-lm backend, stock (HTTP server) | `MLX_OMARCHY_BATCH_WORK=120000 · 4 requests at once (decode = total tok/s)` | 32 | 12.76 | 6.27 | 10.03 | 3 | yes (engines in this run agree) | not gated | `7.2.11-elementary-aurora72-wip4` | Honeykrisp Mesa 9323171c12 | 2026-10-28 | [2026-10-28-engine-compare-qwen4b-short-hk9323171c12-c4-serve-120k+serve-patched-120k+rapid-120k-round2.json](receipts/2026-10-28-engine-compare-qwen4b-short-hk9323171c12-c4-serve-120k+serve-patched-120k+rapid-120k-round2.json) | +## Mac mini M4 (Mac16,10, 10-core GPU, 16 GB), macOS 27.0 26A5421a, Metal + +Measured with the same `engine-compare.py`, models, revisions, prompts, token counts, loads and three rotated rounds, receipts in [`receipts/2026-10-31-m4-mini-macos/`](receipts/2026-10-31-m4-mini-macos/). A separate machine, so never ranked against the M1 Pro table; ★ ranks within this machine only. CSV: [`engine-scoreboard-m4-mini-macos.csv`](engine-scoreboard-m4-mini-macos.csv). + +![First token, streaming speed and typical reply time on the Mac mini M4, macOS (Metal)](engine-scoreboard-m4-mini-macos.svg) + +| ★ | Model | Engine | Settings | Tokens | Decode tok/s | TTFT s | Total s | Runs | Same output as MLX default | Same output as M1 Pro (Linux) | OS | Date | Receipt | +|---|---|---|---|---:|---:|---:|---:|---:|---|---|---|---|---| +| ★ | Granite 4.0 1B MLX 4-bit (IBM, dense) | **MLX on Metal (upstream mlx, in-process)** | `mlx 0.32.3 · mlx-lm 0.31.3` | 32 | **58.59** | 0.22 | 0.76 | 3 | reference | yes | macOS 27.0 (26A5421a) | 2026-10-31 | [2026-10-31-engine-compare-granite1b-short-metal.json](receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-granite1b-short-metal.json) | +| | Granite 4.0 1B MLX 4-bit (IBM, dense) | mlx_lm.server, stock (HTTP server) | `mlx 0.32.3 · mlx-lm 0.31.3` | 32 | 43.01 | 0.22 | 0.94 | 3 | yes | yes | macOS 27.0 (26A5421a) | 2026-10-31 | [2026-10-31-engine-compare-granite1b-short-metal.json](receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-granite1b-short-metal.json) | +| ★ | LFM3-1.7B MLX 8-bit | **MLX on Metal (upstream mlx, in-process)** | `mlx 0.32.3 · mlx-lm 0.31.3` | 32 | **31.53** | 0.24 | 1.22 | 3 | reference | yes | macOS 27.0 (26A5421a) | 2026-10-31 | [2026-10-31-engine-compare-lfm3-short-metal.json](receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-lfm3-short-metal.json) | +| ★ | Qwen3.5-2B MLX 4-bit | **MLX on Metal (upstream mlx, in-process)** | `mlx 0.32.3 · mlx-lm 0.31.3` | 32 | **47.67** | 0.38 | 1.11 | 3 | reference | yes | macOS 27.0 (26A5421a) | 2026-10-31 | [2026-10-31-engine-compare-qwen2b-short-metal.json](receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal.json) | +| | Qwen3.5-2B MLX 4-bit | mlx_lm.server, stock (HTTP server) | `mlx 0.32.3 · mlx-lm 0.31.3` | 32 | 42.36 | 0.42 | 1.17 | 3 | yes | yes | macOS 27.0 (26A5421a) | 2026-10-31 | [2026-10-31-engine-compare-qwen2b-short-metal.json](receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal.json) | +| | Qwen3.5-2B MLX 4-bit | TensorFold Metal v1.1.9 release (HTTP server) | `serve --backend mlx --context 1024 --no-thinking --no-drafts` | 32 | 39.71 | 0.07 | 0.85 | 3 | yes | yes | macOS 27.0 (26A5421a) | 2026-10-31 | [2026-10-31-engine-compare-qwen2b-short-metal.json](receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal.json) | +| ★ | Qwen3.5-2B MLX 4-bit | **mlx_lm.server, stock (HTTP server)** | `mlx 0.32.3 · mlx-lm 0.31.3 · 4 requests at once (decode = total tok/s)` | 32 | **41.63** | 1.86 | 3.07 | 3 | yes (engines in this run agree) | yes | macOS 27.0 (26A5421a) | 2026-10-31 | [2026-10-31-engine-compare-qwen2b-short-metal-c4.json](receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal-c4.json) | +| | Qwen3.5-2B MLX 4-bit | TensorFold Metal v1.1.9 release (HTTP server) | `serve --backend mlx --context 1024 --no-thinking --no-drafts · 4 requests at once (decode = total tok/s)` | 32 | 36.84 | 1.82 | 3.47 | 3 | yes (engines in this run agree) | yes | macOS 27.0 (26A5421a) | 2026-10-31 | [2026-10-31-engine-compare-qwen2b-short-metal-c4.json](receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal-c4.json) | +| ★ | Qwen3.5-2B MLX 4-bit | **TensorFold Metal v1.1.9 release (HTTP server)** | `serve --backend mlx --context 1024 --no-thinking --no-drafts · 8 requests at once (decode = total tok/s)` | 32 | **35.41** | 3.64 | 7.23 | 3 | yes (engines in this run agree) | yes | macOS 27.0 (26A5421a) | 2026-10-31 | [2026-10-31-engine-compare-qwen2b-short-metal-c8.json](receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal-c8.json) | +| ★ | Qwen3.5-2B MLX 4-bit | **MLX on Metal (upstream mlx, in-process)** | `mlx 0.32.3 · mlx-lm 0.31.3` | 256 | **51.26** | 0.52 | 5.37 | 3 | reference | no: diverges after 1207 of 1301 chars | macOS 27.0 (26A5421a) | 2026-10-31 | [2026-10-31-engine-compare-qwen2b-long-metal.json](receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-long-metal.json) | +| | Qwen3.5-2B MLX 4-bit | mlx_lm.server, stock (HTTP server) | `mlx 0.32.3 · mlx-lm 0.31.3` | 256 | 40.64 | 0.55 | 6.83 | 3 | yes | no: diverges after 1207 of 1301 chars | macOS 27.0 (26A5421a) | 2026-10-31 | [2026-10-31-engine-compare-qwen2b-long-metal.json](receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-long-metal.json) | +| | Qwen3.5-2B MLX 4-bit | TensorFold Metal v1.1.9 release (HTTP server) | `serve --backend mlx --context 1024 --no-thinking --no-drafts` | 256 | 36.37 | 0.12 | 7.13 | 3 | no: diverges after 1207 of 1290 chars | yes | macOS 27.0 (26A5421a) | 2026-10-31 | [2026-10-31-engine-compare-qwen2b-long-metal.json](receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-long-metal.json) | +| ★ | Qwen3.5-4B MLX 4-bit | **MLX on Metal (upstream mlx, in-process)** | `mlx 0.32.3 · mlx-lm 0.31.3` | 32 | **28.99** | 0.39 | 1.46 | 3 | reference | no: diverges after 55 of 152 chars | macOS 27.0 (26A5421a) | 2026-10-31 | [2026-10-31-engine-compare-qwen4b-short-metal.json](receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen4b-short-metal.json) | +| | Qwen3.5-4B MLX 4-bit | mlx_lm.server, stock (HTTP server) | `mlx 0.32.3 · mlx-lm 0.31.3` | 32 | 25.15 | 0.38 | 1.58 | 3 | no: diverges after 55 of 137 chars | yes | macOS 27.0 (26A5421a) | 2026-10-31 | [2026-10-31-engine-compare-qwen4b-short-metal.json](receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen4b-short-metal.json) | +| ★ | Qwen3.5-4B MLX 4-bit | **mlx_lm.server, stock (HTTP server)** | `mlx 0.32.3 · mlx-lm 0.31.3 · 4 requests at once (decode = total tok/s)` | 32 | **29.23** | 2.29 | 4.38 | 3 | yes (engines in this run agree) | yes | macOS 27.0 (26A5421a) | 2026-10-31 | [2026-10-31-engine-compare-qwen4b-short-metal-c4.json](receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen4b-short-metal-c4.json) | + +Not in the table: mlx_lm.server, Metal, Qwen3.5-2B MLX 4-bit, 32 tokens, 8 requests at once: all 3 runs failed (`RuntimeError: 2 of 8 concurrent requests failed: {'error': 'URLError: '}`; [2026-10-31-engine-compare-qwen2b-short-metal-c8.json](receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal-c8.json)). + +### How to run the same on Linux on the M4 mini + +Same machine, same work, so the two tables compare row for row: + +1. Boot the mini into Linux (Aurora kernel, vulkan-aurora Honeykrisp) and install `omarchy-mlx`, `omarchy-mlx-vulkan-aurora` and `omarchy-mlx-m4-bench` from techmore/omarchy-mlx branch `m4-mini-bench` (its `docs/m4-mini-bench.md`), plus `tensorfold-vulkan` (PR 1544, `b3ab9ba2`) for the TensorFold rows. +2. Fetch the four checkpoints into the Hugging Face cache at the revisions above (engine-compare refuses other weights) and the pinned `lfm3.py` (mlx-lm PR 2016 head `9027ac5b`). +3. Run the commands below with `MSERIES_HF_SNAPSHOTS=1 MSERIES_MLX_PY=/usr/lib/omarchy-mlx/venv/bin/python MSERIES_HK_ICD= MSERIES_TF_VULKAN_BIN= MSERIES_LFM3_MODULE=` and `--arms mlx-5k,mlx-120k,lm031-120k,tensorfold` (in-process pairs with `mlx-metal`, plain `mlx_lm.server` 0.31.3 with `lm-metal`, TensorFold Vulkan with `tensorfold-metal`). Write receipts to `receipts/-m4-mini-linux/`, add that folder to `MACHINES` in `scoreboard.py` and rerun it. +4. For the release-gate speed number, run `omarchy-mlx-m4-bench` on Linux (also `--engine tensorfold` and `--model qwen3.5-2b`) and `scripts/m4_mini_bench.py` on macOS with this venv (mlx 0.32.3, mlx-lm 0.31.3, transformers 5.16.1, the pins that bench names), then `m4_mini_bench.py --compare`. +5. Record the load: this macOS baseline ran beside a busy build VM (see the receipts README). Quote Linux/macOS ratios only from runs with comparable load, or re-run both quiet. + +```bash +cd mseries/local-ai +# macOS (this table): python 3.14 venv with mlx==0.32.3 mlx-lm==0.31.3 transformers==5.16.1 +HF_HUB_CACHE=... MSERIES_TF_BIN=.../tensorfold-1.1.9-macos-arm64/bin/tensorfold-native \ + $VENV/bin/python engine-compare.py --model qwen2b --reserve-gib 1 # --long; --concurrency 4 / 8 +# Linux on the same mini +MSERIES_HF_SNAPSHOTS=1 MSERIES_MLX_PY=/usr/lib/omarchy-mlx/venv/bin/python ... \ + python3 engine-compare.py --model qwen2b --arms mlx-5k,mlx-120k,lm031-120k,tensorfold +``` + +Conditions, versions and the Linux run that pairs with it: [`receipts/2026-10-31-m4-mini-macos/README.md`](receipts/2026-10-31-m4-mini-macos/README.md). + ## Watched or reported elsewhere (not comparable to the table above) | Model | Engine | Settings | Decode | Status | Blocker | Source | diff --git a/mseries/local-ai/engine-compare.py b/mseries/local-ai/engine-compare.py index 3335b2c1..b01bea68 100755 --- a/mseries/local-ai/engine-compare.py +++ b/mseries/local-ai/engine-compare.py @@ -20,12 +20,18 @@ omlx-120k oMLX (jundot/omlx) as omarchy-mlx packages it for Linux (packaging/omlx-linux), its own venv on the mlx-lm 0.32 line, at the 120,000 budget; skipped when not installed +On macOS (Apple-native baseline; run with the venv's python, see ENGINE-SCOREBOARD.md): + mlx-metal upstream MLX on Metal, in-process, same worker as the mlx-* arms (no submit budget) + lm-metal plain `python -m mlx_lm.server` from that venv, none of our patches + tensorfold-metal TensorFold's v1.1.9 macOS release, `--backend mlx` (Metal), --no-drafts +Models come from the Hugging Face cache ($HF_HUB_CACHE) at the same pinned revisions. + All four use the same weights, chat prompt, greedy decoding and token limit, the shared worker lock, the 6 GiB desktop reserve and the 80 C limit. MLX arms time in-process; server arms time a streamed loopback HTTP request. Those boundaries differ, so compare servers with servers and MLX with MLX first. """ -import argparse, fcntl, hashlib, json, os, signal, socket, subprocess, sys, tempfile, threading, time +import argparse, fcntl, hashlib, json, os, platform, signal, socket, subprocess, sys, tempfile, threading, time import urllib.request, urllib.error from pathlib import Path @@ -53,19 +59,34 @@ 'mlx-community/granite-4.0-1b-4bit@29f108a4e2ff99079a173004871604c2f1e12c1f', 3.5, HOME / '.local/share/mlx-omarchy/venv/bin/python', ('mlx-5k', 'mlx-120k', 'rapid-120k', 'serve-120k', 'serve-patched-120k')), } +MACOS = sys.platform == 'darwin' +MAC_ARMS = {'qwen2b': ('mlx-metal', 'lm-metal', 'tensorfold-metal'), 'qwen4b': ('mlx-metal', 'lm-metal'), + 'lfm3': ('mlx-metal',), 'granite1b': ('mlx-metal', 'lm-metal')} +# Another machine (the M4 mini, either OS) reads every model from the Hugging Face cache at the pinned +# revision instead of this M1 Pro's folders; MSERIES_MLX_PY names the venv (macOS: the one running this). +if MACOS or os.environ.get('MSERIES_HF_SNAPSHOTS'): + HF_HUB = Path(os.environ.get('HF_HUB_CACHE') or HOME / '.cache/huggingface/hub') + for _k, (_d, _sha, _id, _gib, _py, _arms) in list(MODELS.items()): + _repo, _rev = _id.split('@') + MODELS[_k] = (HF_HUB / f"models--{_repo.replace('/', '--')}" / 'snapshots' / _rev, _sha, _id, _gib, + Path(os.environ.get('MSERIES_MLX_PY') or (sys.executable if MACOS else _py)), + MAC_ARMS[_k] if MACOS else _arms) MODEL, WEIGHTS_SHA, MODEL_ID, ALLOWANCE_GIB, MLX_PY, MODEL_ARMS = MODELS['qwen2b'] # MSERIES_HK_ICD selects a candidate driver built by honeykrisp-candidate.sh. ICD = Path(os.environ.get('MSERIES_HK_ICD') or HOME / '.local/share/mlx-omarchy/vulkan/honeykrisp_icd.aarch64.json') DRIVER = (ICD.parent / 'mesa-git-sha').read_text().strip() if (ICD.parent / 'mesa-git-sha').exists() else 'unknown' RAPID_PATH = f"{HOME}/.cache/mseries-metal2vk/rapid-python:{HOME}/.cache/mseries-metal2vk/rapid-src" RAPID_REV = '1d09cdb684bbf58fac130f5804b08ee2e1f56558' -TF_BIN = Path('/usr/libexec/mseries/tensorfold-native-vulkan') +TF_BIN = Path(os.environ.get('MSERIES_TF_VULKAN_BIN') or '/usr/libexec/mseries/tensorfold-native-vulkan') TF_REV = 'b3ab9ba217a80c63fcc48f7643ddfe339a8f4bdc' # PR 1544; receipts record the rev they ran +# macOS: the release omarchy-mlx's m4-mini-bench pins (tensorfold-1.1.9-macos-arm64.tar.gz, sha256 f8330102...a9d7). +TF_METAL_BIN = Path(os.environ.get('MSERIES_TF_BIN') or HOME / '.cache/m4-mini-bench/tensorfold-1.1.9-macos-arm64/bin/tensorfold-native') +TF_METAL_SHA = '4b4b5a5ac152eb2dbed7e9d8004805ed9b74f77c64c3de18eb4f35263c7b9eb8' PROMPT = 'Explain in plain language how a rainbow forms. Give four distinct sentences.' LONG_PROMPT = ('Explain in detail how a rainbow forms, including refraction, reflection, color ' 'and viewing angle. Write at least 350 words.') ARMS = ('mlx-5k', 'mlx-120k', 'rapid-120k', 'serve-120k', 'serve-nolp-120k', 'serve-patched-120k', 'tensorfold', 'omlx-120k', - 'lm031-120k', 'lm032-120k', 'lm031-patched-120k', 'lm032-patched-120k') + 'lm031-120k', 'lm032-120k', 'lm031-patched-120k', 'lm032-patched-120k', 'mlx-metal', 'lm-metal', 'tensorfold-metal') # omarchy-mlx's packaging/omlx-linux/install.sh --home target; its venv holds the omlx command. OMLX_HOME = Path(os.environ.get('MSERIES_OMLX_HOME') or HOME / '.local/share/omlx-linux') OMLX_BIN = OMLX_HOME / '.venvs/omlx/bin/omlx' @@ -92,7 +113,7 @@ def mlx_worker(prompt_text, max_tokens, output): # mlx-lm 0.31.3 has no LFM3; load the pinned model file from mlx-lm PR 2016 # (head 9027ac5b), as the 2026-10-27 LFM3 smoke test did. import hashlib as _h, importlib.util as _u - f = HOME / '.local/share/mseries-ai/runtime/lfm3.py' + f = Path(os.environ.get('MSERIES_LFM3_MODULE') or HOME / '.local/share/mseries-ai/runtime/lfm3.py') if _h.sha256(f.read_bytes()).hexdigest() != LFM3_MODULE_SHA: raise SystemExit('unexpected lfm3.py') spec = _u.spec_from_file_location('mlx_lm.models.lfm3', f) @@ -181,8 +202,8 @@ def one(i): def server_command(arm, port, tmp): - if arm.startswith(('lm031-', 'lm032-')): - py = MLX_PY if arm.startswith('lm031-') else OMLX_PY + if arm.startswith(('lm031-', 'lm032-')) or arm == 'lm-metal': + py = OMLX_PY if arm.startswith('lm032-') else MLX_PY return [str(py), '-m', 'mlx_lm.server', '--model', str(MODEL), '--host', '127.0.0.1', '--port', str(port)] if arm == 'omlx-120k': # oMLX serves every model folder under --model-dir; give it just this one, and keep its @@ -200,6 +221,9 @@ def server_command(arm, port, tmp): if arm == 'tensorfold': return [str(TF_BIN), 'serve', str(MODEL), '--backend', 'vulkan', '--host', '127.0.0.1', '--port', str(port), '--context', '1024', '--no-thinking', '--no-update-check'] + if arm == 'tensorfold-metal': # the Vulkan build has no drafter; switch the release's off to match + return [str(TF_METAL_BIN), 'serve', str(MODEL), '--backend', 'mlx', '--host', '127.0.0.1', + '--port', str(port), '--context', '1024', '--no-thinking', '--no-drafts', '--no-update-check'] return [str(MLX_PY), '-m', 'rapid_mlx.server', '--model', str(MODEL), '--no-mllm', '--host', '127.0.0.1', '--port', str(port)] @@ -219,7 +243,10 @@ def kv_bytes_per_token(model_dir): def env_for(arm, tmp): env = {k: v for k, v in os.environ.items() if not k.startswith('MLX_OMARCHY_')} - env.update(VK_DRIVER_FILES=str(ICD), HF_HUB_OFFLINE='1', OMP_NUM_THREADS='2', OPENBLAS_NUM_THREADS='2') + env.update(HF_HUB_OFFLINE='1', OMP_NUM_THREADS='2', OPENBLAS_NUM_THREADS='2') + if MACOS: + return env + env['VK_DRIVER_FILES'] = str(ICD) if arm != 'tensorfold': env['MLX_OMARCHY_BATCH_WORK'] = '5000' if arm == 'mlx-5k' else '120000' if arm in ('serve-nolp-120k', 'serve-patched-120k'): @@ -292,7 +319,8 @@ def drive(): s = snapshot() samples.append({'t': round(time.monotonic() - started, 2), 'available_gib': round(s['available_bytes'] / GIB, 3), 'max_temp_c': max(s['temperatures_c'].values(), default=0), - 'max_fan_rpm': max(s['fans_rpm'].values(), default=0)}) + 'max_fan_rpm': max(s['fans_rpm'].values(), default=0)} + | ({'load_avg': s['load_avg']} if MACOS else {})) reasons = violations(s, POLICY); bad = bad + 1 if reasons else 0 if bad >= 3 or time.monotonic() - started > DEADLINE_S: stopped = reasons or ['deadline']; break @@ -355,6 +383,8 @@ def main(): p.add_argument('--concurrency', type=int, default=1, help='server arms only: requests at once') p.add_argument('--arms', default=','.join(ARMS)) p.add_argument('--admission-wait-seconds', type=int, default=0) + p.add_argument('--reserve-gib', type=float, default=POLICY['reserve_gib'], + help='memory kept free for the desktop (and, on the M4 mini, its build VM); recorded') p.add_argument('--output', type=Path) a = p.parse_args() global MODEL_NAME, CONCURRENCY @@ -368,6 +398,7 @@ def main(): p.error(f'no runnable arms for {a.model}; it supports {",".join(MODEL_ARMS)}') prompt_text, max_tokens = (LONG_PROMPT, 256) if a.long else (PROMPT, a.max_tokens) global DEADLINE_S + POLICY['reserve_gib'] = a.reserve_gib if a.long: DEADLINE_S = 600 # TensorFold's unbatched decode needs ~100 s per 256-token request with (MODEL / 'model.safetensors').open('rb') as f: @@ -385,16 +416,28 @@ def main(): arms.remove('omlx-120k') if 'tensorfold' in arms and not TF_BIN.exists(): raise SystemExit('TensorFold server missing: build and install the mseries-tensorfold-vulkan package') + if 'tensorfold-metal' in arms and (not TF_METAL_BIN.exists() or hashlib.sha256(TF_METAL_BIN.read_bytes()).hexdigest() != TF_METAL_SHA): + raise SystemExit(f'{TF_METAL_BIN} is not the pinned TensorFold 1.1.9 macOS binary (set MSERIES_TF_BIN)') out = a.output or Path(__file__).resolve().parent / 'receipts' / ( f"{time.strftime('%Y-%m-%d')}-engine-compare-{a.model}-{'long' if a.long else 'short'}" - + ('' if DRIVER == '7faf04c065' else f'-hk{DRIVER}') + (f'-c{CONCURRENCY}' if CONCURRENCY > 1 else '') + + ('-metal' if MACOS else '' if DRIVER == '7faf04c065' else f'-hk{DRIVER}') + (f'-c{CONCURRENCY}' if CONCURRENCY > 1 else '') + ('' if tuple(arms) == tuple(x for x in MODEL_ARMS if not x.startswith('serve-')) else '-' + '+'.join(arms)) + '.json') + (HOME / '.local/state/mseries-ai').mkdir(parents=True, exist_ok=True) lock = (HOME / '.local/state/mseries-ai/worker.lock').open('a') fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) report = {'status': 'running', 'model_key': a.model, 'model': MODEL_ID, 'weights_sha256': WEIGHTS_SHA, 'kernel': os.uname().release, 'driver': 'Honeykrisp Mesa ' + DRIVER, 'tensorfold_rev': TF_REV, 'rapid_rev': RAPID_REV, 'serve_concurrency': int(SERVE_CONCURRENCY), 'prompt_sha256': hashlib.sha256(prompt_text.encode()).hexdigest(), 'max_tokens': max_tokens, 'policy': {**POLICY, 'max_fan_rpm': None}, 'allowance_gib': ALLOWANCE_GIB, 'arms': arms, 'concurrency': CONCURRENCY, 'runs': []} + if MACOS: + from machine import host_info + versions = subprocess.run([str(MLX_PY), '-c', 'import json, platform, mlx.core as mx, mlx_lm, transformers; print(json.dumps(' + '{"mlx": mx.__version__, "mlx_lm": mlx_lm.__version__, "transformers": transformers.__version__, ' + '"python": platform.python_version(), "device": mx.device_info()}))'], + capture_output=True, text=True, check=True).stdout + report.update(kernel=f"macOS {platform.mac_ver()[0]} ({subprocess.run(['sw_vers', '-buildVersion'], capture_output=True, text=True).stdout.strip()})", + driver='Metal', runtime=json.loads(versions), tensorfold_rev='v1.1.9 release', tensorfold_sha256=TF_METAL_SHA, + model_dir=str(MODEL), host_start=host_info()) try: with tempfile.TemporaryDirectory(prefix='engine-compare-', dir=HOME / '.cache') as tmp: for r in range(a.rounds): @@ -425,6 +468,9 @@ def main(): 'median_decode_tps': median([x['decode_tps'] for x in ok]), 'texts': sorted({x['text_sha256'][:12] for x in ok})} report['summary'] = summary + if MACOS: + from machine import host_info + report['host_end'] = host_info() out.write_text(json.dumps(report, indent=2) + '\n'); lock.close() print(json.dumps(summary, indent=2)) diff --git a/mseries/local-ai/engine-scoreboard-m4-mini-macos.csv b/mseries/local-ai/engine-scoreboard-m4-mini-macos.csv new file mode 100644 index 00000000..470f7fd1 --- /dev/null +++ b/mseries/local-ai/engine-scoreboard-m4-mini-macos.csv @@ -0,0 +1,16 @@ +machine,best,model,engine,settings,tokens,decode_tok_s,ttft_s,total_s,runs,same_output,gate,kernel,driver,date,status,source,same_as_m1 +"Mac mini M4 (Mac16,10, 10-core GPU, 16 GB), macOS 27.0 26A5421a, Metal",,"Granite 4.0 1B MLX 4-bit (IBM, dense)","mlx_lm.server, stock (HTTP server)",mlx 0.32.3 · mlx-lm 0.31.3,32,43.01,0.22,0.94,3,yes,not gated,macOS 27.0 (26A5421a),Metal,2026-10-31,measured here,receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-granite1b-short-metal.json,yes +"Mac mini M4 (Mac16,10, 10-core GPU, 16 GB), macOS 27.0 26A5421a, Metal",★,"Granite 4.0 1B MLX 4-bit (IBM, dense)","MLX on Metal (upstream mlx, in-process)",mlx 0.32.3 · mlx-lm 0.31.3,32,58.59,0.22,0.76,3,reference,not gated,macOS 27.0 (26A5421a),Metal,2026-10-31,measured here,receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-granite1b-short-metal.json,yes +"Mac mini M4 (Mac16,10, 10-core GPU, 16 GB), macOS 27.0 26A5421a, Metal",★,LFM3-1.7B MLX 8-bit,"MLX on Metal (upstream mlx, in-process)",mlx 0.32.3 · mlx-lm 0.31.3,32,31.53,0.24,1.22,3,reference,not gated,macOS 27.0 (26A5421a),Metal,2026-10-31,measured here,receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-lfm3-short-metal.json,yes +"Mac mini M4 (Mac16,10, 10-core GPU, 16 GB), macOS 27.0 26A5421a, Metal",,Qwen3.5-2B MLX 4-bit,"mlx_lm.server, stock (HTTP server)",mlx 0.32.3 · mlx-lm 0.31.3,32,42.36,0.42,1.17,3,yes,not gated,macOS 27.0 (26A5421a),Metal,2026-10-31,measured here,receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal.json,yes +"Mac mini M4 (Mac16,10, 10-core GPU, 16 GB), macOS 27.0 26A5421a, Metal",★,Qwen3.5-2B MLX 4-bit,"mlx_lm.server, stock (HTTP server)",mlx 0.32.3 · mlx-lm 0.31.3 · 4 requests at once (decode = total tok/s),32,41.63,1.86,3.07,3,yes (engines in this run agree),not gated,macOS 27.0 (26A5421a),Metal,2026-10-31,measured here,receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal-c4.json,yes +"Mac mini M4 (Mac16,10, 10-core GPU, 16 GB), macOS 27.0 26A5421a, Metal",,Qwen3.5-2B MLX 4-bit,"mlx_lm.server, stock (HTTP server)",mlx 0.32.3 · mlx-lm 0.31.3,256,40.64,0.55,6.83,3,yes,not gated,macOS 27.0 (26A5421a),Metal,2026-10-31,measured here,receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-long-metal.json,no: diverges after 1207 of 1301 chars +"Mac mini M4 (Mac16,10, 10-core GPU, 16 GB), macOS 27.0 26A5421a, Metal",★,Qwen3.5-2B MLX 4-bit,"MLX on Metal (upstream mlx, in-process)",mlx 0.32.3 · mlx-lm 0.31.3,32,47.67,0.38,1.11,3,reference,not gated,macOS 27.0 (26A5421a),Metal,2026-10-31,measured here,receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal.json,yes +"Mac mini M4 (Mac16,10, 10-core GPU, 16 GB), macOS 27.0 26A5421a, Metal",★,Qwen3.5-2B MLX 4-bit,"MLX on Metal (upstream mlx, in-process)",mlx 0.32.3 · mlx-lm 0.31.3,256,51.26,0.52,5.37,3,reference,not gated,macOS 27.0 (26A5421a),Metal,2026-10-31,measured here,receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-long-metal.json,no: diverges after 1207 of 1301 chars +"Mac mini M4 (Mac16,10, 10-core GPU, 16 GB), macOS 27.0 26A5421a, Metal",,Qwen3.5-2B MLX 4-bit,TensorFold Metal v1.1.9 release (HTTP server),serve --backend mlx --context 1024 --no-thinking --no-drafts,32,39.71,0.07,0.85,3,yes,not gated,macOS 27.0 (26A5421a),Metal,2026-10-31,measured here,receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal.json,yes +"Mac mini M4 (Mac16,10, 10-core GPU, 16 GB), macOS 27.0 26A5421a, Metal",,Qwen3.5-2B MLX 4-bit,TensorFold Metal v1.1.9 release (HTTP server),serve --backend mlx --context 1024 --no-thinking --no-drafts · 4 requests at once (decode = total tok/s),32,36.84,1.82,3.47,3,yes (engines in this run agree),not gated,macOS 27.0 (26A5421a),Metal,2026-10-31,measured here,receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal-c4.json,yes +"Mac mini M4 (Mac16,10, 10-core GPU, 16 GB), macOS 27.0 26A5421a, Metal",★,Qwen3.5-2B MLX 4-bit,TensorFold Metal v1.1.9 release (HTTP server),serve --backend mlx --context 1024 --no-thinking --no-drafts · 8 requests at once (decode = total tok/s),32,35.41,3.64,7.23,3,yes (engines in this run agree),not gated,macOS 27.0 (26A5421a),Metal,2026-10-31,measured here,receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal-c8.json,yes +"Mac mini M4 (Mac16,10, 10-core GPU, 16 GB), macOS 27.0 26A5421a, Metal",,Qwen3.5-2B MLX 4-bit,TensorFold Metal v1.1.9 release (HTTP server),serve --backend mlx --context 1024 --no-thinking --no-drafts,256,36.37,0.12,7.13,3,no: diverges after 1207 of 1290 chars,not gated,macOS 27.0 (26A5421a),Metal,2026-10-31,measured here,receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-long-metal.json,yes +"Mac mini M4 (Mac16,10, 10-core GPU, 16 GB), macOS 27.0 26A5421a, Metal",,Qwen3.5-4B MLX 4-bit,"mlx_lm.server, stock (HTTP server)",mlx 0.32.3 · mlx-lm 0.31.3,32,25.15,0.38,1.58,3,no: diverges after 55 of 137 chars,not gated,macOS 27.0 (26A5421a),Metal,2026-10-31,measured here,receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen4b-short-metal.json,yes +"Mac mini M4 (Mac16,10, 10-core GPU, 16 GB), macOS 27.0 26A5421a, Metal",★,Qwen3.5-4B MLX 4-bit,"mlx_lm.server, stock (HTTP server)",mlx 0.32.3 · mlx-lm 0.31.3 · 4 requests at once (decode = total tok/s),32,29.23,2.29,4.38,3,yes (engines in this run agree),not gated,macOS 27.0 (26A5421a),Metal,2026-10-31,measured here,receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen4b-short-metal-c4.json,yes +"Mac mini M4 (Mac16,10, 10-core GPU, 16 GB), macOS 27.0 26A5421a, Metal",★,Qwen3.5-4B MLX 4-bit,"MLX on Metal (upstream mlx, in-process)",mlx 0.32.3 · mlx-lm 0.31.3,32,28.99,0.39,1.46,3,reference,not gated,macOS 27.0 (26A5421a),Metal,2026-10-31,measured here,receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen4b-short-metal.json,no: diverges after 55 of 152 chars diff --git a/mseries/local-ai/engine-scoreboard-m4-mini-macos.svg b/mseries/local-ai/engine-scoreboard-m4-mini-macos.svg new file mode 100644 index 00000000..94c8a97a --- /dev/null +++ b/mseries/local-ai/engine-scoreboard-m4-mini-macos.svg @@ -0,0 +1 @@ +First token, streaming speed and reply time by engine on Mac mini M4, macOS (Metal)Granite 4.0 1B MLX 4-bit (IBM, dense) · 32-token test · one requestFirst token, s · lower is betterStreaming, tok/s · higher is betterTypical reply (200 tokens), s · lowerMLX in-process, Metal ★MLX in-process, Metal: First token, s 0.22 s0.22 sMLX in-process, Metal: Streaming, tok/s 58.658.6MLX in-process, Metal: Typical reply (200 tokens), s 3.6 s3.6 smlx_lm.server, Metalmlx_lm.server, Metal: First token, s 0.22 s0.22 smlx_lm.server, Metal: Streaming, tok/s 43.043.0mlx_lm.server, Metal: Typical reply (200 tokens), s 4.9 s4.9 sLFM3-1.7B MLX 8-bit · 32-token test · one requestFirst token, s · lower is betterStreaming, tok/s · higher is betterTypical reply (200 tokens), s · lowerMLX in-process, Metal ★MLX in-process, Metal: First token, s 0.24 s0.24 sMLX in-process, Metal: Streaming, tok/s 31.531.5MLX in-process, Metal: Typical reply (200 tokens), s 6.6 s6.6 sQwen3.5-2B MLX 4-bit · 32-token test · one requestFirst token, s · lower is betterStreaming, tok/s · higher is betterTypical reply (200 tokens), s · lowerMLX in-process, Metal ★MLX in-process, Metal: First token, s 0.38 s0.38 sMLX in-process, Metal: Streaming, tok/s 47.747.7MLX in-process, Metal: Typical reply (200 tokens), s 4.6 s4.6 sTensorFold MetalTensorFold Metal: First token, s 0.07 s0.07 sTensorFold Metal: Streaming, tok/s 39.739.7TensorFold Metal: Typical reply (200 tokens), s 5.1 s5.1 smlx_lm.server, Metalmlx_lm.server, Metal: First token, s 0.42 s0.42 smlx_lm.server, Metal: Streaming, tok/s 42.442.4mlx_lm.server, Metal: Typical reply (200 tokens), s 5.1 s5.1 sQwen3.5-2B MLX 4-bit · 32-token test · 4 requests at once (streaming = total)First token, s · lower is betterStreaming, tok/s · higher is betterTypical reply (200 tokens), s · lowermlx_lm.server, Metal ★mlx_lm.server, Metal: First token, s 1.86 s1.86 smlx_lm.server, Metal: Streaming, tok/s 41.641.6mlx_lm.server, Metal: Typical reply (200 tokens), s 21.1 s21.1 sTensorFold MetalTensorFold Metal: First token, s 1.82 s1.82 sTensorFold Metal: Streaming, tok/s 36.836.8TensorFold Metal: Typical reply (200 tokens), s 23.5 s23.5 sQwen3.5-2B MLX 4-bit · 32-token test · 8 requests at once (streaming = total)First token, s · lower is betterStreaming, tok/s · higher is betterTypical reply (200 tokens), s · lowerTensorFold Metal ★TensorFold Metal: First token, s 3.64 s3.64 sTensorFold Metal: Streaming, tok/s 35.435.4TensorFold Metal: Typical reply (200 tokens), s 48.8 s48.8 sQwen3.5-2B MLX 4-bit · 256-token test · one requestFirst token, s · lower is betterStreaming, tok/s · higher is betterTypical reply (200 tokens), s · lowerMLX in-process, Metal ★MLX in-process, Metal: First token, s 0.52 s0.52 sMLX in-process, Metal: Streaming, tok/s 51.351.3MLX in-process, Metal: Typical reply (200 tokens), s 4.4 s4.4 smlx_lm.server, Metalmlx_lm.server, Metal: First token, s 0.55 s0.55 smlx_lm.server, Metal: Streaming, tok/s 40.640.6mlx_lm.server, Metal: Typical reply (200 tokens), s 5.5 s5.5 sTensorFold MetalTensorFold Metal: First token, s 0.12 s0.12 sTensorFold Metal: Streaming, tok/s 36.436.4TensorFold Metal: Typical reply (200 tokens), s 5.6 s5.6 sQwen3.5-4B MLX 4-bit · 32-token test · one requestFirst token, s · lower is betterStreaming, tok/s · higher is betterTypical reply (200 tokens), s · lowerMLX in-process, Metal ★MLX in-process, Metal: First token, s 0.39 s0.39 sMLX in-process, Metal: Streaming, tok/s 29.029.0MLX in-process, Metal: Typical reply (200 tokens), s 7.3 s7.3 smlx_lm.server, Metalmlx_lm.server, Metal: First token, s 0.38 s0.38 smlx_lm.server, Metal: Streaming, tok/s 25.225.2mlx_lm.server, Metal: Typical reply (200 tokens), s 8.3 s8.3 sQwen3.5-4B MLX 4-bit · 32-token test · 4 requests at once (streaming = total)First token, s · lower is betterStreaming, tok/s · higher is betterTypical reply (200 tokens), s · lowermlx_lm.server, Metal ★mlx_lm.server, Metal: First token, s 2.29 s2.29 smlx_lm.server, Metal: Streaming, tok/s 29.229.2mlx_lm.server, Metal: Typical reply (200 tokens), s 29.7 s29.7 s diff --git a/mseries/local-ai/machine.py b/mseries/local-ai/machine.py index 3e4d9f73..15bb2118 100644 --- a/mseries/local-ai/machine.py +++ b/mseries/local-ai/machine.py @@ -1,6 +1,7 @@ """Read-only machine telemetry and conservative AI admission policy.""" import json from pathlib import Path +import sys import time GIB = 1024 ** 3 @@ -23,7 +24,47 @@ def number(path): return None +def sh(*cmd): + import subprocess + try: + return subprocess.run(cmd, capture_output=True, text=True, timeout=20).stdout.strip() + except (OSError, subprocess.SubprocessError): + return '' + + +def macos_snapshot(): + """macOS has no /proc or hwmon. Available memory is the kernel's own free-able share + (kern.memorystatus_level, what memory_pressure prints); fans and SMC temperatures need root, + so the thermal signal is pmset's warning level instead.""" + total = int(sh('sysctl', '-n', 'hw.memsize') or 0) + level = int(sh('sysctl', '-n', 'kern.memorystatus_level') or 0) + swap = sh('sysctl', '-n', 'vm.swapusage') # total = 3072.00M used = 2222.50M free = ... + used = swap.split('used = ')[1].split('M')[0] if 'used = ' in swap else '0' + therm = sh('pmset', '-g', 'therm') + return {'time': time.time(), 'platform': 'darwin', 'available_bytes': total * level // 100, + 'total_bytes': total, 'swap_used_bytes': int(float(used) * 1024 ** 2), + 'fans_rpm': {}, 'temperatures_c': {}, 'power_w': None, 'thermal': therm, + 'load_avg': sh('sysctl', '-n', 'vm.loadavg').strip('{} ')} + + +def host_info(): + """What a macOS receipt records about the machine and what else was using it.""" + gpu = sh('system_profiler', 'SPDisplaysDataType') + cores = next((l.split(':')[1].strip() for l in gpu.splitlines() if 'Total Number of Cores' in l), '') + top = sh('top', '-l', '2', '-s', '1', '-n', '6', '-o', 'cpu', '-stats', 'pid,command,cpu,mem') + return {'macos': sh('sw_vers', '-productVersion'), 'build': sh('sw_vers', '-buildVersion'), + 'hw_model': sh('sysctl', '-n', 'hw.model'), 'chip': sh('sysctl', '-n', 'machdep.cpu.brand_string'), + 'gpu_cores': cores, 'memory_gib': int(sh('sysctl', '-n', 'hw.memsize') or 0) / GIB, + 'power': sh('pmset', '-g', 'batt').splitlines()[:1], 'low_power_mode': sh('pmset', '-g').count('lowpowermode 1') > 0, + 'thermal': sh('pmset', '-g', 'therm'), 'uptime': sh('uptime'), + 'top_cpu': top.split('PID')[-1].strip().splitlines()[1:] if 'PID' in top else [], + 'memory_free_pct': sh('sysctl', '-n', 'kern.memorystatus_level'), 'swap': sh('sysctl', '-n', 'vm.swapusage'), + 'lima': sh('limactl', 'list')} + + def snapshot(root=Path('/')): + if sys.platform == 'darwin' and root == Path('/'): + return macos_snapshot() mem = {} for line in (root / 'proc/meminfo').read_text().splitlines(): key, value = line.split(':', 1) @@ -57,7 +98,10 @@ def snapshot(root=Path('/')): def violations(sample, policy=None, extra_bytes=0): policy = DEFAULT_POLICY if policy is None else policy reasons = [] - if not sample['fans_rpm'] or not sample['temperatures_c']: + if sample.get('platform') == 'darwin': + if 'warning level' in sample['thermal'] and 'No thermal warning level' not in sample['thermal']: + reasons.append('macOS reports a thermal warning level') + elif not sample['fans_rpm'] or not sample['temperatures_c']: reasons.append('fan or temperature telemetry unavailable; quiet limits cannot be checked') if sample['available_bytes'] - extra_bytes < policy['reserve_gib'] * GIB: reasons.append('desktop memory reserve would be crossed ' diff --git a/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-granite1b-short-metal.json b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-granite1b-short-metal.json new file mode 100644 index 00000000..559fd1f2 --- /dev/null +++ b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-granite1b-short-metal.json @@ -0,0 +1,457 @@ +{ + "status": "complete", + "model_key": "granite1b", + "model": "mlx-community/granite-4.0-1b-4bit@29f108a4e2ff99079a173004871604c2f1e12c1f", + "weights_sha256": "b7a43ebbc86517b8ecfc4cff265a787ea31620f7eb356984c08abe14646617c7", + "kernel": "macOS 27.0 (26A5421a)", + "driver": "Metal", + "tensorfold_rev": "v1.1.9 release", + "rapid_rev": "1d09cdb684bbf58fac130f5804b08ee2e1f56558", + "serve_concurrency": 4, + "prompt_sha256": "af125b15dfba55caedb541e89e59fe1252c0f13553e5ce6bc181e237fada9926", + "max_tokens": 32, + "policy": { + "reserve_gib": 1.0, + "max_fan_rpm": null, + "max_temp_c": 80 + }, + "allowance_gib": 3.5, + "arms": [ + "mlx-metal", + "lm-metal" + ], + "concurrency": 1, + "runs": [ + { + "arm": "mlx-metal", + "stopped": null, + "samples": [ + { + "t": 0.38, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "23.15 18.38 15.52" + }, + { + "t": 1.7, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "23.15 18.38 15.52" + }, + { + "t": 3.26, + "available_gib": 6.56, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "23.30 18.49 15.57" + }, + { + "t": 4.43, + "available_gib": 6.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "23.30 18.49 15.57" + }, + { + "t": 5.63, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "23.30 18.49 15.57" + } + ], + "log_tail": "", + "ready_s": 4.4398056250010995, + "ttft_s": 0.22947020799983875, + "total_s": 0.7585496660012723, + "completion_tokens": 32, + "text": "A rainbow forms when sunlight enters a water droplet in the air, causing it to refract and reflect inside the droplet. As the light refracts,", + "text_sha256": "81588d18f3846cfeedeae583a5c7771c04a5568c3bec5aece292542a8adc7741", + "decode_tps": 58.59233340319179, + "min_available_gib": 6.24, + "peak_temp_c": 0, + "round": 1, + "admission_available_gib": 7.2 + }, + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.09, + "available_gib": 7.52, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "22.47 18.40 15.56" + }, + { + "t": 1.21, + "available_gib": 7.52, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "22.47 18.40 15.56" + }, + { + "t": 2.61, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "22.52 18.47 15.60" + }, + { + "t": 3.74, + "available_gib": 6.4, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "22.52 18.47 15.60" + }, + { + "t": 4.9, + "available_gib": 6.4, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "22.52 18.47 15.60" + } + ], + "log_tail": "/private/tmp/claude-501/bench/venv/lib/python3.14/site-packages/mlx_lm/server.py:1723: UserWarning: mlx_lm.server is not recommended for production as it only implements basic security checks.\n warnings.warn(\n2026-10-31 11:37:02,263 - INFO - Starting httpd at 127.0.0.1 on port 55164...\n127.0.0.1 - - [31/Oct/2026 11:37:03] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:37:03] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:37:03,188 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:37:03,188 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:37:03,188 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:37:03,188 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:37:03] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:37:03,457 - INFO - Prompt processing progress: 44/45\n2026-10-31 11:37:03,568 - INFO - Prompt processing progress: 45/45\n2026-10-31 11:37:04,311 - INFO - Prompt Cache: 1 sequences, 0.01 GB\n2026-10-31 11:37:04,311 - INFO - - assistant: 1 sequences, 0.01 GB\n2026-10-31 11:37:04,311 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:37:04,311 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:37:04] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:37:04,468 - INFO - Prompt processing progress: 1/1\n", + "ready_s": 4.169430291998651, + "ttft_s": 0.18074983400038036, + "total_s": 0.8736849590004567, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "A rainbow forms when sunlight enters a water droplet in the air, causing it to refract and reflect inside the droplet. As the light refracts,", + "served_model": "/private/tmp/claude-501/bench/hf/models--mlx-community--granite-4.0-1b-4bit/snapshots/29f108a4e2ff99079a173004871604c2f1e12c1f", + "text_sha256": "81588d18f3846cfeedeae583a5c7771c04a5568c3bec5aece292542a8adc7741", + "decode_tps": 44.73723279649965, + "min_available_gib": 6.4, + "peak_temp_c": 0, + "round": 1, + "admission_available_gib": 7.52 + }, + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.25, + "available_gib": 7.84, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.83 18.40 15.59" + }, + { + "t": 1.37, + "available_gib": 8.16, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.83 18.40 15.59" + }, + { + "t": 2.87, + "available_gib": 7.36, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.83 18.40 15.59" + }, + { + "t": 4.28, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "22.25 18.54 15.66" + }, + { + "t": 5.51, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "22.25 18.54 15.66" + } + ], + "log_tail": "/private/tmp/claude-501/bench/venv/lib/python3.14/site-packages/mlx_lm/server.py:1723: UserWarning: mlx_lm.server is not recommended for production as it only implements basic security checks.\n warnings.warn(\n2026-10-31 11:37:10,596 - INFO - Starting httpd at 127.0.0.1 on port 55187...\n127.0.0.1 - - [31/Oct/2026 11:37:11] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:37:11] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:37:11,914 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:37:11,973 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:37:11,973 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:37:11,973 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:37:12] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:37:12,490 - INFO - Prompt processing progress: 44/45\n2026-10-31 11:37:12,651 - INFO - Prompt processing progress: 45/45\n2026-10-31 11:37:13,496 - INFO - Prompt Cache: 1 sequences, 0.01 GB\n2026-10-31 11:37:13,497 - INFO - - assistant: 1 sequences, 0.01 GB\n2026-10-31 11:37:13,497 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:37:13,497 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:37:13] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:37:13,697 - INFO - Prompt processing progress: 1/1\n", + "ready_s": 5.174501624998811, + "ttft_s": 0.23406462499951886, + "total_s": 1.0134833749998506, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "A rainbow forms when sunlight enters a water droplet in the air, causing it to refract and reflect inside the droplet. As the light refracts,", + "served_model": "/private/tmp/claude-501/bench/hf/models--mlx-community--granite-4.0-1b-4bit/snapshots/29f108a4e2ff99079a173004871604c2f1e12c1f", + "text_sha256": "81588d18f3846cfeedeae583a5c7771c04a5568c3bec5aece292542a8adc7741", + "decode_tps": 39.773228447463964, + "min_available_gib": 5.76, + "peak_temp_c": 0, + "round": 2, + "admission_available_gib": 7.84 + }, + { + "arm": "mlx-metal", + "stopped": null, + "samples": [ + { + "t": 0.11, + "available_gib": 7.36, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.67 18.48 15.65" + }, + { + "t": 1.33, + "available_gib": 7.52, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.67 18.48 15.65" + }, + { + "t": 2.44, + "available_gib": 7.68, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.67 18.48 15.65" + }, + { + "t": 3.57, + "available_gib": 6.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.67 18.48 15.65" + }, + { + "t": 4.72, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.85 18.57 15.70" + }, + { + "t": 5.77, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.85 18.57 15.70" + } + ], + "log_tail": "", + "ready_s": 4.693515417000526, + "ttft_s": 0.2248917500000971, + "total_s": 0.7808781660005479, + "completion_tokens": 32, + "text": "A rainbow forms when sunlight enters a water droplet in the air, causing it to refract and reflect inside the droplet. As the light refracts,", + "text_sha256": "81588d18f3846cfeedeae583a5c7771c04a5568c3bec5aece292542a8adc7741", + "decode_tps": 55.75675791326323, + "min_available_gib": 5.92, + "peak_temp_c": 0, + "round": 2, + "admission_available_gib": 7.2 + }, + { + "arm": "mlx-metal", + "stopped": null, + "samples": [ + { + "t": 0.23, + "available_gib": 7.52, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.46 18.54 15.71" + }, + { + "t": 1.53, + "available_gib": 7.52, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.46 18.54 15.71" + }, + { + "t": 2.65, + "available_gib": 7.52, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.46 18.54 15.71" + }, + { + "t": 3.78, + "available_gib": 6.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.46 18.54 15.71" + }, + { + "t": 4.9, + "available_gib": 6.08, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.27 18.55 15.73" + }, + { + "t": 6.08, + "available_gib": 6.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.27 18.55 15.73" + } + ], + "log_tail": "", + "ready_s": 4.820296166999469, + "ttft_s": 0.22424950000095123, + "total_s": 0.7486354579996259, + "completion_tokens": 32, + "text": "A rainbow forms when sunlight enters a water droplet in the air, causing it to refract and reflect inside the droplet. As the light refracts,", + "text_sha256": "81588d18f3846cfeedeae583a5c7771c04a5568c3bec5aece292542a8adc7741", + "decode_tps": 59.11676223808866, + "min_available_gib": 6.08, + "peak_temp_c": 0, + "round": 3, + "admission_available_gib": 7.36 + }, + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.13, + "available_gib": 7.52, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.84 18.51 15.73" + }, + { + "t": 1.22, + "available_gib": 7.68, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.84 18.51 15.73" + }, + { + "t": 2.32, + "available_gib": 7.68, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.84 18.51 15.73" + }, + { + "t": 3.57, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.84 18.51 15.73" + }, + { + "t": 4.87, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.18 18.62 15.78" + }, + { + "t": 5.96, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.18 18.62 15.78" + } + ], + "log_tail": "/private/tmp/claude-501/bench/venv/lib/python3.14/site-packages/mlx_lm/server.py:1723: UserWarning: mlx_lm.server is not recommended for production as it only implements basic security checks.\n warnings.warn(\n2026-10-31 11:37:40,583 - INFO - Starting httpd at 127.0.0.1 on port 55226...\n127.0.0.1 - - [31/Oct/2026 11:37:40] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:37:40] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:37:41,831 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:37:41,832 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:37:41,832 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:37:41,832 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:37:41] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:37:42,190 - INFO - Prompt processing progress: 44/45\n2026-10-31 11:37:42,380 - INFO - Prompt processing progress: 45/45\n2026-10-31 11:37:43,185 - INFO - Prompt Cache: 1 sequences, 0.01 GB\n2026-10-31 11:37:43,185 - INFO - - assistant: 1 sequences, 0.01 GB\n2026-10-31 11:37:43,186 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:37:43,186 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:37:43] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:37:43,375 - INFO - Prompt processing progress: 1/1\n", + "ready_s": 5.032218333999481, + "ttft_s": 0.21650420799960557, + "total_s": 0.9372293750002427, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "A rainbow forms when sunlight enters a water droplet in the air, causing it to refract and reflect inside the droplet. As the light refracts,", + "served_model": "/private/tmp/claude-501/bench/hf/models--mlx-community--granite-4.0-1b-4bit/snapshots/29f108a4e2ff99079a173004871604c2f1e12c1f", + "text_sha256": "81588d18f3846cfeedeae583a5c7771c04a5568c3bec5aece292542a8adc7741", + "decode_tps": 43.01223464834633, + "min_available_gib": 5.92, + "peak_temp_c": 0, + "round": 3, + "admission_available_gib": 7.52 + } + ], + "runtime": { + "mlx": "0.32.3", + "mlx_lm": "0.31.3", + "transformers": "5.16.1", + "python": "3.14.3", + "device": { + "device_name": "Apple M4", + "max_recommended_working_set_size": 12713115648, + "memory_size": 17179869184, + "architecture": "applegpu_g16g", + "max_buffer_length": 9534832640, + "resource_limit": 499000 + } + }, + "tensorfold_sha256": "4b4b5a5ac152eb2dbed7e9d8004805ed9b74f77c64c3de18eb4f35263c7b9eb8", + "model_dir": "/private/tmp/claude-501/bench/hf/models--mlx-community--granite-4.0-1b-4bit/snapshots/29f108a4e2ff99079a173004871604c2f1e12c1f", + "host_start": { + "macos": "27.0", + "build": "26A5421a", + "hw_model": "Mac16,10", + "chip": "Apple M4", + "gpu_cores": "10", + "memory_gib": 16.0, + "power": [ + "Now drawing from 'AC Power'" + ], + "low_power_mode": false, + "thermal": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", + "uptime": "11:36 up 2:55, 4 users, load averages: 23.15 18.38 15.52", + "top_cpu": [ + "42290 com.apple.Virtua 391.8 10G ", + "0 kernel_task 78.1 34M- ", + "4939 Claude Helper (R 77.6 893M+", + "433 WindowServer 42.7 593M-", + "2441 com.apple.WebKit 38.9 895M+", + "2479 com.apple.WebKit 32.3 29M+" + ], + "memory_free_pct": "44", + "swap": "total = 4096.00M used = 3217.88M free = 878.12M (encrypted)", + "lima": "NAME STATUS SSH VMTYPE ARCH CPUS MEMORY DISK DIR\nkbuild Running 127.0.0.1:53830 vz aarch64 8 10GiB 60GiB ~/.lima/kbuild" + }, + "summary": { + "mlx-metal": { + "ok_runs": 3, + "median_ttft_s": 0.2248917500000971, + "median_total_s": 0.7585496660012723, + "median_decode_tps": 58.59233340319179, + "texts": [ + "81588d18f384" + ] + }, + "lm-metal": { + "ok_runs": 3, + "median_ttft_s": 0.21650420799960557, + "median_total_s": 0.9372293750002427, + "median_decode_tps": 43.01223464834633, + "texts": [ + "81588d18f384" + ] + } + }, + "host_end": { + "macos": "27.0", + "build": "26A5421a", + "hw_model": "Mac16,10", + "chip": "Apple M4", + "gpu_cores": "10", + "memory_gib": 16.0, + "power": [ + "Now drawing from 'AC Power'" + ], + "low_power_mode": false, + "thermal": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", + "uptime": "11:37 up 2:56, 4 users, load averages: 20.84 18.59 15.79", + "top_cpu": [ + "42290 com.apple.Virtua 540.8 10G ", + "0 kernel_task 34.3 34M- ", + "433 WindowServer 33.7 599M ", + "4939 Claude Helper (R 26.3 911M-", + "88722 Python 19.4 120M-", + "4916 Claude Helper 18.5 477M+" + ], + "memory_free_pct": "45", + "swap": "total = 4096.00M used = 2641.88M free = 1454.12M (encrypted)", + "lima": "NAME STATUS SSH VMTYPE ARCH CPUS MEMORY DISK DIR\nkbuild Running 127.0.0.1:53830 vz aarch64 8 10GiB 60GiB ~/.lima/kbuild" + } +} diff --git a/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-lfm3-short-metal.json b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-lfm3-short-metal.json new file mode 100644 index 00000000..4a151bd4 --- /dev/null +++ b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-lfm3-short-metal.json @@ -0,0 +1,310 @@ +{ + "status": "complete", + "model_key": "lfm3", + "model": "LiquidAI/LFM3-1.7B-MLX-8bit@ee7667f3cfccb2a41725f91820531823fcabc6b7", + "weights_sha256": "2b06cea98d75527c248a0b04383f7cf0a1c72fb4ed7b45096618fc4e59d015ad", + "kernel": "macOS 27.0 (26A5421a)", + "driver": "Metal", + "tensorfold_rev": "v1.1.9 release", + "rapid_rev": "1d09cdb684bbf58fac130f5804b08ee2e1f56558", + "serve_concurrency": 4, + "prompt_sha256": "af125b15dfba55caedb541e89e59fe1252c0f13553e5ce6bc181e237fada9926", + "max_tokens": 32, + "policy": { + "reserve_gib": 1.0, + "max_fan_rpm": null, + "max_temp_c": 80 + }, + "allowance_gib": 4.5, + "arms": [ + "mlx-metal" + ], + "concurrency": 1, + "runs": [ + { + "arm": "mlx-metal", + "stopped": null, + "samples": [ + { + "t": 0.2, + "available_gib": 7.52, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.50 18.76 15.87" + }, + { + "t": 1.34, + "available_gib": 7.36, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.78 18.87 15.92" + }, + { + "t": 2.8, + "available_gib": 7.04, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.78 18.87 15.92" + }, + { + "t": 4.02, + "available_gib": 5.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.78 18.87 15.92" + }, + { + "t": 5.28, + "available_gib": 5.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.95 18.95 15.97" + }, + { + "t": 6.5, + "available_gib": 5.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.95 18.95 15.97" + }, + { + "t": 7.7, + "available_gib": 5.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.95 18.95 15.97" + } + ], + "log_tail": "The repository /private/tmp/claude-501/bench/hf/models--LiquidAI--LFM3-1.7B-MLX-8bit/snapshots/ee7667f3cfccb2a41725f91820531823fcabc6b7 contains custom code which must be executed to correctly load the model. You can inspect the repository content at /private/tmp/claude-501/bench/hf/models--LiquidAI--LFM3-1.7B-MLX-8bit/snapshots/ee7667f3cfccb2a41725f91820531823fcabc6b7 .\n You can inspect the repository content at https://hf.co//private/tmp/claude-501/bench/hf/models--LiquidAI--LFM3-1.7B-MLX-8bit/snapshots/ee7667f3cfccb2a41725f91820531823fcabc6b7.\nYou can avoid this prompt in future by passing the argument `trust_remote_code=True`.\n\nDo you wish to run the custom code? [y/N] [transformers] You are using a model of type `lfm3` to instantiate a model of type ``. This may be expected if you are loading a checkpoint that shares a subset of the architecture (e.g., loading a `sam2_video` checkpoint into `Sam2Model`), but is otherwise not supported and can yield errors. Please verify that the checkpoint is compatible with the model you are instantiating.\n", + "ready_s": 6.922038165999766, + "ttft_s": 0.23748508300013782, + "total_s": 1.220571583000492, + "completion_tokens": 32, + "text": "A rainbow is created when sunlight passes through water droplets in the air and bends (refracts), then reflects off the back of each droplet and bends", + "text_sha256": "b8cc5b9478a05082c75e5ec40e9d3a528f77c91c5ac0d639b827ff94b25a609b", + "decode_tps": 31.533339131387557, + "min_available_gib": 5.28, + "peak_temp_c": 0, + "round": 1, + "admission_available_gib": 7.52 + }, + { + "arm": "mlx-metal", + "stopped": null, + "samples": [ + { + "t": 0.18, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.16 18.84 15.94" + }, + { + "t": 1.27, + "available_gib": 7.36, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.16 18.84 15.94" + }, + { + "t": 2.37, + "available_gib": 7.36, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.16 18.84 15.94" + }, + { + "t": 3.79, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.90 18.82 15.96" + }, + { + "t": 4.99, + "available_gib": 5.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.90 18.82 15.96" + }, + { + "t": 6.13, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.90 18.82 15.96" + }, + { + "t": 7.33, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.90 18.82 15.96" + } + ], + "log_tail": "The repository /private/tmp/claude-501/bench/hf/models--LiquidAI--LFM3-1.7B-MLX-8bit/snapshots/ee7667f3cfccb2a41725f91820531823fcabc6b7 contains custom code which must be executed to correctly load the model. You can inspect the repository content at /private/tmp/claude-501/bench/hf/models--LiquidAI--LFM3-1.7B-MLX-8bit/snapshots/ee7667f3cfccb2a41725f91820531823fcabc6b7 .\n You can inspect the repository content at https://hf.co//private/tmp/claude-501/bench/hf/models--LiquidAI--LFM3-1.7B-MLX-8bit/snapshots/ee7667f3cfccb2a41725f91820531823fcabc6b7.\nYou can avoid this prompt in future by passing the argument `trust_remote_code=True`.\n\nDo you wish to run the custom code? [y/N] [transformers] You are using a model of type `lfm3` to instantiate a model of type ``. This may be expected if you are loading a checkpoint that shares a subset of the architecture (e.g., loading a `sam2_video` checkpoint into `Sam2Model`), but is otherwise not supported and can yield errors. Please verify that the checkpoint is compatible with the model you are instantiating.\n", + "ready_s": 6.672642083000028, + "ttft_s": 0.24641354199957277, + "total_s": 1.1725572499999544, + "completion_tokens": 32, + "text": "A rainbow is created when sunlight passes through water droplets in the air and bends (refracts), then reflects off the back of each droplet and bends", + "text_sha256": "b8cc5b9478a05082c75e5ec40e9d3a528f77c91c5ac0d639b827ff94b25a609b", + "decode_tps": 33.47212720035801, + "min_available_gib": 5.44, + "peak_temp_c": 0, + "round": 2, + "admission_available_gib": 7.2 + }, + { + "arm": "mlx-metal", + "stopped": null, + "samples": [ + { + "t": 0.15, + "available_gib": 8.0, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.75 18.82 15.97" + }, + { + "t": 1.27, + "available_gib": 8.0, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.75 18.82 15.97" + }, + { + "t": 2.39, + "available_gib": 8.64, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.37 18.78 15.97" + }, + { + "t": 3.73, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.37 18.78 15.97" + }, + { + "t": 4.92, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.37 18.78 15.97" + }, + { + "t": 6.09, + "available_gib": 6.08, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.37 18.78 15.97" + }, + { + "t": 7.34, + "available_gib": 5.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.86 18.70 15.96" + }, + { + "t": 8.46, + "available_gib": 7.36, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.86 18.70 15.96" + } + ], + "log_tail": "The repository /private/tmp/claude-501/bench/hf/models--LiquidAI--LFM3-1.7B-MLX-8bit/snapshots/ee7667f3cfccb2a41725f91820531823fcabc6b7 contains custom code which must be executed to correctly load the model. You can inspect the repository content at /private/tmp/claude-501/bench/hf/models--LiquidAI--LFM3-1.7B-MLX-8bit/snapshots/ee7667f3cfccb2a41725f91820531823fcabc6b7 .\n You can inspect the repository content at https://hf.co//private/tmp/claude-501/bench/hf/models--LiquidAI--LFM3-1.7B-MLX-8bit/snapshots/ee7667f3cfccb2a41725f91820531823fcabc6b7.\nYou can avoid this prompt in future by passing the argument `trust_remote_code=True`.\n\nDo you wish to run the custom code? [y/N] [transformers] You are using a model of type `lfm3` to instantiate a model of type ``. This may be expected if you are loading a checkpoint that shares a subset of the architecture (e.g., loading a `sam2_video` checkpoint into `Sam2Model`), but is otherwise not supported and can yield errors. Please verify that the checkpoint is compatible with the model you are instantiating.\n", + "ready_s": 6.575893125000221, + "ttft_s": 0.23207858300156659, + "total_s": 1.3103606670010777, + "completion_tokens": 32, + "text": "A rainbow is created when sunlight passes through water droplets in the air and bends (refracts), then reflects off the back of each droplet and bends", + "text_sha256": "b8cc5b9478a05082c75e5ec40e9d3a528f77c91c5ac0d639b827ff94b25a609b", + "decode_tps": 28.749434364165932, + "min_available_gib": 5.44, + "peak_temp_c": 0, + "round": 3, + "admission_available_gib": 8.0 + } + ], + "runtime": { + "mlx": "0.32.3", + "mlx_lm": "0.31.3", + "transformers": "5.16.1", + "python": "3.14.3", + "device": { + "device_name": "Apple M4", + "max_recommended_working_set_size": 12713115648, + "memory_size": 17179869184, + "architecture": "applegpu_g16g", + "max_buffer_length": 9534832640, + "resource_limit": 499000 + } + }, + "tensorfold_sha256": "4b4b5a5ac152eb2dbed7e9d8004805ed9b74f77c64c3de18eb4f35263c7b9eb8", + "model_dir": "/private/tmp/claude-501/bench/hf/models--LiquidAI--LFM3-1.7B-MLX-8bit/snapshots/ee7667f3cfccb2a41725f91820531823fcabc6b7", + "host_start": { + "macos": "27.0", + "build": "26A5421a", + "hw_model": "Mac16,10", + "chip": "Apple M4", + "gpu_cores": "10", + "memory_gib": 16.0, + "power": [ + "Now drawing from 'AC Power'" + ], + "low_power_mode": false, + "thermal": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", + "uptime": "11:37 up 2:56, 4 users, load averages: 21.50 18.76 15.87", + "top_cpu": [ + "42290 com.apple.Virtua 544.4 10G ", + "0 kernel_task 43.5 34M+ ", + "433 WindowServer 32.8 599M ", + "88301 Python 20.9 27M ", + "88722 Python 14.9 197M-", + "4916 Claude Helper 11.9 446M-" + ], + "memory_free_pct": "46", + "swap": "total = 4096.00M used = 2625.88M free = 1470.12M (encrypted)", + "lima": "NAME STATUS SSH VMTYPE ARCH CPUS MEMORY DISK DIR\nkbuild Running 127.0.0.1:53830 vz aarch64 8 10GiB 60GiB ~/.lima/kbuild" + }, + "summary": { + "mlx-metal": { + "ok_runs": 3, + "median_ttft_s": 0.23748508300013782, + "median_total_s": 1.220571583000492, + "median_decode_tps": 31.533339131387557, + "texts": [ + "b8cc5b9478a0" + ] + } + }, + "host_end": { + "macos": "27.0", + "build": "26A5421a", + "hw_model": "Mac16,10", + "chip": "Apple M4", + "gpu_cores": "10", + "memory_gib": 16.0, + "power": [ + "Now drawing from 'AC Power'" + ], + "low_power_mode": false, + "thermal": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", + "uptime": "11:38 up 2:57, 4 users, load averages: 19.31 18.60 15.94", + "top_cpu": [ + "42290 com.apple.Virtua 536.6 10G ", + "433 WindowServer 35.1 599M+", + "88301 Python 25.2 27M ", + "1 launchd 21.8 26M+ ", + "0 kernel_task 21.3 34M+ ", + "4939 Claude Helper (R 20.6 915M+" + ], + "memory_free_pct": "53", + "swap": "total = 4096.00M used = 2944.25M free = 1151.75M (encrypted)", + "lima": "NAME STATUS SSH VMTYPE ARCH CPUS MEMORY DISK DIR\nkbuild Running 127.0.0.1:53830 vz aarch64 8 10GiB 60GiB ~/.lima/kbuild" + } +} diff --git a/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-long-metal.json b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-long-metal.json new file mode 100644 index 00000000..9d3fce5e --- /dev/null +++ b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-long-metal.json @@ -0,0 +1,1199 @@ +{ + "status": "complete", + "model_key": "qwen2b", + "model": "mlx-community/Qwen3.5-2B-MLX-4bit@93760be4f1f69842a46bc13dbdc0f19e291392a3", + "weights_sha256": "713fe7e5d3c3965f7106b0d0ee17615f7869c23c8d327996df8c1196fbcf07d5", + "kernel": "macOS 27.0 (26A5421a)", + "driver": "Metal", + "tensorfold_rev": "v1.1.9 release", + "rapid_rev": "1d09cdb684bbf58fac130f5804b08ee2e1f56558", + "serve_concurrency": 4, + "prompt_sha256": "2f23962cfdcb7ce0c2b482c4dd71e415f72113b963cfe6594fd5dc1fd4347921", + "max_tokens": 256, + "policy": { + "reserve_gib": 1.0, + "max_fan_rpm": null, + "max_temp_c": 80 + }, + "allowance_gib": 4.5, + "arms": [ + "mlx-metal", + "lm-metal", + "tensorfold-metal" + ], + "concurrency": 1, + "runs": [ + { + "arm": "mlx-metal", + "stopped": null, + "samples": [ + { + "t": 0.28, + "available_gib": 9.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.04 17.50 16.23" + }, + { + "t": 1.46, + "available_gib": 9.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.04 17.50 16.23" + }, + { + "t": 2.79, + "available_gib": 7.84, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.04 17.50 16.23" + }, + { + "t": 4.17, + "available_gib": 7.36, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.11 17.49 16.24" + }, + { + "t": 5.61, + "available_gib": 7.36, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.11 17.49 16.24" + }, + { + "t": 6.88, + "available_gib": 7.36, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.11 17.49 16.24" + }, + { + "t": 8.25, + "available_gib": 7.36, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.34 17.51 16.25" + }, + { + "t": 9.51, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.34 17.51 16.25" + }, + { + "t": 10.89, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.34 17.51 16.25" + }, + { + "t": 12.46, + "available_gib": 7.52, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.34 17.51 16.25" + }, + { + "t": 13.74, + "available_gib": 8.16, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.16 17.45 16.24" + }, + { + "t": 15.38, + "available_gib": 8.16, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.16 17.45 16.24" + }, + { + "t": 16.77, + "available_gib": 7.52, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.16 17.45 16.24" + } + ], + "log_tail": "", + "ready_s": 10.09442295899862, + "ttft_s": 0.6294245839999348, + "total_s": 6.024209750001319, + "completion_tokens": 256, + "text": "A rainbow is a spectacular natural phenomenon that occurs when sunlight interacts with water droplets in the atmosphere, creating a spectrum of colors that we see in the sky. This optical effect is a complex interplay of physics involving refraction, reflection, dispersion, and the viewing angle. To understand how a rainbow forms, one must first consider the properties of light and the behavior of water droplets.\n\nThe process begins with the entry of sunlight into a water droplet. Sunlight is composed of all visible wavelengths of the electromagnetic spectrum, ranging from red to violet. As these rays enter the droplet, they encounter a medium with a higher refractive index than air. Consequently, the light slows down and bends, a process known as refraction. Because different wavelengths of light travel at different speeds, they refract at slightly different angles. Red light, having the longest wavelength, bends the least, while violet light, with the shortest wavelength, bends the most. This separation of colors is called dispersion.\n\nUpon entering the droplet, the light ray undergoes a second refraction as it exits the back of the droplet. However, the path of the light ray is not a simple straight line; it is bent by the curvature of the droplet. The light ray then", + "text_sha256": "93e110ad5d5c0397fa2df84ca65075a8dc1f844f1d579ceb9b2cca1597c71d76", + "decode_tps": 47.267869276248874, + "min_available_gib": 7.2, + "peak_temp_c": 0, + "round": 1, + "admission_available_gib": 9.44 + }, + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.47, + "available_gib": 9.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.38 17.48 16.25" + }, + { + "t": 1.8, + "available_gib": 9.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.38 17.48 16.25" + }, + { + "t": 3.09, + "available_gib": 9.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.35 17.45 16.25" + }, + { + "t": 4.5, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.35 17.45 16.25" + }, + { + "t": 5.83, + "available_gib": 7.04, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.35 17.45 16.25" + }, + { + "t": 7.05, + "available_gib": 7.04, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.35 17.45 16.25" + }, + { + "t": 8.2, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.89 17.55 16.29" + }, + { + "t": 9.69, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.89 17.55 16.29" + }, + { + "t": 11.03, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.89 17.55 16.29" + }, + { + "t": 12.49, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.13 17.59 16.31" + }, + { + "t": 13.75, + "available_gib": 7.04, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.13 17.59 16.31" + }, + { + "t": 15.21, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.13 17.59 16.31" + }, + { + "t": 16.43, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.13 17.59 16.31" + }, + { + "t": 17.73, + "available_gib": 7.04, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.04 17.56 16.31" + }, + { + "t": 18.83, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.04 17.56 16.31" + }, + { + "t": 20.11, + "available_gib": 7.04, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.04 17.56 16.31" + } + ], + "log_tail": "/private/tmp/claude-501/bench/venv/lib/python3.14/site-packages/mlx_lm/server.py:1723: UserWarning: mlx_lm.server is not recommended for production as it only implements basic security checks.\n warnings.warn(\n2026-10-31 11:43:58,588 - INFO - Starting httpd at 127.0.0.1 on port 56102...\n127.0.0.1 - - [31/Oct/2026 11:43:59] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:43:59] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:44:01,729 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:44:01,730 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:44:01,730 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:44:01,730 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:44:01] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:44:02,330 - INFO - Prompt processing progress: 37/41\n2026-10-31 11:44:02,363 - INFO - Prompt processing progress: 40/41\n2026-10-31 11:44:02,434 - INFO - Prompt processing progress: 41/41\n2026-10-31 11:44:09,225 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:44:09,227 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:44:09,227 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:44:09,227 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:44:09] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:44:09,782 - INFO - Prompt processing progress: 1/1\n", + "ready_s": 13.880224542001088, + "ttft_s": 0.5945795839998027, + "total_s": 6.933723250000185, + "completion_tokens": 256, + "token_count_source": "usage", + "text": "A rainbow is a spectacular natural phenomenon that occurs when sunlight interacts with water droplets in the atmosphere, creating a spectrum of colors that we see in the sky. This optical effect is a complex interplay of physics involving refraction, reflection, dispersion, and the viewing angle. To understand how a rainbow forms, one must first consider the properties of light and the behavior of water droplets.\n\nThe process begins with the entry of sunlight into a water droplet. Sunlight is composed of all visible wavelengths of the electromagnetic spectrum, ranging from red to violet. As these rays enter the droplet, they encounter a medium with a higher refractive index than air. Consequently, the light slows down and bends, a process known as refraction. Because different wavelengths of light travel at different speeds, they refract at slightly different angles. Red light, having the longest wavelength, bends the least, while violet light, with the shortest wavelength, bends the most. This separation of colors is called dispersion.\n\nUpon entering the droplet, the light ray undergoes a second refraction as it exits the back of the droplet. However, the path of the light ray is not a simple straight line; it is bent by the curvature of the droplet. The light ray then", + "served_model": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-2B-MLX-4bit/snapshots/93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "93e110ad5d5c0397fa2df84ca65075a8dc1f844f1d579ceb9b2cca1597c71d76", + "decode_tps": 40.226253487151155, + "min_available_gib": 6.72, + "peak_temp_c": 0, + "round": 1, + "admission_available_gib": 9.12 + }, + { + "arm": "tensorfold-metal", + "stopped": null, + "samples": [ + { + "t": 0.11, + "available_gib": 9.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.40 17.42 16.27" + }, + { + "t": 1.65, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.40 17.42 16.27" + }, + { + "t": 3.21, + "available_gib": 4.48, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.53 17.43 16.28" + }, + { + "t": 4.44, + "available_gib": 5.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.53 17.43 16.28" + }, + { + "t": 5.6, + "available_gib": 6.08, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.53 17.43 16.28" + }, + { + "t": 6.82, + "available_gib": 6.08, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.53 17.43 16.28" + }, + { + "t": 7.87, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.53 17.43 16.28" + }, + { + "t": 9.16, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.32 17.37 16.26" + }, + { + "t": 10.9, + "available_gib": 5.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.32 17.37 16.26" + }, + { + "t": 12.51, + "available_gib": 5.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.32 17.37 16.26" + }, + { + "t": 13.87, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.30 17.35 16.26" + }, + { + "t": 15.01, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.30 17.35 16.26" + }, + { + "t": 16.25, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.30 17.35 16.26" + }, + { + "t": 17.39, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.30 17.35 16.26" + } + ], + "log_tail": "info: qwen3.5: 2.3 GiB kept wired between requests, inside 70% of 16 GiB of RAM\nwarning: memory pressure: warning: one more stream only when none runs, no new kept prompt states, the prompt cache sheds toward 0 MiB\n[tensorfold] prompt chunks of up to 128 tokens, cut at replies 256+ tokens apart\n[tensorfold] serving 93760be4f1f69842a46bc13dbdc0f19e291392a3 at http://127.0.0.1:56174/v1 (sampling: greedy; drafts: off; context: 1024; loaded in 3.8s)\n[tensorfold] 127.0.0.1 \"GET /health HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"GET /v1/models HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-b5f108dc10811be93e4c2b1b318c37a2 prompt=41 cached=0 thinking=False effort=none tokens=256 sha=371840ba123c finish=length rounds=255 accepted=0/0 ttft=0.102 decode=6.748 elapsed=6.850 queued=0.000 tok/s=37.8\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-b03a388d9386b154bab70f3afb3efd69 prompt=41 cached=0 thinking=False effort=none tokens=256 sha=371840ba123c finish=length rounds=255 accepted=0/0 ttft=0.094 decode=6.726 elapsed=6.820 queued=0.000 tok/s=37.9\n[tensorfold] shutting down (SIGTERM): draining: new requests get 503; 0 request(s) in progress may finish within 20.0 s\n[tensorfold] shutting down: drained\n[tensorfold] shutting down: stopped\n", + "ready_s": 10.889959250000175, + "ttft_s": 0.09530137500041747, + "total_s": 6.821432000000641, + "completion_tokens": 256, + "token_count_source": "usage", + "text": "A rainbow is a spectacular natural phenomenon that occurs when sunlight interacts with water droplets in the atmosphere, creating a spectrum of colors that we see in the sky. This optical effect is a complex interplay of physics involving refraction, reflection, dispersion, and the viewing angle. To understand how a rainbow forms, one must first consider the properties of light and the behavior of water droplets.\n\nThe process begins with the entry of sunlight into a water droplet. Sunlight is composed of all visible wavelengths of the electromagnetic spectrum, ranging from red to violet. As these rays enter the droplet, they encounter a medium with a higher refractive index than air. Consequently, the light slows down and bends, a process known as refraction. Because different wavelengths of light travel at different speeds, they refract at slightly different angles. Red light, having the longest wavelength, bends the least, while violet light, with the shortest wavelength, bends the most. This separation of colors is called dispersion.\n\nUpon entering the droplet, the light ray undergoes a second refraction as it exits the back of the droplet. However, the path of the light ray is not a straight line; it curves inward as it travels through the droplet. This curvature is the result", + "served_model": "93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "3f3ac70c5d72eda0b31229d8da4d90ca6fce34ca69e6ebc24a3d6502fc816990", + "decode_tps": 37.91184177306868, + "min_available_gib": 4.48, + "peak_temp_c": 0, + "round": 1, + "admission_available_gib": 9.44 + }, + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.11, + "available_gib": 9.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.43 17.36 16.27" + }, + { + "t": 1.16, + "available_gib": 9.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.43 17.36 16.27" + }, + { + "t": 2.23, + "available_gib": 9.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.32 17.32 16.26" + }, + { + "t": 3.66, + "available_gib": 7.52, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.32 17.32 16.26" + }, + { + "t": 4.8, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.32 17.32 16.26" + }, + { + "t": 6.16, + "available_gib": 7.52, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.32 17.32 16.26" + }, + { + "t": 7.25, + "available_gib": 7.36, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "15.89 17.21 16.23" + }, + { + "t": 8.34, + "available_gib": 7.36, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "15.89 17.21 16.23" + }, + { + "t": 9.56, + "available_gib": 7.36, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "15.89 17.21 16.23" + }, + { + "t": 10.81, + "available_gib": 7.36, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "15.89 17.21 16.23" + }, + { + "t": 12.3, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.30 17.27 16.26" + }, + { + "t": 13.54, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.30 17.27 16.26" + }, + { + "t": 14.67, + "available_gib": 7.84, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.30 17.27 16.26" + }, + { + "t": 15.82, + "available_gib": 8.32, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.30 17.27 16.26" + }, + { + "t": 17.03, + "available_gib": 8.48, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.30 17.27 16.26" + }, + { + "t": 18.11, + "available_gib": 8.32, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.44 17.29 16.27" + }, + { + "t": 19.25, + "available_gib": 8.48, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.44 17.29 16.27" + } + ], + "log_tail": "/private/tmp/claude-501/bench/venv/lib/python3.14/site-packages/mlx_lm/server.py:1723: UserWarning: mlx_lm.server is not recommended for production as it only implements basic security checks.\n warnings.warn(\n2026-10-31 11:44:42,885 - INFO - Starting httpd at 127.0.0.1 on port 56228...\n127.0.0.1 - - [31/Oct/2026 11:44:43] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:44:43] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:44:45,870 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:44:45,870 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:44:45,870 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:44:45,870 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:44:45] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:44:46,358 - INFO - Prompt processing progress: 37/41\n2026-10-31 11:44:46,397 - INFO - Prompt processing progress: 40/41\n2026-10-31 11:44:46,514 - INFO - Prompt processing progress: 41/41\n2026-10-31 11:44:52,828 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:44:52,828 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:44:52,828 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:44:52,828 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:44:52] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:44:53,336 - INFO - Prompt processing progress: 1/1\n", + "ready_s": 12.430404124999768, + "ttft_s": 0.5513630000004923, + "total_s": 6.825524292000409, + "completion_tokens": 256, + "token_count_source": "usage", + "text": "A rainbow is a spectacular natural phenomenon that occurs when sunlight interacts with water droplets in the atmosphere, creating a spectrum of colors that we see in the sky. This optical effect is a complex interplay of physics involving refraction, reflection, dispersion, and the viewing angle. To understand how a rainbow forms, one must first consider the properties of light and the behavior of water droplets.\n\nThe process begins with the entry of sunlight into a water droplet. Sunlight is composed of all visible wavelengths of the electromagnetic spectrum, ranging from red to violet. As these rays enter the droplet, they encounter a medium with a higher refractive index than air. Consequently, the light slows down and bends, a process known as refraction. Because different wavelengths of light travel at different speeds, they refract at slightly different angles. Red light, having the longest wavelength, bends the least, while violet light, with the shortest wavelength, bends the most. This separation of colors is called dispersion.\n\nUpon entering the droplet, the light ray undergoes a second refraction as it exits the back of the droplet. However, the path of the light ray is not a simple straight line; it is bent by the curvature of the droplet. The light ray then", + "served_model": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-2B-MLX-4bit/snapshots/93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "93e110ad5d5c0397fa2df84ca65075a8dc1f844f1d579ceb9b2cca1597c71d76", + "decode_tps": 40.642882471820805, + "min_available_gib": 7.2, + "peak_temp_c": 0, + "round": 2, + "admission_available_gib": 9.28 + }, + { + "arm": "tensorfold-metal", + "stopped": null, + "samples": [ + { + "t": 0.48, + "available_gib": 10.56, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.56 17.30 16.28" + }, + { + "t": 1.87, + "available_gib": 6.08, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.56 17.30 16.28" + }, + { + "t": 3.48, + "available_gib": 4.32, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.56 17.30 16.28" + }, + { + "t": 4.76, + "available_gib": 5.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.60 17.29 16.28" + }, + { + "t": 6.34, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.60 17.29 16.28" + }, + { + "t": 8.02, + "available_gib": 5.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.60 17.29 16.28" + }, + { + "t": 10.09, + "available_gib": 4.48, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.59 17.49 16.36" + }, + { + "t": 11.68, + "available_gib": 4.64, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.59 17.49 16.36" + }, + { + "t": 13.53, + "available_gib": 5.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.59 17.49 16.36" + }, + { + "t": 15.53, + "available_gib": 4.48, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.54 17.48 16.36" + }, + { + "t": 16.75, + "available_gib": 4.64, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.54 17.48 16.36" + }, + { + "t": 18.71, + "available_gib": 4.8, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.54 17.48 16.36" + } + ], + "log_tail": "info: qwen3.5: 2.3 GiB kept wired between requests, inside 70% of 16 GiB of RAM\nwarning: memory pressure: warning: one more stream only when none runs, no new kept prompt states, the prompt cache sheds toward 0 MiB\n[tensorfold] prompt chunks of up to 128 tokens, cut at replies 256+ tokens apart\n[tensorfold] serving 93760be4f1f69842a46bc13dbdc0f19e291392a3 at http://127.0.0.1:56299/v1 (sampling: greedy; drafts: off; context: 1024; loaded in 4.0s)\n[tensorfold] 127.0.0.1 \"GET /health HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"GET /v1/models HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-4b4932f2bb07f48f734ef9a257ac5b92 prompt=41 cached=0 thinking=False effort=none tokens=256 sha=371840ba123c finish=length rounds=255 accepted=0/0 ttft=0.097 decode=7.496 elapsed=7.593 queued=0.000 tok/s=34.0\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-ea7e6b1dd17dbba67aaa660cc37cf00b prompt=41 cached=0 thinking=False effort=none tokens=256 sha=371840ba123c finish=length rounds=255 accepted=0/0 ttft=0.147 decode=7.360 elapsed=7.507 queued=0.001 tok/s=34.6\n[tensorfold] shutting down (SIGTERM): draining: new requests get 503; 0 request(s) in progress may finish within 20.0 s\n[tensorfold] shutting down: drained\n[tensorfold] shutting down: stopped\n", + "ready_s": 11.856769375001022, + "ttft_s": 0.1485493750005844, + "total_s": 7.508524124999894, + "completion_tokens": 256, + "token_count_source": "usage", + "text": "A rainbow is a spectacular natural phenomenon that occurs when sunlight interacts with water droplets in the atmosphere, creating a spectrum of colors that we see in the sky. This optical effect is a complex interplay of physics involving refraction, reflection, dispersion, and the viewing angle. To understand how a rainbow forms, one must first consider the properties of light and the behavior of water droplets.\n\nThe process begins with the entry of sunlight into a water droplet. Sunlight is composed of all visible wavelengths of the electromagnetic spectrum, ranging from red to violet. As these rays enter the droplet, they encounter a medium with a higher refractive index than air. Consequently, the light slows down and bends, a process known as refraction. Because different wavelengths of light travel at different speeds, they refract at slightly different angles. Red light, having the longest wavelength, bends the least, while violet light, with the shortest wavelength, bends the most. This separation of colors is called dispersion.\n\nUpon entering the droplet, the light ray undergoes a second refraction as it exits the back of the droplet. However, the path of the light ray is not a straight line; it curves inward as it travels through the droplet. This curvature is the result", + "served_model": "93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "3f3ac70c5d72eda0b31229d8da4d90ca6fce34ca69e6ebc24a3d6502fc816990", + "decode_tps": 34.646857993639706, + "min_available_gib": 4.32, + "peak_temp_c": 0, + "round": 2, + "admission_available_gib": 10.4 + }, + { + "arm": "mlx-metal", + "stopped": null, + "samples": [ + { + "t": 0.17, + "available_gib": 8.64, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.34 17.44 16.35" + }, + { + "t": 1.25, + "available_gib": 8.8, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.34 17.44 16.35" + }, + { + "t": 2.39, + "available_gib": 8.8, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.23 17.41 16.35" + }, + { + "t": 3.61, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.23 17.41 16.35" + }, + { + "t": 4.82, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.23 17.41 16.35" + }, + { + "t": 6.01, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.23 17.41 16.35" + }, + { + "t": 7.33, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.29 17.42 16.36" + }, + { + "t": 8.69, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.29 17.42 16.36" + }, + { + "t": 10.02, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.29 17.42 16.36" + }, + { + "t": 11.26, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.29 17.42 16.36" + }, + { + "t": 12.51, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.79 17.32 16.33" + }, + { + "t": 13.7, + "available_gib": 7.52, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.79 17.32 16.33" + }, + { + "t": 14.96, + "available_gib": 8.0, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.79 17.32 16.33" + }, + { + "t": 16.13, + "available_gib": 8.32, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.79 17.32 16.33" + } + ], + "log_tail": "", + "ready_s": 10.54756124999949, + "ttft_s": 0.3743491670011281, + "total_s": 5.349249124999915, + "completion_tokens": 256, + "text": "A rainbow is a spectacular natural phenomenon that occurs when sunlight interacts with water droplets in the atmosphere, creating a spectrum of colors that we see in the sky. This optical effect is a complex interplay of physics involving refraction, reflection, dispersion, and the viewing angle. To understand how a rainbow forms, one must first consider the properties of light and the behavior of water droplets.\n\nThe process begins with the entry of sunlight into a water droplet. Sunlight is composed of all visible wavelengths of the electromagnetic spectrum, ranging from red to violet. As these rays enter the droplet, they encounter a medium with a higher refractive index than air. Consequently, the light slows down and bends, a process known as refraction. Because different wavelengths of light travel at different speeds, they refract at slightly different angles. Red light, having the longest wavelength, bends the least, while violet light, with the shortest wavelength, bends the most. This separation of colors is called dispersion.\n\nUpon entering the droplet, the light ray undergoes a second refraction as it exits the back of the droplet. However, the path of the light ray is not a simple straight line; it is bent by the curvature of the droplet. The light ray then", + "text_sha256": "93e110ad5d5c0397fa2df84ca65075a8dc1f844f1d579ceb9b2cca1597c71d76", + "decode_tps": 51.25731213750413, + "min_available_gib": 6.72, + "peak_temp_c": 0, + "round": 2, + "admission_available_gib": 8.64 + }, + { + "arm": "tensorfold-metal", + "stopped": null, + "samples": [ + { + "t": 0.1, + "available_gib": 10.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.24 17.19 16.29" + }, + { + "t": 1.7, + "available_gib": 6.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.54 17.24 16.31" + }, + { + "t": 3.0, + "available_gib": 4.32, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.54 17.24 16.31" + }, + { + "t": 4.26, + "available_gib": 5.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.54 17.24 16.31" + }, + { + "t": 5.6, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.54 17.24 16.31" + }, + { + "t": 8.59, + "available_gib": 4.96, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.14 17.35 16.36" + }, + { + "t": 9.82, + "available_gib": 5.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.14 17.35 16.36" + }, + { + "t": 11.12, + "available_gib": 5.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.14 17.35 16.36" + }, + { + "t": 12.22, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.73 17.26 16.33" + }, + { + "t": 13.62, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.73 17.26 16.33" + }, + { + "t": 15.66, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.73 17.26 16.33" + }, + { + "t": 17.08, + "available_gib": 4.96, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.95 17.51 16.42" + } + ], + "log_tail": "info: qwen3.5: 2.3 GiB kept wired between requests, inside 70% of 16 GiB of RAM\nwarning: memory pressure: warning: one more stream only when none runs, no new kept prompt states, the prompt cache sheds toward 0 MiB\n[tensorfold] prompt chunks of up to 128 tokens, cut at replies 256+ tokens apart\n[tensorfold] serving 93760be4f1f69842a46bc13dbdc0f19e291392a3 at http://127.0.0.1:56364/v1 (sampling: greedy; drafts: off; context: 1024; loaded in 3.5s)\n[tensorfold] 127.0.0.1 \"GET /health HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"GET /v1/models HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-bb7383cdafae37173160ad01564127ff prompt=41 cached=0 thinking=False effort=none tokens=256 sha=371840ba123c finish=length rounds=255 accepted=0/0 ttft=0.089 decode=7.003 elapsed=7.092 queued=0.000 tok/s=36.4\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-5a5e16e2834222adf019ccff1c1b0ef7 prompt=41 cached=0 thinking=False effort=none tokens=256 sha=371840ba123c finish=length rounds=255 accepted=0/0 ttft=0.117 decode=7.012 elapsed=7.129 queued=0.000 tok/s=36.4\n[tensorfold] shutting down (SIGTERM): draining: new requests get 503; 0 request(s) in progress may finish within 20.0 s\n[tensorfold] shutting down: drained\n[tensorfold] shutting down: stopped\n", + "ready_s": 10.699577833000149, + "ttft_s": 0.12207645800117461, + "total_s": 7.133113416000924, + "completion_tokens": 256, + "token_count_source": "usage", + "text": "A rainbow is a spectacular natural phenomenon that occurs when sunlight interacts with water droplets in the atmosphere, creating a spectrum of colors that we see in the sky. This optical effect is a complex interplay of physics involving refraction, reflection, dispersion, and the viewing angle. To understand how a rainbow forms, one must first consider the properties of light and the behavior of water droplets.\n\nThe process begins with the entry of sunlight into a water droplet. Sunlight is composed of all visible wavelengths of the electromagnetic spectrum, ranging from red to violet. As these rays enter the droplet, they encounter a medium with a higher refractive index than air. Consequently, the light slows down and bends, a process known as refraction. Because different wavelengths of light travel at different speeds, they refract at slightly different angles. Red light, having the longest wavelength, bends the least, while violet light, with the shortest wavelength, bends the most. This separation of colors is called dispersion.\n\nUpon entering the droplet, the light ray undergoes a second refraction as it exits the back of the droplet. However, the path of the light ray is not a straight line; it curves inward as it travels through the droplet. This curvature is the result", + "served_model": "93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "3f3ac70c5d72eda0b31229d8da4d90ca6fce34ca69e6ebc24a3d6502fc816990", + "decode_tps": 36.371224617356965, + "min_available_gib": 4.32, + "peak_temp_c": 0, + "round": 3, + "admission_available_gib": 10.24 + }, + { + "arm": "mlx-metal", + "stopped": null, + "samples": [ + { + "t": 0.24, + "available_gib": 8.96, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.80 17.48 16.42" + }, + { + "t": 1.35, + "available_gib": 9.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.80 17.48 16.42" + }, + { + "t": 2.52, + "available_gib": 9.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.80 17.48 16.42" + }, + { + "t": 3.94, + "available_gib": 7.84, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.80 17.48 16.42" + }, + { + "t": 5.12, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.13 17.56 16.45" + }, + { + "t": 6.29, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.13 17.56 16.45" + }, + { + "t": 7.65, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.13 17.56 16.45" + }, + { + "t": 8.93, + "available_gib": 7.04, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.13 17.56 16.45" + }, + { + "t": 10.18, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.52 17.65 16.49" + }, + { + "t": 11.45, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.52 17.65 16.49" + }, + { + "t": 12.72, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.52 17.65 16.49" + }, + { + "t": 14.02, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.52 17.65 16.49" + }, + { + "t": 15.12, + "available_gib": 7.52, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.60 17.89 16.58" + }, + { + "t": 16.45, + "available_gib": 7.68, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.60 17.89 16.58" + } + ], + "log_tail": "", + "ready_s": 11.319369417000416, + "ttft_s": 0.5211798750005983, + "total_s": 5.368391708001582, + "completion_tokens": 256, + "text": "A rainbow is a spectacular natural phenomenon that occurs when sunlight interacts with water droplets in the atmosphere, creating a spectrum of colors that we see in the sky. This optical effect is a complex interplay of physics involving refraction, reflection, dispersion, and the viewing angle. To understand how a rainbow forms, one must first consider the properties of light and the behavior of water droplets.\n\nThe process begins with the entry of sunlight into a water droplet. Sunlight is composed of all visible wavelengths of the electromagnetic spectrum, ranging from red to violet. As these rays enter the droplet, they encounter a medium with a higher refractive index than air. Consequently, the light slows down and bends, a process known as refraction. Because different wavelengths of light travel at different speeds, they refract at slightly different angles. Red light, having the longest wavelength, bends the least, while violet light, with the shortest wavelength, bends the most. This separation of colors is called dispersion.\n\nUpon entering the droplet, the light ray undergoes a second refraction as it exits the back of the droplet. However, the path of the light ray is not a simple straight line; it is bent by the curvature of the droplet. The light ray then", + "text_sha256": "93e110ad5d5c0397fa2df84ca65075a8dc1f844f1d579ceb9b2cca1597c71d76", + "decode_tps": 52.607562612365875, + "min_available_gib": 6.88, + "peak_temp_c": 0, + "round": 3, + "admission_available_gib": 8.96 + }, + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.11, + "available_gib": 9.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.63 17.92 16.60" + }, + { + "t": 1.5, + "available_gib": 9.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.63 17.92 16.60" + }, + { + "t": 2.69, + "available_gib": 9.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.63 17.92 16.60" + }, + { + "t": 3.95, + "available_gib": 7.04, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.63 17.92 16.60" + }, + { + "t": 5.18, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.50 17.92 16.61" + }, + { + "t": 6.48, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.50 17.92 16.61" + }, + { + "t": 7.81, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.50 17.92 16.61" + }, + { + "t": 9.03, + "available_gib": 6.56, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.50 17.92 16.61" + }, + { + "t": 10.28, + "available_gib": 6.56, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.94 18.04 16.66" + }, + { + "t": 11.57, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.94 18.04 16.66" + }, + { + "t": 12.76, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.94 18.04 16.66" + }, + { + "t": 14.16, + "available_gib": 7.04, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.94 18.04 16.66" + }, + { + "t": 15.61, + "available_gib": 7.68, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.75 18.24 16.74" + }, + { + "t": 16.81, + "available_gib": 7.84, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.75 18.24 16.74" + }, + { + "t": 17.97, + "available_gib": 8.0, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.75 18.24 16.74" + }, + { + "t": 19.12, + "available_gib": 7.84, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.75 18.24 16.74" + }, + { + "t": 20.38, + "available_gib": 8.0, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.17 18.37 16.79" + } + ], + "log_tail": "/private/tmp/claude-501/bench/venv/lib/python3.14/site-packages/mlx_lm/server.py:1723: UserWarning: mlx_lm.server is not recommended for production as it only implements basic security checks.\n warnings.warn(\n2026-10-31 11:46:30,850 - INFO - Starting httpd at 127.0.0.1 on port 56427...\n127.0.0.1 - - [31/Oct/2026 11:46:31] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:46:31] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:46:34,287 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:46:34,420 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:46:34,420 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:46:34,420 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:46:34] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:46:35,077 - INFO - Prompt processing progress: 37/41\n2026-10-31 11:46:35,114 - INFO - Prompt processing progress: 40/41\n2026-10-31 11:46:35,191 - INFO - Prompt processing progress: 41/41\n2026-10-31 11:46:42,017 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:46:42,018 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:46:42,018 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:46:42,018 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:46:42] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:46:42,412 - INFO - Prompt processing progress: 1/1\n", + "ready_s": 14.103719083001124, + "ttft_s": 0.42370525000114867, + "total_s": 6.484818832999736, + "completion_tokens": 256, + "token_count_source": "usage", + "text": "A rainbow is a spectacular natural phenomenon that occurs when sunlight interacts with water droplets in the atmosphere, creating a spectrum of colors that we see in the sky. This optical effect is a complex interplay of physics involving refraction, reflection, dispersion, and the viewing angle. To understand how a rainbow forms, one must first consider the properties of light and the behavior of water droplets.\n\nThe process begins with the entry of sunlight into a water droplet. Sunlight is composed of all visible wavelengths of the electromagnetic spectrum, ranging from red to violet. As these rays enter the droplet, they encounter a medium with a higher refractive index than air. Consequently, the light slows down and bends, a process known as refraction. Because different wavelengths of light travel at different speeds, they refract at slightly different angles. Red light, having the longest wavelength, bends the least, while violet light, with the shortest wavelength, bends the most. This separation of colors is called dispersion.\n\nUpon entering the droplet, the light ray undergoes a second refraction as it exits the back of the droplet. However, the path of the light ray is not a simple straight line; it is bent by the curvature of the droplet. The light ray then", + "served_model": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-2B-MLX-4bit/snapshots/93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "93e110ad5d5c0397fa2df84ca65075a8dc1f844f1d579ceb9b2cca1597c71d76", + "decode_tps": 42.07147688426011, + "min_available_gib": 6.56, + "peak_temp_c": 0, + "round": 3, + "admission_available_gib": 9.28 + } + ], + "runtime": { + "mlx": "0.32.3", + "mlx_lm": "0.31.3", + "transformers": "5.16.1", + "python": "3.14.3", + "device": { + "device_name": "Apple M4", + "max_recommended_working_set_size": 12713115648, + "memory_size": 17179869184, + "architecture": "applegpu_g16g", + "max_buffer_length": 9534832640, + "resource_limit": 499000 + } + }, + "tensorfold_sha256": "4b4b5a5ac152eb2dbed7e9d8004805ed9b74f77c64c3de18eb4f35263c7b9eb8", + "model_dir": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-2B-MLX-4bit/snapshots/93760be4f1f69842a46bc13dbdc0f19e291392a3", + "host_start": { + "macos": "27.0", + "build": "26A5421a", + "hw_model": "Mac16,10", + "chip": "Apple M4", + "gpu_cores": "10", + "memory_gib": 16.0, + "power": [ + "Now drawing from 'AC Power'" + ], + "low_power_mode": false, + "thermal": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", + "uptime": "11:43 up 3:02, 4 users, load averages: 16.04 17.50 16.23", + "top_cpu": [ + "42290 com.apple.Virtua 588.2 10G ", + "0 kernel_task 60.9 36M+ ", + "433 WindowServer 33.1 601M ", + "4939 Claude Helper (R 15.8 972M+ ", + "4916 Claude Helper 12.9 445M ", + "33508 top 5.7 3840K+" + ], + "memory_free_pct": "59", + "swap": "total = 4096.00M used = 2802.69M free = 1293.31M (encrypted)", + "lima": "NAME STATUS SSH VMTYPE ARCH CPUS MEMORY DISK DIR\nkbuild Running 127.0.0.1:53830 vz aarch64 8 10GiB 60GiB ~/.lima/kbuild" + }, + "summary": { + "mlx-metal": { + "ok_runs": 3, + "median_ttft_s": 0.5211798750005983, + "median_total_s": 5.368391708001582, + "median_decode_tps": 51.25731213750413, + "texts": [ + "93e110ad5d5c" + ] + }, + "lm-metal": { + "ok_runs": 3, + "median_ttft_s": 0.5513630000004923, + "median_total_s": 6.825524292000409, + "median_decode_tps": 40.642882471820805, + "texts": [ + "93e110ad5d5c" + ] + }, + "tensorfold-metal": { + "ok_runs": 3, + "median_ttft_s": 0.12207645800117461, + "median_total_s": 7.133113416000924, + "median_decode_tps": 36.371224617356965, + "texts": [ + "3f3ac70c5d72" + ] + } + }, + "host_end": { + "macos": "27.0", + "build": "26A5421a", + "hw_model": "Mac16,10", + "chip": "Apple M4", + "gpu_cores": "10", + "memory_gib": 16.0, + "power": [ + "Now drawing from 'AC Power'" + ], + "low_power_mode": false, + "thermal": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", + "uptime": "11:46 up 3:05, 4 users, load averages: 20.83 18.34 16.79", + "top_cpu": [ + "42290 com.apple.Virtua 578.0 10G ", + "433 WindowServer 29.6 603M+", + "0 kernel_task 24.7 34M- ", + "42198 limactl 15.2 98M+ ", + "4939 Claude Helper (R 15.2 997M-", + "4916 Claude Helper 9.7 429M+" + ], + "memory_free_pct": "63", + "swap": "total = 4096.00M used = 2701.44M free = 1394.56M (encrypted)", + "lima": "NAME STATUS SSH VMTYPE ARCH CPUS MEMORY DISK DIR\nkbuild Running 127.0.0.1:53830 vz aarch64 8 10GiB 60GiB ~/.lima/kbuild" + } +} diff --git a/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal-c4.json b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal-c4.json new file mode 100644 index 00000000..bb872b1d --- /dev/null +++ b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal-c4.json @@ -0,0 +1,759 @@ +{ + "status": "complete", + "model_key": "qwen2b", + "model": "mlx-community/Qwen3.5-2B-MLX-4bit@93760be4f1f69842a46bc13dbdc0f19e291392a3", + "weights_sha256": "713fe7e5d3c3965f7106b0d0ee17615f7869c23c8d327996df8c1196fbcf07d5", + "kernel": "macOS 27.0 (26A5421a)", + "driver": "Metal", + "tensorfold_rev": "v1.1.9 release", + "rapid_rev": "1d09cdb684bbf58fac130f5804b08ee2e1f56558", + "serve_concurrency": 4, + "prompt_sha256": "af125b15dfba55caedb541e89e59fe1252c0f13553e5ce6bc181e237fada9926", + "max_tokens": 32, + "policy": { + "reserve_gib": 1.0, + "max_fan_rpm": null, + "max_temp_c": 80 + }, + "allowance_gib": 4.5, + "arms": [ + "lm-metal", + "tensorfold-metal" + ], + "concurrency": 4, + "runs": [ + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.15, + "available_gib": 8.16, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.73 17.94 16.02" + }, + { + "t": 1.33, + "available_gib": 8.0, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.75 17.93 16.03" + }, + { + "t": 2.51, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.75 17.93 16.03" + }, + { + "t": 3.67, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.75 17.93 16.03" + }, + { + "t": 4.94, + "available_gib": 6.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.75 17.93 16.03" + }, + { + "t": 6.44, + "available_gib": 5.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.17 17.99 16.06" + }, + { + "t": 7.79, + "available_gib": 5.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.17 17.99 16.06" + }, + { + "t": 9.0, + "available_gib": 5.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.17 17.99 16.06" + } + ], + "log_tail": "/private/tmp/claude-501/bench/venv/lib/python3.14/site-packages/mlx_lm/server.py:1723: UserWarning: mlx_lm.server is not recommended for production as it only implements basic security checks.\n warnings.warn(\n2026-10-31 11:40:33,244 - INFO - Starting httpd at 127.0.0.1 on port 55549...\n127.0.0.1 - - [31/Oct/2026 11:40:34] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:40:34] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:40:35,756 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:40:35,756 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:40:35,756 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:40:35,756 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:40:35] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:40:36,588 - INFO - Prompt processing progress: 23/27\n2026-10-31 11:40:36,634 - INFO - Prompt processing progress: 26/27\n2026-10-31 11:40:36,680 - INFO - Prompt processing progress: 27/27\n2026-10-31 11:40:37,543 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:40:37,543 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:40:37,543 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:40:37,543 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:40:37] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:40:37,897 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:40:37,897 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:40:37,897 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:40:37,897 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:40:38] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:40:38,296 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:40:38,297 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:40:38,297 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:40:38,297 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:40:38] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:40:38,684 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:40:38,684 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:40:38,685 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:40:38,685 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:40:38] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:40:39,871 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:40:39,871 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:40:39,871 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:40:39,872 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:40:39,964 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:40:39,964 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:40:39,964 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:40:39,965 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:40:40,058 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:40:40,059 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:40:40,059 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:40:40,060 - INFO - Prompt processing progress: 33/33\n", + "ready_s": 6.396151041000849, + "concurrency": 4, + "wall_s": 3.5451855000010255, + "aggregate_tok_s": 36.105303939656466, + "per_request_decode_median": 32.36483594150255, + "ttft_s": 2.552983582998422, + "total_s": 3.5451855000010255, + "completion_tokens": 128, + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors,", + "requests": [ + { + "ttft_s": 2.553943832999721, + "total_s": 3.4822847909999837, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 2.552983582998422, + "total_s": 3.535876499998267, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "Sunlight travels through the atmosphere and gets refracted (bent) as it enters a raindrop.\nThe light then reflects off the back of the rain" + }, + { + "ttft_s": 2.551819082998918, + "total_s": 3.5360424579994287, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 2.5470707090007636, + "total_s": 3.5049003750009433, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + } + ], + "served_model": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-2B-MLX-4bit/snapshots/93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "97af50af7f7909e68d8670dd08652563bf801d48f9186dcd506215b7d652c5db", + "decode_tps": 36.105303939656466, + "min_available_gib": 5.12, + "peak_temp_c": 0, + "round": 1, + "admission_available_gib": 8.16 + }, + { + "arm": "tensorfold-metal", + "stopped": null, + "samples": [ + { + "t": 0.13, + "available_gib": 7.68, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.72 18.09 16.11" + }, + { + "t": 1.47, + "available_gib": 4.64, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.72 18.09 16.11" + }, + { + "t": 2.56, + "available_gib": 4.16, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.72 18.09 16.11" + }, + { + "t": 3.9, + "available_gib": 4.96, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.50 18.04 16.10" + }, + { + "t": 5.02, + "available_gib": 5.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.50 18.04 16.10" + }, + { + "t": 6.18, + "available_gib": 5.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.50 18.04 16.10" + }, + { + "t": 7.57, + "available_gib": 4.96, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.50 18.04 16.10" + } + ], + "log_tail": "info: qwen3.5: 2.3 GiB kept wired between requests, inside 70% of 16 GiB of RAM\nwarning: memory pressure: warning: one more stream only when none runs, no new kept prompt states, the prompt cache sheds toward 0 MiB\n[tensorfold] prompt chunks of up to 128 tokens, cut at replies 256+ tokens apart\n[tensorfold] serving 93760be4f1f69842a46bc13dbdc0f19e291392a3 at http://127.0.0.1:55588/v1 (sampling: greedy; drafts: off; context: 1024; loaded in 2.9s)\n[tensorfold] 127.0.0.1 \"GET /health HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"GET /v1/models HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-f00737caa82cf3785159ea27a29793bf prompt=27 cached=0 thinking=False effort=none tokens=32 sha=778f678f6c4f finish=length rounds=31 accepted=0/0 ttft=0.069 decode=0.846 elapsed=0.915 queued=0.000 tok/s=36.6\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-600447f2a0887ac2780018d210af5b33 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=0.104 decode=0.806 elapsed=0.910 queued=0.000 tok/s=38.5\n[tensorfold] done chatcmpl-02685e8c18c7851be53fcfb2ae66d65e prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=0.971 decode=0.689 elapsed=1.660 queued=0.903 tok/s=45.0\n[tensorfold] done chatcmpl-cb800ba5131f1e9787e65ab355ac2ad0 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=1.718 decode=0.801 elapsed=2.518 queued=1.645 tok/s=38.7\n[tensorfold] done chatcmpl-5accd234a5db3a769dff5e3645ed9c27 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=2.594 decode=0.861 elapsed=3.455 queued=2.509 tok/s=36.0\n[tensorfold] shutting down (SIGTERM): draining: new requests get 503; 0 request(s) in progress may finish within 20.0 s\n[tensorfold] shutting down: drained\n[tensorfold] shutting down: stopped\n", + "ready_s": 4.023085541999535, + "concurrency": 4, + "wall_s": 3.4747828750005283, + "aggregate_tok_s": 36.83683401368799, + "per_request_decode_median": 38.7009264030017, + "ttft_s": 1.7217433750010969, + "total_s": 3.4747828750005283, + "completion_tokens": 128, + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors,", + "requests": [ + { + "ttft_s": 0.11420504200032155, + "total_s": 0.9185534580010426, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 0.9753734170008102, + "total_s": 1.664373584000714, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 2.6047381249991304, + "total_s": 3.466402374999234, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 1.7217433750010969, + "total_s": 2.5227577919995383, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + } + ], + "served_model": "93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "97af50af7f7909e68d8670dd08652563bf801d48f9186dcd506215b7d652c5db", + "decode_tps": 36.83683401368799, + "min_available_gib": 4.16, + "peak_temp_c": 0, + "round": 1, + "admission_available_gib": 7.68 + }, + { + "arm": "tensorfold-metal", + "stopped": null, + "samples": [ + { + "t": 0.15, + "available_gib": 9.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.46 18.02 16.11" + }, + { + "t": 1.33, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.46 18.02 16.11" + }, + { + "t": 2.43, + "available_gib": 4.16, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.94 17.91 16.08" + }, + { + "t": 3.52, + "available_gib": 5.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.94 17.91 16.08" + }, + { + "t": 5.05, + "available_gib": 5.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.94 17.91 16.08" + }, + { + "t": 6.32, + "available_gib": 5.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.94 17.91 16.08" + }, + { + "t": 7.84, + "available_gib": 5.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.87 17.88 16.08" + } + ], + "log_tail": "info: qwen3.5: 2.3 GiB kept wired between requests, inside 70% of 16 GiB of RAM\nwarning: memory pressure: warning: one more stream only when none runs, no new kept prompt states, the prompt cache sheds toward 0 MiB\n[tensorfold] prompt chunks of up to 128 tokens, cut at replies 256+ tokens apart\n[tensorfold] serving 93760be4f1f69842a46bc13dbdc0f19e291392a3 at http://127.0.0.1:55625/v1 (sampling: greedy; drafts: off; context: 1024; loaded in 3.2s)\n[tensorfold] 127.0.0.1 \"GET /health HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"GET /v1/models HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-97b57091d7f03d12112e1d3d93e602f2 prompt=27 cached=0 thinking=False effort=none tokens=32 sha=778f678f6c4f finish=length rounds=31 accepted=0/0 ttft=0.069 decode=0.891 elapsed=0.960 queued=0.000 tok/s=34.8\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-4c458c77a137f38f99ccf502b8b71af3 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=0.104 decode=0.899 elapsed=1.003 queued=0.001 tok/s=34.5\n[tensorfold] done chatcmpl-6453e49cd87aea1ab0cf38c5665f7b2a prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=1.084 decode=0.856 elapsed=1.939 queued=0.996 tok/s=36.2\n[tensorfold] done chatcmpl-0b0d753557e31713209015f9b5dbaaf9 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=2.011 decode=0.830 elapsed=2.841 queued=1.933 tok/s=37.4\n[tensorfold] done chatcmpl-42aa5f9088217675924055bfae6c749f prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=2.927 decode=0.866 elapsed=3.793 queued=2.835 tok/s=35.8\n[tensorfold] shutting down (SIGTERM): draining: new requests get 503; 0 request(s) in progress may finish within 20.0 s\n[tensorfold] shutting down: drained\n[tensorfold] shutting down: stopped\n", + "ready_s": 4.558609333000277, + "concurrency": 4, + "wall_s": 3.80615724999916, + "aggregate_tok_s": 33.629719318619394, + "per_request_decode_median": 36.219591790598194, + "ttft_s": 2.0125052090006648, + "total_s": 3.80615724999916, + "completion_tokens": 128, + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors,", + "requests": [ + { + "ttft_s": 0.11318312500043248, + "total_s": 1.0092080829999759, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 1.0868276670007617, + "total_s": 1.9427180420007062, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 2.0125052090006648, + "total_s": 2.849204459000248, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 2.930152208000436, + "total_s": 3.796254749999207, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + } + ], + "served_model": "93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "97af50af7f7909e68d8670dd08652563bf801d48f9186dcd506215b7d652c5db", + "decode_tps": 33.629719318619394, + "min_available_gib": 4.16, + "peak_temp_c": 0, + "round": 2, + "admission_available_gib": 8.96 + }, + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.15, + "available_gib": 8.8, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.87 17.88 16.08" + }, + { + "t": 1.26, + "available_gib": 9.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.56 17.79 16.06" + }, + { + "t": 2.41, + "available_gib": 9.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.56 17.79 16.06" + }, + { + "t": 3.67, + "available_gib": 8.64, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.56 17.79 16.06" + }, + { + "t": 4.88, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.56 17.79 16.06" + }, + { + "t": 6.11, + "available_gib": 6.56, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.15 17.90 16.10" + }, + { + "t": 7.21, + "available_gib": 6.56, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.15 17.90 16.10" + }, + { + "t": 8.42, + "available_gib": 6.56, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.15 17.90 16.10" + }, + { + "t": 9.52, + "available_gib": 6.4, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.15 17.90 16.10" + }, + { + "t": 10.78, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.14 17.88 16.11" + } + ], + "log_tail": "/private/tmp/claude-501/bench/venv/lib/python3.14/site-packages/mlx_lm/server.py:1723: UserWarning: mlx_lm.server is not recommended for production as it only implements basic security checks.\n warnings.warn(\n2026-10-31 11:41:10,010 - INFO - Starting httpd at 127.0.0.1 on port 55681...\n127.0.0.1 - - [31/Oct/2026 11:41:10] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:41:10] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:41:13,291 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:41:13,291 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:41:13,291 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:41:13,291 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:41:13] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:41:14,550 - INFO - Prompt processing progress: 23/27\n2026-10-31 11:41:14,596 - INFO - Prompt processing progress: 26/27\n2026-10-31 11:41:14,638 - INFO - Prompt processing progress: 27/27\n2026-10-31 11:41:15,461 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:41:15,461 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:41:15,461 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:41:15,461 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:41:15] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:41:15,754 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:41:15,754 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:41:15,754 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:41:15,754 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:41:15] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:41:16,127 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:41:16,127 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:41:16,127 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:41:16,127 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:41:16] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:41:16,446 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:41:16,447 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:41:16,447 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:41:16,447 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:41:16] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:41:17,126 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:41:17,126 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:41:17,127 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:41:17,127 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:41:17,192 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:41:17,192 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:41:17,192 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:41:17,193 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:41:17,274 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:41:17,275 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:41:17,276 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:41:17,278 - INFO - Prompt processing progress: 33/33\n", + "ready_s": 8.610221208999064, + "concurrency": 4, + "wall_s": 3.0747535420014174, + "aggregate_tok_s": 41.62935280877253, + "per_request_decode_median": 26.165432635901997, + "ttft_s": 1.8614164169994183, + "total_s": 3.0747535420014174, + "completion_tokens": 128, + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors,", + "requests": [ + { + "ttft_s": 1.8580361669992271, + "total_s": 3.0209809579991997, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 1.8614164169994183, + "total_s": 3.0674874579999596, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "Sunlight travels through the atmosphere and gets refracted (bent) as it enters a raindrop.\nThe light then reflects off the back of the rain" + }, + { + "ttft_s": 1.8563908329997503, + "total_s": 3.0411600830011594, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 1.8688334159996884, + "total_s": 3.0692934159997094, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + } + ], + "served_model": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-2B-MLX-4bit/snapshots/93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "97af50af7f7909e68d8670dd08652563bf801d48f9186dcd506215b7d652c5db", + "decode_tps": 41.62935280877253, + "min_available_gib": 5.76, + "peak_temp_c": 0, + "round": 2, + "admission_available_gib": 8.8 + }, + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.18, + "available_gib": 8.0, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.14 17.88 16.11" + }, + { + "t": 1.29, + "available_gib": 8.32, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.21 17.88 16.12" + }, + { + "t": 2.59, + "available_gib": 8.8, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.21 17.88 16.12" + }, + { + "t": 3.94, + "available_gib": 6.08, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.21 17.88 16.12" + }, + { + "t": 5.28, + "available_gib": 6.08, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.21 17.88 16.12" + }, + { + "t": 6.52, + "available_gib": 6.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.35 17.90 16.14" + }, + { + "t": 7.67, + "available_gib": 6.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.35 17.90 16.14" + }, + { + "t": 9.01, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.35 17.90 16.14" + } + ], + "log_tail": "/private/tmp/claude-501/bench/venv/lib/python3.14/site-packages/mlx_lm/server.py:1723: UserWarning: mlx_lm.server is not recommended for production as it only implements basic security checks.\n warnings.warn(\n2026-10-31 11:41:24,248 - INFO - Starting httpd at 127.0.0.1 on port 55726...\n127.0.0.1 - - [31/Oct/2026 11:41:25] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:41:25] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:41:27,159 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:41:27,159 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:41:27,159 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:41:27,159 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:41:27] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:41:27,706 - INFO - Prompt processing progress: 23/27\n2026-10-31 11:41:27,762 - INFO - Prompt processing progress: 26/27\n2026-10-31 11:41:27,839 - INFO - Prompt processing progress: 27/27\n2026-10-31 11:41:28,591 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:41:28,591 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:41:28,591 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:41:28,591 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:41:28] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:41:28,863 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:41:28,863 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:41:28,863 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:41:28,863 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:41:28] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:41:29,141 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:41:29,141 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:41:29,141 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:41:29,141 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:41:29] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:41:29,395 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:41:29,395 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:41:29,395 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:41:29,395 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:41:29] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:41:29,846 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:41:29,848 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:41:29,848 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:41:29,848 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:41:29,912 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:41:29,912 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:41:29,912 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:41:29,913 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:41:29,981 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:41:29,983 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:41:29,984 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:41:29,984 - INFO - Prompt processing progress: 33/33\n", + "ready_s": 6.918552000001, + "concurrency": 4, + "wall_s": 2.7385153750001336, + "aggregate_tok_s": 46.74065414001693, + "per_request_decode_median": 24.50884472817889, + "ttft_s": 1.428902207999272, + "total_s": 2.7385153750001336, + "completion_tokens": 128, + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors,", + "requests": [ + { + "ttft_s": 1.4276031669996883, + "total_s": 2.6601889169996866, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 1.428598291000526, + "total_s": 2.693447790999926, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "Sunlight travels through the atmosphere and gets refracted (bent) as it enters a raindrop.\nThe light then reflects off the back of the rain" + }, + { + "ttft_s": 1.428902207999272, + "total_s": 2.7305033749998984, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 1.4306371250004304, + "total_s": 2.732846333999987, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + } + ], + "served_model": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-2B-MLX-4bit/snapshots/93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "97af50af7f7909e68d8670dd08652563bf801d48f9186dcd506215b7d652c5db", + "decode_tps": 46.74065414001693, + "min_available_gib": 5.6, + "peak_temp_c": 0, + "round": 3, + "admission_available_gib": 7.84 + }, + { + "arm": "tensorfold-metal", + "stopped": null, + "samples": [ + { + "t": 0.38, + "available_gib": 8.32, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.08 17.84 16.12" + }, + { + "t": 1.81, + "available_gib": 4.96, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.08 17.84 16.12" + }, + { + "t": 2.95, + "available_gib": 4.48, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.08 17.82 16.13" + }, + { + "t": 4.07, + "available_gib": 5.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.08 17.82 16.13" + }, + { + "t": 5.17, + "available_gib": 5.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.08 17.82 16.13" + }, + { + "t": 6.39, + "available_gib": 5.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.08 17.82 16.13" + }, + { + "t": 7.49, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.08 17.82 16.13" + } + ], + "log_tail": "info: qwen3.5: 2.3 GiB kept wired between requests, inside 70% of 16 GiB of RAM\nwarning: memory pressure: warning: one more stream only when none runs, no new kept prompt states, the prompt cache sheds toward 0 MiB\n[tensorfold] prompt chunks of up to 128 tokens, cut at replies 256+ tokens apart\n[tensorfold] serving 93760be4f1f69842a46bc13dbdc0f19e291392a3 at http://127.0.0.1:55774/v1 (sampling: greedy; drafts: off; context: 1024; loaded in 2.9s)\n[tensorfold] 127.0.0.1 \"GET /health HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"GET /v1/models HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-b37844e5ba669c71f732671960e5400c prompt=27 cached=0 thinking=False effort=none tokens=32 sha=778f678f6c4f finish=length rounds=31 accepted=0/0 ttft=0.064 decode=0.763 elapsed=0.827 queued=0.000 tok/s=40.6\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-1fb115660c3f424937932624a65edbe0 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=0.074 decode=0.787 elapsed=0.862 queued=0.001 tok/s=39.4\n[tensorfold] done chatcmpl-751e69d08d623a7155bfa994fd7c4607 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=0.929 decode=0.817 elapsed=1.746 queued=0.854 tok/s=37.9\n[tensorfold] done chatcmpl-c12b039195f211cd2187b6e8c8d59d08 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=1.818 decode=0.802 elapsed=2.620 queued=1.742 tok/s=38.7\n[tensorfold] done chatcmpl-1c3270bb633dc76dfcc44aae49726af9 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=2.690 decode=0.770 elapsed=3.460 queued=2.614 tok/s=40.2\n[tensorfold] shutting down (SIGTERM): draining: new requests get 503; 0 request(s) in progress may finish within 20.0 s\n[tensorfold] shutting down: drained\n[tensorfold] shutting down: stopped\n", + "ready_s": 3.943455457998425, + "concurrency": 4, + "wall_s": 3.467200041999604, + "aggregate_tok_s": 36.91739687629325, + "per_request_decode_median": 39.352875825868615, + "ttft_s": 1.8212079160002759, + "total_s": 3.467200041999604, + "completion_tokens": 128, + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors,", + "requests": [ + { + "ttft_s": 0.07550454199918022, + "total_s": 0.8632487499999115, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 1.8212079160002759, + "total_s": 2.6220207910009776, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 0.9307119169989164, + "total_s": 1.748270333999244, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 2.6914230000002135, + "total_s": 3.4622863330005202, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + } + ], + "served_model": "93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "97af50af7f7909e68d8670dd08652563bf801d48f9186dcd506215b7d652c5db", + "decode_tps": 36.91739687629325, + "min_available_gib": 4.48, + "peak_temp_c": 0, + "round": 3, + "admission_available_gib": 8.32 + } + ], + "runtime": { + "mlx": "0.32.3", + "mlx_lm": "0.31.3", + "transformers": "5.16.1", + "python": "3.14.3", + "device": { + "device_name": "Apple M4", + "max_recommended_working_set_size": 12713115648, + "memory_size": 17179869184, + "architecture": "applegpu_g16g", + "max_buffer_length": 9534832640, + "resource_limit": 499000 + } + }, + "tensorfold_sha256": "4b4b5a5ac152eb2dbed7e9d8004805ed9b74f77c64c3de18eb4f35263c7b9eb8", + "model_dir": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-2B-MLX-4bit/snapshots/93760be4f1f69842a46bc13dbdc0f19e291392a3", + "host_start": { + "macos": "27.0", + "build": "26A5421a", + "hw_model": "Mac16,10", + "chip": "Apple M4", + "gpu_cores": "10", + "memory_gib": 16.0, + "power": [ + "Now drawing from 'AC Power'" + ], + "low_power_mode": false, + "thermal": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", + "uptime": "11:40 up 2:59, 4 users, load averages: 16.73 17.94 16.02", + "top_cpu": [ + "42290 com.apple.Virtua 549.6 10G ", + "0 kernel_task 66.8 34M- ", + "433 WindowServer 33.7 601M ", + "88301 Python 22.9 27M ", + "4916 Claude Helper 12.3 431M-", + "42198 limactl 11.3 81M" + ], + "memory_free_pct": "51", + "swap": "total = 4096.00M used = 2882.69M free = 1213.31M (encrypted)", + "lima": "NAME STATUS SSH VMTYPE ARCH CPUS MEMORY DISK DIR\nkbuild Running 127.0.0.1:53830 vz aarch64 8 10GiB 60GiB ~/.lima/kbuild" + }, + "summary": { + "lm-metal": { + "ok_runs": 3, + "median_ttft_s": 1.8614164169994183, + "median_total_s": 3.0747535420014174, + "median_decode_tps": 41.62935280877253, + "texts": [ + "97af50af7f79" + ] + }, + "tensorfold-metal": { + "ok_runs": 3, + "median_ttft_s": 1.8212079160002759, + "median_total_s": 3.4747828750005283, + "median_decode_tps": 36.83683401368799, + "texts": [ + "97af50af7f79" + ] + } + }, + "host_end": { + "macos": "27.0", + "build": "26A5421a", + "hw_model": "Mac16,10", + "chip": "Apple M4", + "gpu_cores": "10", + "memory_gib": 16.0, + "power": [ + "Now drawing from 'AC Power'" + ], + "low_power_mode": false, + "thermal": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", + "uptime": "11:41 up 3 hrs, 4 users, load averages: 17.85 17.96 16.20", + "top_cpu": [ + "42290 com.apple.Virtua 565.5 10G ", + "26733 Python 48.0 64M+ ", + "433 WindowServer 34.4 608M+", + "0 kernel_task 26.7 34M+ ", + "4939 Claude Helper (R 13.6 944M+", + "42198 limactl 12.9 82M" + ], + "memory_free_pct": "53", + "swap": "total = 4096.00M used = 3378.75M free = 717.25M (encrypted)", + "lima": "NAME STATUS SSH VMTYPE ARCH CPUS MEMORY DISK DIR\nkbuild Running 127.0.0.1:53830 vz aarch64 8 10GiB 60GiB ~/.lima/kbuild" + } +} diff --git a/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal-c8.json b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal-c8.json new file mode 100644 index 00000000..329fd489 --- /dev/null +++ b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal-c8.json @@ -0,0 +1,805 @@ +{ + "status": "complete", + "model_key": "qwen2b", + "model": "mlx-community/Qwen3.5-2B-MLX-4bit@93760be4f1f69842a46bc13dbdc0f19e291392a3", + "weights_sha256": "713fe7e5d3c3965f7106b0d0ee17615f7869c23c8d327996df8c1196fbcf07d5", + "kernel": "macOS 27.0 (26A5421a)", + "driver": "Metal", + "tensorfold_rev": "v1.1.9 release", + "rapid_rev": "1d09cdb684bbf58fac130f5804b08ee2e1f56558", + "serve_concurrency": 4, + "prompt_sha256": "af125b15dfba55caedb541e89e59fe1252c0f13553e5ce6bc181e237fada9926", + "max_tokens": 32, + "policy": { + "reserve_gib": 1.0, + "max_fan_rpm": null, + "max_temp_c": 80 + }, + "allowance_gib": 4.5, + "arms": [ + "lm-metal", + "tensorfold-metal" + ], + "concurrency": 8, + "runs": [ + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.15, + "available_gib": 9.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.78 17.94 16.20" + }, + { + "t": 1.39, + "available_gib": 9.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.78 17.94 16.20" + }, + { + "t": 2.78, + "available_gib": 8.64, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.52 18.09 16.26" + }, + { + "t": 4.11, + "available_gib": 6.56, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.52 18.09 16.26" + }, + { + "t": 5.31, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.52 18.09 16.26" + }, + { + "t": 6.56, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.52 18.09 16.26" + }, + { + "t": 8.0, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.71 18.14 16.29" + }, + { + "t": 9.23, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.71 18.14 16.29" + }, + { + "t": 10.67, + "available_gib": 6.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.71 18.14 16.29" + }, + { + "t": 12.06, + "available_gib": 6.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.90 18.19 16.32" + } + ], + "log_tail": "-31 11:41:58,012 - INFO - Starting httpd at 127.0.0.1 on port 55815...\n127.0.0.1 - - [31/Oct/2026 11:41:58] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:41:58] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:42:01,018 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:42:01,018 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:42:01,018 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:42:01,018 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:42:01] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:42:01,722 - INFO - Prompt processing progress: 23/27\n2026-10-31 11:42:01,773 - INFO - Prompt processing progress: 26/27\n2026-10-31 11:42:01,820 - INFO - Prompt processing progress: 27/27\n2026-10-31 11:42:02,689 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:42:02,690 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:42:02,690 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:42:02,690 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:42:03] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:42:03,073 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:42:03,073 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:42:03,073 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:42:03,073 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:42:03] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:42:03,449 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:42:03,452 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:42:03,453 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:42:03,453 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:42:03] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:42:03,850 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:42:03,850 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:42:03,850 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:42:03,850 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:42:04] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:42:04,218 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:42:04,219 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:42:04,219 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:42:04,219 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:42:04] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:42:04,540 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:42:04,542 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:42:04,542 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:42:04,542 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:42:04] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:42:05,248 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:42:05,248 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:42:05,248 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:42:05,248 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:42:05,248 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:42:05,249 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:42:05,366 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:42:05,375 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:42:05,375 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:42:05,375 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:42:05,381 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:42:05,381 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:42:06,016 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:42:06,016 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:42:06,016 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:42:06,016 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:42:06,016 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:42:06,016 - INFO - Prompt processing progress: 33/33\n", + "ready_s": 7.023564417000671, + "error": "RuntimeError: 2 of 8 concurrent requests failed: {'error': 'URLError: '}", + "min_available_gib": 6.24, + "peak_temp_c": 0, + "round": 1, + "admission_available_gib": 9.44 + }, + { + "arm": "tensorfold-metal", + "stopped": null, + "samples": [ + { + "t": 0.52, + "available_gib": 9.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.90 18.19 16.32" + }, + { + "t": 1.96, + "available_gib": 5.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.99 18.22 16.34" + }, + { + "t": 3.14, + "available_gib": 4.32, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.99 18.22 16.34" + }, + { + "t": 4.38, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.99 18.22 16.34" + }, + { + "t": 5.71, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.99 18.22 16.34" + }, + { + "t": 6.97, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.67 18.17 16.33" + }, + { + "t": 8.56, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.67 18.17 16.33" + }, + { + "t": 9.74, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.67 18.17 16.33" + }, + { + "t": 10.91, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.67 18.17 16.33" + } + ], + "log_tail": "info: qwen3.5: 2.3 GiB kept wired between requests, inside 70% of 16 GiB of RAM\nwarning: memory pressure: warning: one more stream only when none runs, no new kept prompt states, the prompt cache sheds toward 0 MiB\n[tensorfold] prompt chunks of up to 128 tokens, cut at replies 256+ tokens apart\n[tensorfold] serving 93760be4f1f69842a46bc13dbdc0f19e291392a3 at http://127.0.0.1:55878/v1 (sampling: greedy; drafts: off; context: 1024; loaded in 3.1s)\n[tensorfold] 127.0.0.1 \"GET /health HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"GET /v1/models HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-6285ad7db93d8dea0133cd7c670115ee prompt=27 cached=0 thinking=False effort=none tokens=32 sha=778f678f6c4f finish=length rounds=31 accepted=0/0 ttft=0.071 decode=0.798 elapsed=0.868 queued=0.000 tok/s=38.9\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-9c341e538972fc941a05ab64dfda05fd prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=0.086 decode=0.824 elapsed=0.910 queued=0.004 tok/s=37.6\n[tensorfold] done chatcmpl-973e3cef765e5fde86f252e2181f0e3f prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=0.980 decode=0.797 elapsed=1.777 queued=0.909 tok/s=38.9\n[tensorfold] done chatcmpl-73cb4cb489e6e5e4de8a2330eb7f3a5a prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=1.852 decode=0.794 elapsed=2.646 queued=1.762 tok/s=39.0\n[tensorfold] done chatcmpl-ba7a7ad4ea49e873f6faa01a13ec1969 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=2.711 decode=0.797 elapsed=3.508 queued=2.639 tok/s=38.9\n[tensorfold] done chatcmpl-39ab5f14565904b19600042264248a56 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=3.581 decode=0.826 elapsed=4.407 queued=3.501 tok/s=37.5\n[tensorfold] done chatcmpl-18031215aa4ebf2daa84bda27df8d403 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=4.476 decode=0.789 elapsed=5.265 queued=4.399 tok/s=39.3\n[tensorfold] done chatcmpl-d8215bf055d352076105230fa857a171 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=5.337 decode=0.798 elapsed=6.135 queued=5.259 tok/s=38.9\n[tensorfold] done chatcmpl-fb7e4aed7e52ddda0423394c98fba7e8 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=6.214 decode=0.813 elapsed=7.026 queued=6.125 tok/s=38.2\n[tensorfold] shutting down (SIGTERM): draining: new requests get 503; 0 request(s) in progress may finish within 20.0 s\n[tensorfold] shutting down: drained\n[tensorfold] shutting down: stopped\n", + "ready_s": 3.9982705419988633, + "concurrency": 8, + "wall_s": 7.05099941699882, + "aggregate_tok_s": 36.30690982370887, + "per_request_decode_median": 38.86114895805225, + "ttft_s": 3.582970707999266, + "total_s": 7.05099941699882, + "completion_tokens": 256, + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors,", + "requests": [ + { + "ttft_s": 0.09037741700012702, + "total_s": 0.9140904169998976, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 1.8601234159996238, + "total_s": 2.652992124998491, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 0.9810153750004247, + "total_s": 1.7794618339994486, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 2.715409209000427, + "total_s": 3.5126865000002, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 3.582970707999266, + "total_s": 4.410181666999051, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 4.4780874169991876, + "total_s": 5.267083666998587, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 5.338783166000212, + "total_s": 6.136495041000671, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 6.215644792000603, + "total_s": 7.028856625000117, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + } + ], + "served_model": "93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "97af50af7f7909e68d8670dd08652563bf801d48f9186dcd506215b7d652c5db", + "decode_tps": 36.30690982370887, + "min_available_gib": 4.32, + "peak_temp_c": 0, + "round": 1, + "admission_available_gib": 9.76 + }, + { + "arm": "tensorfold-metal", + "stopped": null, + "samples": [ + { + "t": 0.13, + "available_gib": 9.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.29 18.10 16.32" + }, + { + "t": 1.47, + "available_gib": 6.08, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.29 18.10 16.32" + }, + { + "t": 2.84, + "available_gib": 4.48, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.79 17.99 16.29" + }, + { + "t": 4.16, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.79 17.99 16.29" + }, + { + "t": 5.34, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.79 17.99 16.29" + }, + { + "t": 6.53, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.79 17.99 16.29" + }, + { + "t": 7.75, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.56 17.94 16.28" + }, + { + "t": 8.98, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.56 17.94 16.28" + }, + { + "t": 10.27, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.56 17.94 16.28" + }, + { + "t": 11.42, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.56 17.94 16.28" + } + ], + "log_tail": "info: qwen3.5: 2.3 GiB kept wired between requests, inside 70% of 16 GiB of RAM\nwarning: memory pressure: warning: one more stream only when none runs, no new kept prompt states, the prompt cache sheds toward 0 MiB\n[tensorfold] prompt chunks of up to 128 tokens, cut at replies 256+ tokens apart\n[tensorfold] serving 93760be4f1f69842a46bc13dbdc0f19e291392a3 at http://127.0.0.1:55914/v1 (sampling: greedy; drafts: off; context: 1024; loaded in 3.1s)\n[tensorfold] 127.0.0.1 \"GET /health HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"GET /v1/models HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-e68eb09d423a9011cc838143d97f7446 prompt=27 cached=0 thinking=False effort=none tokens=32 sha=778f678f6c4f finish=length rounds=31 accepted=0/0 ttft=0.072 decode=0.850 elapsed=0.922 queued=0.000 tok/s=36.5\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-8698b736b047a7e1b5fe81262f2518c7 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=0.087 decode=0.806 elapsed=0.892 queued=0.000 tok/s=38.5\n[tensorfold] done chatcmpl-c5f0b6a1b18409a5798e2eadc1c4ae30 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=0.965 decode=0.803 elapsed=1.768 queued=0.881 tok/s=38.6\n[tensorfold] done chatcmpl-2403b2e11c8a04bf71fba873e7099fc5 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=1.853 decode=0.813 elapsed=2.666 queued=1.756 tok/s=38.1\n[tensorfold] done chatcmpl-ff43121e4e865a5aca4a2ab9503d3607 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=2.740 decode=0.821 elapsed=3.561 queued=2.658 tok/s=37.8\n[tensorfold] done chatcmpl-e834b16854fac701a43e611fd0d8875c prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=3.634 decode=0.830 elapsed=4.464 queued=3.551 tok/s=37.4\n[tensorfold] done chatcmpl-edbcaeade563af3fb941e6996f4da8f1 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=4.545 decode=0.884 elapsed=5.429 queued=4.454 tok/s=35.1\n[tensorfold] done chatcmpl-8bf5bb23ea80211512e5e6dbe3298dcd prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=5.502 decode=0.853 elapsed=6.355 queued=5.417 tok/s=36.3\n[tensorfold] done chatcmpl-0e7d7c0f40e935f35c7d6ff37bacd0d4 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=6.424 decode=0.780 elapsed=7.203 queued=6.350 tok/s=39.8\n[tensorfold] shutting down (SIGTERM): draining: new requests get 503; 0 request(s) in progress may finish within 20.0 s\n[tensorfold] shutting down: drained\n[tensorfold] shutting down: stopped\n", + "ready_s": 4.499658458998965, + "concurrency": 8, + "wall_s": 7.229936832998646, + "aggregate_tok_s": 35.408331485217545, + "per_request_decode_median": 38.21138469228694, + "ttft_s": 3.63799412499975, + "total_s": 7.229936832998646, + "completion_tokens": 256, + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors,", + "requests": [ + { + "ttft_s": 0.09214125000107742, + "total_s": 0.8992856670010951, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 0.9706634169997415, + "total_s": 1.7733162500007893, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 1.8604420839983504, + "total_s": 2.6717186249989027, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 2.744795084001453, + "total_s": 3.5658916250013135, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 3.63799412499975, + "total_s": 4.472471499999301, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 4.548832708998816, + "total_s": 5.432783874999586, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 5.510349790998589, + "total_s": 6.363711082998634, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 6.429924458001551, + "total_s": 7.21046700000079, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + } + ], + "served_model": "93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "97af50af7f7909e68d8670dd08652563bf801d48f9186dcd506215b7d652c5db", + "decode_tps": 35.408331485217545, + "min_available_gib": 4.48, + "peak_temp_c": 0, + "round": 2, + "admission_available_gib": 9.12 + }, + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.09, + "available_gib": 9.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.20 17.86 16.27" + }, + { + "t": 1.23, + "available_gib": 9.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.20 17.86 16.27" + }, + { + "t": 2.36, + "available_gib": 9.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.58 17.93 16.30" + }, + { + "t": 3.5, + "available_gib": 7.36, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.58 17.93 16.30" + }, + { + "t": 4.88, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.58 17.93 16.30" + }, + { + "t": 6.11, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.58 17.93 16.30" + }, + { + "t": 7.36, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.34 18.08 16.36" + }, + { + "t": 8.88, + "available_gib": 6.56, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.34 18.08 16.36" + }, + { + "t": 10.17, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.34 18.08 16.36" + }, + { + "t": 11.4, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.34 18.08 16.36" + } + ], + "log_tail": "00 -\n2026-10-31 11:42:46,425 - INFO - Prompt processing progress: 23/27\n2026-10-31 11:42:46,463 - INFO - Prompt processing progress: 26/27\n2026-10-31 11:42:46,499 - INFO - Prompt processing progress: 27/27\n2026-10-31 11:42:47,281 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:42:47,281 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:42:47,281 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:42:47,281 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:42:47] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:42:47,576 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:42:47,576 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:42:47,576 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:42:47,576 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:42:47] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:42:47,947 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:42:47,948 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:42:47,948 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:42:47,948 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:42:48] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:42:48,214 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:42:48,215 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:42:48,215 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:42:48,215 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:42:48] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:42:48,466 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:42:48,466 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:42:48,466 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:42:48,466 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:42:48] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:42:48,788 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:42:48,789 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:42:48,789 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:42:48,789 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:42:49] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:42:49,205 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:42:49,206 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:42:49,206 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:42:49,206 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:42:49] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:42:49,961 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:42:49,962 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:42:49,962 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:42:49,962 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:42:49,962 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:42:49,962 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:42:49,962 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:42:50,111 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:42:50,111 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:42:50,111 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:42:50,113 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:42:50,113 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:42:50,113 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:42:50,113 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:42:50,290 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:42:50,292 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:42:50,293 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:42:50,293 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:42:50,293 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:42:50,293 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:42:50,294 - INFO - Prompt processing progress: 33/33\n", + "ready_s": 6.880575958999543, + "error": "RuntimeError: 1 of 8 concurrent requests failed: {'error': 'URLError: '}", + "min_available_gib": 5.76, + "peak_temp_c": 0, + "round": 2, + "admission_available_gib": 9.76 + }, + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.07, + "available_gib": 9.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.75 17.96 16.33" + }, + { + "t": 1.18, + "available_gib": 10.08, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.75 17.96 16.33" + }, + { + "t": 2.41, + "available_gib": 10.08, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.69 17.95 16.33" + }, + { + "t": 3.61, + "available_gib": 7.04, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.69 17.95 16.33" + }, + { + "t": 4.78, + "available_gib": 7.36, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.69 17.95 16.33" + }, + { + "t": 6.05, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.69 17.95 16.33" + }, + { + "t": 7.23, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.31 17.86 16.31" + }, + { + "t": 8.46, + "available_gib": 7.04, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.31 17.86 16.31" + }, + { + "t": 10.01, + "available_gib": 6.56, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.31 17.86 16.31" + }, + { + "t": 11.22, + "available_gib": 6.08, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.31 17.86 16.31" + } + ], + "log_tail": "00 -\n2026-10-31 11:43:01,269 - INFO - Prompt processing progress: 23/27\n2026-10-31 11:43:01,321 - INFO - Prompt processing progress: 26/27\n2026-10-31 11:43:01,381 - INFO - Prompt processing progress: 27/27\n2026-10-31 11:43:02,199 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:43:02,200 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:43:02,200 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:43:02,200 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:43:02] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:43:02,512 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:43:02,512 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:43:02,513 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:43:02,513 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:43:02] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:43:02,801 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:43:02,801 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:43:02,801 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:43:02,801 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:43:02] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:43:03,177 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:43:03,177 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:43:03,177 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:43:03,177 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:43:03] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:43:03,543 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:43:03,543 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:43:03,543 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:43:03,543 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:43:03] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:43:03,888 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:43:03,888 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:43:03,888 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:43:03,888 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:43:04] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:43:04,202 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:43:04,202 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:43:04,202 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:43:04,202 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:43:04] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:43:05,294 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:43:05,298 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:43:05,298 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:43:05,298 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:43:05,299 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:43:05,299 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:43:05,300 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:43:05,439 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:43:05,440 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:43:05,440 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:43:05,441 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:43:05,441 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:43:05,442 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:43:05,442 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:43:05,557 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:43:05,558 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:43:05,559 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:43:05,559 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:43:05,559 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:43:05,560 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:43:05,560 - INFO - Prompt processing progress: 33/33\n", + "ready_s": 6.766519959001016, + "error": "RuntimeError: 1 of 8 concurrent requests failed: {'error': 'URLError: '}", + "min_available_gib": 6.08, + "peak_temp_c": 0, + "round": 3, + "admission_available_gib": 9.76 + }, + { + "arm": "tensorfold-metal", + "stopped": null, + "samples": [ + { + "t": 0.35, + "available_gib": 9.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.13 17.82 16.31" + }, + { + "t": 1.54, + "available_gib": 6.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.13 17.82 16.31" + }, + { + "t": 2.72, + "available_gib": 4.8, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.96 17.77 16.30" + }, + { + "t": 3.83, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.96 17.77 16.30" + }, + { + "t": 4.91, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.96 17.77 16.30" + }, + { + "t": 6.11, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.96 17.77 16.30" + }, + { + "t": 7.48, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.56 17.67 16.27" + }, + { + "t": 8.58, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.56 17.67 16.27" + }, + { + "t": 9.99, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.56 17.67 16.27" + }, + { + "t": 11.28, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.56 17.67 16.27" + } + ], + "log_tail": "info: qwen3.5: 2.3 GiB kept wired between requests, inside 70% of 16 GiB of RAM\nwarning: memory pressure: warning: one more stream only when none runs, no new kept prompt states, the prompt cache sheds toward 0 MiB\n[tensorfold] prompt chunks of up to 128 tokens, cut at replies 256+ tokens apart\n[tensorfold] serving 93760be4f1f69842a46bc13dbdc0f19e291392a3 at http://127.0.0.1:56037/v1 (sampling: greedy; drafts: off; context: 1024; loaded in 3.2s)\n[tensorfold] 127.0.0.1 \"GET /health HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"GET /v1/models HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-4bbe47916833b291a15d6be278893cc7 prompt=27 cached=0 thinking=False effort=none tokens=32 sha=778f678f6c4f finish=length rounds=31 accepted=0/0 ttft=0.080 decode=0.780 elapsed=0.860 queued=0.002 tok/s=39.7\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-69b60c451435c66ef797722511821b0a prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=0.079 decode=0.789 elapsed=0.868 queued=0.000 tok/s=39.3\n[tensorfold] done chatcmpl-77f925c09224d43018e3298792bc831b prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=0.944 decode=0.800 elapsed=1.744 queued=0.865 tok/s=38.8\n[tensorfold] done chatcmpl-163960cada20f98a3a71aa4e4ac1e852 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=1.825 decode=0.820 elapsed=2.646 queued=1.742 tok/s=37.8\n[tensorfold] done chatcmpl-794061abd06866a6fda7e20e3d32909d prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=2.714 decode=0.915 elapsed=3.629 queued=2.635 tok/s=33.9\n[tensorfold] done chatcmpl-5fb35a29be942eb92eb812cf27e27a6c prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=3.717 decode=0.856 elapsed=4.573 queued=3.620 tok/s=36.2\n[tensorfold] done chatcmpl-56b8b531bcb17e4ca7be78727ffe31ac prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=4.644 decode=0.812 elapsed=5.456 queued=4.563 tok/s=38.2\n[tensorfold] done chatcmpl-b5f93934e082403c12a193f1b044d70f prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=5.531 decode=0.846 elapsed=6.377 queued=5.446 tok/s=36.6\n[tensorfold] done chatcmpl-9083e366eb96954ce2cde2c4a9e300f9 prompt=33 cached=0 thinking=False effort=none tokens=32 sha=fe08a186ad83 finish=length rounds=31 accepted=0/0 ttft=6.450 decode=0.817 elapsed=7.267 queued=6.374 tok/s=37.9\n[tensorfold] shutting down (SIGTERM): draining: new requests get 503; 0 request(s) in progress may finish within 20.0 s\n[tensorfold] shutting down: drained\n[tensorfold] shutting down: stopped\n", + "ready_s": 4.568960917000368, + "concurrency": 8, + "wall_s": 7.281992791000448, + "aggregate_tok_s": 35.15521195192354, + "per_request_decode_median": 37.991123322548475, + "ttft_s": 3.722208250001131, + "total_s": 7.281992791000448, + "completion_tokens": 256, + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors,", + "requests": [ + { + "ttft_s": 0.07998091700028453, + "total_s": 0.869935708000412, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 1.8261953750006796, + "total_s": 2.6466377090000606, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 0.9460138329995971, + "total_s": 1.7457400000002963, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 2.7215807090015005, + "total_s": 3.6365350840005704, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 3.722208250001131, + "total_s": 4.579589375000069, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 4.649204792000091, + "total_s": 5.461762334000014, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 5.534465791999537, + "total_s": 6.380469666999488, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + }, + { + "ttft_s": 6.452803125001083, + "total_s": 7.268783209001413, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "1. Sunlight travels through the atmosphere and gets scattered by water droplets in the air.\n2. This scattering separates the light into its different colors," + } + ], + "served_model": "93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "97af50af7f7909e68d8670dd08652563bf801d48f9186dcd506215b7d652c5db", + "decode_tps": 35.15521195192354, + "min_available_gib": 4.8, + "peak_temp_c": 0, + "round": 3, + "admission_available_gib": 9.6 + } + ], + "runtime": { + "mlx": "0.32.3", + "mlx_lm": "0.31.3", + "transformers": "5.16.1", + "python": "3.14.3", + "device": { + "device_name": "Apple M4", + "max_recommended_working_set_size": 12713115648, + "memory_size": 17179869184, + "architecture": "applegpu_g16g", + "max_buffer_length": 9534832640, + "resource_limit": 499000 + } + }, + "tensorfold_sha256": "4b4b5a5ac152eb2dbed7e9d8004805ed9b74f77c64c3de18eb4f35263c7b9eb8", + "model_dir": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-2B-MLX-4bit/snapshots/93760be4f1f69842a46bc13dbdc0f19e291392a3", + "host_start": { + "macos": "27.0", + "build": "26A5421a", + "hw_model": "Mac16,10", + "chip": "Apple M4", + "gpu_cores": "10", + "memory_gib": 16.0, + "power": [ + "Now drawing from 'AC Power'" + ], + "low_power_mode": false, + "thermal": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", + "uptime": "11:41 up 3 hrs, 4 users, load averages: 17.78 17.94 16.20", + "top_cpu": [ + "42290 com.apple.Virtua 517.4 10G ", + "0 kernel_task 55.6 35M+ ", + "433 WindowServer 47.0 607M ", + "4916 Claude Helper 13.7 433M-", + "27418 Python 12.8 31M ", + "4939 Claude Helper (R 8.9 959M+" + ], + "memory_free_pct": "59", + "swap": "total = 4096.00M used = 3330.75M free = 765.25M (encrypted)", + "lima": "NAME STATUS SSH VMTYPE ARCH CPUS MEMORY DISK DIR\nkbuild Running 127.0.0.1:53830 vz aarch64 8 10GiB 60GiB ~/.lima/kbuild" + }, + "summary": { + "lm-metal": { + "ok_runs": 0, + "median_ttft_s": null, + "median_total_s": null, + "median_decode_tps": null, + "texts": [] + }, + "tensorfold-metal": { + "ok_runs": 3, + "median_ttft_s": 3.63799412499975, + "median_total_s": 7.229936832998646, + "median_decode_tps": 35.408331485217545, + "texts": [ + "97af50af7f79" + ] + } + }, + "host_end": { + "macos": "27.0", + "build": "26A5421a", + "hw_model": "Mac16,10", + "chip": "Apple M4", + "gpu_cores": "10", + "memory_gib": 16.0, + "power": [ + "Now drawing from 'AC Power'" + ], + "low_power_mode": false, + "thermal": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", + "uptime": "11:43 up 3:02, 4 users, load averages: 15.69 17.45 16.21", + "top_cpu": [ + "42290 com.apple.Virtua 649.8 10G ", + "433 WindowServer 33.0 601M-", + "4939 Claude Helper (R 20.0 976M+", + "0 kernel_task 13.4 35M+ ", + "4916 Claude Helper 12.6 447M-", + "4951 claude 7.7 437M+" + ], + "memory_free_pct": "62", + "swap": "total = 4096.00M used = 2802.69M free = 1293.31M (encrypted)", + "lima": "NAME STATUS SSH VMTYPE ARCH CPUS MEMORY DISK DIR\nkbuild Running 127.0.0.1:53830 vz aarch64 8 10GiB 60GiB ~/.lima/kbuild" + } +} diff --git a/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal.json b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal.json new file mode 100644 index 00000000..8cfb0ea8 --- /dev/null +++ b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen2b-short-metal.json @@ -0,0 +1,702 @@ +{ + "status": "complete", + "model_key": "qwen2b", + "model": "mlx-community/Qwen3.5-2B-MLX-4bit@93760be4f1f69842a46bc13dbdc0f19e291392a3", + "weights_sha256": "713fe7e5d3c3965f7106b0d0ee17615f7869c23c8d327996df8c1196fbcf07d5", + "kernel": "macOS 27.0 (26A5421a)", + "driver": "Metal", + "tensorfold_rev": "v1.1.9 release", + "rapid_rev": "1d09cdb684bbf58fac130f5804b08ee2e1f56558", + "serve_concurrency": 4, + "prompt_sha256": "af125b15dfba55caedb541e89e59fe1252c0f13553e5ce6bc181e237fada9926", + "max_tokens": 32, + "policy": { + "reserve_gib": 1.0, + "max_fan_rpm": null, + "max_temp_c": 80 + }, + "allowance_gib": 4.5, + "arms": [ + "mlx-metal", + "lm-metal", + "tensorfold-metal" + ], + "concurrency": 1, + "runs": [ + { + "arm": "mlx-metal", + "stopped": null, + "samples": [ + { + "t": 0.09, + "available_gib": 8.16, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.04 18.56 15.94" + }, + { + "t": 1.31, + "available_gib": 8.16, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.88 18.53 15.95" + }, + { + "t": 2.78, + "available_gib": 7.68, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.88 18.53 15.95" + }, + { + "t": 3.96, + "available_gib": 6.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.88 18.53 15.95" + }, + { + "t": 5.27, + "available_gib": 6.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.88 18.53 15.95" + }, + { + "t": 6.45, + "available_gib": 6.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.97 18.56 15.97" + }, + { + "t": 7.67, + "available_gib": 6.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.97 18.56 15.97" + } + ], + "log_tail": "", + "ready_s": 7.34068520800065, + "ttft_s": 0.35663558399937756, + "total_s": 0.9155917089992727, + "completion_tokens": 32, + "text": "Sunlight travels through the atmosphere and gets refracted (bent) as it enters a raindrop.\nThe light then reflects off the back of the dro", + "text_sha256": "ffbf23cb27bb50106bf151750276ab30f61683b0bec1ceb077bab1d61844393e", + "decode_tps": 55.4605247415543, + "min_available_gib": 6.24, + "peak_temp_c": 0, + "round": 1, + "admission_available_gib": 8.32 + }, + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.05, + "available_gib": 7.68, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.49 18.46 15.96" + }, + { + "t": 1.24, + "available_gib": 7.68, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.49 18.46 15.96" + }, + { + "t": 2.45, + "available_gib": 8.0, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.49 18.46 15.96" + }, + { + "t": 3.65, + "available_gib": 6.4, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.49 18.46 15.96" + }, + { + "t": 4.89, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.37 18.44 15.96" + }, + { + "t": 6.62, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.37 18.44 15.96" + }, + { + "t": 7.91, + "available_gib": 6.08, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.37 18.44 15.96" + }, + { + "t": 9.25, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.50 18.47 15.99" + } + ], + "log_tail": "/private/tmp/claude-501/bench/venv/lib/python3.14/site-packages/mlx_lm/server.py:1723: UserWarning: mlx_lm.server is not recommended for production as it only implements basic security checks.\n warnings.warn(\n2026-10-31 11:38:56,384 - INFO - Starting httpd at 127.0.0.1 on port 55294...\n127.0.0.1 - - [31/Oct/2026 11:38:56] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:38:57] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:38:58,966 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:38:58,968 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:38:58,968 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:38:58,968 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:38:59] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:39:00,355 - INFO - Prompt processing progress: 23/27\n2026-10-31 11:39:00,590 - INFO - Prompt processing progress: 26/27\n2026-10-31 11:39:00,722 - INFO - Prompt processing progress: 27/27\n2026-10-31 11:39:01,512 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:39:01,512 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:39:01,512 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:39:01,512 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:39:01] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:39:01,930 - INFO - Prompt processing progress: 1/1\n", + "ready_s": 8.008932082999308, + "ttft_s": 0.44301358299890126, + "total_s": 1.1748576659992978, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "Sunlight travels through the atmosphere and gets refracted (bent) as it enters a raindrop.\nThe light then reflects off the back of the dro", + "served_model": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-2B-MLX-4bit/snapshots/93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "ffbf23cb27bb50106bf151750276ab30f61683b0bec1ceb077bab1d61844393e", + "decode_tps": 42.35874924739018, + "min_available_gib": 5.6, + "peak_temp_c": 0, + "round": 1, + "admission_available_gib": 7.68 + }, + { + "arm": "tensorfold-metal", + "stopped": null, + "samples": [ + { + "t": 0.25, + "available_gib": 7.36, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.50 18.47 15.99" + }, + { + "t": 1.7, + "available_gib": 4.64, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.46 18.46 16.00" + }, + { + "t": 2.88, + "available_gib": 4.16, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.46 18.46 16.00" + }, + { + "t": 4.08, + "available_gib": 5.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.46 18.46 16.00" + } + ], + "log_tail": "info: qwen3.5: 2.3 GiB kept wired between requests, inside 70% of 16 GiB of RAM\nwarning: memory pressure: warning: one more stream only when none runs, no new kept prompt states, the prompt cache sheds toward 0 MiB\n[tensorfold] prompt chunks of up to 128 tokens, cut at replies 256+ tokens apart\n[tensorfold] serving 93760be4f1f69842a46bc13dbdc0f19e291392a3 at http://127.0.0.1:55322/v1 (sampling: greedy; drafts: off; context: 1024; loaded in 2.9s)\n[tensorfold] 127.0.0.1 \"GET /health HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"GET /v1/models HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-c6fab373fe2d2322e4223888611d2d38 prompt=27 cached=0 thinking=False effort=none tokens=32 sha=778f678f6c4f finish=length rounds=31 accepted=0/0 ttft=0.065 decode=0.773 elapsed=0.838 queued=0.000 tok/s=40.1\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-1569f8381212b50080e6e78596bcf726 prompt=27 cached=0 thinking=False effort=none tokens=32 sha=778f678f6c4f finish=length rounds=31 accepted=0/0 ttft=0.067 decode=0.781 elapsed=0.848 queued=0.000 tok/s=39.7\n[tensorfold] shutting down (SIGTERM): draining: new requests get 503; 0 request(s) in progress may finish within 20.0 s\n[tensorfold] shutting down: drained\n[tensorfold] shutting down: stopped\n", + "ready_s": 3.9757489999992686, + "ttft_s": 0.07205679099934059, + "total_s": 0.85273262499868, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "Sunlight travels through the atmosphere and gets refracted (bent) as it enters a raindrop.\nThe light then reflects off the back of the dro", + "served_model": "93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "ffbf23cb27bb50106bf151750276ab30f61683b0bec1ceb077bab1d61844393e", + "decode_tps": 39.70918356879257, + "min_available_gib": 4.16, + "peak_temp_c": 0, + "round": 1, + "admission_available_gib": 7.36 + }, + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.13, + "available_gib": 7.84, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.10 18.38 15.99" + }, + { + "t": 1.24, + "available_gib": 8.0, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.10 18.38 15.99" + }, + { + "t": 2.33, + "available_gib": 8.16, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.10 18.38 15.99" + }, + { + "t": 3.62, + "available_gib": 8.32, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.85 18.33 15.98" + }, + { + "t": 4.99, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.85 18.33 15.98" + }, + { + "t": 6.23, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.85 18.33 15.98" + }, + { + "t": 7.4, + "available_gib": 7.2, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.85 18.33 15.98" + }, + { + "t": 8.56, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.79 18.30 15.99" + } + ], + "log_tail": "/private/tmp/claude-501/bench/venv/lib/python3.14/site-packages/mlx_lm/server.py:1723: UserWarning: mlx_lm.server is not recommended for production as it only implements basic security checks.\n warnings.warn(\n2026-10-31 11:39:18,682 - INFO - Starting httpd at 127.0.0.1 on port 55348...\n127.0.0.1 - - [31/Oct/2026 11:39:19] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:39:19] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:39:21,462 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:39:21,462 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:39:21,462 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:39:21,462 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:39:21] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:39:21,946 - INFO - Prompt processing progress: 23/27\n2026-10-31 11:39:21,983 - INFO - Prompt processing progress: 26/27\n2026-10-31 11:39:22,027 - INFO - Prompt processing progress: 27/27\n2026-10-31 11:39:22,833 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:39:22,833 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:39:22,833 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:39:22,833 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:39:22] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:39:23,133 - INFO - Prompt processing progress: 1/1\n", + "ready_s": 8.411539082999298, + "ttft_s": 0.3237621250009397, + "total_s": 1.0199038330010808, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "Sunlight travels through the atmosphere and gets refracted (bent) as it enters a raindrop.\nThe light then reflects off the back of the dro", + "served_model": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-2B-MLX-4bit/snapshots/93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "ffbf23cb27bb50106bf151750276ab30f61683b0bec1ceb077bab1d61844393e", + "decode_tps": 44.531163186667904, + "min_available_gib": 6.72, + "peak_temp_c": 0, + "round": 2, + "admission_available_gib": 7.84 + }, + { + "arm": "tensorfold-metal", + "stopped": null, + "samples": [ + { + "t": 0.41, + "available_gib": 8.32, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.04 18.35 16.02" + }, + { + "t": 1.88, + "available_gib": 5.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.04 18.35 16.02" + }, + { + "t": 3.32, + "available_gib": 4.96, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.04 18.35 16.02" + }, + { + "t": 4.56, + "available_gib": 5.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.04 18.35 16.02" + } + ], + "log_tail": "info: qwen3.5: 2.3 GiB kept wired between requests, inside 70% of 16 GiB of RAM\nwarning: memory pressure: warning: one more stream only when none runs, no new kept prompt states, the prompt cache sheds toward 0 MiB\n[tensorfold] prompt chunks of up to 128 tokens, cut at replies 256+ tokens apart\n[tensorfold] serving 93760be4f1f69842a46bc13dbdc0f19e291392a3 at http://127.0.0.1:55389/v1 (sampling: greedy; drafts: off; context: 1024; loaded in 2.9s)\n[tensorfold] 127.0.0.1 \"GET /health HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"GET /v1/models HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-b654905480934c4910a4124833f0c590 prompt=27 cached=0 thinking=False effort=none tokens=32 sha=778f678f6c4f finish=length rounds=31 accepted=0/0 ttft=0.065 decode=0.824 elapsed=0.890 queued=0.000 tok/s=37.6\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-c330dd3a2241136a5a9b4573f30270bc prompt=27 cached=0 thinking=False effort=none tokens=32 sha=778f678f6c4f finish=length rounds=31 accepted=0/0 ttft=0.068 decode=0.774 elapsed=0.842 queued=0.001 tok/s=40.1\n[tensorfold] shutting down (SIGTERM): draining: new requests get 503; 0 request(s) in progress may finish within 20.0 s\n[tensorfold] shutting down: drained\n[tensorfold] shutting down: stopped\n", + "ready_s": 4.002329500000997, + "ttft_s": 0.07251733399971272, + "total_s": 0.8465579589992558, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "Sunlight travels through the atmosphere and gets refracted (bent) as it enters a raindrop.\nThe light then reflects off the back of the dro", + "served_model": "93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "ffbf23cb27bb50106bf151750276ab30f61683b0bec1ceb077bab1d61844393e", + "decode_tps": 40.04957750120454, + "min_available_gib": 4.96, + "peak_temp_c": 0, + "round": 2, + "admission_available_gib": 8.32 + }, + { + "arm": "mlx-metal", + "stopped": null, + "samples": [ + { + "t": 0.22, + "available_gib": 8.48, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.12 18.36 16.03" + }, + { + "t": 1.4, + "available_gib": 8.96, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.12 18.36 16.03" + }, + { + "t": 2.75, + "available_gib": 10.08, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.47 18.22 16.00" + }, + { + "t": 4.17, + "available_gib": 10.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.47 18.22 16.00" + }, + { + "t": 5.39, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.47 18.22 16.00" + }, + { + "t": 6.67, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.47 18.22 16.00" + }, + { + "t": 8.22, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.27 18.17 15.99" + }, + { + "t": 9.75, + "available_gib": 7.04, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.27 18.17 15.99" + } + ], + "log_tail": "", + "ready_s": 8.368872375000137, + "ttft_s": 0.46368174999952316, + "total_s": 1.1139279999988503, + "completion_tokens": 32, + "text": "Sunlight travels through the atmosphere and gets refracted (bent) as it enters a raindrop.\nThe light then reflects off the back of the dro", + "text_sha256": "ffbf23cb27bb50106bf151750276ab30f61683b0bec1ceb077bab1d61844393e", + "decode_tps": 47.67424648743776, + "min_available_gib": 6.72, + "peak_temp_c": 0, + "round": 2, + "admission_available_gib": 8.48 + }, + { + "arm": "tensorfold-metal", + "stopped": null, + "samples": [ + { + "t": 0.7, + "available_gib": 9.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.05 18.31 16.06" + }, + { + "t": 2.07, + "available_gib": 5.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.05 18.31 16.06" + }, + { + "t": 3.28, + "available_gib": 4.8, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.53 18.41 16.10" + }, + { + "t": 4.58, + "available_gib": 5.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.53 18.41 16.10" + }, + { + "t": 5.76, + "available_gib": 5.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "18.53 18.41 16.10" + } + ], + "log_tail": "info: qwen3.5: 2.3 GiB kept wired between requests, inside 70% of 16 GiB of RAM\nwarning: memory pressure: warning: one more stream only when none runs, no new kept prompt states, the prompt cache sheds toward 0 MiB\n[tensorfold] prompt chunks of up to 128 tokens, cut at replies 256+ tokens apart\n[tensorfold] serving 93760be4f1f69842a46bc13dbdc0f19e291392a3 at http://127.0.0.1:55428/v1 (sampling: greedy; drafts: off; context: 1024; loaded in 3.3s)\n[tensorfold] 127.0.0.1 \"GET /health HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"GET /v1/models HTTP/1.1\" 200 -\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-10b221343a21db440478a92fdb38d248 prompt=27 cached=0 thinking=False effort=none tokens=32 sha=778f678f6c4f finish=length rounds=31 accepted=0/0 ttft=0.076 decode=0.895 elapsed=0.971 queued=0.001 tok/s=34.6\n[tensorfold] 127.0.0.1 \"POST /v1/chat/completions HTTP/1.1\" 200 -\n[tensorfold] done chatcmpl-4ed1f8d6defc490525b04f3ed3473bb6 prompt=27 cached=0 thinking=False effort=none tokens=32 sha=778f678f6c4f finish=length rounds=31 accepted=0/0 ttft=0.073 decode=0.983 elapsed=1.056 queued=0.000 tok/s=31.5\n[tensorfold] shutting down (SIGTERM): draining: new requests get 503; 0 request(s) in progress may finish within 20.0 s\n[tensorfold] shutting down: drained\n[tensorfold] shutting down: stopped\n", + "ready_s": 4.578620874999615, + "ttft_s": 0.0764893329996994, + "total_s": 1.0597247910009173, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "Sunlight travels through the atmosphere and gets refracted (bent) as it enters a raindrop.\nThe light then reflects off the back of the dro", + "served_model": "93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "ffbf23cb27bb50106bf151750276ab30f61683b0bec1ceb077bab1d61844393e", + "decode_tps": 31.52856190013603, + "min_available_gib": 4.8, + "peak_temp_c": 0, + "round": 3, + "admission_available_gib": 9.12 + }, + { + "arm": "mlx-metal", + "stopped": null, + "samples": [ + { + "t": 0.11, + "available_gib": 8.16, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.84 18.27 16.07" + }, + { + "t": 1.23, + "available_gib": 8.32, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.84 18.27 16.07" + }, + { + "t": 2.44, + "available_gib": 8.48, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.84 18.27 16.07" + }, + { + "t": 3.84, + "available_gib": 7.04, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.46 18.18 16.05" + }, + { + "t": 5.09, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.46 18.18 16.05" + }, + { + "t": 6.4, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.46 18.18 16.05" + }, + { + "t": 7.67, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.46 18.18 16.05" + } + ], + "log_tail": "", + "ready_s": 6.862831500000539, + "ttft_s": 0.3782165000011446, + "total_s": 1.117459291999694, + "completion_tokens": 32, + "text": "Sunlight travels through the atmosphere and gets refracted (bent) as it enters a raindrop.\nThe light then reflects off the back of the dro", + "text_sha256": "ffbf23cb27bb50106bf151750276ab30f61683b0bec1ceb077bab1d61844393e", + "decode_tps": 41.93480184797099, + "min_available_gib": 6.72, + "peak_temp_c": 0, + "round": 3, + "admission_available_gib": 8.16 + }, + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.27, + "available_gib": 9.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.26 18.13 16.04" + }, + { + "t": 1.41, + "available_gib": 9.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "17.26 18.13 16.04" + }, + { + "t": 2.76, + "available_gib": 9.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.68 17.99 16.01" + }, + { + "t": 4.03, + "available_gib": 6.4, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.68 17.99 16.01" + }, + { + "t": 5.45, + "available_gib": 6.08, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.68 17.99 16.01" + }, + { + "t": 6.75, + "available_gib": 6.08, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.78 17.99 16.02" + }, + { + "t": 8.13, + "available_gib": 6.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "16.78 17.99 16.02" + } + ], + "log_tail": "/private/tmp/claude-501/bench/venv/lib/python3.14/site-packages/mlx_lm/server.py:1723: UserWarning: mlx_lm.server is not recommended for production as it only implements basic security checks.\n warnings.warn(\n2026-10-31 11:40:13,206 - INFO - Starting httpd at 127.0.0.1 on port 55488...\n127.0.0.1 - - [31/Oct/2026 11:40:13] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:40:13] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:40:16,733 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:40:16,733 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:40:16,733 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:40:16,733 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:40:16] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:40:17,306 - INFO - Prompt processing progress: 23/27\n2026-10-31 11:40:17,342 - INFO - Prompt processing progress: 26/27\n2026-10-31 11:40:17,391 - INFO - Prompt processing progress: 27/27\n2026-10-31 11:40:18,175 - INFO - Prompt Cache: 3 sequences, 0.06 GB\n2026-10-31 11:40:18,175 - INFO - - assistant: 2 sequences, 0.04 GB\n2026-10-31 11:40:18,175 - INFO - - user: 1 sequences, 0.02 GB\n2026-10-31 11:40:18,175 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:40:18] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:40:18,562 - INFO - Prompt processing progress: 1/1\n", + "ready_s": 7.5053422919991135, + "ttft_s": 0.4174112079999759, + "total_s": 1.2334240000000136, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "Sunlight travels through the atmosphere and gets refracted (bent) as it enters a raindrop.\nThe light then reflects off the back of the dro", + "served_model": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-2B-MLX-4bit/snapshots/93760be4f1f69842a46bc13dbdc0f19e291392a3", + "text_sha256": "ffbf23cb27bb50106bf151750276ab30f61683b0bec1ceb077bab1d61844393e", + "decode_tps": 37.98960053557416, + "min_available_gib": 6.08, + "peak_temp_c": 0, + "round": 3, + "admission_available_gib": 8.96 + } + ], + "runtime": { + "mlx": "0.32.3", + "mlx_lm": "0.31.3", + "transformers": "5.16.1", + "python": "3.14.3", + "device": { + "device_name": "Apple M4", + "max_recommended_working_set_size": 12713115648, + "memory_size": 17179869184, + "architecture": "applegpu_g16g", + "max_buffer_length": 9534832640, + "resource_limit": 499000 + } + }, + "tensorfold_sha256": "4b4b5a5ac152eb2dbed7e9d8004805ed9b74f77c64c3de18eb4f35263c7b9eb8", + "model_dir": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-2B-MLX-4bit/snapshots/93760be4f1f69842a46bc13dbdc0f19e291392a3", + "host_start": { + "macos": "27.0", + "build": "26A5421a", + "hw_model": "Mac16,10", + "chip": "Apple M4", + "gpu_cores": "10", + "memory_gib": 16.0, + "power": [ + "Now drawing from 'AC Power'" + ], + "low_power_mode": false, + "thermal": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", + "uptime": "11:38 up 2:57, 4 users, load averages: 19.04 18.56 15.94", + "top_cpu": [ + "42290 com.apple.Virtua 578.6 10G ", + "0 kernel_task 34.6 34M- ", + "433 WindowServer 32.9 601M ", + "4939 Claude Helper (R 21.4 918M+", + "4916 Claude Helper 16.2 476M+", + "88301 Python 15.7 27M" + ], + "memory_free_pct": "52", + "swap": "total = 4096.00M used = 2944.25M free = 1151.75M (encrypted)", + "lima": "NAME STATUS SSH VMTYPE ARCH CPUS MEMORY DISK DIR\nkbuild Running 127.0.0.1:53830 vz aarch64 8 10GiB 60GiB ~/.lima/kbuild" + }, + "summary": { + "mlx-metal": { + "ok_runs": 3, + "median_ttft_s": 0.3782165000011446, + "median_total_s": 1.1139279999988503, + "median_decode_tps": 47.67424648743776, + "texts": [ + "ffbf23cb27bb" + ] + }, + "lm-metal": { + "ok_runs": 3, + "median_ttft_s": 0.4174112079999759, + "median_total_s": 1.1748576659992978, + "median_decode_tps": 42.35874924739018, + "texts": [ + "ffbf23cb27bb" + ] + }, + "tensorfold-metal": { + "ok_runs": 3, + "median_ttft_s": 0.07251733399971272, + "median_total_s": 0.85273262499868, + "median_decode_tps": 39.70918356879257, + "texts": [ + "ffbf23cb27bb" + ] + } + }, + "host_end": { + "macos": "27.0", + "build": "26A5421a", + "hw_model": "Mac16,10", + "chip": "Apple M4", + "gpu_cores": "10", + "memory_gib": 16.0, + "power": [ + "Now drawing from 'AC Power'" + ], + "low_power_mode": false, + "thermal": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", + "uptime": "11:40 up 2:59, 4 users, load averages: 16.88 17.99 16.03", + "top_cpu": [ + "42290 com.apple.Virtua 568.4 10G ", + "0 kernel_task 50.2 34M- ", + "433 WindowServer 34.7 602M+", + "88301 Python 24.7 27M+ ", + "4916 Claude Helper 12.1 442M-", + "42198 limactl 7.7 135M+" + ], + "memory_free_pct": "53", + "swap": "total = 4096.00M used = 2890.69M free = 1205.31M (encrypted)", + "lima": "NAME STATUS SSH VMTYPE ARCH CPUS MEMORY DISK DIR\nkbuild Running 127.0.0.1:53830 vz aarch64 8 10GiB 60GiB ~/.lima/kbuild" + } +} diff --git a/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen4b-short-metal-c4.json b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen4b-short-metal-c4.json new file mode 100644 index 00000000..32161a6c --- /dev/null +++ b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen4b-short-metal-c4.json @@ -0,0 +1,492 @@ +{ + "status": "complete", + "model_key": "qwen4b", + "model": "mlx-community/Qwen3.5-4B-MLX-4bit@32f3e8ecf65426fc3306969496342d504bfa13f3", + "weights_sha256": "5fb9acd0246866381cf8c5c354c6db1019f6498eec4ccb4f5edcc71ffeacb2db", + "kernel": "macOS 27.0 (26A5421a)", + "driver": "Metal", + "tensorfold_rev": "v1.1.9 release", + "rapid_rev": "1d09cdb684bbf58fac130f5804b08ee2e1f56558", + "serve_concurrency": 4, + "prompt_sha256": "af125b15dfba55caedb541e89e59fe1252c0f13553e5ce6bc181e237fada9926", + "max_tokens": 32, + "policy": { + "reserve_gib": 1.0, + "max_fan_rpm": null, + "max_temp_c": 80 + }, + "allowance_gib": 6.5, + "arms": [ + "lm-metal" + ], + "concurrency": 4, + "runs": [ + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.13, + "available_gib": 9.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.81 19.31 17.32" + }, + { + "t": 1.31, + "available_gib": 9.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.81 19.31 17.32" + }, + { + "t": 2.86, + "available_gib": 7.52, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.27 19.24 17.31" + }, + { + "t": 4.21, + "available_gib": 5.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.27 19.24 17.31" + }, + { + "t": 5.8, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.27 19.24 17.31" + }, + { + "t": 7.29, + "available_gib": 5.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.27 19.24 17.31" + }, + { + "t": 8.48, + "available_gib": 5.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.56 19.33 17.36" + }, + { + "t": 9.7, + "available_gib": 5.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.56 19.33 17.36" + }, + { + "t": 10.96, + "available_gib": 4.64, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.56 19.33 17.36" + }, + { + "t": 12.27, + "available_gib": 4.16, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.56 19.33 17.36" + }, + { + "t": 13.4, + "available_gib": 4.48, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.20 19.29 17.35" + } + ], + "log_tail": "/private/tmp/claude-501/bench/venv/lib/python3.14/site-packages/mlx_lm/server.py:1723: UserWarning: mlx_lm.server is not recommended for production as it only implements basic security checks.\n warnings.warn(\n2026-10-31 11:48:32,119 - INFO - Starting httpd at 127.0.0.1 on port 56559...\n127.0.0.1 - - [31/Oct/2026 11:48:32] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:48:32] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:48:36,229 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:48:36,231 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:48:36,231 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:48:36,231 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:48:36] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:48:37,308 - INFO - Prompt processing progress: 23/27\n2026-10-31 11:48:37,368 - INFO - Prompt processing progress: 26/27\n2026-10-31 11:48:37,438 - INFO - Prompt processing progress: 27/27\n2026-10-31 11:48:38,949 - INFO - Prompt Cache: 3 sequences, 0.16 GB\n2026-10-31 11:48:38,950 - INFO - - assistant: 2 sequences, 0.11 GB\n2026-10-31 11:48:38,950 - INFO - - user: 1 sequences, 0.05 GB\n2026-10-31 11:48:38,950 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:48:39] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:48:39,335 - INFO - Prompt Cache: 3 sequences, 0.16 GB\n2026-10-31 11:48:39,335 - INFO - - assistant: 2 sequences, 0.11 GB\n2026-10-31 11:48:39,335 - INFO - - user: 1 sequences, 0.05 GB\n2026-10-31 11:48:39,335 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:48:39] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:48:39,719 - INFO - Prompt Cache: 3 sequences, 0.16 GB\n2026-10-31 11:48:39,719 - INFO - - assistant: 2 sequences, 0.11 GB\n2026-10-31 11:48:39,719 - INFO - - user: 1 sequences, 0.05 GB\n2026-10-31 11:48:39,719 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:48:39] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:48:40,119 - INFO - Prompt Cache: 3 sequences, 0.16 GB\n2026-10-31 11:48:40,119 - INFO - - assistant: 2 sequences, 0.11 GB\n2026-10-31 11:48:40,119 - INFO - - user: 1 sequences, 0.05 GB\n2026-10-31 11:48:40,119 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:48:40] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:48:40,916 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:48:40,917 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:48:40,917 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:48:40,917 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:48:41,056 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:48:41,057 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:48:41,059 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:48:41,060 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:48:41,221 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:48:41,223 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:48:41,222 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:48:41,222 - INFO - Prompt processing progress: 33/33\n", + "ready_s": 8.91568183399977, + "concurrency": 4, + "wall_s": 4.454901332999725, + "aggregate_tok_s": 28.732398415165505, + "per_request_decode_median": 14.827650089328234, + "ttft_s": 2.338562875000207, + "total_s": 4.454901332999725, + "completion_tokens": 128, + "text": "A rainbow forms when sunlight hits raindrops in the air and gets split into its seven colors, a process called refraction. As the light enters each water dro", + "requests": [ + { + "ttft_s": 2.334610416999567, + "total_s": 4.404055624998364, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "A rainbow forms when sunlight hits raindrops in the air and gets split into its seven colors, a process called refraction. As the light enters each water dro" + }, + { + "ttft_s": 2.336000833000071, + "total_s": 4.426689500000066, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "A rainbow forms when sunlight hits raindrops in the air and gets split into its seven colors, a process called refraction. As the light enters each water dro" + }, + { + "ttft_s": 2.352317499999117, + "total_s": 4.451542834000065, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "A rainbow forms when sunlight hits raindrops in the air and gets split into its seven colors, a process called refraction. As the light enters each water dro" + }, + { + "ttft_s": 2.338562875000207, + "total_s": 4.449884292000206, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "A rainbow forms when sunlight hits raindrops in the air and gets split into its seven colors, a process called refraction. As the light enters each water dro" + } + ], + "served_model": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-4B-MLX-4bit/snapshots/32f3e8ecf65426fc3306969496342d504bfa13f3", + "text_sha256": "9095cd4ace32bd52edcf7f4b54be83cc498d29c92c3cb20973dfb5364f46626d", + "decode_tps": 28.732398415165505, + "min_available_gib": 4.16, + "peak_temp_c": 0, + "round": 1, + "admission_available_gib": 9.28 + }, + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.23, + "available_gib": 8.96, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.20 19.29 17.35" + }, + { + "t": 1.47, + "available_gib": 9.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.26 19.34 17.38" + }, + { + "t": 2.57, + "available_gib": 9.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.26 19.34 17.38" + }, + { + "t": 4.11, + "available_gib": 6.88, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.26 19.34 17.38" + }, + { + "t": 5.29, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.26 19.34 17.38" + }, + { + "t": 6.55, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.84 19.28 17.37" + }, + { + "t": 8.09, + "available_gib": 5.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.84 19.28 17.37" + }, + { + "t": 9.63, + "available_gib": 5.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.84 19.28 17.37" + }, + { + "t": 10.78, + "available_gib": 5.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.84 19.28 17.37" + }, + { + "t": 11.92, + "available_gib": 4.32, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.25 19.40 17.42" + }, + { + "t": 13.59, + "available_gib": 4.16, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.25 19.40 17.42" + } + ], + "log_tail": "/private/tmp/claude-501/bench/venv/lib/python3.14/site-packages/mlx_lm/server.py:1723: UserWarning: mlx_lm.server is not recommended for production as it only implements basic security checks.\n warnings.warn(\n2026-10-31 11:48:49,627 - INFO - Starting httpd at 127.0.0.1 on port 56586...\n127.0.0.1 - - [31/Oct/2026 11:48:49] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:48:49] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:48:53,506 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:48:53,507 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:48:53,507 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:48:53,507 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:48:53] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:48:54,530 - INFO - Prompt processing progress: 23/27\n2026-10-31 11:48:54,601 - INFO - Prompt processing progress: 26/27\n2026-10-31 11:48:54,700 - INFO - Prompt processing progress: 27/27\n2026-10-31 11:48:56,465 - INFO - Prompt Cache: 3 sequences, 0.16 GB\n2026-10-31 11:48:56,466 - INFO - - assistant: 2 sequences, 0.11 GB\n2026-10-31 11:48:56,466 - INFO - - user: 1 sequences, 0.05 GB\n2026-10-31 11:48:56,466 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:48:56] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:48:56,862 - INFO - Prompt Cache: 3 sequences, 0.16 GB\n2026-10-31 11:48:56,862 - INFO - - assistant: 2 sequences, 0.11 GB\n2026-10-31 11:48:56,863 - INFO - - user: 1 sequences, 0.05 GB\n2026-10-31 11:48:56,863 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:48:57] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:48:57,208 - INFO - Prompt Cache: 3 sequences, 0.16 GB\n2026-10-31 11:48:57,208 - INFO - - assistant: 2 sequences, 0.11 GB\n2026-10-31 11:48:57,208 - INFO - - user: 1 sequences, 0.05 GB\n2026-10-31 11:48:57,208 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:48:57] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:48:57,545 - INFO - Prompt Cache: 3 sequences, 0.16 GB\n2026-10-31 11:48:57,545 - INFO - - assistant: 2 sequences, 0.11 GB\n2026-10-31 11:48:57,545 - INFO - - user: 1 sequences, 0.05 GB\n2026-10-31 11:48:57,546 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:48:57] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:48:58,380 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:48:58,381 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:48:58,382 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:48:58,382 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:48:58,539 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:48:58,539 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:48:58,539 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:48:58,539 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:48:58,684 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:48:58,689 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:48:58,689 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:48:58,690 - INFO - Prompt processing progress: 33/33\n", + "ready_s": 9.777324124999723, + "concurrency": 4, + "wall_s": 4.379749916999572, + "aggregate_tok_s": 29.22541296323347, + "per_request_decode_median": 15.077156436141971, + "ttft_s": 2.2939162500006205, + "total_s": 4.379749916999572, + "completion_tokens": 128, + "text": "A rainbow forms when sunlight hits raindrops in the air and gets split into its seven colors, a process called refraction. As the light enters each water dro", + "requests": [ + { + "ttft_s": 2.2939162500006205, + "total_s": 4.350006874999963, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "A rainbow forms when sunlight hits raindrops in the air and gets split into its seven colors, a process called refraction. As the light enters each water dro" + }, + { + "ttft_s": 2.293037542000093, + "total_s": 4.318919667000955, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "A rainbow forms when sunlight hits raindrops in the air and gets split into its seven colors, a process called refraction. As the light enters each water dro" + }, + { + "ttft_s": 2.29091999999946, + "total_s": 4.3700114999992365, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "A rainbow forms when sunlight hits raindrops in the air and gets split into its seven colors, a process called refraction. As the light enters each water dro" + }, + { + "ttft_s": 2.3000021249990823, + "total_s": 4.37303754200002, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "A rainbow forms when sunlight hits raindrops in the air and gets split into its seven colors, a process called refraction. As the light enters each water dro" + } + ], + "served_model": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-4B-MLX-4bit/snapshots/32f3e8ecf65426fc3306969496342d504bfa13f3", + "text_sha256": "9095cd4ace32bd52edcf7f4b54be83cc498d29c92c3cb20973dfb5364f46626d", + "decode_tps": 29.22541296323347, + "min_available_gib": 4.16, + "peak_temp_c": 0, + "round": 2, + "admission_available_gib": 8.96 + }, + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.22, + "available_gib": 8.48, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.23 19.42 17.44" + }, + { + "t": 1.37, + "available_gib": 8.48, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.23 19.42 17.44" + }, + { + "t": 2.78, + "available_gib": 8.64, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.23 19.42 17.44" + }, + { + "t": 4.11, + "available_gib": 5.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.93 19.60 17.52" + }, + { + "t": 5.81, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.93 19.60 17.52" + }, + { + "t": 7.12, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.93 19.60 17.52" + }, + { + "t": 8.33, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.93 19.60 17.52" + }, + { + "t": 9.61, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.46 19.54 17.51" + }, + { + "t": 10.85, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.46 19.54 17.51" + }, + { + "t": 12.11, + "available_gib": 4.32, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.46 19.54 17.51" + }, + { + "t": 13.33, + "available_gib": 4.16, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.46 19.54 17.51" + } + ], + "log_tail": "/private/tmp/claude-501/bench/venv/lib/python3.14/site-packages/mlx_lm/server.py:1723: UserWarning: mlx_lm.server is not recommended for production as it only implements basic security checks.\n warnings.warn(\n2026-10-31 11:49:06,950 - INFO - Starting httpd at 127.0.0.1 on port 56613...\n127.0.0.1 - - [31/Oct/2026 11:49:07] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:49:07] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:49:11,039 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:49:11,039 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:49:11,039 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:49:11,040 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:49:11] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:49:12,059 - INFO - Prompt processing progress: 23/27\n2026-10-31 11:49:12,140 - INFO - Prompt processing progress: 26/27\n2026-10-31 11:49:12,203 - INFO - Prompt processing progress: 27/27\n2026-10-31 11:49:13,792 - INFO - Prompt Cache: 3 sequences, 0.16 GB\n2026-10-31 11:49:13,792 - INFO - - assistant: 2 sequences, 0.11 GB\n2026-10-31 11:49:13,792 - INFO - - user: 1 sequences, 0.05 GB\n2026-10-31 11:49:13,792 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:49:13] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:49:14,149 - INFO - Prompt Cache: 3 sequences, 0.16 GB\n2026-10-31 11:49:14,149 - INFO - - assistant: 2 sequences, 0.11 GB\n2026-10-31 11:49:14,149 - INFO - - user: 1 sequences, 0.05 GB\n2026-10-31 11:49:14,149 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:49:14] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:49:14,445 - INFO - Prompt Cache: 3 sequences, 0.16 GB\n2026-10-31 11:49:14,445 - INFO - - assistant: 2 sequences, 0.11 GB\n2026-10-31 11:49:14,445 - INFO - - user: 1 sequences, 0.05 GB\n2026-10-31 11:49:14,445 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:49:14] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:49:14,762 - INFO - Prompt Cache: 3 sequences, 0.16 GB\n2026-10-31 11:49:14,763 - INFO - - assistant: 2 sequences, 0.11 GB\n2026-10-31 11:49:14,763 - INFO - - user: 1 sequences, 0.05 GB\n2026-10-31 11:49:14,763 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:49:14] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:49:15,496 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:49:15,498 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:49:15,499 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:49:15,499 - INFO - Prompt processing progress: 29/33\n2026-10-31 11:49:15,644 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:49:15,644 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:49:15,645 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:49:15,646 - INFO - Prompt processing progress: 32/33\n2026-10-31 11:49:15,769 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:49:15,770 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:49:15,771 - INFO - Prompt processing progress: 33/33\n2026-10-31 11:49:15,773 - INFO - Prompt processing progress: 33/33\n", + "ready_s": 9.677016290999745, + "concurrency": 4, + "wall_s": 4.0450695420004195, + "aggregate_tok_s": 31.643460927176, + "per_request_decode_median": 15.663040179893985, + "ttft_s": 2.0373866249992716, + "total_s": 4.0450695420004195, + "completion_tokens": 128, + "text": "A rainbow forms when sunlight hits raindrops in the air and gets split into its seven colors, a process called refraction. As the light enters each water dro", + "requests": [ + { + "ttft_s": 2.0373866249992716, + "total_s": 3.990782124999896, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "A rainbow forms when sunlight hits raindrops in the air and gets split into its seven colors, a process called refraction. As the light enters each water dro" + }, + { + "ttft_s": 2.035592042000644, + "total_s": 4.014773583001443, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "A rainbow forms when sunlight hits raindrops in the air and gets split into its seven colors, a process called refraction. As the light enters each water dro" + }, + { + "ttft_s": 2.035128125000483, + "total_s": 4.035999875000925, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "A rainbow forms when sunlight hits raindrops in the air and gets split into its seven colors, a process called refraction. As the light enters each water dro" + }, + { + "ttft_s": 2.0382880409997597, + "total_s": 4.037934790998406, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "A rainbow forms when sunlight hits raindrops in the air and gets split into its seven colors, a process called refraction. As the light enters each water dro" + } + ], + "served_model": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-4B-MLX-4bit/snapshots/32f3e8ecf65426fc3306969496342d504bfa13f3", + "text_sha256": "9095cd4ace32bd52edcf7f4b54be83cc498d29c92c3cb20973dfb5364f46626d", + "decode_tps": 31.643460927176, + "min_available_gib": 4.16, + "peak_temp_c": 0, + "round": 3, + "admission_available_gib": 8.32 + } + ], + "runtime": { + "mlx": "0.32.3", + "mlx_lm": "0.31.3", + "transformers": "5.16.1", + "python": "3.14.3", + "device": { + "device_name": "Apple M4", + "max_recommended_working_set_size": 12713115648, + "memory_size": 17179869184, + "architecture": "applegpu_g16g", + "max_buffer_length": 9534832640, + "resource_limit": 499000 + } + }, + "tensorfold_sha256": "4b4b5a5ac152eb2dbed7e9d8004805ed9b74f77c64c3de18eb4f35263c7b9eb8", + "model_dir": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-4B-MLX-4bit/snapshots/32f3e8ecf65426fc3306969496342d504bfa13f3", + "host_start": { + "macos": "27.0", + "build": "26A5421a", + "hw_model": "Mac16,10", + "chip": "Apple M4", + "gpu_cores": "10", + "memory_gib": 16.0, + "power": [ + "Now drawing from 'AC Power'" + ], + "low_power_mode": false, + "thermal": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", + "uptime": "11:48 up 3:07, 4 users, load averages: 21.81 19.31 17.32", + "top_cpu": [ + "42290 com.apple.Virtua 543.2 10G ", + "4939 Claude Helper (R 57.0 995M-", + "433 WindowServer 31.2 584M-", + "0 kernel_task 25.3 35M- ", + "4916 Claude Helper 12.1 474M-", + "42198 limactl 8.6 81M-" + ], + "memory_free_pct": "58", + "swap": "total = 5120.00M used = 3710.12M free = 1409.88M (encrypted)", + "lima": "NAME STATUS SSH VMTYPE ARCH CPUS MEMORY DISK DIR\nkbuild Running 127.0.0.1:53830 vz aarch64 8 10GiB 60GiB ~/.lima/kbuild" + }, + "summary": { + "lm-metal": { + "ok_runs": 3, + "median_ttft_s": 2.2939162500006205, + "median_total_s": 4.379749916999572, + "median_decode_tps": 29.22541296323347, + "texts": [ + "9095cd4ace32" + ] + } + }, + "host_end": { + "macos": "27.0", + "build": "26A5421a", + "hw_model": "Mac16,10", + "chip": "Apple M4", + "gpu_cores": "10", + "memory_gib": 16.0, + "power": [ + "Now drawing from 'AC Power'" + ], + "low_power_mode": false, + "thermal": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", + "uptime": "11:49 up 3:08, 4 users, load averages: 21.83 19.68 17.58", + "top_cpu": [ + "42290 com.apple.Virtua 581.9 10G ", + "433 WindowServer 32.0 592M ", + "0 kernel_task 21.9 34M+ ", + "42198 limactl 15.3 81M ", + "4916 Claude Helper 12.1 456M+", + "4939 Claude Helper (R 4.8 997M-" + ], + "memory_free_pct": "56", + "swap": "total = 5120.00M used = 3686.94M free = 1433.06M (encrypted)", + "lima": "NAME STATUS SSH VMTYPE ARCH CPUS MEMORY DISK DIR\nkbuild Running 127.0.0.1:53830 vz aarch64 8 10GiB 60GiB ~/.lima/kbuild" + } +} diff --git a/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen4b-short-metal.json b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen4b-short-metal.json new file mode 100644 index 00000000..dd4e2c15 --- /dev/null +++ b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/2026-10-31-engine-compare-qwen4b-short-metal.json @@ -0,0 +1,541 @@ +{ + "status": "complete", + "model_key": "qwen4b", + "model": "mlx-community/Qwen3.5-4B-MLX-4bit@32f3e8ecf65426fc3306969496342d504bfa13f3", + "weights_sha256": "5fb9acd0246866381cf8c5c354c6db1019f6498eec4ccb4f5edcc71ffeacb2db", + "kernel": "macOS 27.0 (26A5421a)", + "driver": "Metal", + "tensorfold_rev": "v1.1.9 release", + "rapid_rev": "1d09cdb684bbf58fac130f5804b08ee2e1f56558", + "serve_concurrency": 4, + "prompt_sha256": "af125b15dfba55caedb541e89e59fe1252c0f13553e5ce6bc181e237fada9926", + "max_tokens": 32, + "policy": { + "reserve_gib": 1.0, + "max_fan_rpm": null, + "max_temp_c": 80 + }, + "allowance_gib": 6.5, + "arms": [ + "mlx-metal", + "lm-metal" + ], + "concurrency": 1, + "runs": [ + { + "arm": "mlx-metal", + "stopped": null, + "samples": [ + { + "t": 0.31, + "available_gib": 8.48, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.68 18.40 16.83" + }, + { + "t": 1.9, + "available_gib": 8.32, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.68 18.40 16.83" + }, + { + "t": 3.52, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.68 18.40 16.83" + }, + { + "t": 4.83, + "available_gib": 4.48, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.68 18.40 16.83" + }, + { + "t": 6.13, + "available_gib": 6.56, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.38 18.37 16.83" + }, + { + "t": 7.36, + "available_gib": 6.4, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.38 18.37 16.83" + }, + { + "t": 8.59, + "available_gib": 6.72, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.38 18.37 16.83" + }, + { + "t": 9.98, + "available_gib": 6.4, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.43 18.42 16.85" + } + ], + "log_tail": "", + "ready_s": 8.163030041998354, + "ttft_s": 0.39044933399964066, + "total_s": 1.4598310420005873, + "completion_tokens": 32, + "text": "A rainbow forms when sunlight hits raindrops in the air, which act like tiny prisms. As the light enters a drop, it slows down and bends,", + "text_sha256": "b76ba6df80bbbd57c3a015773b0c0787655dbe8b193e3438b573a4428393b1de", + "decode_tps": 28.98871354172495, + "min_available_gib": 4.48, + "peak_temp_c": 0, + "round": 1, + "admission_available_gib": 8.48 + }, + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.19, + "available_gib": 9.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.43 18.42 16.85" + }, + { + "t": 1.32, + "available_gib": 9.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.40 18.44 16.87" + }, + { + "t": 2.9, + "available_gib": 7.52, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.40 18.44 16.87" + }, + { + "t": 4.25, + "available_gib": 6.4, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.40 18.44 16.87" + }, + { + "t": 5.8, + "available_gib": 6.08, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.40 18.44 16.87" + }, + { + "t": 7.07, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.13 18.42 16.87" + }, + { + "t": 8.31, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.13 18.42 16.87" + } + ], + "log_tail": "/private/tmp/claude-501/bench/venv/lib/python3.14/site-packages/mlx_lm/server.py:1723: UserWarning: mlx_lm.server is not recommended for production as it only implements basic security checks.\n warnings.warn(\n2026-10-31 11:47:18,547 - INFO - Starting httpd at 127.0.0.1 on port 56485...\n127.0.0.1 - - [31/Oct/2026 11:47:18] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:47:18] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:47:21,649 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:47:21,649 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:47:21,649 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:47:21,649 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:47:21] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:47:22,329 - INFO - Prompt processing progress: 23/27\n2026-10-31 11:47:22,446 - INFO - Prompt processing progress: 26/27\n2026-10-31 11:47:22,505 - INFO - Prompt processing progress: 27/27\n2026-10-31 11:47:23,801 - INFO - Prompt Cache: 3 sequences, 0.16 GB\n2026-10-31 11:47:23,801 - INFO - - assistant: 2 sequences, 0.11 GB\n2026-10-31 11:47:23,801 - INFO - - user: 1 sequences, 0.05 GB\n2026-10-31 11:47:23,801 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:47:23] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:47:24,132 - INFO - Prompt processing progress: 1/1\n", + "ready_s": 7.126807624999856, + "ttft_s": 0.37735616699865204, + "total_s": 1.5606610830000136, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "A rainbow forms when sunlight hits raindrops in the air and gets split into its seven colors, creating a spectrum. Each raindrop acts like a tiny prism,", + "served_model": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-4B-MLX-4bit/snapshots/32f3e8ecf65426fc3306969496342d504bfa13f3", + "text_sha256": "bdb76d67e763d4e8ae7176616718338bafdab9ecab8cea1f66e0a73e706abe84", + "decode_tps": 26.197812229797524, + "min_available_gib": 5.6, + "peak_temp_c": 0, + "round": 1, + "admission_available_gib": 9.28 + }, + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.17, + "available_gib": 8.64, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.96 18.41 16.88" + }, + { + "t": 1.52, + "available_gib": 8.8, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.96 18.41 16.88" + }, + { + "t": 3.04, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "19.96 18.41 16.88" + }, + { + "t": 4.28, + "available_gib": 6.24, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.08 18.67 16.98" + }, + { + "t": 5.77, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.08 18.67 16.98" + }, + { + "t": 6.92, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.08 18.67 16.98" + }, + { + "t": 8.08, + "available_gib": 5.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.08 18.67 16.98" + } + ], + "log_tail": "/private/tmp/claude-501/bench/venv/lib/python3.14/site-packages/mlx_lm/server.py:1723: UserWarning: mlx_lm.server is not recommended for production as it only implements basic security checks.\n warnings.warn(\n2026-10-31 11:47:30,327 - INFO - Starting httpd at 127.0.0.1 on port 56502...\n127.0.0.1 - - [31/Oct/2026 11:47:30] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:47:30] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:47:33,456 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:47:33,457 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:47:33,457 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:47:33,457 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:47:33] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:47:34,248 - INFO - Prompt processing progress: 23/27\n2026-10-31 11:47:34,292 - INFO - Prompt processing progress: 26/27\n2026-10-31 11:47:34,333 - INFO - Prompt processing progress: 27/27\n2026-10-31 11:47:35,623 - INFO - Prompt Cache: 3 sequences, 0.16 GB\n2026-10-31 11:47:35,623 - INFO - - assistant: 2 sequences, 0.11 GB\n2026-10-31 11:47:35,623 - INFO - - user: 1 sequences, 0.05 GB\n2026-10-31 11:47:35,623 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:47:35] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:47:35,924 - INFO - Prompt processing progress: 1/1\n", + "ready_s": 7.037619959000949, + "ttft_s": 0.34289787499983504, + "total_s": 1.575447458000781, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "A rainbow forms when sunlight hits raindrops in the air and gets split into its seven colors, creating a spectrum. Each raindrop acts like a tiny prism,", + "served_model": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-4B-MLX-4bit/snapshots/32f3e8ecf65426fc3306969496342d504bfa13f3", + "text_sha256": "bdb76d67e763d4e8ae7176616718338bafdab9ecab8cea1f66e0a73e706abe84", + "decode_tps": 25.151117997641002, + "min_available_gib": 5.28, + "peak_temp_c": 0, + "round": 2, + "admission_available_gib": 8.64 + }, + { + "arm": "mlx-metal", + "stopped": null, + "samples": [ + { + "t": 0.15, + "available_gib": 8.8, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.15 18.72 17.01" + }, + { + "t": 1.46, + "available_gib": 8.96, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.15 18.72 17.01" + }, + { + "t": 3.34, + "available_gib": 7.52, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.34 18.60 16.97" + }, + { + "t": 4.63, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.34 18.60 16.97" + }, + { + "t": 5.79, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "20.34 18.60 16.97" + }, + { + "t": 7.14, + "available_gib": 5.92, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "22.23 19.02 17.13" + }, + { + "t": 8.35, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "22.23 19.02 17.13" + } + ], + "log_tail": "", + "ready_s": 6.914567000001625, + "ttft_s": 0.36697812500096916, + "total_s": 1.4193650830002298, + "completion_tokens": 32, + "text": "A rainbow forms when sunlight hits raindrops in the air, which act like tiny prisms. As the light enters a drop, it slows down and bends,", + "text_sha256": "b76ba6df80bbbd57c3a015773b0c0787655dbe8b193e3438b573a4428393b1de", + "decode_tps": 29.45684547339457, + "min_available_gib": 5.6, + "peak_temp_c": 0, + "round": 2, + "admission_available_gib": 8.64 + }, + { + "arm": "mlx-metal", + "stopped": null, + "samples": [ + { + "t": 0.17, + "available_gib": 8.8, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.97 19.02 17.14" + }, + { + "t": 1.3, + "available_gib": 9.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.97 19.02 17.14" + }, + { + "t": 3.21, + "available_gib": 8.16, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.97 19.02 17.14" + }, + { + "t": 4.3, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.97 19.02 17.14" + }, + { + "t": 5.43, + "available_gib": 5.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "22.06 19.08 17.18" + }, + { + "t": 6.67, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "22.06 19.08 17.18" + }, + { + "t": 7.84, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "22.06 19.08 17.18" + }, + { + "t": 9.17, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "22.06 19.08 17.18" + } + ], + "log_tail": "", + "ready_s": 7.243086082999071, + "ttft_s": 0.45752133300084097, + "total_s": 1.566815250000218, + "completion_tokens": 32, + "text": "A rainbow forms when sunlight hits raindrops in the air, which act like tiny prisms. As the light enters a drop, it slows down and bends,", + "text_sha256": "b76ba6df80bbbd57c3a015773b0c0787655dbe8b193e3438b573a4428393b1de", + "decode_tps": 27.945704492687135, + "min_available_gib": 5.44, + "peak_temp_c": 0, + "round": 3, + "admission_available_gib": 8.8 + }, + { + "arm": "lm-metal", + "stopped": null, + "samples": [ + { + "t": 0.51, + "available_gib": 8.96, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "22.13 19.15 17.21" + }, + { + "t": 1.69, + "available_gib": 9.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.96 19.16 17.23" + }, + { + "t": 3.1, + "available_gib": 9.12, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.96 19.16 17.23" + }, + { + "t": 4.41, + "available_gib": 5.44, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.96 19.16 17.23" + }, + { + "t": 5.9, + "available_gib": 6.08, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.96 19.16 17.23" + }, + { + "t": 7.45, + "available_gib": 5.76, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.72 19.16 17.24" + }, + { + "t": 8.7, + "available_gib": 5.6, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.72 19.16 17.24" + }, + { + "t": 9.81, + "available_gib": 5.28, + "max_temp_c": 0, + "max_fan_rpm": 0, + "load_avg": "21.72 19.16 17.24" + } + ], + "log_tail": "/private/tmp/claude-501/bench/venv/lib/python3.14/site-packages/mlx_lm/server.py:1723: UserWarning: mlx_lm.server is not recommended for production as it only implements basic security checks.\n warnings.warn(\n2026-10-31 11:48:08,925 - INFO - Starting httpd at 127.0.0.1 on port 56530...\n127.0.0.1 - - [31/Oct/2026 11:48:09] \"GET /health HTTP/1.1\" 200 -\n127.0.0.1 - - [31/Oct/2026 11:48:09] \"GET /v1/models HTTP/1.1\" 200 -\n2026-10-31 11:48:12,802 - INFO - Prompt Cache: 0 sequences, 0.00 GB\n2026-10-31 11:48:12,803 - INFO - - assistant: 0 sequences, 0.00 GB\n2026-10-31 11:48:12,803 - INFO - - user: 0 sequences, 0.00 GB\n2026-10-31 11:48:12,803 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:48:12] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:48:13,806 - INFO - Prompt processing progress: 23/27\n2026-10-31 11:48:13,863 - INFO - Prompt processing progress: 26/27\n2026-10-31 11:48:13,948 - INFO - Prompt processing progress: 27/27\n2026-10-31 11:48:15,438 - INFO - Prompt Cache: 3 sequences, 0.16 GB\n2026-10-31 11:48:15,438 - INFO - - assistant: 2 sequences, 0.11 GB\n2026-10-31 11:48:15,438 - INFO - - user: 1 sequences, 0.05 GB\n2026-10-31 11:48:15,438 - INFO - - system: 0 sequences, 0.00 GB\n127.0.0.1 - - [31/Oct/2026 11:48:15] \"POST /v1/chat/completions HTTP/1.1\" 200 -\n2026-10-31 11:48:15,831 - INFO - Prompt processing progress: 1/1\n", + "ready_s": 9.052705916999912, + "ttft_s": 0.437413417001153, + "total_s": 1.7525714170005813, + "completion_tokens": 32, + "token_count_source": "usage", + "text": "A rainbow forms when sunlight hits raindrops in the air and gets split into its seven colors, creating a spectrum. Each raindrop acts like a tiny prism,", + "served_model": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-4B-MLX-4bit/snapshots/32f3e8ecf65426fc3306969496342d504bfa13f3", + "text_sha256": "bdb76d67e763d4e8ae7176616718338bafdab9ecab8cea1f66e0a73e706abe84", + "decode_tps": 23.571312344230485, + "min_available_gib": 5.28, + "peak_temp_c": 0, + "round": 3, + "admission_available_gib": 8.96 + } + ], + "runtime": { + "mlx": "0.32.3", + "mlx_lm": "0.31.3", + "transformers": "5.16.1", + "python": "3.14.3", + "device": { + "device_name": "Apple M4", + "max_recommended_working_set_size": 12713115648, + "memory_size": 17179869184, + "architecture": "applegpu_g16g", + "max_buffer_length": 9534832640, + "resource_limit": 499000 + } + }, + "tensorfold_sha256": "4b4b5a5ac152eb2dbed7e9d8004805ed9b74f77c64c3de18eb4f35263c7b9eb8", + "model_dir": "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-4B-MLX-4bit/snapshots/32f3e8ecf65426fc3306969496342d504bfa13f3", + "host_start": { + "macos": "27.0", + "build": "26A5421a", + "hw_model": "Mac16,10", + "chip": "Apple M4", + "gpu_cores": "10", + "memory_gib": 16.0, + "power": [ + "Now drawing from 'AC Power'" + ], + "low_power_mode": false, + "thermal": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", + "uptime": "11:47 up 3:05, 4 users, load averages: 21.09 18.44 16.83", + "top_cpu": [ + "42290 com.apple.Virtua 571.6 10G ", + "0 kernel_task 62.5 34M ", + "433 WindowServer 32.2 600M ", + "4916 Claude Helper 15.0 495M+", + "4939 Claude Helper (R 14.7 997M+", + "42198 limactl 11.6 82M-" + ], + "memory_free_pct": "53", + "swap": "total = 4096.00M used = 2701.44M free = 1394.56M (encrypted)", + "lima": "NAME STATUS SSH VMTYPE ARCH CPUS MEMORY DISK DIR\nkbuild Running 127.0.0.1:53830 vz aarch64 8 10GiB 60GiB ~/.lima/kbuild" + }, + "summary": { + "mlx-metal": { + "ok_runs": 3, + "median_ttft_s": 0.39044933399964066, + "median_total_s": 1.4598310420005873, + "median_decode_tps": 28.98871354172495, + "texts": [ + "b76ba6df80bb" + ] + }, + "lm-metal": { + "ok_runs": 3, + "median_ttft_s": 0.37735616699865204, + "median_total_s": 1.575447458000781, + "median_decode_tps": 25.151117997641002, + "texts": [ + "bdb76d67e763" + ] + } + }, + "host_end": { + "macos": "27.0", + "build": "26A5421a", + "hw_model": "Mac16,10", + "chip": "Apple M4", + "gpu_cores": "10", + "memory_gib": 16.0, + "power": [ + "Now drawing from 'AC Power'" + ], + "low_power_mode": false, + "thermal": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", + "uptime": "11:48 up 3:07, 4 users, load averages: 21.79 19.27 17.30", + "top_cpu": [ + "42290 com.apple.Virtua 554.3 10G ", + "433 WindowServer 31.3 592M ", + "0 kernel_task 23.0 35M- ", + "4939 Claude Helper (R 13.6 995M-", + "4916 Claude Helper 11.7 529M-", + "42198 limactl 7.4 89M" + ], + "memory_free_pct": "57", + "swap": "total = 5120.00M used = 3726.12M free = 1393.88M (encrypted)", + "lima": "NAME STATUS SSH VMTYPE ARCH CPUS MEMORY DISK DIR\nkbuild Running 127.0.0.1:53830 vz aarch64 8 10GiB 60GiB ~/.lima/kbuild" + } +} diff --git a/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/README.md b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/README.md new file mode 100644 index 00000000..9bc8ec7e --- /dev/null +++ b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/README.md @@ -0,0 +1,102 @@ +# Mac mini M4, macOS native (Metal) baseline, 2026-10-31 + +The Apple-native half of a same-machine comparison: these are the scoreboard's models, prompts, token counts and loads +run under macOS on the Mac mini M4, so the same `engine-compare.py` commands can later run under Linux on this mini +and be compared row for row. Table and chart: the "Mac mini M4" section of +[`ENGINE-SCOREBOARD.md`](../../ENGINE-SCOREBOARD.md), `engine-scoreboard-m4-mini-macos.csv` / `.svg`. + +## Machine + +| Item | Value | +|---|---| +| Hardware | Mac mini, `Mac16,10`, Apple M4, 10 CPU cores, 10-core GPU (`applegpu_g16g`, Metal 4), 16 GB | +| OS | macOS 27.0, build `26A5421a` | +| Power | AC power, low power mode off; `pmset -g therm` recorded no thermal or performance warning in any run | +| MLX's view of the GPU | `Apple M4`, max recommended working set 11.84 GiB (from `mx.device_info()`, in every receipt's `runtime`) | + +## Load during the runs (read this before quoting a ratio) + +The Lima VM `kbuild` (8 vCPU, 10 GiB, running CI) was busy for the whole session and was left alone. `top` put +`com.apple.Virtualization` at **390-590 % CPU** (about 4-6 of the 10 cores) at the start of every receipt, the +1-minute load average was **15.7-23.2**, and the VM held 10 GiB of the 16 GB, so macOS reported only 44-59 % of +memory free-able (`kern.memorystatus_level`) and 2.1-2.2 GiB of swap in use. Each receipt records `uptime`, the top +six CPU users, `limactl list`, free memory and swap at its start and end (`host_start`, `host_end`) and the load +average every second of every run (`samples[].load_avg`). The granite run also overlapped the last model download. + +Decode is mostly GPU- and memory-bandwidth-bound, but per-token dispatch runs on the CPU, so these numbers are a +floor for this machine. Compare them with a Linux run under similar load, or re-run both with the VM stopped. + +## Software + +| Item | Value | +|---|---| +| Python venv | CPython 3.14.3, `uv venv` at `/private/tmp/claude-501/bench/venv`; full list in [`venv-freeze.txt`](venv-freeze.txt) | +| MLX | `mlx==0.32.3` (`mlx-metal==0.32.3`), `mlx-lm==0.31.3`, `transformers==5.16.1`: the pins omarchy-mlx's `m4-mini-bench` names for macOS (the MLX release mlx-omarchy is built from, and the Linux venv's mlx-lm) | +| LFM3 model code | mlx-lm 0.31.3 has no LFM3: `mlx_lm/models/lfm3.py` from mlx-lm PR 2016 head `9027ac5b062175fba890c612c3905d5d8d5b5294`, sha256 `a4096cdd…9b0c4` (checked by engine-compare, same file as the M1 Pro) | +| TensorFold | v1.1.9 release `tensorfold-1.1.9-macos-arm64.tar.gz` (sha256 `f8330102962550beea7d312eb78553f01539452fd442b7b195d63bc0f3ffa9d7`), `bin/tensorfold-native` sha256 `4b4b5a5ac152eb2dbed7e9d8004805ed9b74f77c64c3de18eb4f35263c7b9eb8`, the release `m4-mini-bench` pins; `serve --backend mlx --context 1024 --no-thinking --no-drafts` (drafts off to match the Vulkan build, which has none) | +| Models | the scoreboard's pins, from Hugging Face at these revisions; `model.safetensors` sha256 checked by engine-compare before each run | + +| Model | Revision | `model.safetensors` sha256 | +|---|---|---| +| `mlx-community/granite-4.0-1b-4bit` | `29f108a4e2ff99079a173004871604c2f1e12c1f` | `b7a43ebbc86517b8ecfc4cff265a787ea31620f7eb356984c08abe14646617c7` | +| `LiquidAI/LFM3-1.7B-MLX-8bit` | `ee7667f3cfccb2a41725f91820531823fcabc6b7` | `2b06cea98d75527c248a0b04383f7cf0a1c72fb4ed7b45096618fc4e59d015ad` | +| `mlx-community/Qwen3.5-2B-MLX-4bit` | `93760be4f1f69842a46bc13dbdc0f19e291392a3` | `713fe7e5d3c3965f7106b0d0ee17615f7869c23c8d327996df8c1196fbcf07d5` | +| `mlx-community/Qwen3.5-4B-MLX-4bit` | `32f3e8ecf65426fc3306969496342d504bfa13f3` | `5fb9acd0246866381cf8c5c354c6db1019f6498eec4ccb4f5edcc71ffeacb2db` | + +## Engines (macOS arms of `engine-compare.py`) + +- `mlx-metal`: upstream MLX on Metal, in-process; the same worker as the M1 Pro's `mlx-*` arms (load, one warmup on + the same prompt, one timed greedy generation). There is no submit budget on Metal, so there is one MLX arm, not + two. It is the "same output" reference within each receipt, like `mlx-5k` on the M1 Pro. +- `lm-metal`: plain `python -m mlx_lm.server` from the same venv, none of our server patches (the macOS twin of the + M1 Pro's `lm031-120k`). Our patched server is not run here: its small-rows patch targets omarchy's Vulkan kernels. +- `tensorfold-metal`: the TensorFold release above, Qwen3.5-2B only (the one checkpoint its Vulkan engine serves). + +Guards: shared worker lock, one engine at a time, fresh process per run, three rotated rounds. macOS exposes no fan +or SMC temperature without root, so the thermal guard is `pmset -g therm`'s warning level, and the memory guard +uses `kern.memorystatus_level` with `--reserve-gib 1` (the M1 Pro's 6 GiB cannot be met beside a 10 GiB VM). + +## Outcomes worth knowing + +- Every completed row produced the same greedy text across its three runs. +- Against the M1 Pro (Linux, Honeykrisp Vulkan) reference text: identical for Granite 1B, LFM3, Qwen3.5-2B at + 32 tokens and all its concurrent rows. Qwen3.5-2B at 256 tokens: MLX and mlx_lm.server on Metal diverge from the + M1 Pro after 1,207 of 1,301 characters, while TensorFold Metal matches the M1 Pro exactly. Qwen3.5-4B at 32 + tokens: in-process MLX on Metal diverges after 55 characters, while mlx_lm.server on Metal matches the M1 Pro. + Both look like greedy near-ties resolved differently by different kernels (each output is fluent), not defects. +- `mlx_lm.server` at 8 requests at once failed all three rounds: 1-2 of the 8 simultaneous connections were reset + (`ECONNRESET`) while the server kept serving the rest. Python's `ThreadingHTTPServer` listens with a backlog of + 5; macOS resets connections past it where Linux retries them. Not a model or GPU fault; at 4 requests it is fine. + TensorFold served all 8. + +## Fixed-work speed gate (omarchy-mlx `m4_mini_bench.py`) + +The release-gate bench from techmore/omarchy-mlx branch `m4-mini-bench` (`1690652a`), macOS legs, same venv: +`m4-mini-bench-macos-*.json` here. Pair them with `omarchy-mlx-m4-bench` on Linux and +`m4_mini_bench.py --compare`. 512-token prompt, 128 generated tokens, 1 warmup + 5 runs, medians; all three +started at load average 19.8-21.5 (the script flags anything above 0.5). + +| Leg | Prompt tok/s | Generation tok/s | Peak memory | Generated IDs | +|---|---:|---:|---:|---| +| MLX, Qwen3-1.7B 4-bit (default work) | 897.26 | 57.82 | 1.59 GB | `a9fec43a07727b81` | +| MLX, Qwen3.5-2B 4-bit | 852.90 | 54.36 | 1.81 GB | `1f20f523be2f3cfd` | +| TensorFold Metal v1.1.9 (pinned binary), Qwen3.5-2B 4-bit | 586.46 | 37.90 (spread 23 %) | 2.76 GB | `1f20f523be2f3cfd` (same as MLX) | + +## Reproduce + +```bash +python3.14 -m venv venv # or: uv venv --python 3.14 venv +venv/bin/pip install mlx==0.32.3 mlx-lm==0.31.3 transformers==5.16.1 huggingface_hub +export HF_HUB_CACHE=$PWD/hf MSERIES_LFM3_MODULE=$PWD/lfm3.py \ + MSERIES_TF_BIN=$PWD/tensorfold-1.1.9-macos-arm64/bin/tensorfold-native +cd mseries/local-ai; O=receipts/2026-10-31-m4-mini-macos; E="../../venv/bin/python engine-compare.py --reserve-gib 1" +$E --model granite1b --output $O/2026-10-31-engine-compare-granite1b-short-metal.json +$E --model lfm3 --output $O/2026-10-31-engine-compare-lfm3-short-metal.json +$E --model qwen2b --output $O/2026-10-31-engine-compare-qwen2b-short-metal.json +$E --model qwen2b --concurrency 4 --output $O/2026-10-31-engine-compare-qwen2b-short-metal-c4.json +$E --model qwen2b --concurrency 8 --output $O/2026-10-31-engine-compare-qwen2b-short-metal-c8.json +$E --model qwen2b --long --output $O/2026-10-31-engine-compare-qwen2b-long-metal.json +$E --model qwen4b --output $O/2026-10-31-engine-compare-qwen4b-short-metal.json +$E --model qwen4b --concurrency 4 --output $O/2026-10-31-engine-compare-qwen4b-short-metal-c4.json +python3 scoreboard.py +``` diff --git a/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/m4-mini-bench-macos-mlx-qwen3-1.7b.json b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/m4-mini-bench-macos-mlx-qwen3-1.7b.json new file mode 100644 index 00000000..27b5bc72 --- /dev/null +++ b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/m4-mini-bench-macos-mlx-qwen3-1.7b.json @@ -0,0 +1,213 @@ +{ + "child": { + "default_device": "Device(gpu, 0)", + "device_info": { + "architecture": "applegpu_g16g", + "device_name": "Apple M4", + "max_buffer_length": 9534832640, + "max_recommended_working_set_size": 12713115648, + "memory_size": 17179869184, + "resource_limit": 499000 + }, + "load_s": 0.861, + "masked_eos_ids": [ + 151645 + ], + "mx_version": "0.32.3", + "prompt_ids_sha256": "85c492a86b67c993e50b7b5a2ec0541a07a2e956134cb22465a92b09257d3d93", + "runs": [ + { + "finish_reason": "length", + "generated_ids_sha256": "a9fec43a07727b810d7574afcf028b8dd0eac94aef4dadb558ce2511dd9ea5a7", + "generation_tokens": 128, + "generation_tps": 58.35706399407152, + "peak_memory_gb": 1.591, + "prompt_tokens": 512, + "prompt_tps": 920.9440575943783, + "ttft_s": 0.7641, + "wall_s": 2.9793 + }, + { + "finish_reason": "length", + "generated_ids_sha256": "a9fec43a07727b810d7574afcf028b8dd0eac94aef4dadb558ce2511dd9ea5a7", + "generation_tokens": 128, + "generation_tps": 58.544427731058484, + "peak_memory_gb": 1.591, + "prompt_tokens": 512, + "prompt_tps": 899.3318459720762, + "ttft_s": 0.6811, + "wall_s": 2.8886 + }, + { + "finish_reason": "length", + "generated_ids_sha256": "a9fec43a07727b810d7574afcf028b8dd0eac94aef4dadb558ce2511dd9ea5a7", + "generation_tokens": 128, + "generation_tps": 57.36200967390368, + "peak_memory_gb": 1.591, + "prompt_tokens": 512, + "prompt_tps": 897.2590886853911, + "ttft_s": 0.7013, + "wall_s": 2.9505 + }, + { + "finish_reason": "length", + "generated_ids_sha256": "a9fec43a07727b810d7574afcf028b8dd0eac94aef4dadb558ce2511dd9ea5a7", + "generation_tokens": 128, + "generation_tps": 57.81734878434363, + "peak_memory_gb": 1.591, + "prompt_tokens": 512, + "prompt_tps": 884.7390495601101, + "ttft_s": 0.7085, + "wall_s": 2.9484 + }, + { + "finish_reason": "length", + "generated_ids_sha256": "a9fec43a07727b810d7574afcf028b8dd0eac94aef4dadb558ce2511dd9ea5a7", + "generation_tokens": 128, + "generation_tps": 57.55314596658308, + "peak_memory_gb": 1.503, + "prompt_tokens": 512, + "prompt_tps": 875.8753848419052, + "ttft_s": 0.8038, + "wall_s": 3.0506 + } + ], + "versions": { + "huggingface-hub": "1.33.0", + "mlx": "0.32.3", + "mlx-lm": "0.31.3", + "mlx-metal": "0.32.3", + "transformers": "5.16.1" + }, + "warmup": [ + { + "finish_reason": "length", + "generated_ids_sha256": "a9fec43a07727b810d7574afcf028b8dd0eac94aef4dadb558ce2511dd9ea5a7", + "generation_tokens": 128, + "generation_tps": 57.01058538207689, + "peak_memory_gb": 1.591, + "prompt_tokens": 512, + "prompt_tps": 481.6051369454333, + "ttft_s": 1.2991, + "wall_s": 3.5615 + } + ] + }, + "child_returncode": 0, + "child_stderr_tail": [], + "command": [ + "/private/tmp/claude-501/bench/venv/bin/python", + "scripts/m4_mini_bench.py", + "--out", + "/private/tmp/claude-501/bench/os/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/m4-mini-bench-macos-mlx-qwen3-1.7b.json" + ], + "exit_code": 0, + "fixed": { + "engine": "mlx", + "generation_tokens": 128, + "model": "qwen3-1.7b", + "model_file_sha256": { + "model.safetensors": "0e86d9677e519323849eac1bc272caae88567a481ff188c431f70be543d9995f", + "tokenizer.json": "aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4" + }, + "model_repo": "mlx-community/Qwen3-1.7B-4bit", + "model_revision": "3b1b1768f8f8cf8351c712464f906e86c2b8269e", + "prompt_tokens": 512, + "runs": 5, + "sampler": "greedy, EOS masked", + "warmup": 1 + }, + "host": { + "env": {}, + "load_avg_end": [ + 21.470703125, + 20.0693359375, + 17.9150390625 + ], + "load_avg_start": [ + 21.00048828125, + 19.64501953125, + 17.62841796875 + ], + "machine": "arm64", + "macos": { + "gpu": [ + { + "spdisplays_mtlgpufamilysupport": "spdisplays_metal4", + "sppci_cores": "10", + "sppci_model": "Apple M4" + } + ], + "hw.memsize": "17179869184", + "hw.model": "Mac16,10", + "kern.osversion": "26A5421a", + "machdep.cpu.brand_string": "Apple M4", + "pmset": [ + "lowpowermode 0" + ], + "sw_vers": { + "BuildVersion": "26A5421a", + "ProductName": "macOS", + "ProductVersion": "27.0" + } + }, + "python": "3.14.3", + "quiet": false, + "system": "Darwin", + "utc_end": "2026-10-31T15:50:44Z", + "utc_start": "2026-10-31T15:49:50Z" + }, + "schema": "m4-mini-bench/2", + "script": { + "git_commit": "1690652a0de099ef1fc6120c54c7f92ff39a84c0", + "git_dirty": false, + "path": "/private/tmp/claude-501/bench/omarchy-mlx/scripts/m4_mini_bench.py", + "sha256": "8258e71cee62b80fe516670a8b14cc770badbc35af77923d34386d2ea7a03e61" + }, + "summary": { + "generated_ids_sha256": "a9fec43a07727b810d7574afcf028b8dd0eac94aef4dadb558ce2511dd9ea5a7", + "generation_tps": { + "max": 58.544, + "median": 57.817, + "min": 57.362, + "n": 5, + "spread": 0.0205, + "values": [ + 58.357, + 58.544, + 57.362, + 57.817, + 57.553 + ] + }, + "peak_memory_gb": 1.591, + "prompt_tps": { + "max": 920.944, + "median": 897.259, + "min": 875.875, + "n": 5, + "spread": 0.0502, + "values": [ + 920.944, + 899.332, + 897.259, + 884.739, + 875.875 + ] + }, + "ttft_s": { + "max": 0.804, + "median": 0.709, + "min": 0.681, + "n": 5, + "spread": 0.1732, + "values": [ + 0.764, + 0.681, + 0.701, + 0.709, + 0.804 + ] + } + } +} diff --git a/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/m4-mini-bench-macos-mlx-qwen3.5-2b.json b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/m4-mini-bench-macos-mlx-qwen3.5-2b.json new file mode 100644 index 00000000..5fec5a40 --- /dev/null +++ b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/m4-mini-bench-macos-mlx-qwen3.5-2b.json @@ -0,0 +1,220 @@ +{ + "child": { + "default_device": "Device(gpu, 0)", + "device_info": { + "architecture": "applegpu_g16g", + "device_name": "Apple M4", + "max_buffer_length": 9534832640, + "max_recommended_working_set_size": 12713115648, + "memory_size": 17179869184, + "resource_limit": 499000 + }, + "load_s": 2.361, + "masked_eos_ids": [ + 248046 + ], + "mx_version": "0.32.3", + "prompt_ids_sha256": "dca79144a40c1439caf032f798f5ed28fd1621944c364012c4f82c659c6f2dfe", + "runs": [ + { + "finish_reason": "length", + "generated_ids_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "generation_tokens": 128, + "generation_tps": 55.006332793157235, + "peak_memory_gb": 1.806, + "prompt_tokens": 512, + "prompt_tps": 885.4645668183422, + "ttft_s": 0.8751, + "wall_s": 3.2397 + }, + { + "finish_reason": "length", + "generated_ids_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "generation_tokens": 128, + "generation_tps": 53.53328201532624, + "peak_memory_gb": 1.806, + "prompt_tokens": 512, + "prompt_tps": 835.4310321251695, + "ttft_s": 0.8988, + "wall_s": 3.3271 + }, + { + "finish_reason": "length", + "generated_ids_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "generation_tokens": 128, + "generation_tps": 53.6449508850278, + "peak_memory_gb": 1.806, + "prompt_tokens": 512, + "prompt_tps": 852.9000714685798, + "ttft_s": 0.8883, + "wall_s": 3.3014 + }, + { + "finish_reason": "length", + "generated_ids_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "generation_tokens": 128, + "generation_tps": 55.35627656386269, + "peak_memory_gb": 1.806, + "prompt_tokens": 512, + "prompt_tps": 860.0937894978317, + "ttft_s": 0.8621, + "wall_s": 3.2129 + }, + { + "finish_reason": "length", + "generated_ids_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "generation_tokens": 128, + "generation_tps": 54.35586049336136, + "peak_memory_gb": 1.788, + "prompt_tokens": 512, + "prompt_tps": 765.8293993939131, + "ttft_s": 0.9988, + "wall_s": 3.3794 + } + ], + "versions": { + "huggingface-hub": "1.33.0", + "mlx": "0.32.3", + "mlx-lm": "0.31.3", + "mlx-metal": "0.32.3", + "transformers": "5.16.1" + }, + "warmup": [ + { + "finish_reason": "length", + "generated_ids_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "generation_tokens": 128, + "generation_tps": 55.14529573739597, + "peak_memory_gb": 1.805, + "prompt_tokens": 512, + "prompt_tps": 652.5951192469374, + "ttft_s": 1.1287, + "wall_s": 3.478 + } + ] + }, + "child_returncode": 0, + "child_stderr_tail": [], + "command": [ + "/private/tmp/claude-501/bench/venv/bin/python", + "scripts/m4_mini_bench.py", + "--model", + "qwen3.5-2b", + "--model-dir", + "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-2B-MLX-4bit/snapshots/93760be4f1f69842a46bc13dbdc0f19e291392a3", + "--out", + "/private/tmp/claude-501/bench/os/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/m4-mini-bench-macos-mlx-qwen3.5-2b.json" + ], + "exit_code": 0, + "fixed": { + "engine": "mlx", + "generation_tokens": 128, + "model": "qwen3.5-2b", + "model_file_sha256": { + "chat_template.jinja": "273d8e0e683b885071fb17e08d71e5f2a5ddfb5309756181681de4f5a1822d80", + "config.json": "beb7fc5a6e0405fe332821cf1a8ef7b69bb390a8c8933171647de5579debf949", + "model.safetensors": "713fe7e5d3c3965f7106b0d0ee17615f7869c23c8d327996df8c1196fbcf07d5", + "tokenizer.json": "87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4", + "tokenizer_config.json": "e98f1901ac6f0adff67b1d540bfa0c36ac1a0cf59eb72ed78146ef89aafa1182" + }, + "model_repo": "mlx-community/Qwen3.5-2B-MLX-4bit", + "model_revision": "93760be4f1f69842a46bc13dbdc0f19e291392a3", + "prompt_tokens": 512, + "runs": 5, + "sampler": "greedy, EOS masked", + "warmup": 1 + }, + "host": { + "env": {}, + "load_avg_end": [ + 19.751953125, + 19.76123046875, + 17.86474609375 + ], + "load_avg_start": [ + 21.470703125, + 20.0693359375, + 17.9150390625 + ], + "machine": "arm64", + "macos": { + "gpu": [ + { + "spdisplays_mtlgpufamilysupport": "spdisplays_metal4", + "sppci_cores": "10", + "sppci_model": "Apple M4" + } + ], + "hw.memsize": "17179869184", + "hw.model": "Mac16,10", + "kern.osversion": "26A5421a", + "machdep.cpu.brand_string": "Apple M4", + "pmset": [ + "lowpowermode 0" + ], + "sw_vers": { + "BuildVersion": "26A5421a", + "ProductName": "macOS", + "ProductVersion": "27.0" + } + }, + "python": "3.14.3", + "quiet": false, + "system": "Darwin", + "utc_end": "2026-10-31T15:51:11Z", + "utc_start": "2026-10-31T15:50:44Z" + }, + "schema": "m4-mini-bench/2", + "script": { + "git_commit": "1690652a0de099ef1fc6120c54c7f92ff39a84c0", + "git_dirty": false, + "path": "/private/tmp/claude-501/bench/omarchy-mlx/scripts/m4_mini_bench.py", + "sha256": "8258e71cee62b80fe516670a8b14cc770badbc35af77923d34386d2ea7a03e61" + }, + "summary": { + "generated_ids_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "generation_tps": { + "max": 55.356, + "median": 54.356, + "min": 53.533, + "n": 5, + "spread": 0.0335, + "values": [ + 55.006, + 53.533, + 53.645, + 55.356, + 54.356 + ] + }, + "peak_memory_gb": 1.806, + "prompt_tps": { + "max": 885.465, + "median": 852.9, + "min": 765.829, + "n": 5, + "spread": 0.1403, + "values": [ + 885.465, + 835.431, + 852.9, + 860.094, + 765.829 + ] + }, + "ttft_s": { + "max": 0.999, + "median": 0.888, + "min": 0.862, + "n": 5, + "spread": 0.1539, + "values": [ + 0.875, + 0.899, + 0.888, + 0.862, + 0.999 + ] + } + } +} diff --git a/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/m4-mini-bench-macos-tensorfold-qwen3.5-2b.json b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/m4-mini-bench-macos-tensorfold-qwen3.5-2b.json new file mode 100644 index 00000000..b3d0ed47 --- /dev/null +++ b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/m4-mini-bench-macos-tensorfold-qwen3.5-2b.json @@ -0,0 +1,344 @@ +{ + "child": { + "command": [ + "/private/tmp/claude-501/bench/tfdl/tensorfold-1.1.9-macos-arm64/bin/tensorfold-native", + "serve", + "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-2B-MLX-4bit/snapshots/93760be4f1f69842a46bc13dbdc0f19e291392a3", + "--backend", + "mlx", + "--name", + "m4-mini-bench", + "--host", + "127.0.0.1", + "--port", + "56717", + "--context", + "2048", + "--parallel", + "1", + "--no-drafts", + "--no-thinking", + "--no-update-check" + ], + "engine": "tensorfold-metal", + "health": { + "backend": "metal", + "device": "Apple M4", + "live": { + "connections": 0, + "decode_tokens_per_second": 0.0, + "prefill_tokens_per_second": 0.0, + "waiting": 0 + }, + "max_batch_size": 1, + "memory": { + "active": 2749926088, + "cache": 0, + "peak": 2749926088 + }, + "model": "m4-mini-bench", + "model_ids": [ + "m4-mini-bench" + ], + "status": "ok", + "version": "1.1.9", + "warming": false + }, + "health_end": { + "backend": "metal", + "device": "Apple M4", + "live": { + "connections": 0, + "decode_tokens_per_second": 39.5, + "prefill_tokens_per_second": 0.0, + "waiting": 0 + }, + "max_batch_size": 1, + "memory": { + "active": 2759264968, + "cache": 0, + "peak": 2759264968 + }, + "model": "m4-mini-bench", + "model_ids": [ + "m4-mini-bench" + ], + "status": "ok", + "version": "1.1.9", + "warming": false + }, + "ignore_eos": true, + "load_s": 3.079, + "prompt_ids_sha256": "dca79144a40c1439caf032f798f5ed28fd1621944c364012c4f82c659c6f2dfe", + "runs": [ + { + "cached_tokens": 0, + "decode_s": 2.906999125, + "drafts": false, + "finish_reason": "length", + "generated_ids_count": 128, + "generated_ids_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "generation_tokens": 128, + "generation_tps": 43.68766364867757, + "prefill_s": 0.736039875, + "prompt_tokens": 512, + "prompt_tps": 695.6144869189322, + "prompt_tps_from": "prefill_seconds", + "sampling": "greedy", + "server_token_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "ttft_s": 0.7405, + "wall_s": 3.648 + }, + { + "cached_tokens": 0, + "decode_s": 3.350722417, + "drafts": false, + "finish_reason": "length", + "generated_ids_count": 128, + "generated_ids_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "generation_tokens": 128, + "generation_tps": 37.90227425454921, + "prefill_s": 0.825708333, + "prompt_tokens": 512, + "prompt_tps": 620.0736743685012, + "prompt_tps_from": "prefill_seconds", + "sampling": "greedy", + "server_token_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "ttft_s": 0.8271, + "wall_s": 4.1812 + }, + { + "cached_tokens": 0, + "decode_s": 3.354299334, + "drafts": false, + "finish_reason": "length", + "generated_ids_count": 128, + "generated_ids_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "generation_tokens": 128, + "generation_tps": 37.8618564874926, + "prefill_s": 0.873040875, + "prompt_tokens": 512, + "prompt_tps": 586.4559319745481, + "prompt_tps_from": "prefill_seconds", + "sampling": "greedy", + "server_token_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "ttft_s": 0.8784, + "wall_s": 4.2368 + }, + { + "cached_tokens": 0, + "decode_s": 3.643026333, + "drafts": false, + "finish_reason": "length", + "generated_ids_count": 128, + "generated_ids_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "generation_tokens": 128, + "generation_tps": 34.86112599559955, + "prefill_s": 0.87498125, + "prompt_tokens": 512, + "prompt_tps": 585.1553961870611, + "prompt_tps_from": "prefill_seconds", + "sampling": "greedy", + "server_token_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "ttft_s": 0.88, + "wall_s": 4.5279 + }, + { + "cached_tokens": 0, + "decode_s": 3.266979958, + "drafts": false, + "finish_reason": "length", + "generated_ids_count": 128, + "generated_ids_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "generation_tokens": 128, + "generation_tps": 38.87382280659831, + "prefill_s": 0.874664834, + "prompt_tokens": 512, + "prompt_tps": 585.3670801631886, + "prompt_tps_from": "prefill_seconds", + "sampling": "greedy", + "server_token_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "ttft_s": 0.882, + "wall_s": 4.1544 + } + ], + "server_log": "/var/folders/_9/nnc36lvs7vl189v7cl69586c0000gn/T/m4-mini-bench-tf-biwx0xoc/server.log", + "server_log_tail": [ + "info: qwen3.5: 2.3 GiB kept wired between requests, inside 70% of 16 GiB of RAM", + "warning: memory pressure: warning: one more stream only when none runs, no new kept prompt states, the prompt cache sheds toward 0 MiB", + "[tensorfold] prompt chunks of up to 128 tokens, cut at replies 256+ tokens apart", + "[tensorfold] serving m4-mini-bench at http://127.0.0.1:56717/v1 (sampling: greedy; drafts: off; context: 2048; loaded in 2.5s)", + "[tensorfold] 127.0.0.1 \"GET /health HTTP/1.1\" 200 -", + "[tensorfold] 127.0.0.1 \"POST /v1/tokenize HTTP/1.1\" 200 -", + "[tensorfold] done cmpl-96d09d62bd2aa81c13765c911fdf35d1 prompt=512 cached=0 thinking=False effort=none tokens=128 sha=1f20f523be2f finish=length rounds=127 accepted=0/0 ttft=0.869 decode=2.934 elapsed=3.803 queued=0.000 tok/s=43.3", + "[tensorfold] 127.0.0.1 \"POST /v1/completions HTTP/1.1\" 200 -", + "[tensorfold] done cmpl-de6aadb346a794c0f23098796b7bfe8d prompt=512 cached=0 thinking=False effort=none tokens=128 sha=1f20f523be2f finish=length rounds=127 accepted=0/0 ttft=0.741 decode=2.905 elapsed=3.645 queued=0.000 tok/s=43.7", + "[tensorfold] 127.0.0.1 \"POST /v1/completions HTTP/1.1\" 200 -", + "[tensorfold] done cmpl-6d26da7e142613801f2d78a77d54eeb7 prompt=512 cached=0 thinking=False effort=none tokens=128 sha=1f20f523be2f finish=length rounds=127 accepted=0/0 ttft=0.827 decode=3.350 elapsed=4.178 queued=0.000 tok/s=37.9", + "[tensorfold] 127.0.0.1 \"POST /v1/completions HTTP/1.1\" 200 -", + "[tensorfold] done cmpl-9b6077a6d7aec84c732896c8bd37a3b6 prompt=512 cached=0 thinking=False effort=none tokens=128 sha=1f20f523be2f finish=length rounds=127 accepted=0/0 ttft=0.878 decode=3.352 elapsed=4.231 queued=0.002 tok/s=37.9", + "[tensorfold] 127.0.0.1 \"POST /v1/completions HTTP/1.1\" 200 -", + "[tensorfold] done cmpl-0dfb71d7e3f31a7869d3345f724af9d8 prompt=512 cached=0 thinking=False effort=none tokens=128 sha=1f20f523be2f finish=length rounds=127 accepted=0/0 ttft=0.880 decode=3.642 elapsed=4.522 queued=0.000 tok/s=34.9", + "[tensorfold] 127.0.0.1 \"POST /v1/completions HTTP/1.1\" 200 -", + "[tensorfold] done cmpl-345ab3ed3252d04e26679671e7cd2bb7 prompt=512 cached=0 thinking=False effort=none tokens=128 sha=1f20f523be2f finish=length rounds=127 accepted=0/0 ttft=0.882 decode=3.266 elapsed=4.148 queued=0.001 tok/s=38.9", + "[tensorfold] 127.0.0.1 \"POST /v1/completions HTTP/1.1\" 200 -", + "[tensorfold] 127.0.0.1 \"GET /health HTTP/1.1\" 200 -", + "[tensorfold] shutting down (SIGTERM): draining: new requests get 503; 0 request(s) in progress may finish within 20.0 s", + "[tensorfold] shutting down: drained", + "[tensorfold] shutting down: stopped" + ], + "server_returncode": 0, + "warmup": [ + { + "cached_tokens": 0, + "decode_s": 2.932855042, + "drafts": false, + "finish_reason": "length", + "generated_ids_count": 128, + "generated_ids_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "generation_tokens": 128, + "generation_tps": 43.3025151878611, + "prefill_s": 0.868073875, + "prompt_tokens": 512, + "prompt_tps": 589.8115526169936, + "prompt_tps_from": "prefill_seconds", + "sampling": "greedy", + "server_token_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "ttft_s": 0.8694, + "wall_s": 3.8101 + } + ] + }, + "command": [ + "/private/tmp/claude-501/bench/venv/bin/python", + "scripts/m4_mini_bench.py", + "--engine", + "tensorfold", + "--model-dir", + "/private/tmp/claude-501/bench/hf/models--mlx-community--Qwen3.5-2B-MLX-4bit/snapshots/93760be4f1f69842a46bc13dbdc0f19e291392a3", + "--tensorfold-bin", + "/private/tmp/claude-501/bench/tfdl/tensorfold-1.1.9-macos-arm64/bin/tensorfold-native", + "--out", + "/private/tmp/claude-501/bench/os/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/m4-mini-bench-macos-tensorfold-qwen3.5-2b.json" + ], + "engine_identity": { + "binary": "/private/tmp/claude-501/bench/tfdl/tensorfold-1.1.9-macos-arm64/bin/tensorfold-native", + "binary_sha256": "4b4b5a5ac152eb2dbed7e9d8004805ed9b74f77c64c3de18eb4f35263c7b9eb8", + "engine": "tensorfold-metal", + "pinned": true, + "release": "1.1.9", + "release_url": "https://github.com/ashhart/TensorFold/releases/download/v1.1.9/tensorfold-1.1.9-macos-arm64.tar.gz" + }, + "exit_code": 0, + "fixed": { + "engine": "tensorfold-metal", + "generation_tokens": 128, + "model": "qwen3.5-2b", + "model_file_sha256": { + "chat_template.jinja": "273d8e0e683b885071fb17e08d71e5f2a5ddfb5309756181681de4f5a1822d80", + "config.json": "beb7fc5a6e0405fe332821cf1a8ef7b69bb390a8c8933171647de5579debf949", + "model.safetensors": "713fe7e5d3c3965f7106b0d0ee17615f7869c23c8d327996df8c1196fbcf07d5", + "tokenizer.json": "87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4", + "tokenizer_config.json": "e98f1901ac6f0adff67b1d540bfa0c36ac1a0cf59eb72ed78146ef89aafa1182" + }, + "model_repo": "mlx-community/Qwen3.5-2B-MLX-4bit", + "model_revision": "93760be4f1f69842a46bc13dbdc0f19e291392a3", + "prompt_tokens": 512, + "runs": 5, + "sampler": "greedy (temperature 0), ignore_eos", + "warmup": 1 + }, + "host": { + "env": {}, + "load_avg_end": [ + 18.27099609375, + 19.4140625, + 17.8037109375 + ], + "load_avg_start": [ + 19.751953125, + 19.76123046875, + 17.86474609375 + ], + "machine": "arm64", + "macos": { + "gpu": [ + { + "spdisplays_mtlgpufamilysupport": "spdisplays_metal4", + "sppci_cores": "10", + "sppci_model": "Apple M4" + } + ], + "hw.memsize": "17179869184", + "hw.model": "Mac16,10", + "kern.osversion": "26A5421a", + "machdep.cpu.brand_string": "Apple M4", + "pmset": [ + "lowpowermode 0" + ], + "sw_vers": { + "BuildVersion": "26A5421a", + "ProductName": "macOS", + "ProductVersion": "27.0" + } + }, + "python": "3.14.3", + "quiet": false, + "system": "Darwin", + "utc_end": "2026-10-31T15:51:41Z", + "utc_start": "2026-10-31T15:51:11Z" + }, + "schema": "m4-mini-bench/2", + "script": { + "git_commit": "1690652a0de099ef1fc6120c54c7f92ff39a84c0", + "git_dirty": false, + "path": "/private/tmp/claude-501/bench/omarchy-mlx/scripts/m4_mini_bench.py", + "sha256": "8258e71cee62b80fe516670a8b14cc770badbc35af77923d34386d2ea7a03e61" + }, + "summary": { + "generated_ids_sha256": "1f20f523be2f3cfd775543af315eae2db831fb77e93252c064f635aa7695d244", + "generation_tps": { + "max": 43.688, + "median": 37.902, + "min": 34.861, + "n": 5, + "spread": 0.2329, + "values": [ + 43.688, + 37.902, + 37.862, + 34.861, + 38.874 + ] + }, + "peak_memory_gb": 2.759, + "prompt_tps": { + "max": 695.614, + "median": 586.456, + "min": 585.155, + "n": 5, + "spread": 0.1884, + "values": [ + 695.614, + 620.074, + 586.456, + 585.155, + 585.367 + ] + }, + "ttft_s": { + "max": 0.882, + "median": 0.878, + "min": 0.741, + "n": 5, + "spread": 0.1611, + "values": [ + 0.741, + 0.827, + 0.878, + 0.88, + 0.882 + ] + } + } +} diff --git a/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/venv-freeze.txt b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/venv-freeze.txt new file mode 100644 index 00000000..f4697312 --- /dev/null +++ b/mseries/local-ai/receipts/2026-10-31-m4-mini-macos/venv-freeze.txt @@ -0,0 +1,34 @@ +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.23 +click==8.5.0 +filelock==4.2.6 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.8.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==1.33.0 +idna==3.20 +jinja2==3.1.6 +markdown-it-py==4.2.0 +markupsafe==3.0.4 +mdurl==0.1.2 +mlx==0.32.3 +mlx-lm==0.31.3 +mlx-metal==0.32.3 +numpy==2.5.4 +packaging==26.3 +protobuf==7.36.2 +pygments==2.21.0 +pyyaml==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +sentencepiece==0.2.2 +shellingham==1.5.4 +tokenizers==0.23.3 +tqdm==4.70.1 +transformers==5.16.1 +typer==0.27.3 +typing-extensions==4.16.0 diff --git a/mseries/local-ai/scoreboard.py b/mseries/local-ai/scoreboard.py index cef0816c..d13dc635 100644 --- a/mseries/local-ai/scoreboard.py +++ b/mseries/local-ai/scoreboard.py @@ -5,6 +5,8 @@ (mlx-gate.py) and scoreboard-notes.json (rows measured elsewhere or reported by others), then writes engine-scoreboard.csv and ENGINE-SCOREBOARD.md next to this script. The newest receipt wins for each (model, engine, length). Run it after every new receipt. +Other machines (MACHINES) keep their receipts in their own folder and get their own CSV, chart and +section; they are never ranked against the M1 Pro table. """ import csv, json, statistics from pathlib import Path @@ -20,7 +22,14 @@ 'serve-single-120k': ('experiment: serve-nolp-120k + Rapid-MLX singleton module loaded from its checkout', 'MLX_OMARCHY_BATCH_WORK=120000', None), 'serve-patched-120k': ('mlx-omarchy-serve + our four server patches + batching of 4 (HTTP server; runs the everyday model)', 'MLX_OMARCHY_BATCH_WORK=120000', None), 'tensorfold': ('TensorFold Vulkan (HTTP server)', 'serve --backend vulkan --context 1024 --no-thinking', None), + 'mlx-metal': ('MLX on Metal (upstream mlx, in-process)', 'mlx {mlx} · mlx-lm {mlx_lm}', None), + 'lm-metal': ('mlx_lm.server, stock (HTTP server)', 'mlx {mlx} · mlx-lm {mlx_lm}', None), + 'tensorfold-metal': ('TensorFold Metal v1.1.9 release (HTTP server)', 'serve --backend mlx --context 1024 --no-thinking --no-drafts', None), } +REFERENCE_ARMS = ('mlx-5k', 'mlx-metal') # "same output" is checked against the in-process MLX default +# (key, short name, title, receipts folder): machines measured apart from the M1 Pro, each with its own CSV and chart. +MACHINES = [('m4-mini-macos', 'Mac mini M4, macOS (Metal)', + 'Mac mini M4 (Mac16,10, 10-core GPU, 16 GB), macOS 27.0 26A5421a, Metal', RECEIPTS / '2026-10-31-m4-mini-macos')] # Receipts record the TensorFold revision they ran; name the PR it came from. TF_PRS = {'670902196c47f3e70e3f56ab933805713be31aa8': 'PR 1517', '92bb49db5c313c105cbf4b44638d1e1896615174': 'PR 1544'} MODEL_NAMES = {'qwen2b': 'Qwen3.5-2B MLX 4-bit', 'qwen4b': 'Qwen3.5-4B MLX 4-bit', 'lfm3': 'LFM3-1.7B MLX 8-bit', @@ -53,7 +62,8 @@ def gate_results(): SHORT = {'mlx-5k': 'MLX in-process, default', 'mlx-120k': 'MLX in-process, tuned', 'rapid-120k': 'Rapid-MLX server', 'serve-120k': 'Upstream server, stock', 'serve-nolp-120k': 'Upstream server + logprob patch', 'serve-single-120k': 'Upstream + singleton (trial)', - 'serve-patched-120k': 'Upstream server, patched (ships)', 'tensorfold': 'TensorFold Vulkan'} + 'serve-patched-120k': 'Upstream server, patched (ships)', 'tensorfold': 'TensorFold Vulkan', + 'mlx-metal': 'MLX in-process, Metal', 'lm-metal': 'mlx_lm.server, Metal', 'tensorfold-metal': 'TensorFold Metal'} REPLY_TOKENS = 200 @@ -65,10 +75,33 @@ def reply_s(x): return x['ttft_s'] + REPLY_TOKENS / per -def measured_rows(): +def m1_reference_texts(): + """(model, tokens, load) -> (text, receipt) of the M1 Pro's newest reference output, so another machine's + rows can say whether they produce the same greedy text as the M1 Pro under Linux.""" + out = {} + for f in sorted(RECEIPTS.glob('*engine-compare-*.json'), key=lambda f: f.stem): + r = json.loads(f.read_text()) + if r.get('status') != 'complete': + continue + texts = [x['text'] for x in r['runs'] if x['arm'] == 'mlx-5k' and 'text' in x] + every = {x['text'] for x in r['runs'] if 'text' in x} + if texts or len(every) == 1: + out[(r.get('model_key', 'qwen2b'), r['max_tokens'], r.get('concurrency', 1))] = ((texts or list(every))[0], f.name) + return out + + +def diverges(ref, text): + if ref == text: + return 'yes' + i = next((i for i, (u, v) in enumerate(zip(ref, text)) if u != v), min(len(ref), len(text))) + return f'no: diverges after {i} of {len(ref)} chars' + + +def measured_rows(receipts=RECEIPTS): gates = gate_results() + m1 = m1_reference_texts() if receipts != RECEIPTS else {} latest = {} - for f in sorted(RECEIPTS.glob('*engine-compare-*.json'), key=lambda f: f.stem): # stem: '-round2' sorts after its base + for f in sorted(receipts.glob('*engine-compare-*.json'), key=lambda f: f.stem): # stem: '-round2' sorts after its base r = json.loads(f.read_text()) if r.get('status') != 'complete': continue @@ -83,8 +116,9 @@ def measured_rows(): if not ok: continue texts = {x['text_sha256'] for x in ok} - base = {x['text_sha256'] for x in r['runs'] if x['arm'] == 'mlx-5k' and 'text_sha256' in x} - if arm == 'mlx-5k': + ref = next((a for a in REFERENCE_ARMS if a in r['arms']), 'mlx-5k') + base = {x['text_sha256'] for x in r['runs'] if x['arm'] == ref and 'text_sha256' in x} + if arm == ref: same = 'reference' elif not base: every = {x['text_sha256'] for x in r['runs'] if 'text_sha256' in x} @@ -92,11 +126,9 @@ def measured_rows(): elif texts == base: same = 'yes' else: - mt = next(x['text'] for x in r['runs'] if x['arm'] == 'mlx-5k' and 'text' in x) - at = ok[0]['text'] - i = next((i for i, (u, v) in enumerate(zip(mt, at)) if u != v), min(len(mt), len(at))) - same = f'no: diverges after {i} of {len(mt)} chars' + same = diverges(next(x['text'] for x in r['runs'] if x['arm'] == ref and 'text' in x), ok[0]['text']) engine, settings, gate_key = ARM_SETTINGS[arm] + settings = settings.format(**r.get('runtime', {})) if arm == 'tensorfold': rev = r.get('tensorfold_rev', '') engine = engine.replace('Vulkan', 'Vulkan ' + TF_PRS.get(rev, rev[:10] or 'unknown rev')) @@ -110,7 +142,10 @@ def measured_rows(): 'same_output': same if len(texts) == 1 else f'{same}; {len(texts)} variants across its own runs', 'gate': gate or 'not gated', 'kernel': r.get('kernel', ''), 'driver': drv, 'date': f.name[:10], - 'status': 'measured here', 'source': f'receipts/{f.name}', '_model': model, '_conc': conc, '_arm': arm}) + 'status': 'measured here', 'source': str(f.relative_to(HERE)), '_model': model, '_conc': conc, '_arm': arm}) + if m1: + m1ref = m1.get((model, tokens, conc)) + rows[-1]['same_as_m1'] = diverges(m1ref[0], ok[0]['text']) if m1ref else 'n/a (not run on the M1 Pro)' # Most functional per model, length and load: the shortest typical reply (first token plus # REPLY_TOKENS at its streaming speed) among rows whose output matched and whose gate did not fail. for key in {(x['_model'], x['tokens'], x['_conc']) for x in rows}: @@ -121,7 +156,7 @@ def measured_rows(): return rows -def chart(rows): +def chart(rows, machine='Apple M1 Pro'): """Small multiples, no plotting deps: per (model, length, load) block, one row per engine and three panels on their own scales - first token (lower is better), streaming speed (higher is better) and the wait for a typical reply. Two measures never share an axis. Blue is the best value in its panel; @@ -178,10 +213,78 @@ def chart(rows): '@media (prefers-color-scheme:dark){svg{background:#1a1a19}text{fill:#f0efec}.a,.v{fill:#c3c2b7}' '.bar{fill:#6b6a64}.best{fill:#3987e5}.axis{stroke:#52514e}}') return (f'' - f'First token, streaming speed and reply time by engine on Apple M1 Pro' + f'First token, streaming speed and reply time by engine on {machine}' f'{style}{"".join(parts)}\n') +M4_LINUX_NOTE = [ + '', '### How to run the same on Linux on the M4 mini', '', + 'Same machine, same work, so the two tables compare row for row:', '', + '1. Boot the mini into Linux (Aurora kernel, vulkan-aurora Honeykrisp) and install `omarchy-mlx`, ' + '`omarchy-mlx-vulkan-aurora` and `omarchy-mlx-m4-bench` from techmore/omarchy-mlx branch `m4-mini-bench` ' + '(its `docs/m4-mini-bench.md`), plus `tensorfold-vulkan` (PR 1544, `b3ab9ba2`) for the TensorFold rows.', + '2. Fetch the four checkpoints into the Hugging Face cache at the revisions above (engine-compare refuses other ' + 'weights) and the pinned `lfm3.py` (mlx-lm PR 2016 head `9027ac5b`).', + '3. Run the commands below with `MSERIES_HF_SNAPSHOTS=1 MSERIES_MLX_PY=/usr/lib/omarchy-mlx/venv/bin/python ' + 'MSERIES_HK_ICD= MSERIES_TF_VULKAN_BIN= MSERIES_LFM3_MODULE=` ' + 'and `--arms mlx-5k,mlx-120k,lm031-120k,tensorfold` (in-process pairs with `mlx-metal`, plain `mlx_lm.server` 0.31.3 ' + 'with `lm-metal`, TensorFold Vulkan with `tensorfold-metal`). Write receipts to `receipts/-m4-mini-linux/`, ' + 'add that folder to `MACHINES` in `scoreboard.py` and rerun it.', + '4. For the release-gate speed number, run `omarchy-mlx-m4-bench` on Linux (also `--engine tensorfold` and ' + '`--model qwen3.5-2b`) and `scripts/m4_mini_bench.py` on macOS with this venv (mlx 0.32.3, mlx-lm 0.31.3, ' + 'transformers 5.16.1, the pins that bench names), then `m4_mini_bench.py --compare`.', + '5. Record the load: this macOS baseline ran beside a busy build VM (see the receipts README). Quote Linux/macOS ' + 'ratios only from runs with comparable load, or re-run both quiet.', '', + '```bash', + 'cd mseries/local-ai', + '# macOS (this table): python 3.14 venv with mlx==0.32.3 mlx-lm==0.31.3 transformers==5.16.1', + 'HF_HUB_CACHE=... MSERIES_TF_BIN=.../tensorfold-1.1.9-macos-arm64/bin/tensorfold-native \\', + ' $VENV/bin/python engine-compare.py --model qwen2b --reserve-gib 1 # --long; --concurrency 4 / 8', + '# Linux on the same mini', + 'MSERIES_HF_SNAPSHOTS=1 MSERIES_MLX_PY=/usr/lib/omarchy-mlx/venv/bin/python ... \\', + ' python3 engine-compare.py --model qwen2b --arms mlx-5k,mlx-120k,lm031-120k,tensorfold', + '```'] + + +def machine_section(key, short, title, folder): + """Another machine's own CSV, chart and table; ★ ranks only within that machine.""" + rows = measured_rows(folder) + with (HERE / f'engine-scoreboard-{key}.csv').open('w', newline='') as fh: + w = csv.DictWriter(fh, fieldnames=['machine'] + FIELDS + ['same_as_m1'], extrasaction='ignore') + w.writeheader() + for x in rows: + w.writerow({'machine': title, **{k: (fmt(v) if isinstance(v, float) else v) for k, v in x.items()}}) + (HERE / f'engine-scoreboard-{key}.svg').write_text(chart(rows, short)) + md = ['', f'## {title}', '', + f'Measured with the same `engine-compare.py`, models, revisions, prompts, token counts, loads and three rotated ' + f'rounds, receipts in [`{folder.relative_to(HERE)}/`]({folder.relative_to(HERE)}/). A separate machine, so ' + f'never ranked against the M1 Pro table; ★ ranks within this machine only. CSV: ' + f'[`engine-scoreboard-{key}.csv`](engine-scoreboard-{key}.csv).', '', + f'![First token, streaming speed and typical reply time on the {short}](engine-scoreboard-{key}.svg)', '', + '| ★ | Model | Engine | Settings | Tokens | Decode tok/s | TTFT s | Total s | Runs | Same output as MLX default | ' + 'Same output as M1 Pro (Linux) | OS | Date | Receipt |', + '|---|---|---|---|---:|---:|---:|---:|---:|---|---|---|---|---|'] + for x in sorted(rows, key=lambda x: (x['_model'], x['tokens'], x['_conc'], -(x['decode_tok_s'] or 0))): + b = '**' if x.get('best') else '' + md.append(f"| {x.get('best', '')} | {x['model']} | {b}{x['engine']}{b} | `{x['settings']}` | {x['tokens']} | " + f"{b}{fmt(x['decode_tok_s'])}{b} | {fmt(x['ttft_s'])} | {fmt(x['total_s'])} | {x['runs']} | {x['same_output']} | " + f"{x['same_as_m1']} | {x['kernel']} | {x['date']} | [{Path(x['source']).name}]({x['source']}) |") + for f in sorted(folder.glob('*engine-compare-*.json')): # arms that never finished get no row; say so + r = json.loads(f.read_text()) + for arm in r['arms']: + runs = [x for x in r['runs'] if x['arm'] == arm] + if runs and all(x.get('error') or x.get('stopped') for x in runs): + md.append(f"\nNot in the table: {SHORT.get(arm, arm)}, {MODEL_NAMES.get(r['model_key'], r['model_key'])}, " + f"{r['max_tokens']} tokens, {r.get('concurrency', 1)} requests at once: all {len(runs)} runs failed " + f"(`{runs[0].get('error') or runs[0].get('stopped')}`; [{f.name}]({f.relative_to(HERE)})).") + if key.startswith('m4-mini'): + md += M4_LINUX_NOTE + notes = folder / 'README.md' + if notes.exists(): + md += ['', f'Conditions, versions and the Linux run that pairs with it: [`{notes.relative_to(HERE)}`]({notes.relative_to(HERE)}).'] + return md + + def main(): rows = measured_rows() notes = json.loads((HERE / 'scoreboard-notes.json').read_text())['rows'] @@ -210,6 +313,8 @@ def main(): md.append(f"| {x.get('best', '')} | {x['model']} | {b}{x['engine']}{b} | `{x['settings']}` | {x['tokens']} | " f"{b}{fmt(x['decode_tok_s'])}{b} | {fmt(x['ttft_s'])} | {fmt(x['total_s'])} | {x['runs']} | {x['same_output']} | " f"{x['gate']} | `{x['kernel']}` | {x['driver']} | {x['date']} | [{Path(x['source']).name}]({x['source'].replace('receipts/', 'receipts/')}) |") + for machine in MACHINES: + md += machine_section(*machine) md += ['', '## Watched or reported elsewhere (not comparable to the table above)', '', '| Model | Engine | Settings | Decode | Status | Blocker | Source |', '|---|---|---|---|---|---|---|'] for n in notes: