fix: silent exception swallows

gmlx/cache/kvarn_apc.py

@@ -31,6 +31,10 @@
 
 import hashlib
 import importlib
+import logging
+import secrets
+
+_log = logging.getLogger(__name__)
 
 _FLAG = "_gmlx_kvarn_apc"
 _MODE_STAMP = "_gmlx_kvarn_apc_exact"
@@ -99,15 +103,31 @@ def apply_kvarn_salt(manager, model) -> None:
     pairing, gated on the model actually converting: a kvarn-window boot
     of a zero-conversion arch (deepseek4, recurrent_gemma) runs pure fp16
     caches, and salting its entries would cold-miss every cross-boot
-    lookup. Failures leave the salt at its XOR-identity default."""
+    lookup. The XOR-identity default is correct only for that
+    non-converting case; a converting model whose salt computation fails
+    gets a random per-boot salt instead, so its entries stay out of the
+    fp16 hash space and cold-miss across boots rather than warm-adopt
+    under the wrong wire config."""
+    if manager is None or model is None:
+        return
+    if not kvarn_model_converts(model):
+        return
     try:
-        if manager is None or model is None:
-            return
         salt = kvarn_entry_salt(model)
-        if salt and kvarn_model_converts(model):
-            manager._exact_extra_salt = salt
     except Exception:
-        pass
+        salt = secrets.randbits(63) | 1
+        _log.warning(
+            "kvarn exact-tier salt computation failed; salting with a "
+            "per-boot value (cross-boot APC reuse off for this model)",
+            exc_info=True)
+    if not salt:
+        return
+    try:
+        manager._exact_extra_salt = salt
+    except Exception:
+        _log.warning(
+            "kvarn exact-tier salt could not be applied; kvarn entries "
+            "share the fp16 hash space until the next boot", exc_info=True)
 
 
 def kvarn_entry_salt(model=None) -> int:

gmlx/cache/snapshot.py

@@ -1571,12 +1571,17 @@ def _once():
             len(states), rec.nbytes / (1 << 20))
         return p
     except Exception:
+        # One swallow per call: a failing decline (telemetry) or first
+        # release must not leak the other block set out of the manager.
         try:
             _ckpt_decline(manager, "exception")
-            manager.release(main_blocks)
-            manager.release(bounded_blocks)
         except Exception:
-            pass  # best-effort release on the failure path
+            pass  # decline is telemetry only
+        for blks in (main_blocks, bounded_blocks):
+            try:
+                manager.release(blks)
+            except Exception:
+                pass  # best-effort release on the failure path
         _log.warning("APC ckpt store failed; continuing", exc_info=True)
         return 0
 
@@ -2155,11 +2160,13 @@ def _ckpt_disk_lookup(manager, ids, *, extra_hash, min_prefix_tokens,
         _log.info("APC ckpt hit: prefix=%d (disk skeleton)", p)
         return warm, p
     except Exception:
-        try:
-            manager.release(blocks)
-            manager.release(wblocks)
-        except Exception:
-            pass  # best-effort release on the failure path
+        # One swallow per release: a failing first release must not leak
+        # the second block set out of the manager.
+        for blks in (blocks, wblocks):
+            try:
+                manager.release(blks)
+            except Exception:
+                pass  # best-effort release on the failure path
         _log.warning("APC ckpt disk lookup failed; continuing",
                      exc_info=True)
         return None, 0

gmlx/models/gemma4/batched_sdpa.py

@@ -222,8 +222,8 @@ def _claim(queries, keys, values, cache, scale, mask, sinks):
                                       starts=starts)
             _ONECALL[0] += 1
             return out
-        except Exception:
-            pass  # op-build rejection -> per-row loop
+        except Exception as exc:  # op-build rejection -> per-row loop
+            attn_hd512._warn_fallback_once("g4_batched_decode", exc)
     # qL==1 needs no mask after the tail slice; verify blocks (qL 2..8)
     # occupy the LAST qL key positions, which is exactly mx.fast's
     # end-aligned "causal" semantics on the sliced row.

gmlx/serve/governor.py

@@ -980,6 +980,7 @@ def install_governor() -> bool:
     if getattr(_ar.BatchGenerator._next, _INSTALLED_FLAG, False):
         return True
     _orig = _ar.BatchGenerator._next
+    harvest_warned = [False]
 
     def _governed_next(self, **kwargs):
         try:
@@ -991,7 +992,10 @@ def _governed_next(self, **kwargs):
         try:
             _harvest_tick(self, _state(self), out)
         except Exception:
-            pass
+            if not harvest_warned[0]:
+                harvest_warned[0] = True
+                _log.warning("[governor] harvest failed; band inputs may "
+                             "go stale (warn-once)", exc_info=True)
         return out
 
     setattr(_governed_next, _INSTALLED_FLAG, True)

gmlx/serve/patches/completions.py

@@ -15,6 +15,7 @@
 import importlib
 import inspect
 import json
+import logging
 import time
 import uuid
 from typing import Any, List, Optional
@@ -29,6 +30,8 @@
 
 _COMPLETIONS_PATHS = ("/completions", "/v1/completions")
 
+_log = logging.getLogger(__name__)
+
 # Bound once at install time (the route cannot run before install); the
 # sibling patches bind their upstream modules the same way.
 _app_mod = None
@@ -126,13 +129,20 @@ def _include_usage(request: CompletionRequest) -> bool:
     return bool(getattr(so, "include_usage", False))
 
 
+_RECORD_FAILURE_WARNED = False
+
+
 def _record_failure(runtime, model: str, stream: bool, error: str) -> None:
+    global _RECORD_FAILURE_WARNED
     try:
         runtime.metrics.record_failure(endpoint="/v1/completions",
                                        model=model, stream=stream,
                                        error=error)
     except Exception:
-        pass
+        if not _RECORD_FAILURE_WARNED:
+            _RECORD_FAILURE_WARNED = True
+            _log.warning("failure-metrics record raised; failed requests "
+                         "are missing from /v1/metrics", exc_info=True)
 
 
 def _completion_envelope(gen_mod, *, model, stream, prompt_tokens,

gmlx/serve/patches/request_flow.py

@@ -5,6 +5,7 @@
 from __future__ import annotations
 
 import importlib
+import logging
 
 
 import gmlx.serve.bridge_vlm as serving
@@ -15,6 +16,8 @@
 )
 from . import routes as _routes
 
+_log = logging.getLogger(__name__)
+
 
 # Off-loop model load - keep /health (and siblings) responsive during a load
 _LOAD_OFFLOAD_FLAG = "_kq_gguf_load_offload"
@@ -232,8 +235,11 @@ async def _pump():
         task.cancel()
         try:
             await task
-        except BaseException:
-            pass
+        except asyncio.CancelledError:
+            pass    # the expected unwind of our own cancel
+        except Exception:
+            _log.warning("SSE pump task died with an unreported error",
+                         exc_info=True)
         aclose = getattr(body, "aclose", None)
         if aclose is not None:
             try:

gmlx/serve/patches/routes.py

@@ -6,6 +6,7 @@
 from __future__ import annotations
 
 import importlib
+import logging
 import os
 import time
 
@@ -20,6 +21,8 @@
     _remove_routes,
 )
 
+_log = logging.getLogger(__name__)
+
 
 def _mtime(path) -> int:
     try:
@@ -854,9 +857,17 @@ def _release_preload_holds(pool, path, *, only_evicted: bool = False) -> int:
         try:
             if hasattr(pool, "unmark_retained"):
                 pool.unmark_retained(hold)
+        except Exception:
+            _log.warning("preload hold unmark failed; releasing anyway",
+                         exc_info=True)
+        try:
             hold.release()
         except Exception:
-            pass
+            # A dropped-but-unreleased hold would pin the model resident
+            # for the process lifetime; keep it tracked for a retry.
+            _log.warning("preload hold release failed; keeping it tracked",
+                         exc_info=True)
+            continue
         _PRELOAD_HOLDS.remove(hold)
         released += 1
     return released

gmlx/serve/patches/sampling.py

@@ -4,12 +4,16 @@
 
 from __future__ import annotations
 
+import logging
+
 import gmlx.serve.bridge_vlm as serving
 from ._common import (
     _PATCH_FLAG,
     _install_gen_args_transform,
 )
 
+_log = logging.getLogger(__name__)
+
 
 # Sampling-profile injection
 # GenerationArguments attr <- profile sampling key (1:1 names). Each maps to the
@@ -78,9 +82,22 @@ def _effective_request_param(request, spec, key, default=None):
     return default
 
 
+_NEWLINE_PROBE_WARNED = False
+
+
+def _warn_newline_probe_once(exc: BaseException) -> None:
+    global _NEWLINE_PROBE_WARNED
+    if not _NEWLINE_PROBE_WARNED:
+        _NEWLINE_PROBE_WARNED = True
+        _log.warning(
+            "XTC newline probe failed (%s: %s); newline stays maskable "
+            "by XTC for this process", type(exc).__name__, exc)
+
+
 def _xtc_special_tokens(processor) -> list:
     """Newline + EOS token ids, excluded from XTC masking (the same convention
-    as the run/chat CLI). Defensive: any tokenizer shape miss degrades to []."""
+    as the run/chat CLI). Defensive: any tokenizer shape miss degrades to the
+    ids that did resolve, with a warn-once for the newline probe."""
     tok = getattr(processor, "tokenizer", processor)
     if tok is None:
         return []
@@ -90,10 +107,10 @@ def _xtc_special_tokens(processor) -> list:
     except TypeError:
         try:
             ids.extend(tok.encode("\n"))
-        except Exception:
-            pass
-    except Exception:
-        pass
+        except Exception as exc:
+            _warn_newline_probe_once(exc)
+    except Exception as exc:
+        _warn_newline_probe_once(exc)
     eos = getattr(tok, "eos_token_ids", None)
     if eos is None:
         eos = getattr(tok, "eos_token_id", None)

gmlx/serve/residency.py

@@ -412,7 +412,8 @@ def _stamp_apc_mode(rg) -> None:
         try:
             rg.apc_mode = mode
         except Exception:
-            pass
+            _log.warning("apc_mode stamp failed; the generator keeps its "
+                         "default mode", exc_info=True)
 
 
 class _ResidencyPool:

gmlx/spec/engine.py

@@ -153,7 +153,10 @@ def _init_with_stash(self, model, processor, **kwargs):
         try:
             model._kq_apc_manager = kwargs.get("apc_manager")
         except Exception:
-            pass
+            if kwargs.get("apc_manager") is not None:
+                _log.warning(
+                    "APC OFF for this model: manager stash failed -- "
+                    "every request prefills cold", exc_info=True)
         _orig_init(self, model, processor, **kwargs)
         # Stock admission forms a prompt batch only when free slots >=
         # prefill_batch_size. Stock pairs 32/8 (24 slots stay open); the
@@ -289,7 +292,12 @@ def _ckpt_layout_for(model, block_size: int = 16):
         try:
             tags = tuple(ckpt_layout(lm.make_cache(), block_size) or ())
         except Exception:
+            # An empty stash reads as "no ckpt signature" below, never as
+            # a valid all-empty layout that could sign entries.
             tags = ()
+            _log.warning(
+                "APC ckpt layout probe failed; ckpt tier off for this "
+                "model", exc_info=True)
         try:
             model._kq_apc_ckpt_layout = tags
         except Exception:

gmlx/stream/installs.py

@@ -14,8 +14,11 @@
 
 from __future__ import annotations
 
+import logging
 import weakref
 
+_log = logging.getLogger(__name__)
+
 # The attributes the loader hangs a streaming install on. Each is closeable
 # and each holds host resources (shard fds, staging pools, mlocked ranges)
 # that must not outlive the model.
@@ -136,6 +139,7 @@ def release(model) -> None:
     import gc
 
     owner = streaming_owner(model)
+    close_failed = False
     for attr in STREAM_ATTRS:
         helper = getattr(owner, attr, None)
         if helper is None:
@@ -145,12 +149,18 @@ def release(model) -> None:
             try:
                 close()
             except Exception:
-                pass
+                close_failed = True
+                _log.warning("streaming teardown: %s close failed; keeping "
+                             "it tracked", attr, exc_info=True)
+                continue    # keep the attr so the helper stays reachable
         try:
             object.__setattr__(owner, attr, None)
         except Exception:
             pass
-    _LIVE[:] = [(r, n) for r, n in _LIVE if r() is not model]
+    # A model with an un-closed helper keeps its wired-byte charge; the
+    # arena prune already keeps open feeders via _open.
+    if not close_failed:
+        _LIVE[:] = [(r, n) for r, n in _LIVE if r() is not model]
     _ARENAS[:] = [r for r in _ARENAS if _open(r())]
     del owner, model
     gc.collect()