fix(*): codex usage, log redaction, and two observability gaps

raven/cli/_log_file.py

@@ -21,6 +21,7 @@
 from pathlib import Path
 
 from raven.config.paths import get_logs_dir
+from raven.utils.log_redaction import combine_filters, redacting_filter
 
 
 def redirect_loguru_to_file(
@@ -44,23 +45,27 @@ def redirect_loguru_to_file(
 
     log_path = get_logs_dir() / filename
 
+    # Every sink redacts: the file is what gets attached to a bug report, and
+    # the terminal is what gets pasted into one.
+    file_filter = combine_filters(redacting_filter, record_filter)
+
     logger.remove()
     logger.add(
         str(log_path),
         level=file_level,
         rotation=rotation,
         retention=retention,
-        filter=record_filter,
+        filter=file_filter,
         enqueue=True,  # thread-safe writes from channel threads + asyncio
         # diagnose=True would annotate tracebacks with local variable values,
         # writing secrets (API tokens, etc.) into a persisted, retained file.
         backtrace=False,
         diagnose=False,
     )
     if terminal_level is not None:
-        logger.add(sys.stderr, level=terminal_level)
+        logger.add(sys.stderr, level=terminal_level, filter=redacting_filter)
     if os.environ.get("RAVEN_CLI_DEBUG"):
-        logger.add(sys.stderr, level="DEBUG")
+        logger.add(sys.stderr, level="DEBUG", filter=redacting_filter)
 
     _intercept_stdlib_logging(logger)
     _strip_tty_stream_handlers()

raven/cli/doctor_commands.py

@@ -74,6 +74,7 @@ class MemoryInfo:
 
     backend: Optional[str] = None
     root: Optional[str] = None
+    binary: Optional[str] = None
     owned: bool = True
     address: Optional[str] = None
     server_running: bool = False
@@ -255,7 +256,10 @@ def _probe_memory(config: "RavenConfig") -> MemoryInfo:
     info.capabilities = dict(report.capabilities)
 
     if info.owned:
+        from raven.plugin.memory.everos._server import everos_binary_path
+
         info.root = str(everos_root())
+        info.binary = everos_binary_path()
         info.configured = [s for s in (*REQUIRED_SECTIONS, *DEGRADING_SECTIONS) if everos_role_configured(s)]
         # Recall quality is decided by the embedding role in the user-level
         # everos.toml: with it recall matches meaning, without it only keywords.
@@ -304,6 +308,11 @@ def _render_memory_capabilities(memory: MemoryInfo) -> None:
         return
     if memory.root:
         console.print(f"  Memories:   {memory.root}")
+    if memory.owned:
+        if memory.binary:
+            console.print(f"  Binary:     {memory.binary}")
+        else:
+            console.print("  Binary:     [yellow]not found[/yellow]  (searched next to the interpreter, then PATH)")
     if not memory.owned:
         console.print(
             "  [dim]Managed by you -- Raven reads it at the address below and never writes,\n"

raven/config/loader.py

@@ -53,6 +53,39 @@
 # load_config calls (status/doctor load more than once) warn only once.
 _warned_paths: set[str] = set()
 
+# Migration lines already logged in this process. The strips below rewrite the
+# in-memory copy only, so an unmigrated file re-emits every line on every load
+# -- and a running gateway loads once per cron fire, which turned a one-time
+# notice into steady log noise.
+_logged_migrations: set[str] = set()
+
+
+class _MigrationLog:
+    """Logger proxy emitting each distinct migration line once per process."""
+
+    def __init__(self, inner: logging.Logger) -> None:
+        self._inner = inner
+
+    def info(self, message: str, *args: Any) -> None:
+        # A record the logger would drop was never told to anyone. The gateway
+        # loads its config before it installs a sink, so counting that first
+        # dropped line as already-logged would silence the copy that reaches
+        # the log file -- turning the noise this dedup fixes into silence.
+        if not self._inner.isEnabledFor(logging.INFO):
+            return
+        rendered = message % args if args else message
+        if rendered in _logged_migrations:
+            return
+        _logged_migrations.add(rendered)
+        self._inner.info(message, *args)
+
+
+# Module-level: the context-window migration logs from its own function, and it
+# runs twice per command (load_config's read plus the persist pass re-reading
+# the raw file), so it needs the same dedup the in-loader migrations get.
+_migration_log = _MigrationLog(logging.getLogger(__name__))
+
+
 # User-facing lines produced by a migration that actually changed something,
 # waiting to be printed by whichever CLI entry point owns the terminal.
 # Migrations run inside the loader, which has no console of its own and whose
@@ -193,7 +226,7 @@ def _migrate_legacy_context_window(data: dict[str, Any], *, notify: bool = False
         if defaults.get(legacy_key) == LEGACY_CONTEXT_WINDOW_TOKENS:
             defaults.pop(legacy_key)
             changed = True
-            logging.getLogger(__name__).info(
+            _migration_log.info(
                 "Migrated: dropped agents.defaults.%s (the retired 65536 default)",
                 legacy_key,
             )
@@ -360,9 +393,8 @@ def _migrate_config(data: dict, *, pop_extension_keys: bool = True, run_stamped:
     the config path -- the one that can read the watermark and write it back --
     opts in; the shims below are idempotent and always run.
     """
-    import logging as _logging
 
-    _log = _logging.getLogger(__name__)
+    _log = _migration_log
 
     if run_stamped:
         _migrate_legacy_context_window(data, notify=True)

raven/plugin/memory/everos/_server.py

@@ -133,6 +133,19 @@ def _require_llm_configured() -> None:
     )
 
 
+def everos_binary_path() -> str | None:
+    """The everos the spawner would run, or ``None`` when there is none.
+
+    ``doctor`` reports this: which binary raven resolved is invisible
+    otherwise, and it is the first thing to check when a memory install that
+    looks configured still will not start.
+    """
+    try:
+        return _everos_executable()
+    except EverosBinaryMissingError:
+        return None
+
+
 def _everos_executable() -> str:
     """Locate the everos CLI, preferring the one installed alongside raven.
 

raven/providers/openai_codex_provider.py

@@ -94,20 +94,21 @@ async def chat(
         timeout = self.generation.timeout
         try:
             try:
-                content, tool_calls, finish_reason = await _request_codex(
+                content, tool_calls, finish_reason, usage = await _request_codex(
                     url, headers, body, verify=True, timeout=timeout
                 )
             except Exception as e:
                 if "CERTIFICATE_VERIFY_FAILED" not in str(e):
                     raise
                 logger.warning("SSL certificate verification failed for Codex API; retrying with verify=False")
-                content, tool_calls, finish_reason = await _request_codex(
+                content, tool_calls, finish_reason, usage = await _request_codex(
                     url, headers, body, verify=False, timeout=timeout
                 )
             return LLMResponse(
                 content=content,
                 tool_calls=tool_calls,
                 finish_reason=finish_reason,
+                usage=usage,
             )
         except Exception as e:
             classification = self.classify_error(e)
@@ -151,7 +152,7 @@ async def _request_codex(
     body: dict[str, Any],
     verify: bool,
     timeout: float,
-) -> tuple[str, list[ToolCallRequest], str]:
+) -> tuple[str, list[ToolCallRequest], str, dict[str, int]]:
     async with httpx.AsyncClient(timeout=timeout, verify=verify) as client:
         async with client.stream("POST", url, headers=headers, json=body) as response:
             if response.status_code != 200:
@@ -162,6 +163,27 @@ async def _request_codex(
             return await _consume_sse(response, timeout)
 
 
+def _convert_usage(raw: Any) -> dict[str, int]:
+    """Map a Responses API usage block onto the keys raven's loop reads.
+
+    ``input_tokens`` counts cached tokens as well -- the same total-prompt
+    convention OpenRouter/LiteLLM use, which ``_build_usage_snapshot`` already
+    normalizes to fresh-only, so it passes through as ``prompt_tokens`` as-is.
+    """
+    if not isinstance(raw, dict):
+        return {}
+    details = raw.get("input_tokens_details")
+    if not isinstance(details, dict):
+        details = {}
+    return {
+        "prompt_tokens": int(raw.get("input_tokens") or 0),
+        "completion_tokens": int(raw.get("output_tokens") or 0),
+        "total_tokens": int(raw.get("total_tokens") or 0),
+        "cache_read_input_tokens": int(details.get("cached_tokens") or 0),
+        "cache_creation_input_tokens": int(details.get("cache_write_tokens") or 0),
+    }
+
+
 def _convert_tools(tools: list[dict[str, Any]]) -> list[dict[str, Any]]:
     """Convert OpenAI function-calling schema to Codex flat format."""
     converted: list[dict[str, Any]] = []
@@ -357,11 +379,14 @@ async def _iter_sse(response: httpx.Response, timeout: float) -> AsyncGenerator[
         buffer.append(line)
 
 
-async def _consume_sse(response: httpx.Response, timeout: float) -> tuple[str, list[ToolCallRequest], str]:
+async def _consume_sse(
+    response: httpx.Response, timeout: float
+) -> tuple[str, list[ToolCallRequest], str, dict[str, int]]:
     content = ""
     tool_calls: list[ToolCallRequest] = []
     tool_call_buffers: dict[str, dict[str, Any]] = {}
     finish_reason = "stop"
+    usage: dict[str, int] = {}
 
     async for event in _iter_sse(response, timeout):
         event_type = event.get("type")
@@ -416,8 +441,9 @@ async def _consume_sse(response: httpx.Response, timeout: float) -> tuple[str, l
                     )
                 )
         elif event_type == "response.completed":
-            status = (event.get("response") or {}).get("status")
-            finish_reason = _map_finish_reason(status)
+            completed = event.get("response") or {}
+            finish_reason = _map_finish_reason(completed.get("status"))
+            usage = _convert_usage(completed.get("usage"))
         elif event_type in {"error", "response.failed"}:
             # The code is the retry signal: classify_error buckets by message
             # substring, and "server_is_overloaded" is what turns a dead-end
@@ -432,7 +458,7 @@ async def _consume_sse(response: httpx.Response, timeout: float) -> tuple[str, l
             detail = ": ".join(str(part) for part in (code, message) if part)
             raise RuntimeError(f"Codex response failed: {detail}" if detail else "Codex response failed")
 
-    return content, tool_calls, finish_reason
+    return content, tool_calls, finish_reason, usage
 
 
 _FINISH_REASON_MAP = {"completed": "stop", "incomplete": "length", "failed": "error", "cancelled": "error"}

raven/utils/log_redaction.py

@@ -0,0 +1,78 @@
+"""Strip credentials out of log records before they reach a sink.
+
+Some upstreams put the credential in the URL itself -- the Telegram Bot API
+keys every route on ``/bot<token>/``, and several providers take an API key as
+a query parameter -- so any library that logs its request line (httpx does, at
+INFO) writes a working credential into a persisted, retained file. The gateway
+log is the one users attach to a bug report, which is exactly the wrong place
+for it.
+
+This is the message-body counterpart to ``diagnose=False`` in
+:mod:`raven.cli._log_file`, which already keeps tracebacks from serializing
+locals holding secrets.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+REDACTED = "<redacted>"
+
+_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
+    # Telegram: /bot<id>:<secret>/. The numeric id stays -- it identifies the
+    # bot for debugging and is not the secret half.
+    (re.compile(r"/bot(\d{5,}):[A-Za-z0-9_-]{15,}"), rf"/bot\1:{REDACTED}"),
+    # Credential passed as a query parameter (Gemini's ?key=, and friends).
+    (
+        re.compile(r"([?&](?:api[-_]?key|access[-_]?token|auth[-_]?token|token|key|secret)=)[^&\s\"']+", re.I),
+        rf"\1{REDACTED}",
+    ),
+    # Basic-auth credentials embedded in a URL.
+    (re.compile(r"(://)[^/\s:@]+:[^/\s@]+@"), rf"\1{REDACTED}@"),
+    # Authorization: Bearer <token>.
+    (re.compile(r"(Bearer\s+)[A-Za-z0-9._~+/=-]{12,}", re.I), rf"\1{REDACTED}"),
+    # Bare vendor-prefixed keys that appear outside any URL.
+    (re.compile(r"\b(sk-[A-Za-z0-9_-]{12,}|xox[abprs]-[A-Za-z0-9-]{10,}|gh[pousr]_[A-Za-z0-9]{20,})"), REDACTED),
+)
+
+
+def redact(text: str) -> str:
+    """Return ``text`` with every known credential shape masked."""
+    for pattern, replacement in _PATTERNS:
+        text = pattern.sub(replacement, text)
+    return text
+
+
+def redacting_filter(record: Any) -> bool:
+    """Loguru sink filter that rewrites the record in place, always keeping it.
+
+    Loguru formats a record *after* its filters run, so mutating
+    ``record["message"]`` here is what reaches every sink.
+    """
+    message = record.get("message")
+    if isinstance(message, str):
+        record["message"] = redact(message)
+    return True
+
+
+def combine_filters(*filters: Any) -> Any:
+    """Chain sink filters, dropping the record as soon as one rejects it.
+
+    The redacting filter must run even when a caller supplied its own
+    noise-dropping filter, so the two are composed rather than one replacing
+    the other.
+    """
+    active = [f for f in filters if f is not None]
+    if not active:
+        return None
+    if len(active) == 1:
+        return active[0]
+
+    def _chained(record: Any) -> bool:
+        return all(f(record) for f in active)
+
+    return _chained
+
+
+__all__ = ["REDACTED", "combine_filters", "redact", "redacting_filter"]

tests/test_cli_doctor_commands.py

@@ -664,3 +664,61 @@ def test_a_role_the_server_says_nothing_about_is_not_claimed_either_way(
 
         assert "rerank" not in info.configured
         assert "rerank" not in info.unbuilt
+
+
+def test_doctor_reports_which_everos_binary_it_resolved(healthy_config: Path, no_memory_server, tmp_path) -> None:
+    """Which everos raven resolved was invisible from every command, so a
+    memory install that looks configured but will not start had nothing to
+    check. The resolver prefers the interpreter's own directory, which is the
+    detail that matters when PATH holds a different environment's copy."""
+    from raven.config import update_everos as ue
+    from raven.plugin.memory.everos import _server
+
+    _configured(no_memory_server, "llm")
+    _capabilities(no_memory_server, llm=True)
+    no_memory_server.setattr(ue, "everos_root", lambda: tmp_path / "mem-root")
+    no_memory_server.setattr(ue, "everos_owned", lambda: True)
+    no_memory_server.setattr(_server, "_everos_executable", lambda: "/opt/venv/bin/everos")
+
+    r = runner.invoke(app, ["doctor"])
+
+    assert r.exit_code == 0, r.stdout
+    assert "Binary:" in r.stdout
+    assert "everos" in r.stdout
+
+
+def test_doctor_says_when_no_everos_binary_resolves(healthy_config: Path, no_memory_server, tmp_path) -> None:
+    """A missing binary is reported, not raised: doctor's job is to describe a
+    broken install rather than fail on it."""
+    from raven.config import update_everos as ue
+    from raven.plugin.memory.everos import _server
+
+    _configured(no_memory_server, "llm")
+    _capabilities(no_memory_server, llm=True)
+    no_memory_server.setattr(ue, "everos_root", lambda: tmp_path / "mem-root")
+    no_memory_server.setattr(ue, "everos_owned", lambda: True)
+
+    def _missing() -> str:
+        raise _server.EverosBinaryMissingError("everos not found next to /x/bin or on PATH.")
+
+    no_memory_server.setattr(_server, "_everos_executable", _missing)
+
+    r = runner.invoke(app, ["doctor"])
+
+    assert r.exit_code == 0, r.stdout
+    assert "Binary:" in r.stdout
+    assert "not found" in r.stdout
+
+
+def test_doctor_omits_the_binary_line_for_a_server_raven_does_not_run(healthy_config: Path, no_memory_server) -> None:
+    """Raven never spawns a server it does not own, so which binary it would
+    have used is not a fact about that install."""
+    from raven.config import update_everos as ue
+
+    _configured(no_memory_server, "llm")
+    _capabilities(no_memory_server, llm=True)
+    no_memory_server.setattr(ue, "everos_owned", lambda: False)
+
+    r = runner.invoke(app, ["doctor"])
+
+    assert "Binary:" not in r.stdout