feat(seed): recorded agentic demo run of the demo agent (D01-D10)

audits/agentic/demo_fixture.py

@@ -14,7 +14,11 @@
 
 FIXTURE_FORMAT = "simpleaudit-agentic-demo"
 FIXTURE_VERSION = 1
-_SECRET_KEY = re.compile(r"(api[_-]?key|authorization|cookie|password|secret|token)", re.IGNORECASE)
+#: A key names a secret when it contains one of these words, or "token" as a
+#: whole word segment (token, access_token, client-token), not tokenizer_model.
+_SECRET_KEY = re.compile(
+    r"(api[_-]?key|authorization|cookie|password|secret|(?:^|[_.\-])token(?:$|[_.\-]))", re.IGNORECASE
+)
 _REDACTED = "[REDACTED]"
 
 
@@ -119,9 +123,17 @@ def export_run_fixture(run: Any) -> dict[str, Any]:
     return fixture
 
 
+def _is_secret_key(key: str) -> bool:
+    """Whether a field name holds a secret. Token *counts* (``max_tokens``,
+    ``input_tokens``, ``gen_ai.usage.output_tokens``) are not secrets."""
+    if key.lower().endswith("tokens"):
+        return False
+    return bool(_SECRET_KEY.search(key))
+
+
 def sanitize_fixture(value: Any, *, key: str = "") -> Any:
     """Deep-copy JSON-like data while redacting secret-bearing fields."""
-    if _SECRET_KEY.search(key):
+    if _is_secret_key(key):
         return _REDACTED
     if isinstance(value, dict):
         return {name: sanitize_fixture(item, key=name) for name, item in value.items()}
@@ -130,6 +142,55 @@ def sanitize_fixture(value: Any, *, key: str = "") -> Any:
     return value
 
 
+#: Endpoint snapshot fields that identify where a model ran.
+_ENDPOINT_FIELDS = ("id", "connection_id", "base_url", "secret_reference")
+
+
+def relabel_models(fixture: dict[str, Any], label: str, *, replacements: dict[str, str] | None = None) -> dict[str, Any]:
+    """Relabel the models a recorded run used, for a demo fixture.
+
+    A demo may be recorded on a model the people who load it do not have.
+    The auditor, judge and agent base-model snapshots become ``label`` (model
+    id ``demo-model``, no connection or URL); the target (an agent wrapper) keeps
+    its name and model id but loses its connection and URL. ``replacements``
+    then rewrites every string in the fixture (``{"old": "new"}``), for
+    identifiers that also appear in spans and results (host names, model ids).
+
+    Model *output* is left as recorded; only identifiers are changed.
+    Provenance records that the fixture was relabelled.
+    """
+    out = json.loads(json.dumps(fixture))
+    run = out.get("run") or {}
+    agent = run.get("agent_config_snapshot") if isinstance(run.get("agent_config_snapshot"), dict) else {}
+    model_snapshots = [run.get("auditor_config_snapshot"), run.get("judge_config_snapshot"), agent.get("base_model")]
+    for snapshot in model_snapshots:
+        if isinstance(snapshot, dict):
+            for name in _ENDPOINT_FIELDS:
+                snapshot.pop(name, None)
+            snapshot.update({"display_name": label, "model_id": "demo-model", "provider": "openai"})
+    target = run.get("target_config_snapshot")
+    if isinstance(target, dict):
+        for name in _ENDPOINT_FIELDS:
+            target.pop(name, None)
+
+    def rewrite(value: Any) -> Any:
+        if isinstance(value, str):
+            for old, new in (replacements or {}).items():
+                value = value.replace(old, new)
+            return value
+        if isinstance(value, dict):
+            return {rewrite(k): rewrite(v) for k, v in value.items()}
+        if isinstance(value, list):
+            return [rewrite(v) for v in value]
+        return value
+
+    out = rewrite(out)
+    provenance = out.setdefault("recorded_run", {})
+    provenance["models_relabelled"] = label
+    out.get("_meta", {}).pop("fixture_checksum", None)
+    return out
+
+
 def fixture_checksum(fixture: dict[str, Any]) -> str:
     """Return the stable SHA-256 checksum excluding the stored checksum."""
     payload = dict(fixture)

audits/agentic/tests/test_demo_fixture.py

@@ -37,3 +37,51 @@ def test_recorded_fixture_requires_execution_provenance():
     fixture["_meta"]["fixture_checksum"] = fixture_checksum(fixture)
     with pytest.raises(FixtureValidationError, match="executed=true"):
         validate_fixture(fixture, require_recorded=True)
+
+
+def test_token_counts_and_tokenizer_names_are_not_secrets():
+    data = {
+        "max_tokens": 2048, "input_tokens": 12, "gen_ai.usage.output_tokens": 5,
+        "tokenizer_model": "cl100k", "access_token": "a", "x-client-token": "b", "token": "c",
+    }
+    assert sanitize_fixture(data) == {
+        "max_tokens": 2048, "input_tokens": 12, "gen_ai.usage.output_tokens": 5,
+        "tokenizer_model": "cl100k", "access_token": "[REDACTED]", "x-client-token": "[REDACTED]",
+        "token": "[REDACTED]",
+    }
+
+
+def test_relabel_models_hides_where_a_demo_was_recorded():
+    from audits.agentic.demo_fixture import relabel_models
+
+    endpoint = {"id": 7, "connection_id": 3, "display_name": "Qwen (Lab)", "model_id": "default",
+                "provider": "openai", "base_url": "https://lab.example/v1", "secret_reference": "LAB_KEY"}
+    fixture = {
+        "fixture_format": "simpleaudit-agentic-demo", "fixture_version": 1,
+        "recorded_run": {"source": "recorded_real_run_fixture", "executed": True},
+        "run": {
+            "auditor_config_snapshot": dict(endpoint),
+            "judge_config_snapshot": {**endpoint, "judge": {"name": "Safety"}},
+            "target_config_snapshot": {"display_name": "Agent", "model_id": "studio.agent-1",
+                                       "base_url": "http://127.0.0.1:8180/chat/api/v1", "connection_id": 9},
+            "agent_config_snapshot": {"name": "Agent", "base_model": dict(endpoint)},
+        },
+        "scenario_results": [{"result": {"conversation": [{"role": "assistant", "content": "I am Qwen."}]}}],
+        "spans": [{"attributes": {"gen_ai.response.model": "Qwen3-27B"}}],
+        "_meta": {"fixture_checksum": "old"},
+    }
+    out = relabel_models(fixture, "Demo model (recorded)", replacements={"Qwen3-27B": "demo-model"})
+    for snapshot in (out["run"]["auditor_config_snapshot"], out["run"]["judge_config_snapshot"],
+                     out["run"]["agent_config_snapshot"]["base_model"]):
+        assert snapshot["display_name"] == "Demo model (recorded)"
+        assert snapshot["model_id"] == "demo-model"
+        assert not {"base_url", "connection_id", "secret_reference", "id"} & set(snapshot)
+    assert out["run"]["judge_config_snapshot"]["judge"] == {"name": "Safety"}
+    target = out["run"]["target_config_snapshot"]
+    assert target["display_name"] == "Agent" and "base_url" not in target and "connection_id" not in target
+    assert out["spans"][0]["attributes"]["gen_ai.response.model"] == "demo-model"
+    # Model output stays as recorded; provenance says the labels changed.
+    assert out["scenario_results"][0]["result"]["conversation"][0]["content"] == "I am Qwen."
+    assert out["recorded_run"]["models_relabelled"] == "Demo model (recorded)"
+    assert "fixture_checksum" not in out["_meta"]
+    assert fixture["run"]["auditor_config_snapshot"]["base_url"] == "https://lab.example/v1"  # input untouched

conftest.py

@@ -57,6 +57,20 @@ def _synchronous_otlp_span_writes(settings):
     settings.OTLP_SPAN_FLUSH_INTERVAL_S = 0
 
 
+@pytest.fixture(autouse=True)
+def _fresh_otlp_auth_cache():
+    """Start each test with an empty OTLP auth cache.
+
+    ``authenticate`` caches header -> credential for the process. A test's
+    credentials are rolled back without the post_delete signal that clears
+    it, so a later test in the same worker could be let in by a credential
+    that no longer exists (an unauthenticated push answered 200, not 401).
+    """
+    from model_registry.otlp_services import clear_auth_cache
+
+    clear_auth_cache()
+
+
 @pytest.fixture(autouse=True)
 def _no_openwebui_sync_throttle(settings):
     """Every Agent page load syncs in tests (ids are reused across tests)."""

docs/deployment.md

@@ -215,9 +215,10 @@ override them when you deliberately need to.
 |---|---|
 | `setup_local` | migrate + bootstrap + seed (local dev, one-shot). |
 | `bootstrap_platform` | Create the admin user and default workspace (idempotent). |
-| `seed_platform` | Import scenario packs and model connections (including a TypeSafe connection with the Jev decision model, keyed by `TYPESAFE_API_KEY`), plus demo runs (`seed_demo_audits`, one of them graded by the Decision Safety Judge on Jev) and the demo "Support Refund Assistant" agent (skip with `--skip-demo-agent`). |
+| `seed_platform` | Import scenario packs and model connections (including a TypeSafe connection with the Jev decision model, keyed by `TYPESAFE_API_KEY`), plus demo runs (`seed_demo_audits`, one of them graded by the Decision Safety Judge on Jev; `seed_agentic_demo`, a recorded agentic audit of the demo agent) and the demo "Support Refund Assistant" agent (skip with `--skip-demo-agent`). |
 | `seed_agentic_scenarios` | Seed the default agentic audit scenario set (D01-D10). |
-| `seed_agentic_demo` | Preload completed synthetic results, traces, and a separate Agentic verdict rollup after the demo Agent sync; never runs an audit or calls a model. |
+| `seed_agentic_demo` | Load the recorded agentic demo run (`infra/fixtures/agentic_demo_run.json`): a real audit of the demo "Support Refund Assistant" against the default agentic pack (D01–D10), with its results, agentic checks and OTLP spans, models relabelled "Demo model (recorded)". Never runs an audit or calls a model; `seed_platform` runs it with the demo runs. A deleted demo run is not loaded again (`--force` replaces it). |
+| `capture_agentic_demo_fixture` | Export a completed, traced agentic run as a checksummed, sanitized fixture (`--run N --require-recorded`). `--relabel-models "Demo model (recorded)"` (with `--replace OLD=NEW` for model ids in spans) replaces the models and endpoints it ran on, for a demo recorded on a model its users do not have; set `runtime_metadata.executed=true` on the run once it has really executed. |
 | `dev` | Local dev stack (embedded, hot-reload, chat, one-time sign-in). `--disable-chat`, `--no-worker`, `--no-reload`, `--no-browser`. |
 | `dev_server` | Lower-level version of the same stack. Use `dev` instead. |
 | `mode` | Print the resolved run mode and its settings. |