From b0fc0b061e809af69aec2048ec89da04a351596e Mon Sep 17 00:00:00 2001 From: Sushant Gautam Date: Thu, 5 Nov 2026 00:27:50 +0100 Subject: [PATCH 1/3] chore: pin Open WebUI v0.12.4.10-subpath GenAI spans on every content, tool and model path (fork #11): retrieval spans with documents for every content-reading built-in tool (knowledge files and grep, chat files, notes, chats, channels, memories, web search and fetch) and for memory recall, execute_tool spans in default function calling, and chat spans for pipe models. Fixes checks that counted no retrieved documents when an agent read a file with view_knowledge_file instead of searching. --- chat/config.py | 2 +- chat/proxy.py | 2 +- chat/tests/test_proxy.py | 4 ++-- deploy/openwebui-subpath/Caddyfile.subpath | 2 +- deploy/openwebui-subpath/README.md | 2 +- deploy/openwebui-subpath/UPGRADE.md | 1 + docker-compose.yml | 2 +- 7 files changed, 8 insertions(+), 7 deletions(-) diff --git a/chat/config.py b/chat/config.py index 5783dbc6..ccb3e9e3 100644 --- a/chat/config.py +++ b/chat/config.py @@ -57,7 +57,7 @@ def is_disabled(value: str | None) -> bool: ENABLED = not is_disabled(MODE) #: The base path the Open WebUI subpath build serves at. Must match the -#: WEBUI_SUBPATH baked into the image (0.12.4.9-subpath -> /chat). Defined +#: WEBUI_SUBPATH baked into the image (0.12.4.10-subpath -> /chat). Defined #: first: UPSTREAM below bakes it into the default. SUBPATH = os.environ.get("SIMPLEAUDIT_CHAT_SUBPATH", "/chat") if not SUBPATH.startswith("/"): diff --git a/chat/proxy.py b/chat/proxy.py index 591d7d4c..32b0f7e7 100644 --- a/chat/proxy.py +++ b/chat/proxy.py @@ -585,7 +585,7 @@ def start_open_webui(studio_port: int | None = None) -> subprocess.Popen: OWUI_WHEEL_URL = os.environ.get( "SIMPLEAUDIT_CHAT_WHEEL", "https://github.com/SushantGautam/open-webui/releases/download/" - "v0.12.4.9-subpath/open_webui-0.12.4.9-py3-none-any.whl", + "v0.12.4.10-subpath/open_webui-0.12.4.10-py3-none-any.whl", ) OWUI_PACKAGE = os.environ.get("SIMPLEAUDIT_CHAT_PACKAGE", "open-webui") diff --git a/chat/tests/test_proxy.py b/chat/tests/test_proxy.py index f764f2ac..8223e39a 100644 --- a/chat/tests/test_proxy.py +++ b/chat/tests/test_proxy.py @@ -381,8 +381,8 @@ class WheelMarkerTests(SimpleTestCase): def test_default_wheel_is_content_capture_release(self): # Pinned exactly: a version bump must change this assertion deliberately, # not pass silently via a substring of an older tag. - self.assertIn("v0.12.4.9-subpath", proxy.OWUI_WHEEL_URL) - self.assertIn("open_webui-0.12.4.9-py3-none-any.whl", proxy.OWUI_WHEEL_URL) + self.assertIn("v0.12.4.10-subpath", proxy.OWUI_WHEEL_URL) + self.assertIn("open_webui-0.12.4.10-py3-none-any.whl", proxy.OWUI_WHEEL_URL) def test_install_requirement_includes_observability_extra(self): requirement = proxy._owui_install_requirement() diff --git a/deploy/openwebui-subpath/Caddyfile.subpath b/deploy/openwebui-subpath/Caddyfile.subpath index 90010710..cc36731c 100644 --- a/deploy/openwebui-subpath/Caddyfile.subpath +++ b/deploy/openwebui-subpath/Caddyfile.subpath @@ -3,7 +3,7 @@ # # The routing shape enabled by the subpath fork built via # deploy/openwebui-subpath/build.sh (published as -# ghcr.io/sushantgautam/open-webui:v0.12.4.9-subpath). It replaces the old +# ghcr.io/sushantgautam/open-webui:v0.12.4.10-subpath). It replaces the old # cross-origin iframe model (Caddyfile.chat on :8801) with one origin / one # port: # diff --git a/deploy/openwebui-subpath/README.md b/deploy/openwebui-subpath/README.md index 61055e87..91940ccb 100644 --- a/deploy/openwebui-subpath/README.md +++ b/deploy/openwebui-subpath/README.md @@ -154,7 +154,7 @@ pip caveats): [`DESIGN.md`](./DESIGN.md). publishing to ghcr.io (root on push, `-subpath` on `webui_subpath` dispatch). The cutover is done: `docker-compose.yml` pulls -`ghcr.io/sushantgautam/open-webui:v0.12.4.9-subpath` (set `WEBUI_SUBPATH=/chat`), +`ghcr.io/sushantgautam/open-webui:v0.12.4.10-subpath` (set `WEBUI_SUBPATH=/chat`), `chat-proxy` (Caddy, no profile) is the **only published port** and runs `Caddyfile.subpath`, Studio's wrapper page moved to `/playground/`, and `Caddyfile.chat` + the separate cross-origin port are retired. Embedded diff --git a/deploy/openwebui-subpath/UPGRADE.md b/deploy/openwebui-subpath/UPGRADE.md index 631fc142..a7a86937 100644 --- a/deploy/openwebui-subpath/UPGRADE.md +++ b/deploy/openwebui-subpath/UPGRADE.md @@ -131,4 +131,5 @@ half-port; staying on the proven base is the fallback. | 2026-11-04 | v0.12.4.7 | **published ✓** | Schema-shaped GenAI spans (fork #8). `-subpath` image and wheel release `v0.12.4.7-subpath`. | | 2026-11-04 | v0.12.4.8 | **published ✓** | Agentic knowledge search traced as a GenAI `retrieval` span with `gen_ai.retrieval.documents` (fork #9). | | 2026-11-04 | v0.12.4.9 | **published ✓** | Compact GenAI content by default, each piece recorded once per trace (fork #10; `OTEL_GENAI_CONTENT_MODE=full` for standard spans). Studio pins `v0.12.4.9-subpath`. | +| 2026-11-05 | v0.12.4.10 | **published ✓** | GenAI spans on every content, tool and model path (fork #11): retrieval spans for every content-reading built-in tool (knowledge, chat files, notes, chats, channels, memories, web) and memory recall, `execute_tool` spans in default function calling, `chat` spans for pipe models. Studio pins `v0.12.4.10-subpath`. | | | | | | diff --git a/docker-compose.yml b/docker-compose.yml index 0a7ffe6b..1c8442ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -189,7 +189,7 @@ services: # forward /chat/* WITHOUT stripping the prefix (see the Caddyfile header). open-webui: profiles: ["chat"] - image: ${OPEN_WEBUI_IMAGE:-ghcr.io/sushantgautam/open-webui:v0.12.4.9-subpath} + image: ${OPEN_WEBUI_IMAGE:-ghcr.io/sushantgautam/open-webui:v0.12.4.10-subpath} # Derives the OTLP password, then drops the raw key and password from the # environment before Open WebUI starts, so Open WebUI never holds the Django key. # (`$$` is compose's escape for a literal `$`.) Command is the image's own CMD. From c1f845c4c8ce6badbef462a08021b13ed5130575 Mon Sep 17 00:00:00 2001 From: Sushant Gautam Date: Thu, 5 Nov 2026 00:52:50 +0100 Subject: [PATCH 2/3] fix: traced agent runs survive Open WebUI's cold start Right after Open WebUI starts, its model registry can still miss the agent when Studio's once-a-minute refresh ran before Open WebUI loaded its connections, so concurrent scenarios failed with 'Model not found'. A saved-chat request that gets that answer now reloads the registry immediately and retries once. --- infra/tests/test_engine_integration.py | 40 ++++++++++++++++++++++++++ infra/trace_target.py | 12 +++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/infra/tests/test_engine_integration.py b/infra/tests/test_engine_integration.py index b529a4eb..03ca8a62 100644 --- a/infra/tests/test_engine_integration.py +++ b/infra/tests/test_engine_integration.py @@ -500,6 +500,46 @@ def send(history, trace="1"): self.assertNotIn("chat_id", posts[2]) self.assertIsNone(posts[2]["parent_id"]) + @mock.patch("chat.config.UPSTREAM", "http://127.0.0.1:8080/chat") + @mock.patch("chat.config.ENABLED", True) + def test_model_not_found_reloads_open_webui_models_and_retries_once(self): + """Right after Open WebUI starts its model registry may miss the agent: reload, retry once.""" + import asyncio + from types import SimpleNamespace + + from infra import trace_target + from infra.trace_target import TraceContextModelTarget + + trace_target._owui_conversations.clear() + posts, refreshes = [], [] + + class RawClient: + base_url = "http://127.0.0.1:8080/chat/api/v1/" + api_key = "jwt" + + async def post(self, path, *, cast_to, body, options): + posts.append(body) + self.message_id = body["message_ids"][0]["message_id"] + if len(posts) == 1: + raise RuntimeError("Error code: 400 - {'detail': 'Model not found'}") + return {"status": True, "chat_id": "chat-1"} + + async def get(self, path, *, cast_to, options): + return {"chat": {"history": {"messages": {self.message_id: { + "id": self.message_id, "role": "assistant", "done": True, "content": "ok"}}}}} + + async def refresh(origin, token, *, every=60.0): + refreshes.append((origin, token, every)) + + target = TraceContextModelTarget(client=SimpleNamespace(client=RawClient(), acompletion=None), + model="studio.agent-1") + with mock.patch.object(trace_target, "_owui_refresh_models", refresh), \ + mock.patch.object(trace_target, "_owui_get_jwt", mock.AsyncMock(return_value="jwt")): + response = asyncio.run(target.send(user="hi", history=[{"role": "user", "content": "hi"}])) + self.assertEqual(response.content, "ok") + self.assertEqual(len(posts), 2) + self.assertEqual(refreshes, [("http://127.0.0.1:8080/chat", "jwt", 0.0)]) + def test_streaming_call_never_forwards_studio_only_params_to_openai_client(self): """extra_headers (W3C trace context) must not leak into the OpenAI request.""" import asyncio diff --git a/infra/trace_target.py b/infra/trace_target.py index 84fe1814..72efcbb4 100644 --- a/infra/trace_target.py +++ b/infra/trace_target.py @@ -346,7 +346,17 @@ async def _openwebui_saved_chat_call( if chat_id_to_continue: body["chat_id"] = chat_id_to_continue options = {"headers": trace_headers} if trace_headers else {} - created = await raw_client.post("/chat/completions", cast_to=dict, body=body, options=options) + try: + created = await raw_client.post("/chat/completions", cast_to=dict, body=body, options=options) + except Exception as exc: + # Right after Open WebUI starts, its model registry can still miss the agent + # (the throttled refresh may have run before its connections loaded). Reload + # the registry now and retry once. + if "not found" not in str(exc).lower() or "model" not in str(exc).lower(): + raise + origin = base_url.rsplit("/api/v1", 1)[0].rstrip("/") + await _owui_refresh_models(origin, str(getattr(raw_client, "api_key", "") or ""), every=0.0) + created = await raw_client.post("/chat/completions", cast_to=dict, body=body, options=options) chat_id = created.get("chat_id") if isinstance(created, dict) else None if not chat_id: raise RuntimeError("Open WebUI saved-chat request returned no chat_id") From e3c02daff64e967b93d8357cc97728a983d40391 Mon Sep 17 00:00:00 2001 From: Sushant Gautam Date: Thu, 5 Nov 2026 01:02:03 +0100 Subject: [PATCH 3/3] test: stop the concurrent-auditor test leaking mocks into later tests test_concurrent_runs_keep_their_own_auditor_class called mock.patch from two threads at once on the same attributes (decision_target_for, decision_auditor_class, snapshot_api_key). mock.patch is not thread-safe: one thread could restore the other's mock, leaving it installed for every later test on that xdist worker. That is the CI flake seen on #79 and #81 (snapshot_api_key returning '', decision_target_for returning object(), the auditor class being AuditorA). The test now patches once in the main thread, with side effects that pick each thread's class from its own target. --- infra/tests/test_engine_integration.py | 29 +++++++++++++++----------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/infra/tests/test_engine_integration.py b/infra/tests/test_engine_integration.py index 03ca8a62..9a0ff5a7 100644 --- a/infra/tests/test_engine_integration.py +++ b/infra/tests/test_engine_integration.py @@ -1506,19 +1506,24 @@ async def run_scenario_reps(self, **_kwargs): classes = {"a": type("AuditorA", (), {}), "b": type("AuditorB", (), {})} def run(label): - with mock.patch("infra.engine.decision_target_for", return_value=object()), \ - mock.patch("infra.trace_target.decision_auditor_class", return_value=classes[label]), \ - mock.patch("infra.engine.snapshot_api_key", return_value=""): - engine.run_scenario_repeated( - name=label, description="d", expected_behavior=None, test_prompt=None, - target=_snap(label), auditor=_snap("x"), judge=_snap("y"), n_repetitions=2, trace_config=None, - ) + engine.run_scenario_repeated( + name=label, description="d", expected_behavior=None, test_prompt=None, + target=_snap(label), auditor=_snap("x"), judge=_snap("y"), n_repetitions=2, trace_config=None, + ) - threads = [threading.Thread(target=run, args=(k,)) for k in classes] - for t in threads: - t.start() - for t in threads: - t.join(timeout=30) + # Patched once, here: mock.patch is not thread-safe, and two threads patching + # and restoring the same attribute leave one thread's mock installed for every + # later test in this worker. Each thread's class follows from its own target. + with monkeypatch.context() as patched: + patched.setattr(engine, "decision_target_for", + lambda target, **_kw: ("decision", target["model_id"])) + patched.setattr("infra.trace_target.decision_auditor_class", lambda factory: classes[factory()[1]]) + patched.setattr(engine, "snapshot_api_key", lambda _snapshot: "") + threads = [threading.Thread(target=run, args=(k,)) for k in classes] + for t in threads: + t.start() + for t in threads: + t.join(timeout=30) assert {label.split(" ")[0] for label in seen} == {"a", "b"} for label, cls in seen.items(): assert cls is classes[label.split(" ")[0]]