From 463ea80fe3b9b7b86c7f6ab545391832b525d762 Mon Sep 17 00:00:00 2001 From: Lam Nguyen Date: Wed, 8 Jul 2026 13:32:59 +0700 Subject: [PATCH 1/2] fix(observability): sanitize LangSmith trace payloads --- dana/common/observable.py | 4 + dana/common/trace_sanitizer.py | 103 ++++++++++++++++++ docs/project-changelog.md | 1 + .../test_observable_langsmith_dispatch.py | 48 ++++++++ 4 files changed, 156 insertions(+) create mode 100644 dana/common/trace_sanitizer.py diff --git a/dana/common/observable.py b/dana/common/observable.py index 7d9740f..7596c74 100644 --- a/dana/common/observable.py +++ b/dana/common/observable.py @@ -20,6 +20,8 @@ import os from typing import cast +from dana.common.trace_sanitizer import safe_trace_inputs, safe_trace_outputs + # --- Langfuse (existing backend) --- try: @@ -88,6 +90,8 @@ def _langsmith_kwargs(kwargs: dict) -> dict: meta["user_id"] = kwargs["user_id"] if meta: ls["metadata"] = meta + ls["process_inputs"] = kwargs.get("process_inputs", safe_trace_inputs) + ls["process_outputs"] = kwargs.get("process_outputs", safe_trace_outputs) return ls diff --git a/dana/common/trace_sanitizer.py b/dana/common/trace_sanitizer.py new file mode 100644 index 0000000..b1fab6e --- /dev/null +++ b/dana/common/trace_sanitizer.py @@ -0,0 +1,103 @@ +"""Trace payload sanitizers for observability backends.""" + +from dataclasses import fields, is_dataclass +from datetime import date, datetime +from enum import Enum +from typing import Any, cast + + +def safe_trace_inputs(inputs: dict) -> dict: + """Return a JSON-safe trace input payload without recursive Dana objects.""" + return cast("dict", safe_trace_value(inputs)) + + +def safe_trace_outputs(outputs: Any) -> dict: + """Return a JSON-safe trace output payload without recursive Dana objects.""" + return {"output": safe_trace_value(outputs)} + + +def safe_trace_value(value: Any, *, _seen: set[int] | None = None, _depth: int = 0) -> Any: + """Best-effort JSON-safe value for tracing. + + Dana often passes live agents, timelines, and runtime objects through + observable methods; those graphs can be cyclic. Keep traces useful, but + never recurse through the live object graph. + """ + if _seen is None: + _seen = set() + if value is None or isinstance(value, str | int | float | bool): + return value + if isinstance(value, datetime | date): + return value.isoformat() + if isinstance(value, Enum): + return value.value + + obj_id = id(value) + if obj_id in _seen: + return f"" + + if _is_dana_runtime_object(value): + return _summarize_dana_object(value) + + if _depth >= 4: + return _summarize_object(value) + + if isinstance(value, dict): + _seen.add(obj_id) + try: + return { + str(safe_trace_value(k, _seen=_seen, _depth=_depth + 1)): safe_trace_value( + v, + _seen=_seen, + _depth=_depth + 1, + ) + for k, v in value.items() + } + finally: + _seen.discard(obj_id) + if isinstance(value, list | tuple | set | frozenset): + _seen.add(obj_id) + try: + return [safe_trace_value(item, _seen=_seen, _depth=_depth + 1) for item in list(value)[:50]] + finally: + _seen.discard(obj_id) + if is_dataclass(value) and not isinstance(value, type): + _seen.add(obj_id) + try: + return { + field.name: safe_trace_value(getattr(value, field.name), _seen=_seen, _depth=_depth + 1) + for field in fields(value) + if field.repr + } + finally: + _seen.discard(obj_id) + + return _summarize_object(value) + + +def _is_dana_runtime_object(value: Any) -> bool: + module = value.__class__.__module__ + name = value.__class__.__name__ + return module.startswith("dana.core.agent") or module.startswith("dana.core.runtime") or name.endswith("Timeline") + + +def _summarize_dana_object(value: Any) -> dict: + summary = _summarize_object(value) + timeline = getattr(value, "timeline", None) + if isinstance(timeline, list): + summary["entry_count"] = len(timeline) + for attr in ("max_context_tokens", "agent_type", "object_id", "session_id"): + try: + attr_value = getattr(value, attr) + except Exception: + continue + if isinstance(attr_value, str | int | float | bool): + summary[attr] = attr_value + return summary + + +def _summarize_object(value: Any) -> dict: + return { + "__class__": value.__class__.__name__, + "__module__": value.__class__.__module__, + } diff --git a/docs/project-changelog.md b/docs/project-changelog.md index fa258e8..8a3fc37 100644 --- a/docs/project-changelog.md +++ b/docs/project-changelog.md @@ -16,6 +16,7 @@ ### Changed - `@observable` (`dana/common/observable.py`): when no tracing backend is enabled, the decorator now returns the target function unchanged (identity) instead of wrapping it in a passthrough flush layer. No introspection-sensitive call sites affected; `inspect.signature()` on decorated functions now returns the real signature. Langfuse-path tracing behavior is unchanged. +- `@observable` LangSmith path now sanitizes Dana runtime objects before SDK serialization. Live agents/timelines are logged as compact summaries, avoiding `RecursionError` crashes on cyclic `CompressedTimeline` graphs. - `CompressedTimeline.__init__` default `max_tokens_until_compression` now defers to env trigger (150000) when unset. Explicit value continues to win. - `CompressedTimelineConfig` gains `enable_cheap_shrink_tool_results`, `cheap_shrink_keep_recent`, `enable_reactive_compact` fields. - `star_agent._maybe_compress_timeline` (sync + async) re-raises `PromptTooLongError` from summary path instead of swallowing — lets caller-layer retry kick in. diff --git a/tests/unit/test_observable_langsmith_dispatch.py b/tests/unit/test_observable_langsmith_dispatch.py index 177d412..5ad1df9 100644 --- a/tests/unit/test_observable_langsmith_dispatch.py +++ b/tests/unit/test_observable_langsmith_dispatch.py @@ -8,7 +8,9 @@ import asyncio import builtins import importlib +import json import sys +import types import pytest @@ -32,6 +34,15 @@ def clean_env(monkeypatch): def _install_backend_mocks(monkeypatch, *, traceable=None, observe=None): """Patch the real langsmith/langsmith attributes observable binds at import.""" + if "langsmith" not in sys.modules: + shim = types.ModuleType("langsmith") + shim.traceable = lambda *a, **k: (lambda f: f) + monkeypatch.setitem(sys.modules, "langsmith", shim) + if "langfuse" not in sys.modules: + shim = types.ModuleType("langfuse") + shim.observe = lambda *a, **k: (lambda f: f) + shim.Langfuse = lambda *a, **k: None + monkeypatch.setitem(sys.modules, "langfuse", shim) if traceable is not None: monkeypatch.setattr("langsmith.traceable", traceable) if observe is not None: @@ -303,3 +314,40 @@ async def f(): assert asyncio.run(f()) == 5 assert not observe_calls, "langfuse.observe must not be called under async either" + + +def test_langsmith_processors_sanitize_cyclic_timeline_inputs(monkeypatch, clean_env): + """LangSmith must not receive raw Dana timelines; SDK JSON serialization + recurses through live agent/timeline graphs and can crash the caller. + """ + from dana.core.timeline.timeline import Timeline, TimelineEntry, TimelineEntryType + + monkeypatch.setenv("LANGSMITH_TRACING", "true") + captured: dict = {} + + def fake_traceable(*a, **k): + captured.update(k) + return lambda f: f + + _install_backend_mocks(monkeypatch, traceable=fake_traceable) + obs = _reload_observable() + + @obs.observable(name="x") + def f(timeline): + return timeline + + timeline = Timeline() + entry = TimelineEntry(TimelineEntryType.USER_MESSAGE, "hello") + entry.metadata["timeline"] = timeline + timeline.add_entry(entry) + + assert f(timeline) is timeline + safe_inputs = captured["process_inputs"]({"timeline": timeline, "entry": entry}) + safe_outputs = captured["process_outputs"](timeline) + + json.dumps(safe_inputs) + json.dumps(safe_outputs) + assert safe_inputs["timeline"]["__class__"] == "Timeline" + assert safe_inputs["timeline"]["entry_count"] == 1 + assert safe_inputs["entry"]["metadata"]["timeline"]["__class__"] == "Timeline" + assert safe_outputs["output"]["__class__"] == "Timeline" From b41858145208ac3a68d4c73f30369844df7b7f43 Mon Sep 17 00:00:00 2001 From: Lam Nguyen Date: Wed, 8 Jul 2026 13:33:09 +0700 Subject: [PATCH 2/2] feat(skill): add dana cross-repo debug workflow --- .claude/skills/dana-cross-repo-debug/SKILL.md | 247 ++++++++++++++++++ .../dana-cross-repo-debug/evals/evals.json | 23 ++ 2 files changed, 270 insertions(+) create mode 100644 .claude/skills/dana-cross-repo-debug/SKILL.md create mode 100644 .claude/skills/dana-cross-repo-debug/evals/evals.json diff --git a/.claude/skills/dana-cross-repo-debug/SKILL.md b/.claude/skills/dana-cross-repo-debug/SKILL.md new file mode 100644 index 0000000..608733c --- /dev/null +++ b/.claude/skills/dana-cross-repo-debug/SKILL.md @@ -0,0 +1,247 @@ +--- +name: dana-cross-repo-debug +description: Debug dana-runtime bugs that surface through the sibling dana-os / dana-console integration. Use this skill whenever the user mentions console-bug, mise run console:dev, localhost:5273, Playground, dana-librarian, LangSmith/Langfuse tracing behavior through the console, or a bug where dana-os appears to use the wrong dana-runtime revision. It guides reproducing through tmux/UI/API, checking backend logs, binding dana-os to the local dana-runtime checkout, fixing runtime code, and verifying the cross-repo path end to end. +category: debugging +keywords: [dana-runtime, dana-os, dana-console, tmux, cross-repo, langsmith, playground] +metadata: + author: dana-runtime + version: "1.0.0" +--- + +# Dana Cross-Repo Debug + +Use this skill when a bug is reported in dana-runtime but only appears when the sibling dana-os console is running. The important trap is that dana-os may pin `dana` to a remote git revision, so a fix in `/Users/lam/Desktop/repos/dana-runtime` is invisible until the dana-os workspace points to the local checkout or to a commit containing the fix. + +## Repos And Surfaces + +Assume these local paths unless the user says otherwise: + +- Runtime repo: `/Users/lam/Desktop/repos/dana-runtime` +- Console/integration repo: `/Users/lam/Desktop/repos/dana-os-dana-odb-librarian` +- tmux session: `console-bug` +- Dev command: `mise run console:dev` +- Frontend: `http://localhost:5273/` +- Backend: `http://localhost:8100/` +- UI route: `http://localhost:5273/` -> `Playground` -> `@dana-librarian` +- Runtime librarian agent package: `/Users/lam/Desktop/repos/dana-os-dana-odb-librarian/dana-agents/dana-runtime-librarian` + +## Working Rule + +Debug the path the user actually runs. Do not stop after a unit test if the bug was reported through the console. Confirm: + +1. The console backend imports `dana` from the intended checkout. +2. The failing request reaches dana-runtime code. +3. The tmux log no longer shows the original failure after the fix. + +## Start With The Live Console + +Inspect the tmux pane before changing code: + +```bash +tmux capture-pane -t console-bug -p -S -240 +tmux list-panes -t console-bug -a -F '#{session_name}:#{window_index}.#{pane_index} #{pane_current_path} #{pane_current_command}' +``` + +Check whether the servers are already running: + +```bash +lsof -nP -iTCP:8100 -sTCP:LISTEN || true +lsof -nP -iTCP:5273 -sTCP:LISTEN || true +curl -I --max-time 5 http://localhost:5273/ || true +curl -sS -m 5 http://localhost:8100/docs | head || true +``` + +If `console:dev` refuses because port `8100` is in use, identify the holder. If it is a leaked uvicorn reload pair from the same console session, stop it before retrying. Prefer gentle termination first; use `kill -9` only for a stuck leaked process. + +```bash +lsof -nP -iTCP:8100 -sTCP:LISTEN +kill +sleep 2 +lsof -nP -iTCP:8100 -sTCP:LISTEN || true +``` + +Start or restart the console in the existing tmux session: + +```bash +tmux send-keys -t console-bug 'mise run console:dev' Enter +sleep 8 +tmux capture-pane -t console-bug -p -S -120 +``` + +## Confirm Which Runtime Dana-OS Uses + +This is the most common source of false verification. `uv run` can reinstall the remote pinned `dana` package at startup. + +From the dana-os repo, inspect the source pin: + +```bash +sed -n '1,80p' /Users/lam/Desktop/repos/dana-os-dana-odb-librarian/pyproject.toml +git -C /Users/lam/Desktop/repos/dana-os-dana-odb-librarian diff -- pyproject.toml uv.lock +``` + +Confirm the active interpreter imports local runtime files: + +```bash +/Users/lam/Desktop/repos/dana-os-dana-odb-librarian/.venv/bin/python - <<'PY' +import dana +import dana.common.observable +print(dana.__file__) +print(dana.common.observable.__file__) +PY +``` + +Expected during local debugging: + +```text +/Users/lam/Desktop/repos/dana-runtime/dana/__init__.py +/Users/lam/Desktop/repos/dana-runtime/dana/common/observable.py +``` + +## Bind Dana-OS To Local Dana-Runtime + +If dana-os points at a remote git source: + +```toml +dana = { git = "https://github.com/aitomatic/dana-runtime.git", rev = "..." } +``` + +For local debugging, change only the `dana` source to: + +```toml +dana = { path = "../dana-runtime", editable = true } +``` + +Then refresh the lock in dana-os: + +```bash +cd /Users/lam/Desktop/repos/dana-os-dana-odb-librarian +uv lock +``` + +Restart `console-bug`. Watch the startup log: it should reinstall `dana` from the local path, not fetch the remote git revision. + +If the user wants a commit-bound integration instead of local editable debugging, first commit the dana-runtime fix, then pin dana-os back to: + +```toml +dana = { git = "https://github.com/aitomatic/dana-runtime.git", rev = "" } +``` + +Run `uv lock` after changing the pin. Do not invent a commit hash before the fix is committed and available. + +## Reproduce Through The Backend First + +Use the backend API for fast, deterministic reproduction. It exercises the same backend path the Playground uses, without browser automation noise. + +Find the registered librarian package: + +```bash +curl -sS -m 20 http://localhost:8100/api/sessions \ + -H 'content-type: application/json' \ + --data '{"operation":"agent.find","args":{},"mode":"live"}' \ + | jq '.result.agents[] | select(.name|contains("librarian"))' +``` + +Check readiness and mission id: + +```bash +curl -sS -m 20 http://localhost:8100/api/sessions \ + -H 'content-type: application/json' \ + --data '{"operation":"agent.describe","args":{"agent_dir":"/Users/lam/Desktop/repos/dana-os-dana-odb-librarian/dana-agents/dana-runtime-librarian"},"mode":"live"}' \ + | jq '.result | {name,operation,mission_id,readiness,errors}' +``` + +Run the crashing turn: + +```bash +curl -sS -m 90 http://localhost:8100/api/sessions \ + -H 'content-type: application/json' \ + --data '{"operation":"agent.ask","args":{"mission_id":"mission:bms-librarian:primary","message":"model dump exclude_none=True mode=json"},"mode":"live"}' \ + | jq '{invocation_id, ok:(.result.answer? != null), detail, answer:(.result.answer? // null)}' +``` + +Immediately inspect logs: + +```bash +tmux capture-pane -t console-bug -p -S -160 +``` + +Classify the first real failure, not the final wrapper error. For example: + +- LangSmith serializer failures mention `langsmith/_internal/_serde.py`, `serialize_run_dict`, `Timeline.__repr__`, or `CompressedTimeline`. +- Wrong runtime source shows paths under `.venv/lib/python.../site-packages/dana` rather than `/Users/lam/Desktop/repos/dana-runtime/dana`. +- Event loop failures such as `Cannot run the event loop while another loop is running` may be secondary if they occur after another exception is already unwinding. + +## Use The UI When Needed + +Use the UI when the bug depends on selection state, readiness gating, mention insertion, websocket streaming, or visual behavior. + +Manual path: + +1. Open `http://localhost:5273/`. +2. Click `Playground` in the left rail. +3. In the composer, type `@dana-librarian`. +4. Select the suggestion if needed. +5. Send the prompt. +6. Watch `tmux capture-pane -t console-bug -p -S -160`. + +For automation, use the repo's browser/devtools skill or a focused Puppeteer script. Prefer one script that keeps a single browser session open over many tiny commands, because session state can be lost between helper invocations. + +## Fix In Dana-Runtime + +Once the failure is traced to dana-runtime, edit the runtime repo. Use the existing code structure and add a regression test near the failing subsystem. + +Common locations for this integration path: + +- `dana/common/observable.py`: tracing backend dispatch. +- `dana/common/trace_sanitizer.py`: safe trace payload conversion. +- `dana/common/llm/llm.py`: sync and async LLM boundaries. +- `dana/core/llm/llm_caller.py`: provider invocation, retry, reactive compaction. +- `dana/core/runtime/base.py`: prompt build, native tools, runtime dispatch. +- `dana/core/timeline/*`: timeline and compressed timeline behavior. + +After editing, run focused verification in dana-runtime: + +```bash +cd /Users/lam/Desktop/repos/dana-runtime +uv run python -m py_compile +uv run ruff check +uv run python -m pytest -q +``` + +If the fix is about cross-repo behavior, finish with the live backend call and tmux log check. The final proof is the console operation returning HTTP 200 and the tmux pane showing normal runtime logs instead of the original error. + +## Verification Checklist + +Before reporting success, verify all that apply: + +- `pyproject.toml` in dana-os points to local `../dana-runtime` for local debugging, or to a real fix commit hash for pinned integration. +- `uv.lock` in dana-os reflects the same source. +- The dana-os venv imports `dana` from `/Users/lam/Desktop/repos/dana-runtime/dana`. +- `mise run console:dev` is running in `console-bug`. +- `http://localhost:5273/` and `http://localhost:8100/` respond. +- `agent.describe` for dana-runtime-librarian reports readiness `ready`. +- `agent.ask` reproducer returns an `invocation_id` and `result.answer`. +- `tmux capture-pane` does not show the original crash after the fresh request. +- Focused dana-runtime tests and compile/lint checks pass. + +## Report Format + +Keep the report concise: + +```markdown +Root cause: +Fix: +Cross-repo binding: +Verification: +- : +- : +Unresolved questions: +``` + +## Common Pitfalls + +- Do not trust a successful local dana-runtime test if dana-os still pins a remote git revision. +- Do not keep restarting `console:dev` without checking leaked `8100` listeners. +- Do not treat old tmux scrollback as current. Restart cleanly or compare timestamps. +- Do not debug browser selectors before proving the backend operation fails. +- Do not commit dana-os dependency changes accidentally if they were only for local debugging. diff --git a/.claude/skills/dana-cross-repo-debug/evals/evals.json b/.claude/skills/dana-cross-repo-debug/evals/evals.json new file mode 100644 index 0000000..a960ee2 --- /dev/null +++ b/.claude/skills/dana-cross-repo-debug/evals/evals.json @@ -0,0 +1,23 @@ +{ + "skill_name": "dana-cross-repo-debug", + "evals": [ + { + "id": 1, + "prompt": "The console-bug tmux session is running mise run console:dev. Chatting with dana-librarian in the Playground crashes, but dana-runtime tests pass locally. Debug it end to end and tell me what is actually running.", + "expected_output": "Agent should inspect tmux logs, check ports, verify dana-os dependency source, confirm imported dana path, reproduce through /api/sessions, and avoid claiming success from local tests alone.", + "files": [] + }, + { + "id": 2, + "prompt": "We fixed dana/common/observable.py in dana-runtime, but the Playground still shows the old LangSmith RecursionError. Figure out why and make the console use my local runtime checkout.", + "expected_output": "Agent should identify the remote git pin in dana-os pyproject/uv.lock, switch dana source to local editable ../dana-runtime for debugging, run uv lock, restart tmux console, and verify import paths.", + "files": [] + }, + { + "id": 3, + "prompt": "I need a clean repro for the dana-librarian bug without clicking around the UI. Use the running console backend to trigger the same agent.ask path and inspect the tmux log.", + "expected_output": "Agent should use /api/sessions agent.find, agent.describe, and agent.ask curl calls, then capture tmux logs and classify the first real failure.", + "files": [] + } + ] +}