Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions backend/src/analytics_agent/agent/graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,13 +82,15 @@ def build_graph(

if system_prompt_override:
from analytics_agent.skills.loader import (
get_check_data_quality_prompt_section,
get_improve_context_prompt_section,
get_search_business_context_section,
get_skill_system_prompt_section,
)

system_prompt = system_prompt_override.format(engine_name=engine_name)
system_prompt += get_search_business_context_section()
system_prompt += get_check_data_quality_prompt_section()
system_prompt += get_improve_context_prompt_section()
if enabled_mutations:
system_prompt += get_skill_system_prompt_section(enabled_mutations)
Expand Down
2 changes: 2 additions & 0 deletions backend/src/analytics_agent/prompts/system.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ def build_system_prompt(
enabled_skills: set[str] | None = None,
) -> str:
from analytics_agent.skills.loader import (
get_check_data_quality_prompt_section,
get_improve_context_prompt_section,
get_search_business_context_section,
get_skill_system_prompt_section,
Expand All @@ -23,6 +24,7 @@ def build_system_prompt(

# Always inject always-on meta-skills
base = base + get_search_business_context_section()
base = base + get_check_data_quality_prompt_section()
base = base + get_improve_context_prompt_section()

if enabled_skills:
Expand Down
62 changes: 62 additions & 0 deletions backend/src/analytics_agent/skills/check-data-quality/SKILL.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
---
name: check_data_quality
description: >
Call this whenever the user asks about a dataset's trustworthiness, freshness,
staleness, known data-quality issues (negative values, NULLs, out-of-range), or
before querying a table that may be stale or known-broken. Reads DataHub
assertions for the candidate table(s) and surfaces any FAILING freshness or
quality checks so you reason about stale / planted-bug data before writing SQL —
instead of blindly trusting a table that is known to lag its source.
metadata:
author: analytics-agent
version: "1.0"
---

## Data Quality & Freshness Check

Run this workflow **before writing SQL** any time the question involves staleness,
freshness, trust, or a known data-quality problem on a specific table (e.g. "how many
rows have a negative amount", "is this mart behind its source", "is this table
trustworthy", "how stale is X").

Do **not** skip this to go straight to `list_tables` or `execute_sql`. Assertions are the
DataHub-native signal for known-broken or stale data.

---

### Step 1 — Resolve the candidate dataset URN

From the tables you've already discovered via `search` / `get_entities`, call
`get_dataset_assertions` for each candidate mart/table the question targets:

```
get_dataset_assertions(urn="<dataset_urn>", count=10)
```

`dataset_urn` is the `urn:li:dataset:(...)` from your earlier catalog lookups.

### Step 2 — Read the assertion summary

For each assertion returned, note `type` (e.g. `FRESHNESS`, `VOLUME`, `FIELD`), the
`latestResultType` (`SUCCESS` / `FAILURE` / `NO_RUN`), and the description.

- A **`FRESHNESS` assertion with `FAILURE`** means the table lags its source — the
freshness SLA was breached. For staleness questions this is the signal to compute the
lag between the raw source and the mart.
- A **`VOLUME` / `FIELD` assertion with `FAILURE`** means a known data-quality problem is
planted in this table (e.g. negative amounts, NULL names) — exactly the kind of
"planted bug" question you should answer directly.

### Step 3 — Fold the signal into the query

- For **staleness / days-behind** questions: the failing freshness assertion tells you
*which* table is stale. Query both the raw source and the mart, and compute the
difference (e.g. `julianday(max(raw_ts)) - julianday(max(mart_date))`) — preferring
`julianday`/`date()` over `CAST(... AS DATE)`, which is unreliable for text dates.
- For **data-quality count** questions: use the failing assertion as confirmation that a
filter like `WHERE amount < 0` is the right answer surface.

**Cite what you find.** When you answer a staleness or quality question, reference the
assertion that led you there. If a table has no assertions, say so and note the gap
(suggest `/improve-context`).

14 changes: 14 additions & 0 deletions backend/src/analytics_agent/skills/loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,20 @@ def get_search_business_context_section() -> str:
return f"\n\n## Skill: search_business_context\n\n{body}"


def get_check_data_quality_prompt_section() -> str:
"""Return the always-on check-data-quality skill section for the system prompt.

Instructs the agent to read DataHub assertions (via the native
get_dataset_assertions tool) before querying a table that may be stale or
known-broken.
"""
parsed = _load_skill_md("check-data-quality")
if parsed is None:
return ""
_fm, body = parsed
return f"\n\n## Skill: check_data_quality\n\n{body}"


def get_skill_system_prompt_section(enabled_skills: set[str]) -> str:
"""
Return a markdown section to inject into the system prompt containing
Expand Down
98 changes: 98 additions & 0 deletions tests/unit/test_check_data_quality_skill.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
"""
Tests for the always-on check_data_quality skill (PR #96 + wiring).

Two concerns:
1. Wiring — the skill's SKILL.md body is injected into the system prompt
(the gap that made the skill a no-op: it was never registered).
2. Contract — the tool the skill tells the agent to call
(get_dataset_assertions) exists and, given a FAILING freshness assertion,
surfaces exactly the fields the skill instructs the agent to read
(type / latestResultType / description).
"""

from __future__ import annotations

from unittest.mock import MagicMock, patch

import pytest
from analytics_agent.prompts.system import build_system_prompt
from analytics_agent.skills.loader import get_check_data_quality_prompt_section

# ---------------------------------------------------------------------------
# 1. Wiring — the skill reaches the system prompt
# ---------------------------------------------------------------------------


def test_check_data_quality_section_loads_and_mentions_the_tool() -> None:
section = get_check_data_quality_prompt_section()
assert section.strip(), "check_data_quality section must not be empty"
# It routes through the real native tool and is about assertions/freshness.
assert "get_dataset_assertions" in section
assert "assertion" in section.lower()
assert "fresh" in section.lower()


def test_system_prompt_includes_check_data_quality_section() -> None:
"""The always-on wiring: build_system_prompt must inject the skill body."""
prompt = build_system_prompt("snowflake")
assert "check_data_quality" in prompt
assert "get_dataset_assertions" in prompt


# ---------------------------------------------------------------------------
# 2. Contract — the referenced tool exists and yields the fields the skill reads
# ---------------------------------------------------------------------------


def test_get_dataset_assertions_is_importable_with_compatible_signature() -> None:
import inspect

da = pytest.importorskip("datahub_agent_context.mcp_tools.assertions")
params = inspect.signature(da.get_dataset_assertions).parameters
# The skill calls get_dataset_assertions(urn=..., count=10)
assert "urn" in params
assert "count" in params


def test_get_dataset_assertions_surfaces_failing_freshness() -> None:
"""Mocked assertions response: a FAILING freshness assertion must surface as
type=FRESHNESS + latestResultType=FAILURE — the exact fields the skill reads."""
da = pytest.importorskip("datahub_agent_context.mcp_tools.assertions")

raw_assertion = {
"urn": "urn:li:assertion:orders-mart-freshness",
"info": {
"type": "FRESHNESS",
"description": "orders_mart must be < 24h behind its source",
},
"runEvents": {
"total": 5,
"succeeded": 3,
"failed": 2,
"runEvents": [{"result": {"type": "FAILURE"}}],
},
}
gql_response = {
"searchAcrossEntities": {
"start": 0,
"total": 1,
"searchResults": [{"entity": raw_assertion}],
}
}

with (
patch.object(da, "execute_graphql", return_value=gql_response),
patch.object(da, "get_graph", return_value=MagicMock()),
):
result = da.get_dataset_assertions(
urn="urn:li:dataset:(urn:li:dataPlatform:snowflake,db.schema.orders_mart,PROD)",
count=10,
)

assert result["success"] is True
assertions = result["data"]["assertions"]
assert len(assertions) == 1
a = assertions[0]
assert a["type"] == "FRESHNESS"
assert a["latestResultType"] == "FAILURE"
assert a["description"]