From 9e58f687c0c553eb81a11410a2b68505e9226472 Mon Sep 17 00:00:00 2001 From: Adam Normandin Date: Thu, 6 Aug 2026 01:33:56 -0400 Subject: [PATCH 1/2] Create SKILL.md --- .../skills/check-data-quality/SKILL.md | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 backend/src/analytics_agent/skills/check-data-quality/SKILL.md diff --git a/backend/src/analytics_agent/skills/check-data-quality/SKILL.md b/backend/src/analytics_agent/skills/check-data-quality/SKILL.md new file mode 100644 index 0000000..04ee80d --- /dev/null +++ b/backend/src/analytics_agent/skills/check-data-quality/SKILL.md @@ -0,0 +1,62 @@ +--- +name: check_data_quality +description: > + Call this whenever the user asks about a dataset's trustworthiness, freshness, + staleness, known data-quality issues (negative values, NULLs, out-of-range), or + before querying a table that may be stale or known-broken. Reads DataHub + assertions for the candidate table(s) and surfaces any FAILING freshness or + quality checks so you reason about stale / planted-bug data before writing SQL — + instead of blindly trusting a table that is known to lag its source. +metadata: + author: analytics-agent + version: "1.0" +--- + +## Data Quality & Freshness Check + +Run this workflow **before writing SQL** any time the question involves staleness, +freshness, trust, or a known data-quality problem on a specific table (e.g. "how many +rows have a negative amount", "is this mart behind its source", "is this table +trustworthy", "how stale is X"). + +Do **not** skip this to go straight to `list_tables` or `execute_sql`. Assertions are the +DataHub-native signal for known-broken or stale data. + +--- + +### Step 1 — Resolve the candidate dataset URN + +From the tables you've already discovered via `search` / `get_entities`, call +`get_dataset_assertions` for each candidate mart/table the question targets: + +``` +get_dataset_assertions(urn="", count=10) +``` + +`dataset_urn` is the `urn:li:dataset:(...)` from your earlier catalog lookups. + +### Step 2 — Read the assertion summary + +For each assertion returned, note `type` (e.g. `FRESHNESS`, `VOLUME`, `FIELD`), the +`latestResultType` (`SUCCESS` / `FAILURE` / `NO_RUN`), and the description. + +- A **`FRESHNESS` assertion with `FAILURE`** means the table lags its source — the + freshness SLA was breached. For staleness questions this is the signal to compute the + lag between the raw source and the mart. +- A **`VOLUME` / `FIELD` assertion with `FAILURE`** means a known data-quality problem is + planted in this table (e.g. negative amounts, NULL names) — exactly the kind of + "planted bug" question you should answer directly. + +### Step 3 — Fold the signal into the query + +- For **staleness / days-behind** questions: the failing freshness assertion tells you + *which* table is stale. Query both the raw source and the mart, and compute the + difference (e.g. `julianday(max(raw_ts)) - julianday(max(mart_date))`) — preferring + `julianday`/`date()` over `CAST(... AS DATE)`, which is unreliable for text dates. +- For **data-quality count** questions: use the failing assertion as confirmation that a + filter like `WHERE amount < 0` is the right answer surface. + +**Cite what you find.** When you answer a staleness or quality question, reference the +assertion that led you there. If a table has no assertions, say so and note the gap +(suggest `/improve-context`). + From 86b2ee3bf570b7b9fafa600bec412e464bfee3b9 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Thu, 13 Aug 2026 18:28:12 -0700 Subject: [PATCH 2/2] feat(skills): wire check_data_quality as an always-on skill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SKILL.md alone was a no-op: the loader/graph inject a fixed set of skill sections, and check-data-quality was registered nowhere, so its instructions never reached the system prompt. Register it as an always-on, instruction-only skill (the improve-context / search-business-context pattern): add get_check_data_quality_prompt_section() and inject it in both the default (build_system_prompt) and override (graph.py) prompt paths. No new tool needed — get_dataset_assertions is already a native DataHub tool from datahub_agent_context. Tests: skill body reaches the system prompt; and a mocked assertions response confirms get_dataset_assertions surfaces the type/latestResultType/ description fields the skill instructs the agent to read. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/src/analytics_agent/agent/graph.py | 2 + backend/src/analytics_agent/prompts/system.py | 2 + backend/src/analytics_agent/skills/loader.py | 14 +++ tests/unit/test_check_data_quality_skill.py | 98 +++++++++++++++++++ 4 files changed, 116 insertions(+) create mode 100644 tests/unit/test_check_data_quality_skill.py diff --git a/backend/src/analytics_agent/agent/graph.py b/backend/src/analytics_agent/agent/graph.py index 4e44833..f69903d 100644 --- a/backend/src/analytics_agent/agent/graph.py +++ b/backend/src/analytics_agent/agent/graph.py @@ -82,6 +82,7 @@ def build_graph( if system_prompt_override: from analytics_agent.skills.loader import ( + get_check_data_quality_prompt_section, get_improve_context_prompt_section, get_search_business_context_section, get_skill_system_prompt_section, @@ -89,6 +90,7 @@ def build_graph( system_prompt = system_prompt_override.format(engine_name=engine_name) system_prompt += get_search_business_context_section() + system_prompt += get_check_data_quality_prompt_section() system_prompt += get_improve_context_prompt_section() if enabled_mutations: system_prompt += get_skill_system_prompt_section(enabled_mutations) diff --git a/backend/src/analytics_agent/prompts/system.py b/backend/src/analytics_agent/prompts/system.py index c925791..1d11b54 100644 --- a/backend/src/analytics_agent/prompts/system.py +++ b/backend/src/analytics_agent/prompts/system.py @@ -13,6 +13,7 @@ def build_system_prompt( enabled_skills: set[str] | None = None, ) -> str: from analytics_agent.skills.loader import ( + get_check_data_quality_prompt_section, get_improve_context_prompt_section, get_search_business_context_section, get_skill_system_prompt_section, @@ -23,6 +24,7 @@ def build_system_prompt( # Always inject always-on meta-skills base = base + get_search_business_context_section() + base = base + get_check_data_quality_prompt_section() base = base + get_improve_context_prompt_section() if enabled_skills: diff --git a/backend/src/analytics_agent/skills/loader.py b/backend/src/analytics_agent/skills/loader.py index 78dee91..7e3b865 100644 --- a/backend/src/analytics_agent/skills/loader.py +++ b/backend/src/analytics_agent/skills/loader.py @@ -130,6 +130,20 @@ def get_search_business_context_section() -> str: return f"\n\n## Skill: search_business_context\n\n{body}" +def get_check_data_quality_prompt_section() -> str: + """Return the always-on check-data-quality skill section for the system prompt. + + Instructs the agent to read DataHub assertions (via the native + get_dataset_assertions tool) before querying a table that may be stale or + known-broken. + """ + parsed = _load_skill_md("check-data-quality") + if parsed is None: + return "" + _fm, body = parsed + return f"\n\n## Skill: check_data_quality\n\n{body}" + + def get_skill_system_prompt_section(enabled_skills: set[str]) -> str: """ Return a markdown section to inject into the system prompt containing diff --git a/tests/unit/test_check_data_quality_skill.py b/tests/unit/test_check_data_quality_skill.py new file mode 100644 index 0000000..1e3642d --- /dev/null +++ b/tests/unit/test_check_data_quality_skill.py @@ -0,0 +1,98 @@ +""" +Tests for the always-on check_data_quality skill (PR #96 + wiring). + +Two concerns: + 1. Wiring — the skill's SKILL.md body is injected into the system prompt + (the gap that made the skill a no-op: it was never registered). + 2. Contract — the tool the skill tells the agent to call + (get_dataset_assertions) exists and, given a FAILING freshness assertion, + surfaces exactly the fields the skill instructs the agent to read + (type / latestResultType / description). +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest +from analytics_agent.prompts.system import build_system_prompt +from analytics_agent.skills.loader import get_check_data_quality_prompt_section + +# --------------------------------------------------------------------------- +# 1. Wiring — the skill reaches the system prompt +# --------------------------------------------------------------------------- + + +def test_check_data_quality_section_loads_and_mentions_the_tool() -> None: + section = get_check_data_quality_prompt_section() + assert section.strip(), "check_data_quality section must not be empty" + # It routes through the real native tool and is about assertions/freshness. + assert "get_dataset_assertions" in section + assert "assertion" in section.lower() + assert "fresh" in section.lower() + + +def test_system_prompt_includes_check_data_quality_section() -> None: + """The always-on wiring: build_system_prompt must inject the skill body.""" + prompt = build_system_prompt("snowflake") + assert "check_data_quality" in prompt + assert "get_dataset_assertions" in prompt + + +# --------------------------------------------------------------------------- +# 2. Contract — the referenced tool exists and yields the fields the skill reads +# --------------------------------------------------------------------------- + + +def test_get_dataset_assertions_is_importable_with_compatible_signature() -> None: + import inspect + + da = pytest.importorskip("datahub_agent_context.mcp_tools.assertions") + params = inspect.signature(da.get_dataset_assertions).parameters + # The skill calls get_dataset_assertions(urn=..., count=10) + assert "urn" in params + assert "count" in params + + +def test_get_dataset_assertions_surfaces_failing_freshness() -> None: + """Mocked assertions response: a FAILING freshness assertion must surface as + type=FRESHNESS + latestResultType=FAILURE — the exact fields the skill reads.""" + da = pytest.importorskip("datahub_agent_context.mcp_tools.assertions") + + raw_assertion = { + "urn": "urn:li:assertion:orders-mart-freshness", + "info": { + "type": "FRESHNESS", + "description": "orders_mart must be < 24h behind its source", + }, + "runEvents": { + "total": 5, + "succeeded": 3, + "failed": 2, + "runEvents": [{"result": {"type": "FAILURE"}}], + }, + } + gql_response = { + "searchAcrossEntities": { + "start": 0, + "total": 1, + "searchResults": [{"entity": raw_assertion}], + } + } + + with ( + patch.object(da, "execute_graphql", return_value=gql_response), + patch.object(da, "get_graph", return_value=MagicMock()), + ): + result = da.get_dataset_assertions( + urn="urn:li:dataset:(urn:li:dataPlatform:snowflake,db.schema.orders_mart,PROD)", + count=10, + ) + + assert result["success"] is True + assertions = result["data"]["assertions"] + assert len(assertions) == 1 + a = assertions[0] + assert a["type"] == "FRESHNESS" + assert a["latestResultType"] == "FAILURE" + assert a["description"]