From dfebd0cec3e34499d0189b26249bd0a7e04b2b0c Mon Sep 17 00:00:00 2001 From: igutierrez Date: Fri, 6 Nov 2026 12:41:41 -0600 Subject: [PATCH 1/3] Prepare the agent skills for the NVIDIA skills catalog The NVIDIA/skills catalog syncs product skills from their source repos every hour. It accepts a skill only if NVSkills CI has signed it, and the skill directory must carry an eval dataset and a governance skill card. - request-nvskills-ci.yml is the catalog's request workflow, pinned to a SHA. A maintainer comments /nvskills-ci on a PR that touches skills/ to start a signing run. - skills/{tabular,relational}/evals/evals.json hold 7 and 6 cases. Every assertion is a claim the skill itself makes. - skills/{tabular,relational}/skill-card.md are rendered with the catalog's skill-card-generator. - tests/test_skills.py no longer holds skill-card.md and BENCHMARK.md to its markdown rules. Both use the catalog's HTML-laced template, and NVSkills CI writes BENCHMARK.md. Co-Authored-By: Claude Opus 5.5 Signed-off-by: igutierrez --- .github/workflows/request-nvskills-ci.yml | 40 ++++++++++ skills/relational/evals/evals.json | 75 +++++++++++++++++++ skills/relational/skill-card.md | 46 ++++++++++++ skills/tabular/evals/evals.json | 90 +++++++++++++++++++++++ skills/tabular/skill-card.md | 44 +++++++++++ tests/test_skills.py | 13 +++- 6 files changed, 307 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/request-nvskills-ci.yml create mode 100644 skills/relational/evals/evals.json create mode 100644 skills/relational/skill-card.md create mode 100644 skills/tabular/evals/evals.json create mode 100644 skills/tabular/skill-card.md diff --git a/.github/workflows/request-nvskills-ci.yml b/.github/workflows/request-nvskills-ci.yml new file mode 100644 index 000000000..7acd83fac --- /dev/null +++ b/.github/workflows/request-nvskills-ci.yml @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# NVSkills CI evaluates and signs changes under skills/. The NVIDIA skills +# catalog (github.com/NVIDIA/skills) syncs these skills from main every hour, +# and accepts only files that match the signature (skill.oms.sig) NVSkills CI +# writes into the PR. This is the catalog's request workflow, unchanged except +# that it is pinned to a SHA like every other action here. When the catalog +# changes the template, re-pin to its current main. +# +# A maintainer or admin comments `/nvskills-ci` on a PR to start a run. It needs +# the NVSKILLS_CI_DISPATCH_TOKEN secret. A PR that does not touch skills/ skips +# it. +name: Request NVSkills CI + +on: + issue_comment: + types: [created] + pull_request: + types: [opened, reopened, synchronize, ready_for_review] + push: + +jobs: + request: + if: > + github.event_name == 'pull_request' || + (github.event_name == 'issue_comment' && + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/nvskills-ci')) || + (github.event_name == 'push' && + github.actor == (vars.NVSKILLS_SIGNATURE_PUSH_ACTOR || 'nv-skills-ci[bot]') && + startsWith(github.event.head_commit.message, vars.NVSKILLS_SIGNATURE_COMMIT_TITLE || 'Attach NVSkills validation signatures')) + permissions: + contents: read + pull-requests: read + statuses: read + uses: NVIDIA/skills/.github/workflows/team-request.yml@b707bbe61ff1026dd59a44140610f92c8eae1151 # main, 2026-11-06 + secrets: + NVSKILLS_CI_DISPATCH_TOKEN: ${{ secrets.NVSKILLS_CI_DISPATCH_TOKEN }} diff --git a/skills/relational/evals/evals.json b/skills/relational/evals/evals.json new file mode 100644 index 000000000..7dbb605e5 --- /dev/null +++ b/skills/relational/evals/evals.json @@ -0,0 +1,75 @@ +{ + "skill_name": "sdm-relational", + "evals": [ + { + "id": "r01", + "prompt": "I have three pandas DataFrames: users(user_id, segment), orders(order_id, user_id, item, price, ordered_at) and items(item, kind). Using sdm_client and kumo-relational at http://localhost:8000, predict how many orders users 1, 2 and 3 will place in the next 30 days. Include anything I need to install.", + "expected_output": "Installs structured-data-models-client[relational]. Builds Schema.from_data with primary_keys for all three tables, time_columns={'orders': 'ordered_at'}, and relationships from the many side to the one side: Relationship('orders', 'user_id', 'users', 'user_id') and Relationship('orders', 'item', 'items', 'item'). Asks PredictiveQuery('PREDICT COUNT(orders.*, 0, 30, days) FOR EACH users.user_id', schema).materialize(indices=[1, 2, 3], model=KumoRelational()) and calls client.predict(request, model=KumoRelational()) with a generous timeout. Joins on id (the entity), else row_index, never row order.", + "assertions": [ + "Mentions installing the relational extra (structured-data-models-client[relational]) or MissingExtraError without it", + "Uses Schema.from_data with primary keys and time_columns={'orders': 'ordered_at'}", + "Declares relationships from orders to users and orders to items (many side first)", + "Writes the target as COUNT(orders.*, 0, 30, days) FOR EACH users.user_id", + "Calls materialize(indices=[1, 2, 3], model=KumoRelational()) and client.predict(..., model=KumoRelational())", + "Says to join results on id (or row_index), not on row order", + "Does not pass a task= argument; the task follows from the query" + ] + }, + { + "id": "r02", + "prompt": "My kumo-relational schema has `Relationship('users', 'user_id', 'orders', 'user_id')` because each user has many orders. Predictions look wrong. Is that relationship right?", + "expected_output": "No. Relationship(from_table, from_column, to_table, to_column) points from the many side to the one side: an order names a user, so it should be Relationship('orders', 'user_id', 'users', 'user_id').", + "assertions": [ + "Says the relationship is reversed", + "Explains a Relationship points from the many side to the one side", + "Gives the corrected Relationship('orders', 'user_id', 'users', 'user_id')" + ] + }, + { + "id": "r03", + "prompt": "I want per-entity explanations from kumo-relational for a regression. I added outputs=['prediction', 'explanation'] and our deployment returned `422 UNSUPPORTED_OUTPUT`. Is my code wrong? How should I write this so it works on our deployment and on others?", + "expected_output": "The code is not wrong: the client will send explanation, but outputs narrow at the endpoint by task, and the shipping deployment serves only prediction and quantiles on a regression. Another relational deployment, and SDMClient.in_process(), do serve explanation. So ask for it, catch the refusal (a StructuredDataError), and fall back to prediction only, without assuming an explanation always or never arrives. When one does arrive it is on result.explanations.gradients, not in to_pandas().", + "assertions": [ + "Explains the refusal comes from the deployment, not from a bug in the request", + "Says the shipping deployment serves prediction and quantiles on a regression", + "Recommends requesting explanation, catching the refusal, and falling back", + "Does not claim explanations are never available from kumo-relational (another deployment or SDMClient.in_process() serves them)", + "Says explanations, when returned, are on result.explanations rather than in to_pandas()" + ] + }, + { + "id": "r04", + "prompt": "I'm predicting next-quarter revenue per customer with kumo-relational and then summing the predictions to get a company-wide forecast. Anything wrong with that?", + "expected_output": "A regression's prediction is the median of the predicted distribution, not the mean, and medians do not add up. Ask for the mean with materialize(..., prediction_statistic='mean') when predictions will be summed across entities.", + "assertions": [ + "States that a regression prediction is the median by default", + "Explains that medians cannot be summed to a meaningful total", + "Recommends prediction_statistic='mean' in materialize()" + ] + }, + { + "id": "r05", + "prompt": "I need kumo-relational scores for 50,000 users, and I'd like to pin the context in a session so later calls are cheaper. How does sdm_client handle that many rows, and how do I combine it with fit()?", + "expected_output": "materialize() does not refuse 50,000 entities: it builds batches of at most 10,000 rows per call (the deployment's limit; rows_per_call= only lowers it), and predict() sends one call per batch and returns one result numbered across the whole request. One materialize() gathers at most 100,000 entities in at most 100 calls. fit() takes a request of one call, so fit a one-call request and pass the batched request to the handle's predict(). Pass holdout= with every id the session will score, because the context is fixed once pinned and a later-scored row would otherwise sit in it with its answer.", + "assertions": [ + "Says the client batches automatically rather than refusing 50,000 entities", + "States the per-call limit of 10,000 scored rows", + "Says rows_per_call can only lower the batch size", + "Says fit() refuses a request gathered in several batches, and to fit a one-call request and send the batched request to the handle's predict()", + "Recommends holdout= with the ids to be scored later, because the pinned context cannot be changed" + ] + }, + { + "id": "r06", + "prompt": "Our users and orders tables are in a warehouse and are far too big to pull into pandas. How do I point kumo-relational at them with sdm_client without exporting the tables?", + "expected_output": "Uses a connector source from sdm_connectors (e.g. SnowflakeSource, DatabricksSource, PostgresSource, DuckDBSource) wrapping the driver's own connection, then Schema.from_source(source, {name: table}, primary_keys=..., time_columns=..., relationships=...). materialize() then samples with SQL so only a bounded sample leaves: up to a thousand rows of each table to read its columns, then only the rows a prediction needs. Everything after the schema is the same as with from_data. Installs the relational extra plus the connector extra.", + "assertions": [ + "Uses Schema.from_source rather than Schema.from_data", + "Builds a source from sdm_connectors around a warehouse connection", + "Explains that only a bounded sample leaves the warehouse, not the whole table", + "Keeps the PredictiveQuery / materialize / predict flow the same as with in-memory data", + "Does not export the full tables to pandas as the recommended approach" + ] + } + ] +} diff --git a/skills/relational/skill-card.md b/skills/relational/skill-card.md new file mode 100644 index 000000000..d4b014cfd --- /dev/null +++ b/skills/relational/skill-card.md @@ -0,0 +1,46 @@ +## Description:
+Ask kumo-relational a question about an entity and the records linked to it: write the question as a predictive query over a schema of related tables, covering classification, regression, forecasting and link prediction.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill with coding agents to write Python that describes related tables (in memory or in a warehouse) and asks NVIDIA's kumo-relational structured-data foundation model predictive queries about entities through the structured-data-models client, without training a model.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [pql.md](pql.md)
+- [references/beyond-one-answer.md](references/beyond-one-answer.md)
+- [references/warehouses.md](references/warehouses.md)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown with inline Python code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [None]
+ +## Skill Version(s):
+1.0.0 (source: CHANGELOG, released 2026-10-15; git tag v1.0.0)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/tabular/evals/evals.json b/skills/tabular/evals/evals.json new file mode 100644 index 000000000..4ac4c97d8 --- /dev/null +++ b/skills/tabular/evals/evals.json @@ -0,0 +1,90 @@ +{ + "skill_name": "sdm-tabular", + "evals": [ + { + "id": "t01", + "prompt": "I have a pandas DataFrame `labelled` of past customers with columns age, spend and churn (0/1), and another DataFrame `to_score` of new customers with just age and spend. Using the structured-data-models client and the kumo-tabular model at http://localhost:8000, write Python that predicts churn for the new customers with class probabilities, and tell me how to line the answers back up with my rows.", + "expected_output": "Builds Request(context=labelled, predict=to_score, target='churn', task='classification') and calls client.predict(request, model=KumoTabular(), outputs=['prediction', 'probabilities']) inside `with SDMClient(url='http://localhost:8000') as client:`, then answer.to_pandas(). Explains that row_index is the position in the predict frame and is what to join on, and that probability columns are named for the class (probability_0, probability_1) rather than by position.", + "assertions": [ + "Imports Request, SDMClient and KumoTabular from sdm_client", + "Builds a Request with context=labelled, predict=to_score and target='churn'", + "States task='classification' explicitly", + "Calls client.predict with model=KumoTabular() and outputs including 'prediction' and 'probabilities'", + "Says to join results back on row_index (the position in the predict frame), not on row order", + "Says probability columns are named for the class they belong to (e.g. probability_1), not numbered by position", + "Does not train, fit a scikit-learn model, or call any training API" + ] + }, + { + "id": "t02", + "prompt": "My target column `visits` holds only 0 and 1 -- it's how many times a user came back last week, and next week it could be higher. I'm using kumo-tabular through sdm_client. How should I set up the request, and can I get a prediction interval rather than just a point estimate?", + "expected_output": "Explains that a target of two whole numbers is read as a classification unless told otherwise, and that this one is a quantity, so pass task='regression'. For an interval, ask for outputs=['prediction', 'quantiles'], which gives quantile_0.1 through quantile_0.9 beside the point prediction. Notes that probabilities belong to classification, and asking for them on a regression comes back as 422 UNSUPPORTED_OUTPUT.", + "assertions": [ + "Recommends task='regression' because visits is a quantity, not a label", + "Explains that without task= a target of two whole numbers would be read as a classification", + "Recommends the 'quantiles' output for an interval", + "Says quantiles come back as per-decile columns such as quantile_0.1 through quantile_0.9", + "Does not recommend the 'probabilities' output for this regression, or warns it is refused (422 UNSUPPORTED_OUTPUT)" + ] + }, + { + "id": "t03", + "prompt": "I'm sending kumo-tabular a Request with a context DataFrame of 15,000 labelled rows and it errors before anything is sent. What's going on and how do I fix it without writing my own sampling code?", + "expected_output": "Explains that a direct Request has a 10,000-row context cap, and that the client refuses a larger context with the count and the cap rather than dropping rows. The fix without hand-written sampling is to describe the frame with Schema.from_data and ask a PredictiveQuery, whose materialize(indices=..., model=KumoTabular()) samples the context down to the 10,000-row cap.", + "assertions": [ + "Identifies the 10,000-row context limit as the cause", + "Says the client refuses rather than silently trimming the context", + "Recommends PredictiveQuery(...).materialize(...) as the way to have the context sampled down to the cap", + "Uses Schema.from_data to describe the frame for the predictive query", + "Does not claim the endpoint or client silently truncates the context" + ] + }, + { + "id": "t04", + "prompt": "I have one DataFrame `customers` with customer_id, age, spend and churn for every customer. I want kumo-tabular to score customers 7, 12 and 3 using the rest of the table as context. Write the code with sdm_client.", + "expected_output": "Builds Schema.from_data({'customers': customers}, primary_keys={'customers': 'customer_id'}), then PredictiveQuery('PREDICT customers.churn FOR EACH customers.customer_id', schema).materialize(indices=[7, 12, 3], model=KumoTabular()), and calls client.predict(request, model=KumoTabular(), ...). Explains that the named rows are held out of their own context, so request.context_rows is the table minus the scored rows, and that the answer carries an id column for the entity.", + "assertions": [ + "Uses Schema.from_data with primary_keys={'customers': 'customer_id'}", + "Writes the query as PREDICT customers.churn FOR EACH customers.customer_id", + "Calls materialize(indices=[7, 12, 3], model=KumoTabular())", + "Explains that the scored rows are held out of the context", + "Does not split the DataFrame into two Request frames by hand as the primary approach" + ] + }, + { + "id": "t05", + "prompt": "Same labelled context every time, but rows to score arrive in batches all day from our kumo-tabular service. Re-sending 9,000 labelled rows on every call seems wasteful. Is there a better pattern in sdm_client, and anything to watch for in deployment?", + "expected_output": "Recommends a session: fitted = client.fit(request, model=KumoTabular()), then fitted.predict(next_request) for each batch, then fitted.close(). Notes that the handle's predict() takes a Request, not a bare DataFrame, that fit() is the only way to build one (FittedModel(...) raises), and that the session lives in one process's memory on the endpoint, so a deployment with several replicas needs session affinity.", + "assertions": [ + "Recommends client.fit(...) to pin the context once", + "Calls predict on the fitted handle for each new batch", + "Says the fitted handle's predict() takes a Request rather than a bare DataFrame", + "Closes the session (fitted.close()) when done", + "Warns that a session lives in one replica's memory, so requests for one session must reach the same replica" + ] + }, + { + "id": "t06", + "prompt": "For each customer kumo-tabular scored, I want to know which input columns pushed the churn prediction up or down. How do I get that from sdm_client, and how should I read the numbers?", + "expected_output": "Adds 'explanation' to outputs (or writes EXPLAIN PREDICT in a query). Explanations are not in to_pandas(); they are on result.explanations.gradients, keyed by the row's id when the request has one, otherwise its row_index, and result.explanations.gradients.to_pandas() gives one line per (row, output, column). Cautions that a gradient is the change in the output score per unit change in the column -- a diagnostic, not a causal effect or a feature-importance ranking -- and that columns are as the model preprocessed them, not as sent.", + "assertions": [ + "Requests the 'explanation' output (or uses EXPLAIN PREDICT)", + "Reads explanations from result.explanations.gradients rather than from to_pandas()", + "Says entries are keyed by id when present, otherwise by row_index", + "States that a gradient is not a causal effect or a feature-importance ranking", + "Notes the explained columns are the model's preprocessed columns, which may differ from the columns sent" + ] + }, + { + "id": "t07", + "prompt": "I have a customers table and an orders table that references customers by customer_id. I want to predict how many orders each customer places next month. Can I do this with KumoTabular in sdm_client?", + "expected_output": "Says no: kumo-tabular reads one table with no relationships and no time windows, and refuses forecasts, aggregations and a schema with a second table, naming the model that can answer. The question is about an entity and the records linked to it, so it belongs to kumo-relational (KumoRelational) and its skill.", + "assertions": [ + "Says kumo-tabular cannot answer this", + "Gives the reason: one table only, no relationships, no aggregation or time window", + "Directs the user to kumo-relational / KumoRelational", + "Does not produce KumoTabular code that joins or aggregates the orders table into a feature and presents that as the supported approach" + ] + } + ] +} diff --git a/skills/tabular/skill-card.md b/skills/tabular/skill-card.md new file mode 100644 index 000000000..3f92b09e4 --- /dev/null +++ b/skills/tabular/skill-card.md @@ -0,0 +1,44 @@ +## Description:
+Ask kumo-tabular a question about rows of one table: classification or regression over labelled rows and rows to score, with no relationships and no time windows.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill with coding agents to write Python that asks NVIDIA's kumo-tabular structured-data foundation model to classify or regress rows of a single table through the structured-data-models client, without training a model.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [references/pql.md](references/pql.md)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown with inline Python code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [None]
+ +## Skill Version(s):
+1.0.0 (source: CHANGELOG, released 2026-10-15; git tag v1.0.0)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/tests/test_skills.py b/tests/test_skills.py index 79b5e4d95..b970bf3f7 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -49,8 +49,19 @@ LINK = re.compile(r'\]\(([^)]+)\)') +#: Files the NVIDIA skills catalog requires beside each skill. They are written +#: for the catalog, not for an agent. A skill links to none of them, and none +#: holds an example. The governance card is HTML-laced markdown in the catalog's +#: own template. NVSkills CI writes the benchmark report on each signing run. +#: Holding either to the markdown read here would fail on a file nobody here +#: writes by hand. +CATALOG_FILES = frozenset({'skill-card.md', 'BENCHMARK.md'}) + + def documents() -> list[Path]: - return sorted(SKILLS.rglob('*.md')) + return sorted( + path for path in SKILLS.rglob('*.md') if path.name not in CATALOG_FILES + ) def fences(path: Path, language: str) -> list[str]: From 8fcb3c648cf6f690524938307b44db10fe5193ac Mon Sep 17 00:00:00 2001 From: igutierrez Date: Fri, 6 Nov 2026 13:59:31 -0600 Subject: [PATCH 2/3] Declare the hosted endpoint's API key on the skill cards A hosted NIM takes a key (SDMClient(api_key=...), sent as X-API-Key); a container you run yourself takes none (docs/get-started/prerequisites.md). The cards said no credential was specified. Co-Authored-By: Claude Opus 5.5 Signed-off-by: igutierrez --- skills/relational/skill-card.md | 4 ++-- skills/tabular/skill-card.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/skills/relational/skill-card.md b/skills/relational/skill-card.md index d4b014cfd..00bd8f169 100644 --- a/skills/relational/skill-card.md +++ b/skills/relational/skill-card.md @@ -15,8 +15,8 @@ Developers and data scientists use this skill with coding agents to write Python Global
## Requirements / Dependencies:
-**Requires API Key or External Credential:** [Not Specified]
-**Credential Type(s):** [None identified]
+**Requires API Key or External Credential:** [Optional]
+**Credential Type(s):** [API key]
Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
diff --git a/skills/tabular/skill-card.md b/skills/tabular/skill-card.md index 3f92b09e4..c97af8464 100644 --- a/skills/tabular/skill-card.md +++ b/skills/tabular/skill-card.md @@ -15,8 +15,8 @@ Developers and data scientists use this skill with coding agents to write Python Global
## Requirements / Dependencies:
-**Requires API Key or External Credential:** [Not Specified]
-**Credential Type(s):** [None identified]
+**Requires API Key or External Credential:** [Optional]
+**Credential Type(s):** [API key]
Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
From 22c459dade217a1de4abbf165f459b15c7b09344 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Tue, 17 Nov 2026 12:19:32 -0600 Subject: [PATCH 3/3] Name each skill directory for its skill The skills catalog copies skills/tabular to skills/sdm-tabular and skills/relational to skills/sdm-relational (catalog_dir is the skill's name). Each skill sent an agent to the other as ../tabular/ or ../relational/, which resolve here and nowhere in the catalog, and the tabular skill linked ../../docs/, which the catalog does not copy. The directories now carry the names the frontmatter already gave them, so a sibling link reads the same here, in the catalog and in an install, and name, directory and catalog_dir agree as kumo-engine's do. The docs link is a URL. A test holds every link inside a skill to a skill. Co-Authored-By: Claude Opus 5.5 Signed-off-by: igutierrez --- docs/about/architecture.md | 2 +- skills/README.md | 12 +-- .../{relational => sdm-relational}/SKILL.md | 2 +- .../evals/evals.json | 0 skills/{relational => sdm-relational}/pql.md | 2 +- .../references/beyond-one-answer.md | 0 .../references/warehouses.md | 0 .../skill-card.md | 0 skills/{tabular => sdm-tabular}/SKILL.md | 6 +- .../{tabular => sdm-tabular}/evals/evals.json | 0 .../references/pql.md | 6 +- skills/{tabular => sdm-tabular}/skill-card.md | 0 tests/test_skills.py | 87 ++++++++++++------- 13 files changed, 73 insertions(+), 44 deletions(-) rename skills/{relational => sdm-relational}/SKILL.md (99%) rename skills/{relational => sdm-relational}/evals/evals.json (100%) rename skills/{relational => sdm-relational}/pql.md (99%) rename skills/{relational => sdm-relational}/references/beyond-one-answer.md (100%) rename skills/{relational => sdm-relational}/references/warehouses.md (100%) rename skills/{relational => sdm-relational}/skill-card.md (100%) rename skills/{tabular => sdm-tabular}/SKILL.md (96%) rename skills/{tabular => sdm-tabular}/evals/evals.json (100%) rename skills/{tabular => sdm-tabular}/references/pql.md (93%) rename skills/{tabular => sdm-tabular}/skill-card.md (100%) diff --git a/docs/about/architecture.md b/docs/about/architecture.md index 575f0551d..dbc165a58 100644 --- a/docs/about/architecture.md +++ b/docs/about/architecture.md @@ -577,7 +577,7 @@ is assembled here rather than asked for. served either — a forecast is sent as a regression per step, and a bare `LIST_DISTINCT` is refused outright. -`skills/relational/SKILL.md` carries the same matrix for an agent, and +`skills/sdm-relational/SKILL.md` carries the same matrix for an agent, and `tests/test_skills.py` asks the endpoint for every one of these on every task, so a deployment that gains or loses an output fails the suite rather than leaving the document wrong. diff --git a/skills/README.md b/skills/README.md index 349e8df4d..373f9ba9b 100644 --- a/skills/README.md +++ b/skills/README.md @@ -7,8 +7,8 @@ Start at one of these two. Each is enough on its own to ask a first question. | Skill | Model | Load it when | |---|---|---| -| [`tabular/SKILL.md`](tabular/SKILL.md) | `kumo-tabular` | The question is about rows of one table | -| [`relational/SKILL.md`](relational/SKILL.md) | `kumo-relational` | The question is about an entity and what it is linked to | +| [`sdm-tabular/SKILL.md`](sdm-tabular/SKILL.md) | `kumo-tabular` | The question is about rows of one table | +| [`sdm-relational/SKILL.md`](sdm-relational/SKILL.md) | `kumo-relational` | The question is about an entity and what it is linked to | An agent answering a question about a single table does not need the relational material, and the four task kinds that make the relational model @@ -20,10 +20,10 @@ be loaded without the skill it belongs to: | Reference | | |---|---| -| [`relational/pql.md`](relational/pql.md) | Writing the predictive query | -| [`tabular/references/pql.md`](tabular/references/pql.md) | The same language on one table, and what it refuses | -| [`relational/references/warehouses.md`](relational/references/warehouses.md) | Tables that stay in a warehouse, and metric views | -| [`relational/references/beyond-one-answer.md`](relational/references/beyond-one-answer.md) | Rankings, forecasts and explanations | +| [`sdm-relational/pql.md`](sdm-relational/pql.md) | Writing the predictive query | +| [`sdm-tabular/references/pql.md`](sdm-tabular/references/pql.md) | The same language on one table, and what it refuses | +| [`sdm-relational/references/warehouses.md`](sdm-relational/references/warehouses.md) | Tables that stay in a warehouse, and metric views | +| [`sdm-relational/references/beyond-one-answer.md`](sdm-relational/references/beyond-one-answer.md) | Rankings, forecasts and explanations | ## What these models are diff --git a/skills/relational/SKILL.md b/skills/sdm-relational/SKILL.md similarity index 99% rename from skills/relational/SKILL.md rename to skills/sdm-relational/SKILL.md index 813e4afd4..7ff48bbbc 100644 --- a/skills/relational/SKILL.md +++ b/skills/sdm-relational/SKILL.md @@ -9,7 +9,7 @@ An entity, and the records linked to it. The question is written as a predictive query, the client gathers each entity's neighbourhood, and the model reads that neighbourhood in one pass. -Load [`../tabular/SKILL.md`](../tabular/SKILL.md) instead when the question is +Load [`../sdm-tabular/SKILL.md`](../sdm-tabular/SKILL.md) instead when the question is about rows of one table with no links. Writing the query itself is in [`pql.md`](pql.md). diff --git a/skills/relational/evals/evals.json b/skills/sdm-relational/evals/evals.json similarity index 100% rename from skills/relational/evals/evals.json rename to skills/sdm-relational/evals/evals.json diff --git a/skills/relational/pql.md b/skills/sdm-relational/pql.md similarity index 99% rename from skills/relational/pql.md rename to skills/sdm-relational/pql.md index 091d494bd..df5ac292f 100644 --- a/skills/relational/pql.md +++ b/skills/sdm-relational/pql.md @@ -345,7 +345,7 @@ ASSUMING users.segment = 'a' The same language, cut down to a column of one table as the target. Every clause above is refused there. It has its own reference: -[`../tabular/references/pql.md`](../tabular/references/pql.md). +[`../sdm-tabular/references/pql.md`](../sdm-tabular/references/pql.md). ## When it will not parse diff --git a/skills/relational/references/beyond-one-answer.md b/skills/sdm-relational/references/beyond-one-answer.md similarity index 100% rename from skills/relational/references/beyond-one-answer.md rename to skills/sdm-relational/references/beyond-one-answer.md diff --git a/skills/relational/references/warehouses.md b/skills/sdm-relational/references/warehouses.md similarity index 100% rename from skills/relational/references/warehouses.md rename to skills/sdm-relational/references/warehouses.md diff --git a/skills/relational/skill-card.md b/skills/sdm-relational/skill-card.md similarity index 100% rename from skills/relational/skill-card.md rename to skills/sdm-relational/skill-card.md diff --git a/skills/tabular/SKILL.md b/skills/sdm-tabular/SKILL.md similarity index 96% rename from skills/tabular/SKILL.md rename to skills/sdm-tabular/SKILL.md index f6486c95c..4cb67edfa 100644 --- a/skills/tabular/SKILL.md +++ b/skills/sdm-tabular/SKILL.md @@ -9,7 +9,7 @@ One table. You hold rows whose answer is known and rows whose answer you want. Both go in one request, the model reads them together, and the answer comes back a row at a time. -Load [`../relational/SKILL.md`](../relational/SKILL.md) instead when the +Load [`../sdm-relational/SKILL.md`](../sdm-relational/SKILL.md) instead when the question is about an entity and the records linked to it. The two models share this client's objects -- `Schema`, `PredictiveQuery`, `Request` -- and the language. What differs is almost everything that can be asked in it. @@ -222,7 +222,7 @@ datetimes may appear as derived columns, and a table without numerical features gives an empty list. A gradient is the change in that output score per unit change in that column: a diagnostic, not a causal effect or a feature-importance ranking. The whole shape is under Explanations in -[`docs/reference/prediction-output.md`](../../docs/reference/prediction-output.md). +[`docs/reference/prediction-output.md`](https://github.com/NVIDIA/structured-data-models-client/blob/main/docs/reference/prediction-output.md). ## When it refuses @@ -241,4 +241,4 @@ Catch `StructuredDataError` to catch all of them. ## Then - [`references/pql.md`](references/pql.md) — the query language on one table -- [`../relational/SKILL.md`](../relational/SKILL.md) — when the question is about an entity and what it links to +- [`../sdm-relational/SKILL.md`](../sdm-relational/SKILL.md) — when the question is about an entity and what it links to diff --git a/skills/tabular/evals/evals.json b/skills/sdm-tabular/evals/evals.json similarity index 100% rename from skills/tabular/evals/evals.json rename to skills/sdm-tabular/evals/evals.json diff --git a/skills/tabular/references/pql.md b/skills/sdm-tabular/references/pql.md similarity index 93% rename from skills/tabular/references/pql.md rename to skills/sdm-tabular/references/pql.md index d207fa86e..32fa39e8a 100644 --- a/skills/tabular/references/pql.md +++ b/skills/sdm-tabular/references/pql.md @@ -1,6 +1,6 @@ # Writing a query for kumo-tabular -The same language as [`../../relational/pql.md`](../../relational/pql.md), cut +The same language as [`../../sdm-relational/pql.md`](../../sdm-relational/pql.md), cut down to what one table can answer. Read this one when the schema has a single table; almost every clause the relational reference describes is refused here, and the refusals say so. @@ -63,7 +63,7 @@ name rows already in the table or `predict=` to pass rows you hold. Every one of them is the tabular profile of the language refusing, not the grammar: the relational model takes all seven, within the bounds -[`../../relational/pql.md`](../../relational/pql.md) gives each of them. +[`../../sdm-relational/pql.md`](../../sdm-relational/pql.md) gives each of them. Three are worth seeing here, because all three are queries somebody would reasonably write. @@ -108,4 +108,4 @@ Pass model=KumoRelational() to gather them as a graph If the question really is about one table, build the schema from that table alone. If it is about an entity and the records linked to it, it belongs to -`kumo-relational` and [`../../relational/SKILL.md`](../../relational/SKILL.md). +`kumo-relational` and [`../../sdm-relational/SKILL.md`](../../sdm-relational/SKILL.md). diff --git a/skills/tabular/skill-card.md b/skills/sdm-tabular/skill-card.md similarity index 100% rename from skills/tabular/skill-card.md rename to skills/sdm-tabular/skill-card.md diff --git a/tests/test_skills.py b/tests/test_skills.py index b970bf3f7..d24efe809 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -73,8 +73,8 @@ def fences(path: Path, language: str) -> list[str]: def test_there_is_a_skill_for_each_model(): - assert (SKILLS / 'tabular' / 'SKILL.md').exists() - assert (SKILLS / 'relational' / 'SKILL.md').exists() + assert (SKILLS / 'sdm-tabular' / 'SKILL.md').exists() + assert (SKILLS / 'sdm-relational' / 'SKILL.md').exists() assert (SKILLS / 'README.md').exists() @@ -158,7 +158,7 @@ def test_an_example_that_predicts_shows_what_came_back(path): ) -@pytest.mark.parametrize('name', ['tabular', 'relational']) +@pytest.mark.parametrize('name', ['sdm-tabular', 'sdm-relational']) def test_a_skill_shows_output_it_has_run(name): """Each skill has to stake something on a live endpoint. @@ -171,15 +171,20 @@ def test_a_skill_shows_output_it_has_run(name): assert '**Verified**' in body, f'{name} marks no output as verified' -@pytest.mark.parametrize('name', ['tabular', 'relational']) +@pytest.mark.parametrize('name', ['sdm-tabular', 'sdm-relational']) def test_a_skill_announces_itself(name): - """An agent finds a skill by its frontmatter, so it has to be there.""" + """An agent finds a skill by its frontmatter, so it has to be there. + + The name is the directory's. The skills catalog copies each directory to + one named for the skill, so a skill that links to its sibling as + ``..//`` reaches it only where the two agree. + """ yaml = pytest.importorskip('yaml') body = (SKILLS / name / 'SKILL.md').read_text(encoding='utf-8') assert body.startswith('---\n'), f'{name} opens with no frontmatter' front = yaml.safe_load(body[4 : body.index('\n---\n', 3)]) assert isinstance(front, dict), f'{name} frontmatter is not a mapping' - assert front.get('name') == f'sdm-{name}', ( + assert front.get('name') == name, ( f'{name} is announced as {front.get("name")!r}' ) assert 'Triggers:' in front.get('description', ''), ( @@ -296,26 +301,28 @@ def test_a_document_is_written_in_the_one_markdown_this_file_reads(path): (['Skill', 'Model', 'Load it when'], False), (['Reference', ''], False), ], - 'skills/relational/SKILL.md': [ + 'skills/sdm-relational/SKILL.md': [ (['', '`kumo-relational`'], True), (['Limit', 'Value'], True), (['Refusal', 'Cause'], False), ], - 'skills/relational/references/beyond-one-answer.md': [], - 'skills/relational/references/warehouses.md': [ + 'skills/sdm-relational/references/beyond-one-answer.md': [], + 'skills/sdm-relational/references/warehouses.md': [ (['Source', 'Connection it takes', 'Namespace', 'Extra'], True) ], - 'skills/relational/pql.md': [ + 'skills/sdm-relational/pql.md': [ (['Clause', 'Least', 'Most'], True), (['Target', 'Task', 'Calls'], True), (['Aggregation', ''], True), (['Comparison', ''], True), ], - 'skills/tabular/SKILL.md': [ + 'skills/sdm-tabular/SKILL.md': [ (['', '`kumo-tabular`'], True), (['Refusal', 'Cause'], False), ], - 'skills/tabular/references/pql.md': [(['Clause', 'On one table'], True)], + 'skills/sdm-tabular/references/pql.md': [ + (['Clause', 'On one table'], True) + ], } @@ -345,6 +352,26 @@ def test_every_link_resolves(path): ) +@pytest.mark.parametrize( + 'path', [p for p in documents() if p.parent != SKILLS], ids=str +) +def test_a_skill_links_only_to_skills(path): + """The skills catalog copies each skill directory and nothing around it. + + A relative link out of ``skills/`` into the rest of this repository + resolves here and nowhere the skill is installed. Link to a sibling skill + by its directory, or to anything else by its URL. + """ + for target in LINK.findall(path.read_text(encoding='utf-8')): + if target.startswith(('http://', 'https://', '#')): + continue + resolved = (path.parent / target).resolve() + assert any( + resolved.is_relative_to(skill.parent.resolve()) + for skill in SKILLS.glob('*/SKILL.md') + ), f'{path.relative_to(SKILLS)} links to {target}, outside every skill' + + @pytest.mark.parametrize('path', documents(), ids=str) def test_verified_output_is_marked_one_way(path): """``**Verified**`` at the start of a line labels the block beneath it. @@ -644,7 +671,7 @@ def resolve(query, schema, **extra): ) -PQL = SKILLS / 'relational' / 'pql.md' +PQL = SKILLS / 'sdm-relational' / 'pql.md' FENCED = re.compile(r'^```[a-z]*\n.*?^```$', re.DOTALL | re.MULTILINE) @@ -1652,9 +1679,9 @@ def test_a_temporal_assumption_is_answered(schema): assert request.task.kind == 'regression' -TABULAR_PQL = SKILLS / 'tabular' / 'references' / 'pql.md' +TABULAR_PQL = SKILLS / 'sdm-tabular' / 'references' / 'pql.md' -#: Each row of the clause table in tabular/references/pql.md, written as a +#: Each row of the clause table in sdm-tabular/references/pql.md, written as a #: query over one table. The claim is that every one is refused there, and #: several of them are ordinary relational queries. ON_ONE_TABLE = { @@ -1707,7 +1734,7 @@ def test_the_tabular_clause_table_is_every_clause_it_refuses(one_table): shown = table(TABULAR_PQL, 'Clause') del shown['Clause'] assert set(shown) == set(ON_ONE_TABLE), ( - f'tabular/references/pql.md lists {sorted(set(shown) ^ set(ON_ONE_TABLE))} ' + f'sdm-tabular/references/pql.md lists {sorted(set(shown) ^ set(ON_ONE_TABLE))} ' f'that this does not ask about' ) assert all(cells == ['refused'] for cells in shown.values()), ( @@ -1817,7 +1844,7 @@ def test_a_plain_column_is_what_the_tabular_reference_promises(one_table): ) -WAREHOUSES = SKILLS / 'relational' / 'references' / 'warehouses.md' +WAREHOUSES = SKILLS / 'sdm-relational' / 'references' / 'warehouses.md' def test_the_source_table_is_every_source_there_is(): @@ -2085,7 +2112,7 @@ def test_a_connector_error_is_not_a_structured_data_error(): def test_the_tabular_model_refuses_a_second_table(schema): - """The sentence ``skills/tabular/SKILL.md`` quotes.""" + """The sentence ``skills/sdm-tabular/SKILL.md`` quotes.""" from sdm_client import KumoTabular, PredictiveQuery from sdm_client.errors import StructuredDataError @@ -2210,7 +2237,7 @@ def test_a_quoted_refusal_from_the_endpoint_is_the_refusal(query, said, schema): not TABULAR_URL, reason='set SDM_TABULAR_URL to a kumo-tabular NIM' ) @pytest.mark.parametrize( - 'name', ['tabular/SKILL.md', 'tabular/references/pql.md'] + 'name', ['sdm-tabular/SKILL.md', 'sdm-tabular/references/pql.md'] ) def test_the_tabular_examples_print_what_the_skill_shows(name): assert_every_example_prints_what_it_shows(SKILLS / name, TABULAR_URL) @@ -2223,10 +2250,10 @@ def test_the_tabular_examples_print_what_the_skill_shows(name): @pytest.mark.parametrize( 'name', [ - 'relational/SKILL.md', - 'relational/pql.md', - 'relational/references/beyond-one-answer.md', - 'relational/references/warehouses.md', + 'sdm-relational/SKILL.md', + 'sdm-relational/pql.md', + 'sdm-relational/references/beyond-one-answer.md', + 'sdm-relational/references/warehouses.md', 'README.md', ], ) @@ -2241,7 +2268,7 @@ def test_the_relational_examples_print_what_the_skill_shows(name): @pytest.mark.parametrize( 'name, model_name', - [('tabular', 'KumoTabular'), ('relational', 'KumoRelational')], + [('sdm-tabular', 'KumoTabular'), ('sdm-relational', 'KumoRelational')], ) def test_the_capability_table_is_the_clients_own(name, model_name): """Every row of the table both skills open with, against the client.""" @@ -2263,7 +2290,7 @@ def test_the_capability_table_is_the_clients_own(name, model_name): 'Relationships': capabilities.relationships, 'Sessions': capabilities.sessions, } - if name == 'tabular': + if name == 'sdm-tabular': from sdm_client.models import TABULAR_LIMITS said['Explanations'] = 'explanation' in capabilities.outputs @@ -2291,7 +2318,7 @@ def test_the_capability_table_is_the_clients_own(name, model_name): ) -#: Each row of the limits table in ``skills/relational/SKILL.md``, against the +#: Each row of the limits table in ``skills/sdm-relational/SKILL.md``, against the #: constant it reports. Every key is spoken for in both directions: a limit the #: client enforces and the skill omits is a refusal an agent cannot see coming, #: and a row naming no limit is a number nobody is holding to anything. @@ -2314,7 +2341,7 @@ def test_the_limits_table_is_the_shipped_limits(): assert set(LIMIT_ROWS.values()) == set(RELATIONAL_LIMITS), ( 'the client enforces a limit this table does not account for' ) - path = SKILLS / 'relational' / 'SKILL.md' + path = SKILLS / 'sdm-relational' / 'SKILL.md' assert heading(path, 'Limit') == 'Value' shown = table(path, 'Limit') shown.pop('Limit') @@ -2419,7 +2446,7 @@ def test_the_tabular_outputs_narrow_by_task_as_the_skill_says(): assert len(deciles) == 9, f'quantiles came back as {deciles}' -#: The output matrix skills/relational/SKILL.md gives for the deployment: +#: The output matrix skills/sdm-relational/SKILL.md gives for the deployment: #: what each task is served, and everything the client would send that it #: will not. Asked of the endpoint, so a deployment that gains or loses one #: fails here rather than making the skill quietly wrong. @@ -2447,7 +2474,9 @@ def not_served(outputs: tuple[str, ...]) -> list[str]: the endpoint happens to allow. """ advertised = set( - listed(table(SKILLS / 'relational' / 'SKILL.md', 'Tasks')['Outputs'][0]) + listed( + table(SKILLS / 'sdm-relational' / 'SKILL.md', 'Tasks')['Outputs'][0] + ) ) return sorted(advertised - set(outputs))