From a0fe6e18729b9f2adcc5efef148afacab73e8987 Mon Sep 17 00:00:00 2001 From: igutierrez Date: Tue, 17 Nov 2026 20:16:59 -0600 Subject: [PATCH] Prepare the agent skills for the NVIDIA skills catalog The catalog copies each skill directory to one named for the skill and nothing around it, so each skill now lives in a directory named as its frontmatter names it (sdm-tabular, sdm-relational), links to its sibling by that name, and reaches the repository's docs by URL rather than by a relative path that leaves skills/. A test holds every skill to that. Each skill gains the catalog's governance card and evals for NVSkills CI. The evals follow these skills as they stand on v2-candidate: no extra for kumo-relational, no client-side size limits, unique-id keys for explanations, and one eval each for running the model in this process. The request workflow is the catalog's, pinned to a SHA, the same file as on main's PR, so the NVSkills CI gate runs on PRs into this branch. This is the v2-candidate port of #82. Co-Authored-By: Claude Opus 5.5 Signed-off-by: igutierrez --- .github/workflows/request-nvskills-ci.yml | 40 +++++++ scripts/check_repo_hygiene.py | 4 +- skills/README.md | 16 +-- .../{relational => sdm-relational}/SKILL.md | 2 +- skills/sdm-relational/evals/evals.json | 88 ++++++++++++++ skills/{relational => sdm-relational}/pql.md | 2 +- .../references/beyond-one-answer.md | 2 +- .../references/in-process.md | 2 +- .../references/warehouses.md | 0 skills/sdm-relational/skill-card.md | 47 ++++++++ skills/{tabular => sdm-tabular}/SKILL.md | 6 +- skills/sdm-tabular/evals/evals.json | 104 ++++++++++++++++ .../references/in-process.md | 0 .../references/pql.md | 6 +- skills/sdm-tabular/skill-card.md | 45 +++++++ tests/test_skills.py | 113 ++++++++++++------ 16 files changed, 422 insertions(+), 55 deletions(-) create mode 100644 .github/workflows/request-nvskills-ci.yml rename skills/{relational => sdm-relational}/SKILL.md (99%) create mode 100644 skills/sdm-relational/evals/evals.json rename skills/{relational => sdm-relational}/pql.md (99%) rename skills/{relational => sdm-relational}/references/beyond-one-answer.md (98%) rename skills/{relational => sdm-relational}/references/in-process.md (98%) rename skills/{relational => sdm-relational}/references/warehouses.md (100%) create mode 100644 skills/sdm-relational/skill-card.md rename skills/{tabular => sdm-tabular}/SKILL.md (97%) create mode 100644 skills/sdm-tabular/evals/evals.json rename skills/{tabular => sdm-tabular}/references/in-process.md (100%) rename skills/{tabular => sdm-tabular}/references/pql.md (93%) create mode 100644 skills/sdm-tabular/skill-card.md diff --git a/.github/workflows/request-nvskills-ci.yml b/.github/workflows/request-nvskills-ci.yml new file mode 100644 index 000000000..7acd83fac --- /dev/null +++ b/.github/workflows/request-nvskills-ci.yml @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# NVSkills CI evaluates and signs changes under skills/. The NVIDIA skills +# catalog (github.com/NVIDIA/skills) syncs these skills from main every hour, +# and accepts only files that match the signature (skill.oms.sig) NVSkills CI +# writes into the PR. This is the catalog's request workflow, unchanged except +# that it is pinned to a SHA like every other action here. When the catalog +# changes the template, re-pin to its current main. +# +# A maintainer or admin comments `/nvskills-ci` on a PR to start a run. It needs +# the NVSKILLS_CI_DISPATCH_TOKEN secret. A PR that does not touch skills/ skips +# it. +name: Request NVSkills CI + +on: + issue_comment: + types: [created] + pull_request: + types: [opened, reopened, synchronize, ready_for_review] + push: + +jobs: + request: + if: > + github.event_name == 'pull_request' || + (github.event_name == 'issue_comment' && + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/nvskills-ci')) || + (github.event_name == 'push' && + github.actor == (vars.NVSKILLS_SIGNATURE_PUSH_ACTOR || 'nv-skills-ci[bot]') && + startsWith(github.event.head_commit.message, vars.NVSKILLS_SIGNATURE_COMMIT_TITLE || 'Attach NVSkills validation signatures')) + permissions: + contents: read + pull-requests: read + statuses: read + uses: NVIDIA/skills/.github/workflows/team-request.yml@b707bbe61ff1026dd59a44140610f92c8eae1151 # main, 2026-11-06 + secrets: + NVSKILLS_CI_DISPATCH_TOKEN: ${{ secrets.NVSKILLS_CI_DISPATCH_TOKEN }} diff --git a/scripts/check_repo_hygiene.py b/scripts/check_repo_hygiene.py index bc250351a..5a00736bb 100644 --- a/scripts/check_repo_hygiene.py +++ b/scripts/check_repo_hygiene.py @@ -369,8 +369,8 @@ def check_repository_url() -> int: IN_PROCESS_REFERENCES = ( - 'skills/tabular/references/in-process.md', - 'skills/relational/references/in-process.md', + 'skills/sdm-tabular/references/in-process.md', + 'skills/sdm-relational/references/in-process.md', ) # The release that first carries `in_process()`. Until the index has it the diff --git a/skills/README.md b/skills/README.md index 3468990c6..f60308986 100644 --- a/skills/README.md +++ b/skills/README.md @@ -7,8 +7,8 @@ Start at one of these two. Each is enough on its own to ask a first question. | Skill | Model | Load it when | |---|---|---| -| [`tabular/SKILL.md`](tabular/SKILL.md) | `kumo-tabular` | The question is about rows of one table | -| [`relational/SKILL.md`](relational/SKILL.md) | `kumo-relational` | The question is about an entity and what it is linked to | +| [`sdm-tabular/SKILL.md`](sdm-tabular/SKILL.md) | `kumo-tabular` | The question is about rows of one table | +| [`sdm-relational/SKILL.md`](sdm-relational/SKILL.md) | `kumo-relational` | The question is about an entity and what it is linked to | An agent answering a question about a single table does not need the relational material. What makes the relational model what it is, the two @@ -20,12 +20,12 @@ be loaded without the skill it belongs to: | Reference | | |---|---| -| [`relational/pql.md`](relational/pql.md) | Writing the predictive query | -| [`tabular/references/pql.md`](tabular/references/pql.md) | The same language on one table, and what it refuses | -| [`relational/references/warehouses.md`](relational/references/warehouses.md) | Tables that stay in a warehouse, and metric views | -| [`relational/references/beyond-one-answer.md`](relational/references/beyond-one-answer.md) | Rankings, forecasts and explanations | -| [`tabular/references/in-process.md`](tabular/references/in-process.md) | Running kumo-tabular in this process, with no NIM | -| [`relational/references/in-process.md`](relational/references/in-process.md) | Running kumo-relational in this process, with no NIM | +| [`sdm-relational/pql.md`](sdm-relational/pql.md) | Writing the predictive query | +| [`sdm-tabular/references/pql.md`](sdm-tabular/references/pql.md) | The same language on one table, and what it refuses | +| [`sdm-relational/references/warehouses.md`](sdm-relational/references/warehouses.md) | Tables that stay in a warehouse, and metric views | +| [`sdm-relational/references/beyond-one-answer.md`](sdm-relational/references/beyond-one-answer.md) | Rankings, forecasts and explanations | +| [`sdm-tabular/references/in-process.md`](sdm-tabular/references/in-process.md) | Running kumo-tabular in this process, with no NIM | +| [`sdm-relational/references/in-process.md`](sdm-relational/references/in-process.md) | Running kumo-relational in this process, with no NIM | ## What these models are diff --git a/skills/relational/SKILL.md b/skills/sdm-relational/SKILL.md similarity index 99% rename from skills/relational/SKILL.md rename to skills/sdm-relational/SKILL.md index 6ec73bfbd..2de6db926 100644 --- a/skills/relational/SKILL.md +++ b/skills/sdm-relational/SKILL.md @@ -9,7 +9,7 @@ An entity, and the records linked to it. The question is written as a predictive query, the client gathers each entity's neighbourhood, and the model reads that neighbourhood in one pass. -Load [`../tabular/SKILL.md`](../tabular/SKILL.md) instead when the question is +Load [`../sdm-tabular/SKILL.md`](../sdm-tabular/SKILL.md) instead when the question is about rows of one table with no links. Writing the query itself is in [`pql.md`](pql.md). diff --git a/skills/sdm-relational/evals/evals.json b/skills/sdm-relational/evals/evals.json new file mode 100644 index 000000000..2ee6db9bd --- /dev/null +++ b/skills/sdm-relational/evals/evals.json @@ -0,0 +1,88 @@ +{ + "skill_name": "sdm-relational", + "evals": [ + { + "id": "r01", + "prompt": "I have three pandas DataFrames: users(user_id, segment), orders(order_id, user_id, item, price, ordered_at) and items(item, kind). Using sdm_client and kumo-relational at http://localhost:8000, predict how many orders users 1, 2 and 3 will place in the next 30 days. Include anything I need to install.", + "expected_output": "Installs structured-data-models-client; KumoRelational needs no extra. Builds Schema.from_data with primary_keys for all three tables, time_columns={'orders': 'ordered_at'}, and relationships from the many side to the one side: Relationship('orders', 'user_id', 'users', 'user_id') and Relationship('orders', 'item', 'items', 'item'). Asks PredictiveQuery('PREDICT COUNT(orders.*, 0, 30, days) FOR EACH users.user_id', schema).materialize(indices=[1, 2, 3], model=KumoRelational()) and calls client.predict(request, model=KumoRelational()) with a generous timeout. Joins on id (the entity), else row_index, never row order.", + "assertions": [ + "Installs structured-data-models-client without requiring a relational extra", + "Uses Schema.from_data with primary keys and time_columns={'orders': 'ordered_at'}", + "Declares relationships from orders to users and orders to items (many side first)", + "Writes the target as COUNT(orders.*, 0, 30, days) FOR EACH users.user_id", + "Calls materialize(indices=[1, 2, 3], model=KumoRelational()) and client.predict(..., model=KumoRelational())", + "Says to join results on id (or row_index), not on row order", + "Does not pass a task= argument; the task follows from the query" + ] + }, + { + "id": "r02", + "prompt": "My kumo-relational schema has `Relationship('users', 'user_id', 'orders', 'user_id')` because each user has many orders. Predictions look wrong. Is that relationship right?", + "expected_output": "No. Relationship(from_table, from_column, to_table, to_column) points from the many side to the one side: an order names a user, so it should be Relationship('orders', 'user_id', 'users', 'user_id').", + "assertions": [ + "Says the relationship is reversed", + "Explains a Relationship points from the many side to the one side", + "Gives the corrected Relationship('orders', 'user_id', 'users', 'user_id')" + ] + }, + { + "id": "r03", + "prompt": "I want per-entity explanations from kumo-relational for a regression. I added outputs=['prediction', 'explanation'] and our deployment returned `422 UNSUPPORTED_OUTPUT`. Is my code wrong? How should I write this so it works on our deployment and on others?", + "expected_output": "The code is not wrong: the client will send explanation, but outputs narrow at the endpoint by task, and the shipping deployment serves only prediction and quantiles on a regression. Another relational deployment, and SDMClient.in_process(), do serve explanation. So ask for it, catch the refusal (a StructuredDataError), and fall back to prediction only, without assuming an explanation always or never arrives. When one does arrive it is on result.explanations.gradients, not in to_pandas().", + "assertions": [ + "Explains the refusal comes from the deployment, not from a bug in the request", + "Says the shipping deployment serves prediction and quantiles on a regression", + "Recommends requesting explanation, catching the refusal, and falling back", + "Does not claim explanations are never available from kumo-relational (another deployment or SDMClient.in_process() serves them)", + "Says explanations, when returned, are on result.explanations rather than in to_pandas()" + ] + }, + { + "id": "r04", + "prompt": "I'm predicting next-quarter revenue per customer with kumo-relational and then summing the predictions to get a company-wide forecast. Anything wrong with that?", + "expected_output": "A regression's prediction is the median of the predicted distribution, not the mean, and medians do not add up. Ask for the mean with materialize(..., prediction_statistic='mean') when predictions will be summed across entities.", + "assertions": [ + "States that a regression prediction is the median by default", + "Explains that medians cannot be summed to a meaningful total", + "Recommends prediction_statistic='mean' in materialize()" + ] + }, + { + "id": "r05", + "prompt": "I need kumo-relational scores for 50,000 users, and I'd like to pin the context in a session so later calls are cheaper. How does sdm_client handle that many rows, and how do I combine it with fit()?", + "expected_output": "materialize() does not refuse 50,000 entities: it gathers them in batches of 10,000 rows by default (rows_per_call= asks for another size), and predict() sends one call per batch and returns one result numbered across the whole request, so to_pandas() gives the indices back in the order asked. The client holds a request to no size of its own; a batch larger than the endpoint takes is the endpoint's refusal. Every batch is built before the first is sent, so the whole population is held in memory at once. fit() takes a request of one call, so fit a one-call request and pass the batched request to the handle's predict(). Pass holdout= with every id the session will score, because the context is fixed once pinned and a later-scored row would otherwise sit in it with its answer.", + "assertions": [ + "Says the client batches automatically rather than refusing 50,000 entities", + "Says the default batch is 10,000 rows and rows_per_call= sets another size", + "Says the client imposes no size limit of its own; a limit is the endpoint's", + "Says fit() refuses a request gathered in several batches, and to fit a one-call request and send the batched request to the handle's predict()", + "Recommends holdout= with the ids to be scored later, because the pinned context cannot be changed" + ] + }, + { + "id": "r06", + "prompt": "Our users and orders tables are in a warehouse and are far too big to pull into pandas. How do I point kumo-relational at them with sdm_client without exporting the tables?", + "expected_output": "Uses a connector source from sdm_connectors (e.g. SnowflakeSource, DatabricksSource, PostgresSource, DuckDBSource) wrapping the driver's own connection, then Schema.from_source(source, {name: table}, primary_keys=..., time_columns=..., relationships=...). materialize() then samples with SQL so only a bounded sample leaves: up to a thousand rows of each table to read its columns, then only the rows a prediction needs. Everything after the schema is the same as with from_data. Installs the connector's extra, e.g. structured-data-models-client[snowflake].", + "assertions": [ + "Uses Schema.from_source rather than Schema.from_data", + "Builds a source from sdm_connectors around a warehouse connection", + "Explains that only a bounded sample leaves the warehouse, not the whole table", + "Keeps the PredictiveQuery / materialize / predict flow the same as with in-memory data", + "Does not export the full tables to pandas as the recommended approach" + ] + }, + { + "id": "r07", + "prompt": "We have no kumo-relational NIM, and our users and orders tables must not leave this machine. Can sdm_client still answer 'how many orders will each user place in the next 30 days'? What changes in my code and install?", + "expected_output": "Yes: open the client with SDMClient.in_process() in place of a URL; the schema, the query and materialize() are unchanged, and gathering still happens in this client. It needs the in-process extra and the connectors package (gathering imports it even for in-memory frames), and Python 3.11 or later; until a published release carries in_process(), both are installed from a checkout of v2-candidate as the reference describes. The first prediction fetches the published weights (nvidia/Kumo-Relational) into the Hugging Face cache with no account or token. On a CPU its numbers can differ from a GPU-hosted NIM's. Join the answer back on id or row_index, never row order.", + "assertions": [ + "Uses SDMClient.in_process() in place of SDMClient(url=...)", + "Keeps Schema.from_data, PredictiveQuery and materialize() unchanged", + "Names the in-process extra and the structured-data-models-connectors package", + "Says the weights are fetched once from Hugging Face, with no token required", + "Says CPU results can differ from a GPU-hosted NIM's", + "Does not send the tables to any remote endpoint" + ] + } + ] +} diff --git a/skills/relational/pql.md b/skills/sdm-relational/pql.md similarity index 99% rename from skills/relational/pql.md rename to skills/sdm-relational/pql.md index c2b36130d..c14668bbc 100644 --- a/skills/relational/pql.md +++ b/skills/sdm-relational/pql.md @@ -345,7 +345,7 @@ ASSUMING users.segment = 'a' The same language, cut down to a column of one table as the target. Every clause above is refused there. It has its own reference: -[`../tabular/references/pql.md`](../tabular/references/pql.md). +[`../sdm-tabular/references/pql.md`](../sdm-tabular/references/pql.md). ## When it will not parse diff --git a/skills/relational/references/beyond-one-answer.md b/skills/sdm-relational/references/beyond-one-answer.md similarity index 98% rename from skills/relational/references/beyond-one-answer.md rename to skills/sdm-relational/references/beyond-one-answer.md index e2e1e1e41..83b39083e 100644 --- a/skills/relational/references/beyond-one-answer.md +++ b/skills/sdm-relational/references/beyond-one-answer.md @@ -167,4 +167,4 @@ if analysis is not None: cohorts[cohorts['table'] == 'orders'] # the cohorts of each column of orders ``` -See [the result contract](../../../docs/reference/prediction-output.md). +See [the result contract](https://github.com/NVIDIA/structured-data-models-client/blob/main/docs/reference/prediction-output.md). diff --git a/skills/relational/references/in-process.md b/skills/sdm-relational/references/in-process.md similarity index 98% rename from skills/relational/references/in-process.md rename to skills/sdm-relational/references/in-process.md index d4ddcccdc..f4581cd2d 100644 --- a/skills/relational/references/in-process.md +++ b/skills/sdm-relational/references/in-process.md @@ -192,7 +192,7 @@ model already loaded with structured-data-models is used as it is: `sdm.models.KumoRelational`, shares its weights with every context the client fits, and refuses `device=` beside it, since that model is already somewhere. The quickstart's section on a model in this process, in -[`docs/get-started/quickstart.md`](../../../docs/get-started/quickstart.md), +[`docs/get-started/quickstart.md`](https://github.com/NVIDIA/structured-data-models-client/blob/main/docs/get-started/quickstart.md), shows it. A model loaded that way arrives in evaluation mode and loaded for both tasks, diff --git a/skills/relational/references/warehouses.md b/skills/sdm-relational/references/warehouses.md similarity index 100% rename from skills/relational/references/warehouses.md rename to skills/sdm-relational/references/warehouses.md diff --git a/skills/sdm-relational/skill-card.md b/skills/sdm-relational/skill-card.md new file mode 100644 index 000000000..6000bf481 --- /dev/null +++ b/skills/sdm-relational/skill-card.md @@ -0,0 +1,47 @@ +## Description:
+Ask kumo-relational a question about an entity and the records linked to it: write the question as a predictive query over a schema of related tables, covering classification, regression, forecasting and ranking the candidates an entity may link to.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill with coding agents to write Python that describes related tables (in memory or in a warehouse) and asks NVIDIA's kumo-relational structured-data foundation model predictive queries about entities through the structured-data-models client, without training a model.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Optional]
+**Credential Type(s):** [API key]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [pql.md](pql.md)
+- [references/beyond-one-answer.md](references/beyond-one-answer.md)
+- [references/warehouses.md](references/warehouses.md)
+- [references/in-process.md](references/in-process.md)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown with inline Python code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [None]
+ +## Skill Version(s):
+2.0.0rc1 (source: CHANGELOG, unreleased release candidate on v2-candidate)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/tabular/SKILL.md b/skills/sdm-tabular/SKILL.md similarity index 97% rename from skills/tabular/SKILL.md rename to skills/sdm-tabular/SKILL.md index c69a197d0..f18703191 100644 --- a/skills/tabular/SKILL.md +++ b/skills/sdm-tabular/SKILL.md @@ -9,7 +9,7 @@ One table. You hold rows whose answer is known and rows whose answer you want. Both go in one request, the model reads them together, and the answer comes back a row at a time. -Load [`../relational/SKILL.md`](../relational/SKILL.md) instead when the +Load [`../sdm-relational/SKILL.md`](../sdm-relational/SKILL.md) instead when the question is about an entity and the records linked to it. The two models share this client's objects -- `Schema`, `PredictiveQuery`, `Request` -- and the language. What differs is almost everything that can be asked in it. @@ -52,7 +52,7 @@ Keys are unique entity IDs, or `row_index` when IDs are missing or repeated. The client imposes no separate explanation row limit; the endpoint must support gradients. The optional Python model runtime also supports them. `EXPLAIN` adds explanation output without dropping ordinary outputs; `outputs=[]` remains invalid. -See [the result contract](../../docs/reference/prediction-output.md). +See [the result contract](https://github.com/NVIDIA/structured-data-models-client/blob/main/docs/reference/prediction-output.md). ```python # With missing or repeated IDs, keys are row_index values. @@ -256,4 +256,4 @@ Catch `StructuredDataError` to catch all of them. - [`references/pql.md`](references/pql.md) — the query language on one table - [`references/in-process.md`](references/in-process.md) — the model in this process, with no NIM -- [`../relational/SKILL.md`](../relational/SKILL.md) — when the question is about an entity and what it links to +- [`../sdm-relational/SKILL.md`](../sdm-relational/SKILL.md) — when the question is about an entity and what it links to diff --git a/skills/sdm-tabular/evals/evals.json b/skills/sdm-tabular/evals/evals.json new file mode 100644 index 000000000..57abb952a --- /dev/null +++ b/skills/sdm-tabular/evals/evals.json @@ -0,0 +1,104 @@ +{ + "skill_name": "sdm-tabular", + "evals": [ + { + "id": "t01", + "prompt": "I have a pandas DataFrame `labelled` of past customers with columns age, spend and churn (0/1), and another DataFrame `to_score` of new customers with just age and spend. Using the structured-data-models client and the kumo-tabular model at http://localhost:8000, write Python that predicts churn for the new customers with class probabilities, and tell me how to line the answers back up with my rows.", + "expected_output": "Builds Request(context=labelled, predict=to_score, target='churn', task='classification') and calls client.predict(request, model=KumoTabular(), outputs=['prediction', 'probabilities']) inside `with SDMClient(url='http://localhost:8000') as client:`, then answer.to_pandas(). Explains that row_index is the position in the predict frame and is what to join on, and that probability columns are named for the class (probability_0, probability_1) rather than by position.", + "assertions": [ + "Imports Request, SDMClient and KumoTabular from sdm_client", + "Builds a Request with context=labelled, predict=to_score and target='churn'", + "States task='classification' explicitly", + "Calls client.predict with model=KumoTabular() and outputs including 'prediction' and 'probabilities'", + "Says to join results back on row_index (the position in the predict frame), not on row order", + "Says probability columns are named for the class they belong to (e.g. probability_1), not numbered by position", + "Does not train, fit a scikit-learn model, or call any training API" + ] + }, + { + "id": "t02", + "prompt": "My target column `visits` holds only 0 and 1 -- it's how many times a user came back last week, and next week it could be higher. I'm using kumo-tabular through sdm_client. How should I set up the request, and can I get a prediction interval rather than just a point estimate?", + "expected_output": "Explains that a target of two whole numbers is read as a classification unless told otherwise, and that this one is a quantity, so pass task='regression'. For an interval, ask for outputs=['prediction', 'quantiles'], which gives quantile_0.1 through quantile_0.9 beside the point prediction. Notes that probabilities belong to classification, and asking for them on a regression comes back as 422 UNSUPPORTED_OUTPUT.", + "assertions": [ + "Recommends task='regression' because visits is a quantity, not a label", + "Explains that without task= a target of two whole numbers would be read as a classification", + "Recommends the 'quantiles' output for an interval", + "Says quantiles come back as per-decile columns such as quantile_0.1 through quantile_0.9", + "Does not recommend the 'probabilities' output for this regression, or warns it is refused (422 UNSUPPORTED_OUTPUT)" + ] + }, + { + "id": "t03", + "prompt": "I'm sending kumo-tabular a Request with a context DataFrame of 15,000 labelled rows and our endpoint refuses it as too large. Does sdm_client trim it for me, and how do I fix it without writing my own sampling code?", + "expected_output": "A direct Request sends exactly the rows it is given: it trims neither frame and holds the context to no size of its own, so a context larger than the endpoint takes is the endpoint's refusal. The fix without hand-written sampling is to describe the frame with Schema.from_data and ask a PredictiveQuery, whose materialize(indices=..., model=KumoTabular()) samples the context to 10,000 rows unless num_context_samples says otherwise.", + "assertions": [ + "Says a direct Request sends exactly the rows given and does not trim the context", + "Attributes the refusal to the endpoint's limit rather than to a bug in the request", + "Recommends PredictiveQuery(...).materialize(...) as the way to have the context sampled", + "Says materialize() samples the context to 10,000 rows by default, adjustable with num_context_samples", + "Uses Schema.from_data to describe the frame for the predictive query", + "Does not claim the client silently truncates the context" + ] + }, + { + "id": "t04", + "prompt": "I have one DataFrame `customers` with customer_id, age, spend and churn for every customer. I want kumo-tabular to score customers 7, 12 and 3 using the rest of the table as context. Write the code with sdm_client.", + "expected_output": "Builds Schema.from_data({'customers': customers}, primary_keys={'customers': 'customer_id'}), then PredictiveQuery('PREDICT customers.churn FOR EACH customers.customer_id', schema).materialize(indices=[7, 12, 3], model=KumoTabular()), and calls client.predict(request, model=KumoTabular(), ...). Explains that the named rows are held out of their own context, so request.context_rows is the table minus the scored rows, and that the answer carries an id column for the entity.", + "assertions": [ + "Uses Schema.from_data with primary_keys={'customers': 'customer_id'}", + "Writes the query as PREDICT customers.churn FOR EACH customers.customer_id", + "Calls materialize(indices=[7, 12, 3], model=KumoTabular())", + "Explains that the scored rows are held out of the context", + "Does not split the DataFrame into two Request frames by hand as the primary approach" + ] + }, + { + "id": "t05", + "prompt": "Same labelled context every time, but rows to score arrive in batches all day from our kumo-tabular service. Re-sending 9,000 labelled rows on every call seems wasteful. Is there a better pattern in sdm_client, and anything to watch for in deployment?", + "expected_output": "Recommends a session: fitted = client.fit(request, model=KumoTabular()), then fitted.predict(next_request) for each batch, then fitted.close(). Notes that the handle's predict() takes a Request, not a bare DataFrame, that fit() is the only way to build one (FittedModel(...) raises), and that the session lives in one process's memory on the endpoint, so a deployment with several replicas needs session affinity.", + "assertions": [ + "Recommends client.fit(...) to pin the context once", + "Calls predict on the fitted handle for each new batch", + "Says the fitted handle's predict() takes a Request rather than a bare DataFrame", + "Closes the session (fitted.close()) when done", + "Warns that a session lives in one replica's memory, so requests for one session must reach the same replica" + ] + }, + { + "id": "t06", + "prompt": "For each customer kumo-tabular scored, I want to know which input columns pushed the churn prediction up or down. How do I get that from sdm_client, and how should I read the numbers?", + "expected_output": "Adds 'explanation' to outputs (or writes EXPLAIN PREDICT in a query). Explanations are not in to_pandas(); they are on result.explanations.gradients, keyed by the row's id when IDs are unique, otherwise its row_index, and result.explanations.gradients.to_pandas() gives one line per (row, output, column). Cautions that a gradient is the change in the output score per unit change in the column -- a diagnostic, not a causal effect or a feature-importance ranking -- and that columns are as the model preprocessed them, not as sent.", + "assertions": [ + "Requests the 'explanation' output (or uses EXPLAIN PREDICT)", + "Reads explanations from result.explanations.gradients rather than from to_pandas()", + "Says entries are keyed by unique ids, otherwise by row_index", + "States that a gradient is not a causal effect or a feature-importance ranking", + "Notes the explained columns are the model's preprocessed columns, which may differ from the columns sent" + ] + }, + { + "id": "t07", + "prompt": "I have a customers table and an orders table that references customers by customer_id. I want to predict how many orders each customer places next month. Can I do this with KumoTabular in sdm_client?", + "expected_output": "Says no: kumo-tabular reads one table with no relationships and no time windows, and refuses forecasts, aggregations and a schema with a second table, naming the model that can answer. The question is about an entity and the records linked to it, so it belongs to kumo-relational (KumoRelational) and its skill.", + "assertions": [ + "Says kumo-tabular cannot answer this", + "Gives the reason: one table only, no relationships, no aggregation or time window", + "Directs the user to kumo-relational / KumoRelational", + "Does not produce KumoTabular code that joins or aggregates the orders table into a feature and presents that as the supported approach" + ] + }, + { + "id": "t08", + "prompt": "There's no NIM deployed for us, and the customer rows can't leave my laptop. Can I still score them with kumo-tabular through sdm_client? What do I install, and will the numbers match what a GPU-hosted NIM would give?", + "expected_output": "Yes: open the client with SDMClient.in_process() in place of a URL, and the request, model handle and result are the same as against a NIM. It needs the in-process extra (structured-data-models-client[in-process]) and Python 3.11 or later; until a published release carries in_process(), it is installed from a checkout of v2-candidate as the reference describes. The first prediction fetches the published weights (nvidia/Kumo-Tabular) into the Hugging Face cache with no account or token. On a GPU the answers are a NIM's exactly; on a CPU (including Apple silicon) a probability can differ from a GPU NIM's by more than rounding, so compare on the same device or compare shapes. A refusal is a StructuredDataError with the NIM's code, not an HTTP error. Joins on row_index, the position in the predict frame.", + "assertions": [ + "Uses SDMClient.in_process() in place of SDMClient(url=...)", + "Keeps the Request / predict / to_pandas flow the same as against a NIM", + "Names the in-process extra and Python 3.11 or later", + "Says the weights are fetched once from Hugging Face into its cache, with no token required", + "Says CPU results can differ from a GPU-hosted NIM's, and that GPU results match", + "Does not send the rows to any remote endpoint" + ] + } + ] +} diff --git a/skills/tabular/references/in-process.md b/skills/sdm-tabular/references/in-process.md similarity index 100% rename from skills/tabular/references/in-process.md rename to skills/sdm-tabular/references/in-process.md diff --git a/skills/tabular/references/pql.md b/skills/sdm-tabular/references/pql.md similarity index 93% rename from skills/tabular/references/pql.md rename to skills/sdm-tabular/references/pql.md index 0874fb082..0377dc2d4 100644 --- a/skills/tabular/references/pql.md +++ b/skills/sdm-tabular/references/pql.md @@ -1,6 +1,6 @@ # Writing a query for kumo-tabular -The same language as [`../../relational/pql.md`](../../relational/pql.md), cut +The same language as [`../../sdm-relational/pql.md`](../../sdm-relational/pql.md), cut down to what one table can answer. Read this one when the schema has a single table; almost every clause the relational reference describes is refused here, and the refusals say so. @@ -63,7 +63,7 @@ name rows already in the table or `predict=` to pass rows you hold. Every one of them is the tabular profile of the language refusing, not the grammar: the relational model takes all seven, within the bounds -[`../../relational/pql.md`](../../relational/pql.md) gives each of them. +[`../../sdm-relational/pql.md`](../../sdm-relational/pql.md) gives each of them. Three are worth seeing here, because all three are queries somebody would reasonably write. @@ -108,4 +108,4 @@ Pass model=KumoRelational() to gather them as a graph If the question really is about one table, build the schema from that table alone. If it is about an entity and the records linked to it, it belongs to -`kumo-relational` and [`../../relational/SKILL.md`](../../relational/SKILL.md). +`kumo-relational` and [`../../sdm-relational/SKILL.md`](../../sdm-relational/SKILL.md). diff --git a/skills/sdm-tabular/skill-card.md b/skills/sdm-tabular/skill-card.md new file mode 100644 index 000000000..5ba5b580b --- /dev/null +++ b/skills/sdm-tabular/skill-card.md @@ -0,0 +1,45 @@ +## Description:
+Ask kumo-tabular a question about rows of one table: classification or regression over labelled rows and rows to score, with no relationships and no time windows.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and data scientists use this skill with coding agents to write Python that asks NVIDIA's kumo-tabular structured-data foundation model to classify or regress rows of a single table through the structured-data-models client, without training a model.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Optional]
+**Credential Type(s):** [API key]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate. See skill body for more details.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [references/pql.md](references/pql.md)
+- [references/in-process.md](references/in-process.md)
+ + +## Skill Output:
+**Output Type(s):** [Code, Analysis]
+**Output Format:** [Markdown with inline Python code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [None]
+ +## Skill Version(s):
+2.0.0rc1 (source: CHANGELOG, unreleased release candidate on v2-candidate)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/tests/test_skills.py b/tests/test_skills.py index e8c406c24..535609703 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -51,8 +51,19 @@ LINK = re.compile(r'\]\(([^)]+)\)') +#: Files the NVIDIA skills catalog requires beside each skill. They are written +#: for the catalog, not for an agent. A skill links to none of them, and none +#: holds an example. The governance card is HTML-laced markdown in the catalog's +#: own template. NVSkills CI writes the benchmark report on each signing run. +#: Holding either to the markdown read here would fail on a file nobody here +#: writes by hand. +CATALOG_FILES = frozenset({'skill-card.md', 'BENCHMARK.md'}) + + def documents() -> list[Path]: - return sorted(SKILLS.rglob('*.md')) + return sorted( + path for path in SKILLS.rglob('*.md') if path.name not in CATALOG_FILES + ) def fences(path: Path, language: str) -> list[str]: @@ -64,8 +75,8 @@ def fences(path: Path, language: str) -> list[str]: def test_there_is_a_skill_for_each_model(): - assert (SKILLS / 'tabular' / 'SKILL.md').exists() - assert (SKILLS / 'relational' / 'SKILL.md').exists() + assert (SKILLS / 'sdm-tabular' / 'SKILL.md').exists() + assert (SKILLS / 'sdm-relational' / 'SKILL.md').exists() assert (SKILLS / 'README.md').exists() @@ -149,7 +160,7 @@ def test_an_example_that_predicts_shows_what_came_back(path): ) -@pytest.mark.parametrize('name', ['tabular', 'relational']) +@pytest.mark.parametrize('name', ['sdm-tabular', 'sdm-relational']) def test_a_skill_shows_output_it_has_run(name): """Each skill has to stake something on a live endpoint. @@ -162,15 +173,20 @@ def test_a_skill_shows_output_it_has_run(name): assert '**Verified**' in body, f'{name} marks no output as verified' -@pytest.mark.parametrize('name', ['tabular', 'relational']) +@pytest.mark.parametrize('name', ['sdm-tabular', 'sdm-relational']) def test_a_skill_announces_itself(name): - """An agent finds a skill by its frontmatter, so it has to be there.""" + """An agent finds a skill by its frontmatter, so it has to be there. + + The name is the directory's. The skills catalog copies each directory to + one named for the skill, so a skill that links to its sibling as + ``..//`` reaches it only where the two agree. + """ yaml = pytest.importorskip('yaml') body = (SKILLS / name / 'SKILL.md').read_text(encoding='utf-8') assert body.startswith('---\n'), f'{name} opens with no frontmatter' front = yaml.safe_load(body[4 : body.index('\n---\n', 3)]) assert isinstance(front, dict), f'{name} frontmatter is not a mapping' - assert front.get('name') == f'sdm-{name}', ( + assert front.get('name') == name, ( f'{name} is announced as {front.get("name")!r}' ) assert 'Triggers:' in front.get('description', ''), ( @@ -178,7 +194,7 @@ def test_a_skill_announces_itself(name): ) -@pytest.mark.parametrize('name', ['tabular', 'relational']) +@pytest.mark.parametrize('name', ['sdm-tabular', 'sdm-relational']) def test_a_skill_says_how_to_run_its_model_here(name): """A skill assumes a URL. Where there is none, the model runs in the agent's own process, and the skill has to say so where it is found: in @@ -301,28 +317,30 @@ def test_a_document_is_written_in_the_one_markdown_this_file_reads(path): (['Skill', 'Model', 'Load it when'], False), (['Reference', ''], False), ], - 'skills/relational/SKILL.md': [ + 'skills/sdm-relational/SKILL.md': [ (['', '`kumo-relational`'], True), (['Limit', 'Value'], True), (['Refusal', 'Cause'], False), ], - 'skills/relational/references/beyond-one-answer.md': [], - 'skills/relational/references/in-process.md': [], - 'skills/relational/references/warehouses.md': [ + 'skills/sdm-relational/references/beyond-one-answer.md': [], + 'skills/sdm-relational/references/in-process.md': [], + 'skills/sdm-relational/references/warehouses.md': [ (['Source', 'Connection it takes', 'Namespace', 'Extra'], True) ], - 'skills/relational/pql.md': [ + 'skills/sdm-relational/pql.md': [ (['Clause', 'Least', 'Most'], True), (['Target', 'Task', 'Calls'], True), (['Aggregation', ''], True), (['Comparison', ''], True), ], - 'skills/tabular/SKILL.md': [ + 'skills/sdm-tabular/SKILL.md': [ (['', '`kumo-tabular`'], True), (['Refusal', 'Cause'], False), ], - 'skills/tabular/references/in-process.md': [], - 'skills/tabular/references/pql.md': [(['Clause', 'On one table'], True)], + 'skills/sdm-tabular/references/in-process.md': [], + 'skills/sdm-tabular/references/pql.md': [ + (['Clause', 'On one table'], True) + ], } @@ -352,6 +370,26 @@ def test_every_link_resolves(path): ) +@pytest.mark.parametrize( + 'path', [p for p in documents() if p.parent != SKILLS], ids=str +) +def test_a_skill_links_only_to_skills(path): + """The skills catalog copies each skill directory and nothing around it. + + A relative link out of ``skills/`` into the rest of this repository + resolves here and nowhere the skill is installed. Link to a sibling skill + by its directory, or to anything else by its URL. + """ + for target in LINK.findall(path.read_text(encoding='utf-8')): + if target.startswith(('http://', 'https://', '#')): + continue + resolved = (path.parent / target).resolve() + assert any( + resolved.is_relative_to(skill.parent.resolve()) + for skill in SKILLS.glob('*/SKILL.md') + ), f'{path.relative_to(SKILLS)} links to {target}, outside every skill' + + @pytest.mark.parametrize('path', documents(), ids=str) def test_verified_output_is_marked_one_way(path): """``**Verified**`` at the start of a line labels the block beneath it. @@ -649,7 +687,7 @@ def resolve(query, schema, **extra): ) -PQL = SKILLS / 'relational' / 'pql.md' +PQL = SKILLS / 'sdm-relational' / 'pql.md' FENCED = re.compile(r'^```[a-z]*\n.*?^```$', re.DOTALL | re.MULTILINE) @@ -1656,9 +1694,9 @@ def test_a_temporal_assumption_is_answered(schema): assert request.task.kind == 'regression' -TABULAR_PQL = SKILLS / 'tabular' / 'references' / 'pql.md' +TABULAR_PQL = SKILLS / 'sdm-tabular' / 'references' / 'pql.md' -#: Each row of the clause table in tabular/references/pql.md, written as a +#: Each row of the clause table in sdm-tabular/references/pql.md, written as a #: query over one table. The claim is that every one is refused there, and #: several of them are ordinary relational queries. ON_ONE_TABLE = { @@ -1711,7 +1749,7 @@ def test_the_tabular_clause_table_is_every_clause_it_refuses(one_table): shown = table(TABULAR_PQL, 'Clause') del shown['Clause'] assert set(shown) == set(ON_ONE_TABLE), ( - f'tabular/references/pql.md lists {sorted(set(shown) ^ set(ON_ONE_TABLE))} ' + f'sdm-tabular/references/pql.md lists {sorted(set(shown) ^ set(ON_ONE_TABLE))} ' f'that this does not ask about' ) assert all(cells == ['refused'] for cells in shown.values()), ( @@ -1799,7 +1837,7 @@ def test_a_plain_column_is_what_the_tabular_reference_promises(one_table): ) -WAREHOUSES = SKILLS / 'relational' / 'references' / 'warehouses.md' +WAREHOUSES = SKILLS / 'sdm-relational' / 'references' / 'warehouses.md' def test_the_source_table_is_every_source_there_is(): @@ -2071,7 +2109,7 @@ def test_a_connector_error_is_not_a_structured_data_error(): def test_the_tabular_model_refuses_a_second_table(schema): - """The sentence ``skills/tabular/SKILL.md`` quotes.""" + """The sentence ``skills/sdm-tabular/SKILL.md`` quotes.""" from sdm_client import KumoTabular, PredictiveQuery from sdm_client.errors import StructuredDataError @@ -2203,7 +2241,7 @@ def test_a_quoted_refusal_from_the_endpoint_is_the_refusal(query, said, schema): not TABULAR_URL, reason='set SDM_TABULAR_URL to a kumo-tabular NIM' ) @pytest.mark.parametrize( - 'name', ['tabular/SKILL.md', 'tabular/references/pql.md'] + 'name', ['sdm-tabular/SKILL.md', 'sdm-tabular/references/pql.md'] ) def test_the_tabular_examples_print_what_the_skill_shows(name): assert_every_example_prints_what_it_shows(SKILLS / name, TABULAR_URL) @@ -2216,10 +2254,10 @@ def test_the_tabular_examples_print_what_the_skill_shows(name): @pytest.mark.parametrize( 'name', [ - 'relational/SKILL.md', - 'relational/pql.md', - 'relational/references/beyond-one-answer.md', - 'relational/references/warehouses.md', + 'sdm-relational/SKILL.md', + 'sdm-relational/pql.md', + 'sdm-relational/references/beyond-one-answer.md', + 'sdm-relational/references/warehouses.md', 'README.md', ], ) @@ -2246,7 +2284,10 @@ def installed(module: str, extra: str) -> None: ) @pytest.mark.parametrize( 'name', - ['tabular/references/in-process.md', 'relational/references/in-process.md'], + [ + 'sdm-tabular/references/in-process.md', + 'sdm-relational/references/in-process.md', + ], ) def test_the_in_process_examples_print_what_the_skill_shows(name): """These examples open no connection: they fetch the published weights @@ -2259,7 +2300,7 @@ def test_the_in_process_examples_print_what_the_skill_shows(name): package even when the tables are frames already in memory. """ installed('sdm', 'structured-data-models-client[in-process]') - if name.startswith('relational/'): + if name.startswith('sdm-relational/'): installed('sdm_connectors', 'structured-data-models-connectors') assert_every_example_prints_what_it_shows(SKILLS / name) @@ -2269,7 +2310,7 @@ def test_the_in_process_examples_print_what_the_skill_shows(name): @pytest.mark.parametrize( 'name, model_name', - [('tabular', 'KumoTabular'), ('relational', 'KumoRelational')], + [('sdm-tabular', 'KumoTabular'), ('sdm-relational', 'KumoRelational')], ) def test_the_capability_table_is_the_clients_own(name, model_name): """Every row of the table both skills open with, against the client.""" @@ -2291,7 +2332,7 @@ def test_the_capability_table_is_the_clients_own(name, model_name): 'Relationships': capabilities.relationships, 'Sessions': capabilities.sessions, } - if name == 'tabular': + if name == 'sdm-tabular': said['Explanations'] = 'explanation' in capabilities.outputs assert set(shown) == set(said), ( f'{name} shows rows nothing checks: {sorted(set(shown) ^ set(said))}' @@ -2316,7 +2357,7 @@ def test_the_capability_table_is_the_clients_own(name, model_name): ) -#: Each row of the limits table in ``skills/relational/SKILL.md``, against the +#: Each row of the limits table in ``skills/sdm-relational/SKILL.md``, against the #: constant it reports. Every key is spoken for in both directions: a limit the #: client enforces and the skill omits is a refusal an agent cannot see coming, #: and a row naming no limit is a number nobody is holding to anything. @@ -2332,7 +2373,7 @@ def test_the_limits_table_is_the_shipped_limits(): assert set(LIMIT_ROWS.values()) == set(RELATIONAL_LIMITS), ( 'the client enforces a limit this table does not account for' ) - path = SKILLS / 'relational' / 'SKILL.md' + path = SKILLS / 'sdm-relational' / 'SKILL.md' assert heading(path, 'Limit') == 'Value' shown = table(path, 'Limit') shown.pop('Limit') @@ -2437,7 +2478,7 @@ def test_the_tabular_outputs_narrow_by_task_as_the_skill_says(): assert len(deciles) == 9, f'quantiles came back as {deciles}' -#: The output matrix skills/relational/SKILL.md gives for the deployment: +#: The output matrix skills/sdm-relational/SKILL.md gives for the deployment: #: what each task is served, and everything the client would send that it #: will not. Asked of the endpoint, so a deployment that gains or loses one #: fails here rather than making the skill quietly wrong. @@ -2465,7 +2506,9 @@ def not_served(outputs: tuple[str, ...]) -> list[str]: the endpoint happens to allow. """ advertised = set( - listed(table(SKILLS / 'relational' / 'SKILL.md', 'Tasks')['Outputs'][0]) + listed( + table(SKILLS / 'sdm-relational' / 'SKILL.md', 'Tasks')['Outputs'][0] + ) ) return sorted(advertised - set(outputs))